"""Supervisor task queue, persistence, timeouts, and evolution scheduling.""" from __future__ import annotations import logging import math import pathlib import queue as _stdqueue # noqa: F401 — re-exported for the test suite's reap-queue isolation import threading import time # noqa: F401 -- facade name tests read (queue.time) import uuid from typing import Any, Dict, List, Optional, Tuple from supervisor.state import ( load_state, append_jsonl, # noqa: F401 -- queue_snapshot leaf reads it via the _queue() handle atomic_write_text, # noqa: F401 -- queue_snapshot leaf reads it via the _queue() handle budget_remaining, EVOLUTION_BUDGET_RESERVE, reconstruct_task_cost as reconstruct_task_cost, ) from supervisor.message_bus import ( coerce_chat_identity, # noqa: F401 -- queue_timeouts leaf reads it via the _queue() handle notification_chat_route, send_with_budget, ) from ouroboros.config import ( DATA_DIR, FINALIZATION_GRACE_DEFAULT_SEC, get_finalization_grace_sec, get_per_call_timeout_ceiling_sec, # noqa: F401 -- queue_timeouts leaf reads it via the _queue() handle get_task_abs_ceiling_sec, # noqa: F401 -- queue_timeouts leaf reads it via the _queue() handle get_task_idle_timeout_sec, # noqa: F401 -- queue_timeouts leaf reads it via the _queue() handle ) from ouroboros.consciousness_authority import apply_consciousness_authority, is_consciousness_origin from ouroboros.contracts.task_contract import attach_task_contract, build_task_contract, normalize_allowed_resources # noqa: F401 from ouroboros.schedule_contract import RESERVED_TEMPLATE_FIELDS, schedule_slug # noqa: F401 from ouroboros.skill_loader import skill_identity_collision_names # noqa: F401 from ouroboros.outcomes import terminal_outcome_axes from ouroboros.utils import atomic_write_json, read_json_dict, utc_now_iso # noqa: F401 from supervisor.evolution_lifecycle import ( # noqa: F401 -- public queue API and lazy scheduler dependencies _deliver_pending_owner_report, _read_evolution_campaign, begin_evolution_transaction, build_evolution_task_text, disable_evolution_authority, disable_evolution_projection, deliver_pending_owner_report, enqueue_evolution_task_if_needed, evolution_block_reason, notify_owner_cycle_outcome, pause_evolution_campaign, start_evolution_campaign, # noqa: F401 -- historical queue API re-export ) from supervisor.task_lifecycle import ( # noqa: F401 -- public queue API re-exports BUDGET_ROOT_FENCES, apply_budget_root_admission_fence, cancel_task_by_id, clear_acceptance_fence_for_root, resume_budget_paused_task, restore_queue_fences, transition_acceptance_fence, ) log = logging.getLogger(__name__) DRIVE_ROOT: pathlib.Path = pathlib.Path(DATA_DIR) # The queue snapshot path has ONE authority (MIGRATION row 1030, D18): this # module. init() rebinds it per drive root; the queue_snapshot leaf reads it # through the _queue() handle. QUEUE_SNAPSHOT_PATH: pathlib.Path = DRIVE_ROOT / "state" / "queue_snapshot.json" HEARTBEAT_STALE_SEC: int = 120 QUEUE_MAX_RETRIES: int = 1 FINALIZATION_GRACE_SEC: int = FINALIZATION_GRACE_DEFAULT_SEC SCHEDULED_TASKS_FILE = pathlib.Path("state") / "scheduled_tasks.json" # BUG3: pause a campaign whose objective fails to absorb after this many reviewed cycles. # Mirrors the consecutive-failures threshold; keyed on the objective fingerprint, not failures. OBJECTIVE_REPEAT_CAP: int = 3 # Whether THIS process owns the supervisor's live maps (``init`` ran): a process that merely # imports the module sees empty maps, which prove nothing (``task_settlement_liveness``). INITIALIZED = False def init(drive_root: pathlib.Path) -> None: global DRIVE_ROOT, FINALIZATION_GRACE_SEC, INITIALIZED, QUEUE_SNAPSHOT_PATH DRIVE_ROOT = drive_root INITIALIZED = True QUEUE_SNAPSHOT_PATH = drive_root / "state" / "queue_snapshot.json" FINALIZATION_GRACE_SEC = get_finalization_grace_sec() BUDGET_ROOT_FENCES.clear() # A previous process's direct-chat turns must not outlive it in the roster, # and this clear is the last moment their ids exist: the roster is taken over # here and handed to snapshot restore below, which fences them like any other # row the stop caught. from supervisor.direct_roots import take_direct_roots PRIOR_DIRECT_ROOTS.clear() PRIOR_DIRECT_ROOTS.update(take_direct_roots(drive_root)) def refresh_timeouts_from_settings(settings: dict) -> None: """Hot-reload the active liveness settings. The flat wall-clock pair this once also had to absorb (soft/hard) is retired in 7.0: load_settings strips the keys, so there is no stored value left to accept, warn about, or lie about honoring. """ global FINALIZATION_GRACE_SEC FINALIZATION_GRACE_SEC = get_finalization_grace_sec(settings) # The previous process's direct-chat roots, taken from `state/direct_roots.json` # by init above and consumed once by snapshot restore. A process-local handover # of the SAME fragment, never a second store. PRIOR_DIRECT_ROOTS: Dict[str, Any] = {} # Set by workers.init_queue_refs(). PENDING: List[Dict[str, Any]] = [] RUNNING: Dict[str, Dict[str, Any]] = {} QUEUE_SEQ_COUNTER_REF: Dict[str, int] = {"value": 0} ACCEPTANCE_FENCES: Dict[str, Dict[str, Any]] = {} ADMISSION_RESERVATIONS: Dict[str, str] = {} # Guards PENDING/RUNNING mutations across main loop, direct chat, watchdog. _queue_lock = threading.RLock() from supervisor.task_admission import ( # noqa: E402,F401 - public queue API coerce_queue_order, prefer_terminalization_retry_rows, record_scheduled_admission, reject_invalid_task_depth, release_task_admission, restore_invalid_depth_admission, restore_terminalization_retry, restore_terminalization_retry_rows, reserve_task_admission, ) # Variant A off-loop worker reaper lives in supervisor/task_reaper.py (module size); re-export # the thin names the enforce path and tests use — monkeypatching these queue names still works. from supervisor.task_reaper import ( # noqa: E402,F401 — re-exported for enforce path + tests ensure_reaper_started as _ensure_reaper_started, reap_queue as _reap_queue, reap_timed_out_task as _reap_timed_out_task, request_finalization_grace as _request_finalization_grace, resolve_grace_episode_for_spared_task as _resolve_grace_episode_for_spared_task, ) def init_queue_refs(pending: List[Dict[str, Any]], running: Dict[str, Dict[str, Any]], seq_counter_ref: Dict[str, int]) -> None: """Bind queue structures owned by workers.py.""" global PENDING, RUNNING, QUEUE_SEQ_COUNTER_REF PENDING = pending RUNNING = running QUEUE_SEQ_COUNTER_REF = seq_counter_ref ADMISSION_RESERVATIONS.clear() def _task_priority(task_type: str) -> int: t = str(task_type or "").strip().lower() if t in ("skill_publish", "task", "review", "deep_self_review"): return 0 if t == "evolution": return 1 return 2 def _queue_sort_key(task: Dict[str, Any]) -> Tuple[int, int]: pr = coerce_queue_order(task.get("priority"), _task_priority(str(task.get("type") or ""))) seq = coerce_queue_order(task.get("_queue_seq")) return pr, seq def sort_pending() -> None: """Sort pending queue by priority and insertion sequence.""" PENDING.sort(key=_queue_sort_key) def drain_all_pending(*, persist: bool = True) -> list: """Drain pending tasks; optionally defer snapshot persistence until custody settles.""" drained = list(PENDING) PENDING.clear() if persist: persist_queue_snapshot(reason="drain_all_pending") return drained def enqueue_task( task: Dict[str, Any], front: bool = False, *, restoring_snapshot: bool = False, ) -> Dict[str, Any]: """Add task to PENDING (thread-safe: HTTP handlers enqueue concurrently with the supervisor main loop, so the mutation must hold the queue lock).""" t = dict(task) attach_task_contract(apply_consciousness_authority(t)) # The allowance read takes the cross-process ledger lock: read it BEFORE the queue # lock so a contended ledger never stalls every queue reader; only the live-root # count and the append must be one transaction with the lock (the window gates # starts — a window stale by milliseconds changes nothing). consciousness_window = None if not restoring_snapshot and _consciousness_root(t): from ouroboros.consciousness_allowance import allowance_window consciousness_window = allowance_window(DRIVE_ROOT) with _queue_lock: require_unique_id = bool(t.pop("_require_unique_task_id", False)) require_worker_pool = bool(t.pop("_require_worker_pool", False)) admission_token = str(t.pop("_admission_token", "") or "") task_id = str(t.get("id") or "").strip() reserved_token = str(ADMISSION_RESERVATIONS.get(task_id) or "") if reserved_token and admission_token != reserved_token: # A reservation owns this id until its request either enqueues or releases it. # Tokenless internal callers and competing ingress must not consume/collide with it. t["_admission_blocked"] = "admission_reservation_owned" return t if require_unique_id and task_id: # Exact-id ownership wins over malformed-depth replay. live_duplicate = task_id in RUNNING or any( isinstance(row, dict) and str(row.get("id") or "") == task_id for row in PENDING ) if live_duplicate: if ADMISSION_RESERVATIONS.get(task_id) == admission_token: ADMISSION_RESERVATIONS.pop(task_id, None) t["_admission_blocked"] = "duplicate_task_id" return t try: from ouroboros.routing_wait import is_own_admission_stub from ouroboros.task_results import load_task_result stored = load_task_result(DRIVE_ROOT, task_id, strict=True) # The emitted promote stub (#1160) belongs to THIS admission token: # its own enqueue reads around it, any other row still owns the id. if stored and not is_own_admission_stub(stored, admission_token): if ADMISSION_RESERVATIONS.get(task_id) == admission_token: ADMISSION_RESERVATIONS.pop(task_id, None) t["_admission_blocked"] = "duplicate_task_id" return t except Exception: log.warning("Fresh task-id lookup failed for %s", task_id, exc_info=True) t["_admission_blocked"] = "task_id_lookup_failed" if ADMISSION_RESERVATIONS.get(task_id) == admission_token: ADMISSION_RESERVATIONS.pop(task_id, None) return t retry = restore_terminalization_retry(t, pending=PENDING, running=RUNNING, queue_seq_counter_ref=QUEUE_SEQ_COUNTER_REF, sort_pending=sort_pending) if restoring_snapshot else None if retry: return retry if reject_invalid_task_depth(t, reservations=ADMISSION_RESERVATIONS, admission_token=admission_token): return t if require_worker_pool: try: from supervisor import workers pool_state = workers._worker_pool_execution_state() except Exception: pool_state = {"available": False, "disabled_reason": "state_unavailable"} if not pool_state["available"]: if ADMISSION_RESERVATIONS.get(task_id) == admission_token: ADMISSION_RESERVATIONS.pop(task_id, None) t["_admission_blocked"] = "worker_pool_unavailable" t["_worker_pool_disabled_reason"] = pool_state["disabled_reason"] return t consciousness_block = None if restoring_snapshot else _consciousness_admission_block(t, consciousness_window) if consciousness_block is not None: t["_admission_blocked"], t["_admission_detail"] = consciousness_block if ADMISSION_RESERVATIONS.get(task_id) == admission_token: ADMISSION_RESERVATIONS.pop(task_id, None) return t if admission_token and reserved_token != admission_token: t["_admission_blocked"] = "admission_reservation_lost" return t project_id = str(t.get("project_id") or "").strip() if project_id: try: from ouroboros.projects_registry import get_reserved_project project = get_reserved_project(DRIVE_ROOT, project_id) lifecycle = str((project or {}).get("lifecycle") or "active") if project is not None and lifecycle != "active": t["_admission_blocked"] = "project_routing_fence" t["_project_lifecycle"] = lifecycle t["_project_id"] = project_id if ADMISSION_RESERVATIONS.get(task_id) == admission_token: ADMISSION_RESERVATIONS.pop(task_id, None) return t except Exception: log.warning("Project admission check failed for %s", project_id, exc_info=True) t["_admission_blocked"] = "project_routing_fence_lookup_failed" t["_project_id"] = project_id if ADMISSION_RESERVATIONS.get(task_id) == admission_token: ADMISSION_RESERVATIONS.pop(task_id, None) return t root_id = str(t.get("root_task_id") or "").strip() if root_id and not restoring_snapshot and apply_budget_root_admission_fence(t, root_id): if ADMISSION_RESERVATIONS.get(task_id) == admission_token: ADMISSION_RESERVATIONS.pop(task_id, None) return t fence = ACCEPTANCE_FENCES.get(root_id) if root_id else None if isinstance(fence, dict) and str(fence.get("status") or "") in {"active", "sealed"}: t["_admission_blocked"] = "task_acceptance_fence" t["_acceptance_fence_token"] = str(fence.get("token") or "") t["_acceptance_fence_status"] = str(fence.get("status") or "active") if ADMISSION_RESERVATIONS.get(task_id) == admission_token: ADMISSION_RESERVATIONS.pop(task_id, None) return t QUEUE_SEQ_COUNTER_REF["value"] += 1 seq = QUEUE_SEQ_COUNTER_REF["value"] t["priority"] = coerce_queue_order(t.get("priority"), _task_priority(str(t.get("type") or ""))) _att = t.get("_attempt") t.setdefault("_attempt", int(_att) if _att is not None else 1) t["_queue_seq"] = -seq if front else seq t["queued_at"] = utc_now_iso() if admission_token: t["_admission_owner_token"] = admission_token PENDING.append(t) sort_pending() if ADMISSION_RESERVATIONS.get(task_id) == admission_token: ADMISSION_RESERVATIONS.pop(task_id, None) return t def live_consciousness_root_count() -> int: """Live PENDING+RUNNING roots that consciousness started (its origin marker on the task metadata; subagents are their root's business). Sibling of ``queue_has_task_type``.""" def _counts(task: Any) -> bool: return ( isinstance(task, dict) and str(task.get("delegation_role") or "root") == "root" and is_consciousness_origin(task.get("metadata")) ) live = sum(1 for task in PENDING if _counts(task)) return live + sum( 1 for meta in RUNNING.values() if isinstance(meta, dict) and _counts(meta.get("task")) ) def _consciousness_root(task: Dict[str, Any]) -> bool: return (is_consciousness_origin(task.get("metadata")) and str(task.get("delegation_role") or "root") == "root") def _consciousness_admission_block(task: Dict[str, Any], window: Optional[Dict[str, Any]] = None) -> Optional[Tuple[str, str]]: """The ONE admission door for the roots consciousness starts (owner decisions В11/В18). Called under the queue lock, so the live count and the admission are one transaction. Returns ``(reason, detail)`` in the queue's existing refusal vocabulary when a consciousness-origin ROOT may not start — the concurrency cap over live roots with the same marker (``OUROBOROS_CONSCIOUSNESS_MAX_TASKS``, 0 = never) or the rolling-24h allowance (``OUROBOROS_CONSCIOUSNESS_DAILY_USD``, 0 = consciousness may not spend; an unreadable ledger refuses honestly as ``allowance_unknown``) — and ``None`` when it may. Subagents are bounded by their root's own cap and the per-root child cap, never counted twice; a snapshot restore re-admits already-admitted work and is not gated here. ``window`` is the allowance the caller read before taking the lock; read here only when absent. """ if not _consciousness_root(task): return None from ouroboros.config import get_consciousness_max_tasks from ouroboros.consciousness_allowance import ( STATUS_AVAILABLE, STATUS_UNKNOWN, allowance_window, ) max_tasks = get_consciousness_max_tasks() live = live_consciousness_root_count() if live >= max_tasks: return ("consciousness_task_limit", ( f"{live} of {max_tasks} consciousness-started tasks already live" if max_tasks else "OUROBOROS_CONSCIOUSNESS_MAX_TASKS=0: consciousness never starts tasks" )) if window is None: window = allowance_window(DRIVE_ROOT) if window["status"] == STATUS_UNKNOWN: return ("consciousness_allowance_unknown", f"the usage ledger could not be read: {window.get('error') or 'unknown error'}") if window["status"] != STATUS_AVAILABLE: if not window["limit_usd"]: return ("consciousness_allowance_exhausted", "OUROBOROS_CONSCIOUSNESS_DAILY_USD=0: consciousness may not spend") at_least = " (at least)" if window["unknown_unmetered"] else "" return ("consciousness_allowance_exhausted", ( f"${window['accounted_usd']:.2f}{at_least} of ${window['limit_usd']:.2f} " f"spent in the last 24 h; resets at {window['resets_at'] or 'unknown'}" )) return None def queue_has_task_type(task_type: str) -> bool: """Return whether this task type is pending or running.""" tt = str(task_type or "") if any(str(t.get("type") or "") == tt for t in PENDING): return True for meta in RUNNING.values(): task = meta.get("task") if isinstance(meta, dict) else None if isinstance(task, dict) and str(task.get("type") or "") == tt: return True return False # Cron/timezone schedule helpers live in supervisor/schedule_time.py (P7 # module-size relief); imported under their historical private names. from supervisor.schedule_time import ( # noqa: E402 next_cron_time as _next_cron_time, # noqa: F401 once_due as _once_due, # noqa: F401 parse_schedule_time as _parse_schedule_time, # noqa: F401 prune_consumed_once_records as _prune_consumed_once, record_last_error as _record_last_error, # noqa: F401 schedule_next_run as _schedule_next_run, # noqa: F401 timezone_for_schedule as _timezone_for_schedule, # noqa: F401 ) def _emit_cancel_task_done( task: Optional[Dict[str, Any]], task_id: str, *, cost_fields: Optional[Dict[str, Any]] = None, status: str = "cancelled", ) -> None: """Emit a task_done event after a cancel so the UI live card resolves. Covers both the agent-tool path (_handle_cancel_task) and the HTTP path. ``status`` carries the STORED terminal truth: when a worker wrote its own natural result just before the kill, the card must resolve to THAT outcome rather than be left unresolved until a reload. ``cost_fields`` is the caller's accounting authority — a reconstructed ledger projection or a CONFIRMED pre-start zero. An absent projection emits an honest nullable unknown; the old default fabricated a final $0 for every cancel (Poltergeist A1.10, owner 10=B).""" try: from supervisor import workers chat_id = int((task or {}).get("chat_id") or 0) if isinstance(task, dict) else 0 workers.get_event_q().put({ "type": "task_done", "task_id": str(task_id), # The tree identity survives even though the row already left # PENDING/RUNNING: the fence-release seam resolves the root # from the event when the queue no longer holds the task. "root_task_id": str((task or {}).get("root_task_id") or "") if isinstance(task, dict) else "", "task_type": str((task or {}).get("type") or ""), "chat_id": chat_id, "status": status, "outcome_axes": terminal_outcome_axes( lifecycle=status, execution=status, reason_code=status, review_trigger="supervisor_terminal", ), **(cost_fields or { "cost_accounting_status": "unavailable", "cost_final": False, # ABI-3: honest name only — the retired alias is read-only. "accounted_upper_bound_usd": None, }), "metadata": (task or {}).get("metadata") if isinstance((task or {}).get("metadata"), dict) else {}, }) except Exception: log.debug("Failed to emit task_done for cancelled task %s", task_id, exc_info=True) # Cancellation custody and the terminal-cancel result-field builder live in # supervisor.task_lifecycle (module-size boundary); re-exported so # `supervisor.queue` stays the single import surface for callers. from supervisor.task_lifecycle import ( # noqa: E402, F401 -- intentional public re-exports CANCEL_ALREADY_SETTLED, CANCEL_CANCELLED, CANCEL_FAILED, CANCEL_NOT_FOUND, _CANCEL_TERMINALIZED, _cancel_result_fields, cancel_task_custody, drive_cancel_intent_scope, task_has_live_ownership, task_subtree_is_live, ) from supervisor.queue_transitions import ( # noqa: E402, F401 -- intentional public re-exports evolution_stop_report, stop_evolution_tasks, sweep_orphaned_budget_fences, task_settlement_interlock, task_settlement_liveness, ) def _cancel_task_by_id_single(task_id: str) -> bool: """Boolean facade for the pre-v6.82 single-task callers.""" return cancel_task_custody(task_id) in {CANCEL_CANCELLED, CANCEL_ALREADY_SETTLED} # Evolution-stop transitions (GR2-13) live in supervisor.queue_transitions # (module-size boundary); re-exported below with the other transition helpers # so `supervisor.queue` stays the single import surface for callers. def queue_deep_self_review_task(reason: str, model: str = "", force: bool = False, chat_id: Optional[int] = None, origin: Optional[Dict[str, Any]] = None) -> Optional[str]: """Queue a deep self-review task. ``chat_id`` targets a specific chat (e.g. the external transport chat that ran ``/review``) so the queued ack and the task results return to the requester instead of always defaulting to the web owner's ``owner_chat_id``. ``origin`` is the requester's consciousness origin (a wake-up or its tree), stamped on the root so the admission door and the ledger see it; empty for the owner. """ # Membership, not truthiness: a review asked for from the hidden partition # is answered there, not silently re-routed to the owner's main chat. target_chat_id = notification_chat_route(chat_id, load_state().get("owner_chat_id")) if target_chat_id is None: return None if (not force) and queue_has_task_type("deep_self_review"): return None tid = uuid.uuid4().hex[:8] admitted = enqueue_task({ "id": tid, "type": "deep_self_review", "chat_id": int(target_chat_id), "text": reason or "Deep self-review", "model": model, "_require_worker_pool": True, **({"metadata": dict(origin)} if origin else {}), }) if admitted.get("_admission_blocked"): reason = admitted.get("_worker_pool_disabled_reason") or admitted["_admission_blocked"] detail = str(admitted.get("_admission_detail") or "") hint = f" {detail}." if detail else " Use /restart to restore the worker pool." send_with_budget( int(target_chat_id), f"Deep self-review could not be queued: {reason}.{hint}", role="system", system_type="deep_self_review_unavailable", ) return None persist_queue_snapshot(reason="deep_self_review_enqueued") # Typed SYSTEM row: an acknowledgement is never a task's answer, and the bench # trajectory reader takes the last UNTYPED outbound row as one. send_with_budget(int(target_chat_id), f"🔎 Deep self-review queued: {tid} ({reason})", role="system", system_type="deep_self_review_queued") return tid def get_evolution_status_snapshot(*, budget_projection: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: """Return a non-mutating evolution scheduling snapshot. ``budget_projection``: optional pre-computed global usage projection from a caller that already replayed the ledger this request (``/api/state``), so the snapshot does not replay it again. Default ``None`` keeps the self-computing, strict fail-closed behavior — a caller whose own computation FAILED must pass nothing, so the paused-evolution disclosure still comes from this snapshot. """ st = load_state() enabled = bool(st.get("evolution_mode_enabled")) owner_chat_id = int(st.get("owner_chat_id") or 0) consecutive_failures = int(st.get("evolution_consecutive_failures") or 0) try: # A status snapshot is a display read: without a supplied projection it rides the last validated snapshot. remaining: Optional[float] = round(float(budget_remaining(st, strict=True, projection=budget_projection, allow_stale=True)), 2) accounting_available = True except Exception: remaining = None accounting_available = False queued_task = next((t for t in PENDING if str(t.get("type") or "") == "evolution"), None) running_task = next( ( (meta.get("task") if isinstance(meta, dict) else None) for meta in RUNNING.values() if isinstance(meta, dict) and isinstance(meta.get("task"), dict) and str(meta["task"].get("type") or "") == "evolution" ), None, ) status = "disabled" detail = "Evolution mode is off." campaign = _read_evolution_campaign() active_tx = campaign.get("active_transaction") if isinstance(campaign.get("active_transaction"), dict) else {} restart_blocked = bool( active_tx and str(active_tx.get("commit_sha") or "").strip() and (bool(active_tx.get("restart_required")) or not bool(active_tx.get("restart_verified"))) ) if restart_blocked: status = "waiting_for_restart_verify" detail = "Waiting for restart verification before the next absorbed evolution cycle." elif isinstance(running_task, dict): status = "running" detail = "Evolution task is running now." elif isinstance(queued_task, dict): status = "queued" detail = "Evolution task is queued and waiting for a worker." elif not accounting_available: status = "accounting_unavailable" detail = "Cost accounting is unavailable; evolution dispatch is paused without changing the campaign." elif consecutive_failures >= 3: status = "paused_failures" detail = ( f"Paused after {consecutive_failures} consecutive failures. " "Use Evolve again after investigating the failure." ) elif enabled and not owner_chat_id: status = "waiting_for_owner_chat" detail = "Waiting for the first owner chat binding before scheduling evolution." elif enabled and remaining is not None and remaining < EVOLUTION_BUDGET_RESERVE: status = "budget_blocked" detail = ( f"Budget reserve active: ${remaining:.2f} remaining, " f"${EVOLUTION_BUDGET_RESERVE:.0f} reserved for conversations." ) elif enabled and (PENDING or RUNNING): status = "waiting_for_idle" detail = "Waiting for active tasks to finish before the next evolution cycle." elif enabled: status = "idle_ready" detail = "Idle and ready to queue the next evolution cycle." elif remaining is not None and remaining < EVOLUTION_BUDGET_RESERVE and str(st.get("last_evolution_task_at") or "").strip(): status = "budget_stopped" detail = ( f"Evolution auto-stopped because only ${remaining:.2f} remains, " f"below the ${EVOLUTION_BUDGET_RESERVE:.0f} conversation reserve." ) return { "enabled": enabled, "status": status, "detail": detail, "campaign": campaign, "cycle": int(st.get("evolution_cycle") or 0), "owner_chat_bound": bool(owner_chat_id), "last_task_at": str(st.get("last_evolution_task_at") or ""), "consecutive_failures": consecutive_failures, "cost_accounting_status": "available" if accounting_available else "unavailable", # Unbounded budget (supervisor not initialized / TOTAL_BUDGET<=0) # is float('inf'), which strict JSON cannot carry — surface None so # /api/state stays serializable on onboarding installs. "budget_remaining_usd": remaining if remaining is not None and math.isfinite(remaining) else None, "budget_reserve_usd": float(EVOLUTION_BUDGET_RESERVE), "pending_count": len(PENDING), "running_count": len(RUNNING), "queued_task_id": str((queued_task or {}).get("id") or ""), "running_task_id": str((running_task or {}).get("id") or ""), } # v7next F1 (D08): moved spans live in their owner leaves; re-exported here # so this facade stays the single import surface for callers and tests. from supervisor.queue_schedules import ( # noqa: E402, F401 -- intentional public re-exports _SKILL_SCHEDULE_SYNC_INTERVAL_SEC, _last_skill_schedule_sync, _schedule_running_or_queued, _scheduled_tasks_path, _task_from_schedule, _write_scheduled_tasks, SCHEDULE_ACTIONS, ScheduleLockTimeout, ScheduleRefused, ScheduleStoreUnreadable, check_scheduled_tasks, list_scheduled_tasks, load_schedule_store, resync_skill_schedules, schedule_activity_projection, schedule_tool_projection, schedule_lifecycle_status, schedule_transaction, sync_skill_schedules, ) from supervisor.schedule_lifecycle import ( # noqa: E402, F401 -- intentional public re-exports mutate_scheduled_task, remove_scheduled_task, upsert_scheduled_task, ) from supervisor.queue_snapshot import ( # noqa: E402, F401 -- intentional public re-exports _kept_service_pids, _retained_daemon_pids, parse_iso_to_ts, persist_queue_snapshot, restore_pending_from_snapshot, ) from supervisor.queue_timeouts import ( # noqa: E402, F401 -- intentional public re-exports _enforce_task_timeouts_locked, _has_live_descendant, _has_pending_descendant, _is_descendant_of, _subtree_progressing, _task_deadline_ts, _task_drive_for_task, enforce_task_timeouts, )