ouroboros/server.py
Anton Razzhigaev a4b78a0dd8 fix(updates): crash-safe stash lifecycle, mandatory managed test gate
Second review wave (triad + scope simulation) surfaced real crash
windows in the Q1=C stash lifecycle and two gate bypasses; all fixed:

- restore_update_stash now applies-then-drops (never pop): dying
  between apply and drop leaves the entry for a self-healing replay.
  rollback writes a durable stash_restored marker BETWEEN apply and
  drop, and a rollback replay that already restored the work resumes
  past the destructive reset instead of wiping it (_finish_rollback
  extracted as the shared tail).
- The fenced clean apply now binds the stash decision to the LIVE
  worktree (not the plan's stale dirty count), writes a durable
  stashing_local_work tx phase BEFORE the stash mutation, and fails
  closed when the tree still reports dirty after an empty stash. Boot
  finalization recovers a crash in that phase by restoring the
  attempt's stash (found by marker when stash_sha never landed) and
  clearing the marker.
- Boot finalize restores the stash BEFORE clearing the tx so the
  durable stash_sha pointer survives a crash in the window; the
  conflicting-restore note now reaches the owner chat from
  _boot_managed_update_tasks (disclosure contract), not only the
  supervisor log.
- The managed post-commit test gate is now mandatory: it ignores the
  caller's skip_tests and the OUROBOROS_PRE_PUSH_TESTS=0 env toggle
  (force flag threaded through _post_commit_result /
  _git_commit_with_tests / _run_pre_push_tests).
- clean(snapshot,target) does NOT imply clean(base,target) for
  file/directory type collisions: the clean-build path (extracted as
  _build_clean_merge_commit) now inventories a conflicting base
  re-merge and routes the plan to the assisted lane instead of
  refusing forever with kind=unknown.
- Docs/UI sync: plan docstring parents, DEVELOPMENT managed-update
  rule, update dialog note. Five new tests cover boot-finalize restore
  + replay, pre-apply crash recovery, rollback resume, the mandatory
  gate, and the file/directory counterexample.

Co-authored-by: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com>
2026-08-04 17:54:49 +03:00

2630 lines
112 KiB
Python

"""Self-editable Starlette/uvicorn entry point for UI and supervisor runtime."""
import asyncio
import base64
import json
import logging
import socket
import subprocess
import os
import pathlib
import sys
import threading
import time
import uuid
from ouroboros.utils import read_json_dict, utc_now_iso
from typing import Any, Dict, Optional
from starlette.applications import Starlette
from starlette.routing import Route, Mount
import uvicorn
from ouroboros.server_control import (
execute_panic_stop as _execute_panic_stop_impl,
restart_current_process as _restart_current_process_impl,
)
from ouroboros.server_auth import (
NetworkAuthGate,
get_network_auth_startup_warning,
validate_network_auth_configuration,
)
from ouroboros.server_entrypoint import find_free_port, parse_server_args, write_port_file
from ouroboros.server_web import NoCacheStaticFiles, make_index_page, resolve_web_dir
from ouroboros.usage_accounting import ensure_legacy_imported
from ouroboros.gateway import collect_routes
from ouroboros.gateway import settings as _gateway_settings
from ouroboros.gateway.ws import (
broadcast_ws,
broadcast_ws_sync,
close_all_ws,
has_ws_clients as _has_ws_clients,
set_event_loop as _set_ws_event_loop,
)
REPO_DIR = pathlib.Path(os.environ.get("OUROBOROS_REPO_DIR", pathlib.Path(__file__).parent))
DATA_DIR = pathlib.Path(os.environ.get("OUROBOROS_DATA_DIR",
pathlib.Path.home() / "Ouroboros" / "data"))
DEFAULT_HOST = os.environ.get("OUROBOROS_SERVER_HOST", "127.0.0.1")
DEFAULT_PORT = int(os.environ.get("OUROBOROS_SERVER_PORT", "8765"))
PORT_FILE = DATA_DIR / "state" / "server_port"
sys.path.insert(0, str(REPO_DIR))
if not os.environ.get("OUROBOROS_AGENT_PYTHON"):
_agent_python = sys.executable
if isinstance(_agent_python, str) and _agent_python:
os.environ["OUROBOROS_AGENT_PYTHON"] = _agent_python
_LOG_FORMAT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
_pytest_default_real_data_dir = (
"pytest" in sys.modules
and not os.environ.get("OUROBOROS_DATA_DIR")
and DATA_DIR == pathlib.Path.home() / "Ouroboros" / "data"
)
if _pytest_default_real_data_dir:
logging.basicConfig(level=logging.INFO, format=_LOG_FORMAT, handlers=[logging.StreamHandler()])
else:
_log_dir = DATA_DIR / "logs"
_log_dir.mkdir(parents=True, exist_ok=True)
from logging.handlers import RotatingFileHandler
_file_handler = RotatingFileHandler(
_log_dir / "server.log", maxBytes=2 * 1024 * 1024, backupCount=3, encoding="utf-8",
)
_file_handler.setFormatter(logging.Formatter(_LOG_FORMAT))
logging.basicConfig(level=logging.INFO, format=_LOG_FORMAT, handlers=[_file_handler, logging.StreamHandler()])
from ouroboros.observability import SecretRedactingLogFilter as _SecretRedactingLogFilter
for _handler in logging.getLogger().handlers:
_handler.addFilter(_SecretRedactingLogFilter())
# httpx logs each request URL at INFO; polling transports put credentials in
# the URL path, so even redacted lines are noise at this level.
logging.getLogger("httpx").setLevel(logging.WARNING)
logging.getLogger("httpcore").setLevel(logging.WARNING)
log = logging.getLogger("server")
RESTART_EXIT_CODE = 42
PANIC_EXIT_CODE = 99
_restart_requested = threading.Event()
_LAUNCHER_MANAGED = str(os.environ.get("OUROBOROS_MANAGED_BY_LAUNCHER", "") or "").strip() == "1"
# Captured in main() for Settings LAN-reachability metadata.
_BIND_HOST = DEFAULT_HOST
def _has_active_evolution_transaction() -> bool:
try:
path = DATA_DIR / "state" / "evolution_campaign.json"
if not path.is_file():
return False
raw = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(raw, dict):
return False
if raw.get("status") not in {"active", "paused"}:
return False
tx = raw.get("active_transaction")
return isinstance(tx, dict) and not str(tx.get("commit_sha") or "").strip()
except Exception:
return False
def _installed_skill_names():
"""Names of skills currently installed ON DISK (disk-derived, not in-memory).
Passed to the process-custody reaper so it can tell which skill-companion
orphans are safe to reap (owner uninstalled). Disk-derived so it is correct
independent of in-memory extension-reload timing; returns None on any failure
so the reaper fails toward KEEP (never mass-kills live skills' companions).
"""
try:
from ouroboros.config import get_skills_repo_path
from ouroboros.skill_loader import discover_skills
names = {s.name for s in discover_skills(DATA_DIR, repo_path=get_skills_repo_path())}
# Coalesce an EMPTY result to None ("unknown"), NOT "everything
# uninstalled": discover_skills returns [] without raising when the skills
# dir is momentarily unavailable; treating that as an empty install set
# would let an enforced reap mass-kill live companions. None ⇒ keep-all.
return names or None
except Exception:
log.debug("Could not compute installed skill names for custody reaper", exc_info=True)
return None
def _restart_current_process(host: str, port: int) -> None:
_restart_current_process_impl(host, port, repo_dir=REPO_DIR, log=log)
from ouroboros.config import (
load_settings, save_settings, apply_settings_to_env as _apply_settings_to_env,
)
from ouroboros.server_runtime import (
apply_runtime_provider_defaults,
has_startup_ready_provider,
needs_local_model_autostart,
setup_remote_if_configured,
ws_heartbeat_loop,
)
_supervisor_ready = threading.Event()
_supervisor_error: Optional[str] = None
_event_loop: Optional[asyncio.AbstractEventLoop] = None
_supervisor_thread: Optional[threading.Thread] = None
_consciousness: Any = None
def _describe_bg_consciousness_state(requested_enabled: bool) -> dict:
snapshot = _consciousness.status_snapshot() if _consciousness else {}
running = bool(snapshot.get("running"))
paused = bool(snapshot.get("paused"))
next_wakeup_sec = int(snapshot.get("next_wakeup_sec") or 0)
idle_reason = str(snapshot.get("last_idle_reason") or "")
detail = "Background consciousness is off."
status = "disabled"
if requested_enabled and running and paused:
status = "paused"
detail = "Paused while another foreground task is active."
elif requested_enabled and running and idle_reason == "thinking":
status = "running"
detail = "Background consciousness is thinking now."
elif requested_enabled and running and idle_reason == "budget_blocked":
status = "budget_blocked"
detail = "Background consciousness hit its budget allocation and is waiting."
elif requested_enabled and running:
status = "running"
detail = (
"Background consciousness is idle between wakeups."
+ (f" Next wakeup in {next_wakeup_sec}s." if next_wakeup_sec > 0 else "")
)
elif requested_enabled:
status = "stopped"
detail = "Enabled in state, but the background thread is not running."
if idle_reason == "error_backoff" and snapshot.get("last_error"):
status = "error_backoff"
detail = f"Waiting to retry after an internal error: {snapshot['last_error']}"
return {
"enabled": requested_enabled,
"status": status,
"detail": detail,
**snapshot,
}
def _start_supervisor_if_needed(settings: dict) -> bool:
"""Start the supervisor once when runtime providers become available."""
global _supervisor_thread, _supervisor_error
if not has_startup_ready_provider(settings):
return False
if _supervisor_thread and _supervisor_thread.is_alive():
return False
_supervisor_error = None
_supervisor_thread = threading.Thread(
target=_run_supervisor,
args=(settings,),
daemon=True,
name="supervisor-main",
)
_supervisor_thread.start()
return True
def _task_belongs_to_chat(ctx: Any, task_id: str, task_obj: Dict[str, Any], chat_id: int) -> bool:
try:
if int(task_obj.get("chat_id") or 0) == int(chat_id or 0):
return True
except (TypeError, ValueError):
pass
try:
from ouroboros.projects_registry import project_chat_for_task
return int(project_chat_for_task(ctx.DRIVE_ROOT, task_id) or 0) == int(chat_id or 0)
except Exception:
return False
def _active_direct_root(ctx: Any) -> Dict[str, Any]:
"""Snapshot the one in-process direct root without creating queue state."""
try:
agent = ctx.get_chat_agent()
lock = getattr(agent, "_owner_message_admission_lock", None)
if lock is None:
return {}
with lock:
task_id = str(getattr(agent, "_current_task_id", "") or "").strip()
if (
not getattr(agent, "_busy", False)
or not getattr(agent, "_accepting_owner_messages", False)
or not task_id
):
return {}
metadata = getattr(agent, "_current_task_metadata", {})
metadata = metadata if isinstance(metadata, dict) else {}
return {
"task_id": task_id,
"status": "running",
"title": _clip_marked(metadata.get("title"), 120),
"objective": _clip_marked(getattr(agent, "_current_task_text", ""), 600),
"project_id": str(metadata.get("project_id") or ""),
"chat_id": int(getattr(agent, "_current_chat_id", 0) or 0),
"started_at": float(getattr(agent, "_task_started_ts", 0.0) or 0.0),
"steerable": True,
"direct_chat": True,
}
except Exception:
return {}
def _addressable_root_tasks(ctx: Any, chat_id: Optional[int] = None) -> list:
"""Compact RUNNING+PENDING owner-root manifest, without choosing a target."""
out: list = []
seen: set[str] = set()
def _add(task_id: Any, task_obj: Any, status: str, started_at: Any = None) -> None:
tid = str(task_id or "").strip()
if not tid or tid in seen or not isinstance(task_obj, dict):
return
if task_obj.get("_is_direct_chat") or str(task_obj.get("delegation_role") or "") == "subagent":
return
if chat_id is not None and not _task_belongs_to_chat(ctx, tid, task_obj, int(chat_id or 0)):
return
objective = str(
task_obj.get("objective") or task_obj.get("description") or task_obj.get("text") or ""
).strip()
out.append({
"task_id": tid,
"status": status,
"title": _clip_marked(task_obj.get("title"), 120),
"objective": _clip_marked(objective, 600),
"project_id": str(task_obj.get("project_id") or ""),
"started_at": started_at,
"steerable": True,
})
seen.add(tid)
for tid, running in list(getattr(ctx, "RUNNING", {}).items()):
if not isinstance(running, dict):
continue
task_obj = running.get("task") if isinstance(running.get("task"), dict) else running
_add(tid, task_obj, "running", running.get("started_at"))
for pending in list(getattr(ctx, "PENDING", []) or []):
if isinstance(pending, dict):
_add(pending.get("id"), pending, "pending", pending.get("queued_at"))
direct = _active_direct_root(ctx)
if direct and str(direct.get("task_id") or "") not in seen:
if chat_id is None or int(direct.get("chat_id") or 0) == int(chat_id or 0):
out.append(direct)
return out
def _stage_mailbox_attachments(
ctx: Any,
task_id: str,
task_metadata: Any,
image_data: Any = None,
) -> str:
"""Stage one routed turn's files into the existing task artifact store."""
metadata = task_metadata if isinstance(task_metadata, dict) else {}
uploads = list(metadata.get("chat_attachment_uploads") or [])
temp_source: Optional[pathlib.Path] = None
if image_data and not uploads:
# Non-Web transports may carry an inline image rather than an uploaded
# path. Materialise it only long enough for the canonical staging helper
# to copy it into the addressed task's artifact store.
try:
raw = base64.b64decode(str(image_data[0] or ""), validate=True)
if raw and len(raw) <= 50 * 1024 * 1024:
mime = str(image_data[1] or "image/jpeg").lower()
suffix = ".png" if "png" in mime else ".webp" if "webp" in mime else ".jpg"
temp_source = pathlib.Path(ctx.DRIVE_ROOT) / "uploads" / f"routed-{uuid.uuid4().hex}{suffix}"
temp_source.parent.mkdir(parents=True, exist_ok=True)
with temp_source.open("xb") as handle:
handle.write(raw)
handle.flush()
os.fsync(handle.fileno())
uploads.append({"path": str(temp_source), "label": "owner image"})
except Exception:
log.warning("Unable to stage routed inline image for task %s", task_id, exc_info=True)
try:
if not uploads:
return ""
from ouroboros.artifacts import stage_task_attachments
from ouroboros.gateway.tasks import _render_attachment_lines
manifest = stage_task_attachments(ctx.DRIVE_ROOT, task_id, uploads)
rendered = _render_attachment_lines(manifest)
return f"\n\n[ATTACHMENTS]\n{rendered}\n[END_ATTACHMENTS]" if rendered else ""
finally:
if temp_source is not None:
try:
temp_source.unlink(missing_ok=True)
except OSError:
log.debug("Unable to remove routed attachment staging source", exc_info=True)
def _route_project_chat_to_running_task(
ctx: Any,
chat_id: int,
message: str,
client_message_id: str = "",
*,
task_metadata: Any = None,
image_data: Any = None,
) -> str:
"""Deliver a Project follow-up to the sole RUNNING/PENDING root mailbox.
Multi-project (v6.32.0): a focused project room with exactly ONE active pooled
task IS that task's context, so a follow-up is delivered to it as a TRANSPORT
invariant (the loop drains the mailbox every round) — there is no routing CHOICE
to make. But when the room has ZERO or MORE THAN ONE steerable task, picking a
target is a JUDGMENT, and code must never make it mechanically (BIBLE P5 LLM-first,
v6.34.0 WS1): this returns "" so the message flows to the decision turn, where the
agent sees `current_chat.running_tasks` and chooses `steer_task` / `promote_chat_to_task`.
Returns the delivered task id, or "" (no delivery — fall through to the decision lane).
A chat is a project thread by REGISTRY membership, not a bare numeric range —
large external-transport (Telegram-style) chat ids must not be misclassified and
have their owner messages swallowed.
"""
try:
if not _project_id_for_registered_chat(ctx, chat_id):
return ""
except Exception:
return ""
try:
steerable = _addressable_root_tasks(ctx, chat_id)
# Exactly one candidate => unambiguous transport. Zero or many => a routing
# decision the AGENT must make (P5/WS1), so do not deliver here.
if len(steerable) != 1:
return ""
candidate = steerable[0]
tid = str(candidate["task_id"])
direct_agent = None
direct_lock = None
if candidate.get("direct_chat"):
direct_agent = ctx.get_chat_agent()
direct_lock = getattr(direct_agent, "_owner_message_admission_lock", None)
if direct_lock is None:
return ""
task_obj: Dict[str, Any] = {}
running = getattr(ctx, "RUNNING", {}).get(tid)
if isinstance(running, dict):
task_obj = running.get("task") if isinstance(running.get("task"), dict) else running
if not task_obj:
task_obj = next(
(row for row in list(getattr(ctx, "PENDING", []) or []) if str(row.get("id") or "") == tid),
{},
)
from ouroboros.owner_mailbox import write_owner_message
from supervisor.queue import (
ACCEPTANCE_FENCES,
_queue_lock,
_task_drive_for_task,
persist_queue_snapshot,
)
# Active drive (child drive for forked/workspace tasks) — mirror
# forward_to_worker / steer_task so the mailbox lands where the task
# actually drains it, not the canonical root. A stable msg_id derived from
# client_message_id makes this 1:1 delivery idempotent — a WebSocket retry of
# the same message can't double-deliver (drain_owner_entries dedups by msg_id),
# matching steer_task's contract.
direct_lock_held = False
queue_lock_held = False
fence_generation_changed = False
active_fence = None
if direct_lock is not None:
direct_lock.acquire()
direct_lock_held = True
if not (
getattr(direct_agent, "_busy", False)
and getattr(direct_agent, "_accepting_owner_messages", False)
and str(getattr(direct_agent, "_current_task_id", "") or "") == tid
):
direct_lock.release()
direct_lock_held = False
return ""
task_drive = pathlib.Path(ctx.DRIVE_ROOT) if direct_lock_held else _task_drive_for_task(task_obj, tid)
msg_id = f"{client_message_id}:{tid}" if client_message_id else None
try:
attachment_note = _stage_mailbox_attachments(ctx, tid, task_metadata, image_data)
if not direct_lock_held:
_queue_lock.acquire()
queue_lock_held = True
live_meta = getattr(ctx, "RUNNING", {}).get(tid)
still_pending = any(
isinstance(row, dict) and str(row.get("id") or "") == tid
for row in list(getattr(ctx, "PENDING", []) or [])
)
if live_meta is None and not still_pending:
return ""
fence_root = str(task_obj.get("root_task_id") or tid)
active_fence = ACCEPTANCE_FENCES.get(fence_root)
if isinstance(active_fence, dict) and str(active_fence.get("status") or "") == "sealed":
return ""
if not write_owner_message(
task_drive, f"{message}{attachment_note}", tid, msg_id=msg_id
):
return ""
if direct_lock_held:
direct_agent._owner_message_generation = int(
getattr(direct_agent, "_owner_message_generation", 0) or 0
) + 1
else:
if isinstance(active_fence, dict) and str(active_fence.get("status") or "") == "active":
active_fence["owner_message_generation"] = int(
active_fence.get("owner_message_generation") or 0
) + 1
fence_generation_changed = True
finally:
if queue_lock_held:
_queue_lock.release()
if direct_lock_held:
direct_lock.release()
if fence_generation_changed:
persist_queue_snapshot(reason="acceptance_fence_owner_message")
return tid
except Exception:
log.debug("Mailbox follow-up routing failed; falling back to direct lane", exc_info=True)
return ""
def _clip_marked(value: str, limit: int) -> str:
"""Clip a routing/recognition string but NEVER silently: an explicit omission
marker keeps a decision-context field honest (no silent ``[:N]`` truncation of a
cognitive/routing artifact — DEVELOPMENT.md). The marker + the full task_id keep
enough signal for the agent to disambiguate the steer target."""
s = str(value or "").strip()
if len(s) <= limit:
return s
return s[:limit] + f" …[+{len(s) - limit} chars omitted]"
def _chat_running_tasks(ctx: Any, chat_id: int) -> list:
"""Structural snapshot of the owner's RUNNING root tasks in THIS chat (id +
objective + recency). The decision turn reads this from runtime context to
pick a steer_task target by its own judgment — code only exposes the state,
it never auto-chooses (BIBLE P5). Direct in-process turns and subagents are
not pooled RUNNING tasks and are excluded."""
return [row for row in _addressable_root_tasks(ctx, chat_id) if row.get("status") == "running"]
def _main_routing_manifest(ctx: Any) -> Dict[str, Any]:
"""Bounded canonical facts for one Main-chat LLM routing decision."""
from ouroboros.projects_registry import list_projects
from ouroboros.task_results import list_task_results
from ouroboros.utils import iter_jsonl_objects
projects = [{
"project_id": str(row.get("id") or ""),
"name": _clip_marked(row.get("name"), 120),
"chat_id": int(row.get("chat_id") or 0),
"lifecycle": str(row.get("lifecycle") or "active"),
} for row in list_projects(ctx.DRIVE_ROOT)]
roots = _addressable_root_tasks(ctx, None)
all_results = list_task_results(ctx.DRIVE_ROOT)
all_results.sort(key=lambda row: str(row.get("ts") or row.get("updated_at") or ""), reverse=True)
finals: list = []
for row in all_results[:16]:
bundle = row.get("artifact_bundle") if isinstance(row.get("artifact_bundle"), dict) else {}
artifacts = bundle.get("artifacts") if isinstance(bundle.get("artifacts"), list) else []
finals.append({
"task_id": str(row.get("task_id") or row.get("id") or ""),
"status": str(row.get("status") or ""),
"title": _clip_marked(row.get("title"), 120),
"objective": _clip_marked(row.get("objective") or row.get("description"), 300),
"project_id": str(row.get("project_id") or ""),
"artifact_status": str(row.get("artifact_status") or ""),
"artifact_refs": [
str(item.get("path") or item.get("name") or "")
for item in artifacts[:8] if isinstance(item, dict)
],
})
dialogue_rows: list = []
chat_paths = sorted(
(pathlib.Path(ctx.DRIVE_ROOT) / "archive").glob("chat_*.jsonl"),
key=lambda path: path.name,
)[-2:] + [pathlib.Path(ctx.DRIVE_ROOT) / "logs" / "chat.jsonl"]
for path in chat_paths:
for row in iter_jsonl_objects(path):
text = str(row.get("text") or "").strip()
if text:
dialogue_rows.append({
"ts": str(row.get("ts") or ""),
"direction": str(row.get("direction") or ""),
"chat_id": int(row.get("chat_id") or 1),
"text": _clip_marked(text, 500),
"task_id": str(row.get("task_id") or ""),
"client_message_id": str(row.get("client_message_id") or ""),
})
dialogue = dialogue_rows[-20:]
return {
"projects": projects[:40],
"root_tasks": roots[:40],
"final_results": finals,
"recent_canonical_dialogue": dialogue,
"omissions": {
"projects": max(0, len(projects) - 40),
"root_tasks": max(0, len(roots) - 40),
"final_results": max(0, len(all_results) - 16),
"dialogue_rows": max(0, len(dialogue_rows) - 20),
},
}
def _decision_turn_metadata(ctx: Any, chat_id: int, client_message_id: str, task_metadata: Any) -> Any:
"""Enrich a chat turn's metadata with the structural facts the decision turn
needs: the RUNNING tasks in THIS chat (so it can steer_task the right one
instead of spawning a duplicate) and the originating message id (for idempotent
steer delivery). P5-clean: surfaces state only; the agent picks the target by
judgment among answer / steer_task / promote_chat_to_task / route_to_project."""
md = dict(task_metadata) if isinstance(task_metadata, dict) else {}
swarm_intent = bool(md.get("force_plan"))
addressable_here = _addressable_root_tasks(ctx, chat_id)
running_here = [row for row in addressable_here if row.get("status") == "running"]
project_id = str(md.get("project_id") or "").strip() or _project_id_for_registered_chat(
ctx, chat_id,
)
is_main_lane = not bool(project_id)
try:
# Every non-Project owner transport is the Main lane. External transports
# commonly use a real provider chat id rather than Web's numeric ``1``;
# keying this decision to ``chat_id == 1`` made their canonical router see
# neither Projects nor globally addressable roots.
main_manifest = _main_routing_manifest(ctx) if is_main_lane else {}
if main_manifest and not (
main_manifest.get("projects") or main_manifest.get("root_tasks")
):
main_manifest = {}
except Exception:
log.warning("Unable to build Main routing manifest", exc_info=True)
main_manifest = {"error": "routing_manifest_unavailable"} if is_main_lane else {}
if not swarm_intent and not addressable_here and not client_message_id and not main_manifest:
return task_metadata
if addressable_here:
md["current_chat"] = {
"chat_id": int(chat_id or 0),
"running_tasks": running_here,
"addressable_root_tasks": addressable_here,
}
if main_manifest:
md["main_routing_manifest"] = main_manifest
if client_message_id:
md["client_message_id"] = client_message_id
option_roots = (
list(main_manifest.get("root_tasks") or [])
if is_main_lane and isinstance(main_manifest, dict)
else addressable_here
)
manual_options = [] if swarm_intent else [
{
"action": "steer_task",
"task_id": row["task_id"],
"status": row["status"],
"title": row.get("title") or row.get("objective"),
"project_id": str(row.get("project_id") or ""),
}
for row in option_roots
if isinstance(row, dict) and row.get("task_id")
]
if not swarm_intent and is_main_lane and isinstance(main_manifest, dict):
manual_options.extend({
"action": "new_task_in_project",
"project_id": str(row.get("project_id") or ""),
"project_name": str(row.get("name") or row.get("project_id") or "Project"),
"label": f"New task in {str(row.get('name') or 'Project')}",
} for row in list(main_manifest.get("projects") or []) if isinstance(row, dict))
elif project_id and not swarm_intent:
manual_options.append({
"action": "new_task_in_project",
"project_id": project_id,
"label": "New task in Project",
})
routing_contract = {
"llm_first": True,
"source_lane": "main" if is_main_lane else "project",
"valid_actions": (
(["promote_chat_to_task", "route_to_project"] if is_main_lane else ["promote_chat_to_task"])
if swarm_intent else
[
"answer_inline", "steer_task", "promote_chat_to_task", "route_to_project",
"needs_manual_target",
]
),
"on_uncertain_or_invalid_target": (
"promote_chat_to_task" if swarm_intent else "needs_manual_target"
),
"manual_options": manual_options,
}
if not swarm_intent:
routing_contract["manual_target_tool"] = {"name": "route_to_project", "project_id": ""}
md["routing_contract"] = routing_contract
return md
def _supervisor_loop_stalled(last_tick: float, now: float, deadline_sec: int) -> bool:
"""True when the supervisor loop has not published a liveness tick within the
deadline (WS3). deadline_sec<=0 disables the watchdog."""
return deadline_sec > 0 and (now - last_tick) > deadline_sec
def _chat_turn_wedged(busy: bool, last_activity_ts, now: float, deadline_sec: int) -> bool:
"""True when an IN-PROCESS direct-chat turn is busy but its liveness tick has been
silent past the deadline (WS3). ``last_activity_ts is None`` => the turn has not
started its liveness loop yet (not wedged). deadline_sec<=0 disables the check."""
if not busy or last_activity_ts is None or deadline_sec <= 0:
return False
return (now - last_activity_ts) > deadline_sec
def _alert_chat_turn_wedge(task_id, gap: float) -> None:
"""WS3: a direct-chat turn is heartbeat-silent. New messages still get answered
(WS10 ephemeral decision turns), but a hung IN-PROCESS turn cannot be killed and
still holds the chat-agent lock, so admission cannot be freed in-process (full
kill-ability via out-of-process direct chat was deferred per owner). Surface it +
recommend /restart, which is the safe full recovery."""
from supervisor.state import append_jsonl, load_state
try:
append_jsonl(DATA_DIR / "logs" / "supervisor.jsonl", {
"ts": utc_now_iso(), "type": "chat_turn_wedge",
"task_id": str(task_id or ""), "silent_sec": round(gap, 1),
})
except Exception:
log.debug("chat-turn wedge log failed", exc_info=True)
try:
owner_chat = int((load_state() or {}).get("owner_chat_id") or 0)
if owner_chat:
from supervisor.message_bus import send_with_budget
send_with_budget(
owner_chat,
f"⚠️ A chat turn looks wedged (~{int(gap)}s with no heartbeat). New messages "
"still get answered, but the stuck turn can't be cleared in-process — /restart "
"to fully recover it.",
is_progress=True,
task_id=str(task_id or ""),
progress_meta={
"task_incident": "chat_turn_wedge",
"toast_once": f"{task_id or 'direct-chat'}:chat_turn_wedge",
},
)
except Exception:
log.debug("chat-turn wedge owner alert failed", exc_info=True)
def _start_supervisor_liveness_watchdog(liveness: list, stop_event=None) -> None:
"""Dedicated daemon thread (NOT inside the supervisor loop, so it fires even when
that loop stalls). It ALERTS the owner on two silent-wedge classes — a supervisor
loop stall (new-message intake starvation) and a heartbeat-silent in-process
direct-chat turn — converting a multi-hour silent wedge into an immediate signal.
It deliberately does NOT kill a hung thread or free the chat-agent lock: the wedged
turn holds that lock for its whole duration, so in-process admission-freeing is
unsafe (out-of-process direct chat for full kill-ability was deferred per owner);
WS10 ephemeral decision turns keep the chat responsive meanwhile. ``stop_event`` is
a PER-GENERATION token: when the supervisor loop that owns ``liveness`` exits (incl.
the crash-storm death path, which never sets the global restart flag), it is set so
this watchdog stops watching a now-stale liveness list (no false post-revival alert)."""
from ouroboros.config import get_supervisor_liveness_deadline_sec
deadline = get_supervisor_liveness_deadline_sec()
if deadline <= 0:
return
def _watch() -> None:
from supervisor.state import append_jsonl, load_state
interval = min(15, max(1, deadline // 3))
loop_alerted = False
wedged_task = None
while not _restart_requested.is_set() and not (stop_event is not None and stop_event.is_set()):
time.sleep(interval)
now = time.time()
# (1) Supervisor loop stall — new-message intake starvation.
if _supervisor_loop_stalled(liveness[0], now, deadline):
if not loop_alerted:
gap = now - liveness[0]
log.error(
"Supervisor loop STALLED ~%.0fs — new-message intake starved (WS10 "
"ephemeral chat still answers); investigate a blocking step.", gap,
)
try:
append_jsonl(DATA_DIR / "logs" / "supervisor.jsonl", {
"ts": utc_now_iso(), "type": "supervisor_loop_stall", "stalled_sec": round(gap, 1),
})
except Exception:
log.debug("loop-stall log failed", exc_info=True)
try:
owner_chat = int((load_state() or {}).get("owner_chat_id") or 0)
if owner_chat:
from supervisor.message_bus import send_with_budget
send_with_budget(
owner_chat,
f"⚠️ My supervisor loop stalled for ~{int(gap)}s — new messages may be "
"delayed. I recover on the next tick or a restart; investigating.",
is_progress=True,
progress_meta={
"task_incident": "supervisor_loop_stall",
"toast_once": f"supervisor-loop-stall:{int(liveness[0])}",
},
)
except Exception:
log.debug("loop-stall owner alert failed", exc_info=True)
loop_alerted = True
else:
loop_alerted = False
# (2) In-process direct-chat turn wedge — a heartbeat-silent busy turn.
try:
from supervisor.workers import chat_turn_liveness
busy, turn_task, turn_ts = chat_turn_liveness()
except Exception:
busy, turn_task, turn_ts = (False, None, None)
if _chat_turn_wedged(busy, turn_ts, now, deadline):
if wedged_task != turn_task: # alert once per wedged turn
_alert_chat_turn_wedge(turn_task, now - (turn_ts or now))
wedged_task = turn_task
elif not busy:
wedged_task = None
threading.Thread(target=_watch, name="supervisor-liveness-watchdog", daemon=True).start()
def _periodic_supervisor_maintenance(last_custody_reap: list, last_review_reconcile: list) -> None:
"""Throttled periodic upkeep extracted from the supervisor loop: custody reap
of orphaned task-scoped processes (every 600s) + review-job zombie reconcile
(every 300s). Each cadence gates itself via its own last-run marker."""
if time.time() - last_custody_reap[0] > 600:
last_custody_reap[0] = time.time()
try:
from ouroboros.process_custody import reap_orphaned_processes
from supervisor.queue import RUNNING as _running_tasks
reap_orphaned_processes(
DATA_DIR, running_task_ids=set(_running_tasks.keys()),
live_owner_skills=_installed_skill_names(),
)
except Exception:
log.debug("Periodic custody reap failed", exc_info=True)
if time.time() - last_review_reconcile[0] > 300:
last_review_reconcile[0] = time.time()
_periodic_zombie_reconcile()
def _scoped_task_metadata(project_id: str, task_metadata: Any) -> Any:
"""Bind a chat frame's task_metadata to the thread's project via chat_id (the
SSOT). A registered project chat scopes to its OWN project, overriding any
client-supplied project_id; a non-project chat DROPS an untrusted client
project_id (work is scoped to a project only via the promote_chat_to_task tool,
never a raw ws frame). Prevents a stale/malformed frame (chat_id A + project_id
B) from rendering in A while loading/writing project B's memory."""
if project_id:
return {**(task_metadata or {}), "project_id": project_id}
if task_metadata and task_metadata.get("project_id"):
return {k: v for k, v in task_metadata.items() if k != "project_id"}
return task_metadata
def _owner_binding_chat_id(ctx: Any, chat_id: int, is_external_transport: bool) -> int:
"""The owner's canonical chat for owner-targeted notices (restart, supervisor
death, consciousness). External transports bind to their own chat; a WEB owner
always binds to MAIN (1), never a project panel — so if the first post-reset
web message lands in a project room, owner notices still reach main."""
if not is_external_transport and _project_id_for_registered_chat(ctx, chat_id):
return 1
try:
return int(chat_id or 0)
except (TypeError, ValueError):
return 0
def _project_id_for_registered_chat(ctx: Any, chat_id: int) -> str:
"""Return the registered project id for a project chat_id, else ``""``.
NOT an isolation gate (full project awareness, v6.32.0): the one mind notices
EVERY human message via inject_observation, project rooms included. This just
classifies a chat as a project thread so the message is scoped to that project
(task_metadata.project_id) and routed to its panel. This active-only lookup is
paired with ``_reserved_project_for_chat`` for deleting/tombstoned IDs, so a
reserved chat cannot be resurrected through ordinary routing.
"""
try:
from ouroboros.projects_registry import list_projects
cid = int(chat_id or 0)
for project in list_projects(ctx.DRIVE_ROOT):
try:
if int(project.get("chat_id") or 0) == cid:
return str(project.get("id") or "").strip()
except (TypeError, ValueError):
continue
except Exception:
log.debug("Project chat_id lookup failed", exc_info=True)
return ""
def _reserved_project_for_chat(ctx: Any, chat_id: int) -> Dict[str, Any]:
try:
from ouroboros.projects_registry import list_reserved_projects
cid = int(chat_id or 0)
for project in list_reserved_projects(ctx.DRIVE_ROOT):
try:
if int(project.get("chat_id") or 0) == cid:
return dict(project)
except (TypeError, ValueError):
continue
except Exception:
log.debug("Reserved Project chat lookup failed", exc_info=True)
return {}
def _record_routing_receipt(
bridge: Any,
ctx: Any,
*,
chat_id: int,
client_message_id: str,
action: str,
target: str = "",
status: str,
persist: bool = True,
options: Optional[list] = None,
) -> None:
"""Emit a typed bubble-free ack and optionally persist its presentation state."""
if persist:
try:
from ouroboros.project_dialogue import append_chat_annotation
append_chat_annotation(
ctx.DRIVE_ROOT,
client_message_id,
action=action,
target=target,
status=status,
)
except Exception:
log.debug("Routing annotation append failed", exc_info=True)
try:
ack = getattr(bridge, "send_routing_ack", None)
if callable(ack):
ack_kwargs = {
"client_message_id": client_message_id,
"action": action,
"target": target,
"status": status,
}
if options is not None:
ack_kwargs["options"] = options
ack(
chat_id,
**ack_kwargs,
)
else:
broadcast = getattr(bridge, "broadcast", None)
if callable(broadcast):
payload = {
"type": "message_annotation",
"annotation_type": "routing_ack",
"chat_id": int(chat_id or 0),
"client_message_id": str(client_message_id or ""),
"action": action,
"target": target,
"status": status,
"suppress_bubble": True,
}
if options is not None:
payload["options"] = options
broadcast(payload)
except Exception:
log.debug("Routing receipt broadcast failed", exc_info=True)
def _route_owner_message(bridge: Any, ctx: Any, incoming: Dict[str, Any]) -> None:
"""Route one non-command owner message through the canonical decision lane."""
chat_id = int(incoming["chat_id"])
text = str(incoming.get("text") or "")
image_caption = str(incoming.get("image_caption") or "")
client_message_id = str(incoming.get("client_message_id") or "")
image_data = incoming.get("image_data")
task_constraint = incoming.get("task_constraint")
task_metadata = incoming.get("task_metadata")
from ouroboros.contracts.task_constraint import normalize_task_constraint
normalized_constraint = normalize_task_constraint(task_constraint)
if normalized_constraint and normalized_constraint.mode == "skill_repair":
# Repair is already a typed, narrowly confined task request. Sending it
# through the conversation decision lane would combine skill_repair with
# _ephemeral_turn: ephemeral hides the repair mutators while heal mode
# blocks promotion. Promote it directly without weakening either policy.
from supervisor.events import _handle_promote_chat_to_task
ctx.consciousness.inject_observation(
f"Message from my human: {incoming.get('log_text') or ''}"
)
task_id = uuid.uuid4().hex[:16]
event = {
"type": "promote_chat_to_task",
"task_id": task_id,
"routing_token": uuid.uuid4().hex,
"objective": text or image_caption,
"chat_id": chat_id,
"client_message_id": client_message_id,
"task_constraint": task_constraint,
"routed_from_main": True,
}
origin_ref = incoming.get("origin_message_ref")
if isinstance(origin_ref, dict) and origin_ref:
event["source_ref"] = origin_ref
event["source_text"] = str(incoming.get("log_text") or "")
else:
event["origin_suppressed"] = True
try:
outcome = _handle_promote_chat_to_task(event, ctx)
except Exception:
log.warning("Direct skill-repair promotion failed", exc_info=True)
outcome = {
"status": "needs_manual_target",
"reason": "repair_promotion_failed",
"task_id": task_id,
}
outcome = outcome if isinstance(outcome, dict) else {"status": "scheduled", "task_id": task_id}
outcome_status = str(outcome.get("status") or "needs_manual_target")
if outcome_status == "scheduled":
try:
ctx.send_with_budget(
chat_id,
f"✅ Repair task {task_id} was accepted and durably scheduled.",
)
except Exception:
log.debug("Repair promotion success notification failed", exc_info=True)
else:
reason = str(outcome.get("reason") or outcome_status)
try:
ctx.send_with_budget(
chat_id,
f"⚠️ Repair task was not started ({reason}). Please retry from the skill card.",
)
except Exception:
log.debug("Repair promotion refusal notification failed", exc_info=True)
return
reserved_project = _reserved_project_for_chat(ctx, chat_id)
project_id = (
str(reserved_project.get("id") or "")
if str((reserved_project or {}).get("lifecycle") or "active") == "active"
else ""
)
if reserved_project and not project_id:
_record_routing_receipt(
bridge,
ctx,
chat_id=chat_id,
client_message_id=client_message_id,
action="project_route",
target=str(reserved_project.get("id") or ""),
status="project_unavailable",
)
return
ctx.consciousness.inject_observation(f"Message from my human: {incoming.get('log_text') or ''}")
task_metadata = _scoped_task_metadata(project_id, task_metadata)
swarm_intent = bool(
isinstance(task_metadata, dict) and task_metadata.get("force_plan")
)
# The turn's origin identity rides UNCONDITIONALLY (not only when the
# decision lane runs): a bare direct turn with no projects/roots yet — the
# first-ever project creation — must still carry it so promote/route/bind
# receive the ref by value.
origin_ref = incoming.get("origin_message_ref")
if isinstance(origin_ref, dict) and origin_ref:
task_metadata = {
**(task_metadata or {}),
"origin_message_ref": origin_ref,
"origin_message_text": str(incoming.get("log_text") or ""),
}
else:
# A suppressed (never-logged) message has a DESIGNED absence of origin;
# downstream binders must not classify it as a producer bug.
task_metadata = {**(task_metadata or {}), "origin_suppressed": True}
if project_id and not swarm_intent:
routed_to_task = _route_project_chat_to_running_task(
ctx,
chat_id,
text or image_caption,
client_message_id,
task_metadata=task_metadata,
image_data=image_data,
)
if routed_to_task:
_record_routing_receipt(
bridge,
ctx,
chat_id=chat_id,
client_message_id=client_message_id,
action="mailbox_delivery",
target=routed_to_task,
status="delivered",
)
return
global_roots = _addressable_root_tasks(ctx, None)
try:
from ouroboros.projects_registry import list_projects
has_projects = bool(list_projects(ctx.DRIVE_ROOT))
except Exception:
log.warning("Unable to inspect Projects for owner routing", exc_info=True)
has_projects = True
needs_decision_lane = swarm_intent or bool(project_id) or has_projects or bool(global_roots)
if needs_decision_lane:
task_metadata = _decision_turn_metadata(ctx, chat_id, client_message_id, task_metadata)
agent = ctx.get_chat_agent()
def _run_direct() -> None:
try:
ctx.handle_chat_direct(
chat_id,
text or image_caption,
image_data,
task_constraint=task_constraint,
task_metadata=task_metadata,
)
finally:
ctx.consciousness.resume()
if needs_decision_lane or agent._busy:
threading.Thread(
target=ctx.handle_chat_ephemeral,
args=(chat_id, text or image_caption, image_data),
kwargs={"task_constraint": task_constraint, "task_metadata": task_metadata},
daemon=True,
).start()
else:
ctx.consciousness.pause()
threading.Thread(target=_run_direct, daemon=True).start()
def _process_bridge_updates(bridge, offset: int, ctx: Any) -> int:
from supervisor.message_bus import coerce_chat_identity
updates = bridge.get_updates(offset=offset, timeout=1)
for upd in updates:
offset = int(upd["update_id"]) + 1
msg = upd.get("message") or {}
if not msg:
continue
chat_id = coerce_chat_identity((msg.get("chat") or {}).get("id"), 1)
user_id = coerce_chat_identity((msg.get("from") or {}).get("id"), chat_id or 1)
text = str(msg.get("text") or "")
source = str(msg.get("source") or "web")
sender_label = str(msg.get("sender_label") or "")
sender_session_id = str(msg.get("sender_session_id") or "")
client_message_id = str(msg.get("client_message_id") or "")
transport = msg.get("transport") if isinstance(msg.get("transport"), dict) else {}
image_base64 = str(msg.get("image_base64") or "")
image_mime = str(msg.get("image_mime") or "image/jpeg")
image_caption = str(msg.get("image_caption") or "")
suppress_chat_log = bool(msg.get("suppress_chat_log"))
task_constraint = msg.get("task_constraint") if isinstance(msg.get("task_constraint"), dict) else None
task_metadata = msg.get("task_metadata") if isinstance(msg.get("task_metadata"), dict) else None
image_data = (image_base64, image_mime, image_caption) if image_base64 else None
log_text = text or image_caption or ("(image attached)" if image_base64 else "")
now_iso = utc_now_iso()
if not client_message_id:
# Some owner transports have no client-generated id. Give the
# canonical row a deterministic host id before logging/routing so a
# typed non-bubble acknowledgement cannot be silently dropped and a
# replay of the same inbound update remains idempotent.
identity = json.dumps(
{
"source": source,
"session": sender_session_id,
"chat_id": chat_id,
"user_id": user_id,
"update_id": int(upd.get("update_id") or 0),
"text": text,
"caption": image_caption,
"transport": transport,
},
ensure_ascii=False,
sort_keys=True,
default=str,
)
client_message_id = f"host-{uuid.uuid5(uuid.NAMESPACE_URL, identity).hex}"
st = ctx.load_state()
owner_id = st.get("owner_id")
lowered = text.strip().lower()
is_slash_command = lowered.startswith("/")
is_external_transport = source != "web"
external_identity_present = (not is_external_transport) or (chat_id > 0 and user_id > 0)
# Global owner = primary chat for outbound notices (web on desktop, the
# first transport on headless Colab). Bound once, on the first message.
if owner_id is None and external_identity_present:
owner_id = user_id
from supervisor.message_bus import log_chat
# Origin identity is captured HERE, where the host writes the canonical
# row (BIBLE P2: identity by value, never re-derived from content
# downstream). Only a row that is actually logged mints a ref — a
# suppressed message must not reference a non-existent canonical row.
origin_message_ref: Optional[Dict[str, Any]] = None
if not suppress_chat_log:
log_chat(
"in",
chat_id,
user_id,
log_text,
ts=now_iso,
source=source,
sender_label=sender_label,
sender_session_id=sender_session_id,
client_message_id=client_message_id,
transport=transport,
)
from ouroboros.project_dialogue import build_owner_message_ref
origin_message_ref = build_owner_message_ref(
chat_id=chat_id,
client_message_id=client_message_id,
ts=now_iso,
text=log_text,
)
if source != "web":
bridge.broadcast({
"type": "photo" if image_base64 else "chat",
"role": "user",
"content": text,
"caption": image_caption,
"image_base64": image_base64,
"mime": image_mime,
"ts": now_iso,
"source": source,
"sender_label": sender_label,
"sender_session_id": sender_session_id,
"client_message_id": client_message_id,
"transport": transport,
"chat_id": chat_id,
})
def _stamp_owner_activity(live: dict) -> None:
if live.get("owner_id") is None and external_identity_present:
live["owner_id"] = user_id
live["owner_chat_id"] = _owner_binding_chat_id(ctx, chat_id, is_external_transport)
live["last_owner_message_at"] = now_iso
ctx.update_state(_stamp_owner_activity)
if not text and not image_base64:
continue
if is_external_transport and is_slash_command:
if not external_identity_present:
ctx.send_with_budget(chat_id, "⚠️ Command ignored: this transport did not provide owner identity.")
continue
owner_ext_id = st.get("owner_external_id")
owner_ext_chat_id = st.get("owner_external_chat_id")
if owner_ext_id is None:
def _bind_external_owner(live: dict) -> None:
if live.get("owner_external_id") is None:
live["owner_external_id"] = user_id
live["owner_external_chat_id"] = chat_id
live["owner_external_bound_at"] = now_iso
ctx.update_state(_bind_external_owner)
ctx.send_with_budget(chat_id, "✅ Owner chat registered. Send the command again to execute it.")
continue
try:
owner_ext_id_int = int(owner_ext_id or 0)
owner_ext_chat_id_int = int(owner_ext_chat_id or 0)
except (TypeError, ValueError):
owner_ext_id_int = 0
owner_ext_chat_id_int = 0
if owner_ext_id_int != user_id or owner_ext_chat_id_int != chat_id:
ctx.send_with_budget(chat_id, "⚠️ Command ignored: this transport is not the bound owner chat.")
continue
if lowered.startswith("/panic"):
ctx.send_with_budget(chat_id, "🛑 PANIC: killing everything. App will close.")
_execute_panic_stop(ctx.consciousness, ctx.kill_workers)
elif lowered.startswith("/restart"):
ctx.send_with_budget(chat_id, "♻️ Restarting.")
ok, restart_msg = _safe_restart_serialized(
ctx.safe_restart,
reason="owner_restart",
unsynced_policy="rescue_and_reset",
)
if not ok:
ctx.send_with_budget(chat_id, f"⚠️ Restart cancelled: {restart_msg}")
continue
state_dir = DATA_DIR / "state"
owner_restart_flag = state_dir / "owner_restart_no_resume.flag"
stable_skip_flag = state_dir / "panic_stop.flag"
try:
state_dir.mkdir(parents=True, exist_ok=True)
owner_restart_flag.write_text("owner_restart", encoding="utf-8")
# Pair owner flag with panic_stop for stable-build auto-resume compatibility.
stable_skip_flag.write_text("owner_restart_no_resume", encoding="utf-8")
except Exception:
owner_restart_flag.unlink(missing_ok=True)
stable_skip_flag.unlink(missing_ok=True)
log.warning("Failed to write owner restart no-resume flag", exc_info=True)
ctx.send_with_budget(chat_id, "⚠️ Restart cancelled: could not write restart state.")
continue
try:
ctx.kill_workers(
force=True,
terminal_status="cancelled",
result_reason="Owner restart stopped this task before process restart.",
**_managed_update_pending_kwargs(),
)
except Exception:
owner_restart_flag.unlink(missing_ok=True)
stable_skip_flag.unlink(missing_ok=True)
log.warning("Restart cancelled because worker shutdown failed", exc_info=True)
try:
ctx.send_with_budget(chat_id, "⚠️ Restart cancelled: failed to stop workers.")
except Exception:
pass
continue
try:
ctx.send_with_budget(chat_id, "Stopping active task. New settings apply to the next message.")
except Exception:
log.warning("Failed to send owner restart stop notice; continuing restart", exc_info=True)
_request_restart_exit()
elif lowered == "/review" or lowered.startswith("/review "):
# Target the requesting chat so the ack and results return to the
# external transport owner, not the default web owner_chat_id.
ctx.queue_deep_self_review_task(reason="owner:/review", force=True, chat_id=chat_id)
elif lowered.startswith("/evolve"):
parts = lowered.split()
action = parts[1] if len(parts) > 1 else "on"
turn_on = action not in ("off", "stop", "0")
objective = ""
if turn_on and len(parts) > 2:
objective = text.split(None, 2)[2].strip()
if turn_on:
from supervisor.evolution_lifecycle import evolution_block_reason, start_evolution_campaign
block = evolution_block_reason()
if block:
ctx.send_with_budget(chat_id, block)
continue
try:
if not start_evolution_campaign(objective, source="owner_chat"):
raise RuntimeError("campaign write was refused")
except Exception:
log.warning("Failed to start evolution campaign", exc_info=True)
ctx.send_with_budget(chat_id, "⚠️ Evolution stayed OFF: campaign state could not be created.")
continue
st2 = ctx.load_state()
st2["evolution_mode_enabled"] = bool(turn_on)
if turn_on:
st2["evolution_consecutive_failures"] = 0
# Owner stop is AUTHORITATIVE against the post-task promotion pipeline: the
# durable evolution_owner_stopped flag (read by apply_pending_request) blocks an
# autonomous re-arm until the owner /evolve starts again. Set True on stop,
# cleared (False) on turn_on — the only owner-authorized clear.
st2["evolution_owner_stopped"] = (not turn_on)
# Owner-initiated evolution must not inherit a stale post-task one-shot
# autostop, which would disable the owner's campaign after one cycle.
st2["post_task_autostop"] = False
ctx.save_state(st2)
if not turn_on:
# Cancel the live evolution worker BEFORE the terminal campaign close below:
# complete_evolution_campaign runs the per-cycle worktree cleanup, which skips
# while a task still holds the shared worktree — so the running cycle must be
# gone first (pruning PENDING alone leaves a mid-cycle task running).
from supervisor.queue import cancel_running_evolution_tasks
from ouroboros.post_task_evolution import drop_pending_request
# Fast path: drop any queued post-task promotion so it cannot re-arm on
# the next boot tick (the evolution_owner_stopped flag is the durable backstop).
drop_pending_request(ctx.DRIVE_ROOT)
cancelled = cancel_running_evolution_tasks("disabled via owner chat")
ctx.PENDING[:] = [t for t in ctx.PENDING if str(t.get("type")) != "evolution"]
ctx.sort_pending()
ctx.persist_queue_snapshot(reason="evolve_off")
if cancelled:
ctx.send_with_budget(
chat_id,
f"🛑 Cancelled running evolution task(s): {', '.join(cancelled)}",
)
try:
from supervisor.evolution_lifecycle import complete_evolution_campaign
if not turn_on:
# Terminal close (not a resumable pause): /evolve start mints a FRESH
# campaign rather than resurrecting this one.
complete_evolution_campaign("disabled via owner chat", status="stopped")
except Exception:
log.warning("Failed to update evolution campaign state", exc_info=True)
_evo_msg = "ON" if turn_on else "OFF — post-task auto-evolution also paused until /evolve start"
ctx.send_with_budget(chat_id, f"🧬 Evolution campaign: {_evo_msg}")
elif lowered.startswith("/bg"):
parts = lowered.split()
action = parts[1] if len(parts) > 1 else "status"
if action in ("start", "on", "1"):
result = ctx.consciousness.start()
_bg_s = ctx.load_state()
_bg_s["bg_consciousness_enabled"] = True
ctx.save_state(_bg_s)
ctx.send_with_budget(chat_id, f"🧠 {result}")
elif action in ("stop", "off", "0"):
result = ctx.consciousness.stop()
_bg_s = ctx.load_state()
_bg_s["bg_consciousness_enabled"] = False
ctx.save_state(_bg_s)
ctx.send_with_budget(chat_id, f"🧠 {result}")
else:
bg_status = "running" if ctx.consciousness.is_running else "stopped"
ctx.send_with_budget(chat_id, f"🧠 Background consciousness: {bg_status}")
elif lowered.startswith("/status"):
from supervisor.state import status_text
from supervisor.queue import SOFT_TIMEOUT_SEC, HARD_TIMEOUT_SEC
status = status_text(ctx.WORKERS, ctx.PENDING, ctx.RUNNING, SOFT_TIMEOUT_SEC, HARD_TIMEOUT_SEC)
ctx.send_with_budget(chat_id, status)
else:
_route_owner_message(
bridge,
ctx,
{
"chat_id": chat_id,
"text": text,
"image_caption": image_caption,
"client_message_id": client_message_id,
"image_data": image_data,
"task_constraint": task_constraint,
"task_metadata": task_metadata,
"log_text": log_text,
"origin_message_ref": origin_message_ref,
},
)
return offset
def _runtime_branch_defaults() -> tuple[str, str]:
branch_dev = "ouroboros"
branch_stable = "ouroboros-stable"
if not _LAUNCHER_MANAGED:
return branch_dev, branch_stable
try:
from supervisor import git_ops as git_ops_module
if hasattr(git_ops_module, "managed_branch_defaults"):
return git_ops_module.managed_branch_defaults(REPO_DIR)
except Exception:
pass
return branch_dev, branch_stable
def _bootstrap_supervisor_repo(settings: dict, git_ops_module=None):
if git_ops_module is None:
from supervisor import git_ops as git_ops_module
branch_dev, branch_stable = _runtime_branch_defaults()
git_ops_module.init(
repo_dir=REPO_DIR,
drive_root=DATA_DIR,
remote_url="",
branch_dev=branch_dev,
branch_stable=branch_stable,
)
git_ops_module.ensure_repo_present()
setup_remote_if_configured(settings, log)
if _LAUNCHER_MANAGED:
# An in-flight managed-update assisted merge intentionally leaves MERGE_HEAD + the partly
# resolved merge in the live worktree (over pre_update_sha). Use the NON-destructive
# rescue_and_block policy so the bootstrap restart does not reset/clean that merge state
# away before finalize_managed_update_on_boot / _recover_assisted_on_boot can resume it.
try:
from supervisor.update_merge import active_update_tx
_managed_update_active = bool(active_update_tx())
except Exception:
_managed_update_active = False
block = _has_active_evolution_transaction() or _managed_update_active
policy = "rescue_and_block" if block else "rescue_and_reset"
ok, msg = _safe_restart_serialized(
git_ops_module.safe_restart,
reason="bootstrap",
unsynced_policy=policy,
)
if not ok and policy == "rescue_and_block":
try:
from supervisor.evolution_lifecycle import pause_evolution_campaign
from supervisor.state import load_state, save_state
st = load_state()
st["evolution_mode_enabled"] = False
save_state(st)
pause_evolution_campaign(f"bootstrap blocked to protect active evolution transaction: {msg}")
except Exception:
log.debug("Failed to pause evolution after blocked bootstrap", exc_info=True)
return ok, msg
log.info("Local-dev server start detected — skipping bootstrap git reset.")
deps_ok, deps_msg = git_ops_module.sync_runtime_dependencies(reason="bootstrap_local_dev")
if not deps_ok:
return False, f"Failed local-dev deps sync: {deps_msg}"
import_result = git_ops_module.import_test()
if import_result.get("ok"):
return True, "OK: local-dev bootstrap"
return False, f"Local-dev import test failed (rc={import_result.get('returncode', -1)})"
def _periodic_zombie_reconcile() -> None:
"""Heal zombie 'running' records on a supervisor cadence.
A worker that died mid-review (crash / SIGKILL / manual stop) leaves
``review_job.json`` at status=running forever in headless/no-UI runs, where
the boot and ``GET /api/extensions`` reconciles never fire; the same death
leaves ``task_results/<id>.json`` at running. Both reconciles are
liveness-gated (pid-dead / queue-empty + worker-boot evidence), so a live
review or task is never touched.
"""
try:
from ouroboros.skill_review_runner import reconcile_stale_review_jobs
reconcile_stale_review_jobs(DATA_DIR)
except Exception:
log.debug("Periodic skill review-job reconcile failed", exc_info=True)
try:
from ouroboros.task_status import reconcile_orphaned_running_tasks
reconcile_orphaned_running_tasks(DATA_DIR)
except Exception:
log.debug("Periodic orphaned running-task reconcile failed", exc_info=True)
try:
from ouroboros.projects_registry import reconcile_projects
reconcile_projects(DATA_DIR)
except Exception:
log.debug("Project registry reconcile failed", exc_info=True)
_resume_interrupted_project_deletions()
def _resume_interrupted_project_deletions() -> None:
try:
from supervisor.task_lifecycle import resume_project_deletions
resume_project_deletions(DATA_DIR)
except Exception:
log.debug("Project deletion recovery failed", exc_info=True)
def _run_supervisor(settings: dict) -> None:
"""Initialize and run the supervisor loop. Called in a background thread."""
global _supervisor_error, _supervisor_thread, _consciousness
_apply_settings_to_env(settings)
# Revival must drop the prior consciousness and cached event-queue binding.
if _consciousness is not None:
try:
_consciousness.stop()
except Exception:
log.debug("Failed to stop previous consciousness instance", exc_info=True)
_consciousness = None
try:
from supervisor import workers as _workers_mod
_workers_mod._chat_agent = None
except Exception:
log.debug("Failed to reset cached chat agent", exc_info=True)
try:
ensure_legacy_imported(pathlib.Path(DATA_DIR))
from supervisor.message_bus import init as bus_init
from supervisor.message_bus import LocalChatBridge
bridge = LocalChatBridge(settings)
bridge._broadcast_fn = broadcast_ws_sync
from ouroboros.utils import set_log_sink
set_log_sink(bridge.push_log)
bus_init(
drive_root=DATA_DIR,
total_budget_limit=float(settings.get("TOTAL_BUDGET", 10.0)),
budget_report_every=10,
chat_bridge=bridge,
)
from supervisor.state import init as state_init, init_state, load_state, save_state, update_state
from supervisor.state import append_jsonl, update_budget_from_usage, rotate_chat_log_if_needed
state_init(DATA_DIR, float(settings.get("TOTAL_BUDGET", 10.0)))
init_state()
from supervisor.git_ops import safe_restart
ok, msg = _bootstrap_supervisor_repo(settings)
if not ok:
log.error("Supervisor bootstrap failed: %s", msg)
from supervisor.queue import (
enqueue_task, enforce_task_timeouts, enqueue_evolution_task_if_needed,
persist_queue_snapshot, restore_pending_from_snapshot,
cancel_task_by_id, queue_deep_self_review_task, sort_pending,
)
from supervisor.workers import (
init as workers_init, get_event_q, WORKERS, PENDING, RUNNING,
spawn_workers, kill_workers, assign_tasks, ensure_workers_healthy,
handle_chat_direct, handle_chat_ephemeral, _get_chat_agent, auto_resume_after_restart,
)
max_workers = int(settings.get("OUROBOROS_MAX_WORKERS", 10))
soft_timeout = int(settings.get("OUROBOROS_SOFT_TIMEOUT_SEC", 600))
hard_timeout = int(settings.get("OUROBOROS_HARD_TIMEOUT_SEC", 1800))
# Managed manifest branch defaults must drive worker commit/restart flows too.
_workers_branch_dev, _workers_branch_stable = _runtime_branch_defaults()
workers_init(
repo_dir=REPO_DIR, drive_root=DATA_DIR, max_workers=max_workers,
soft_timeout=soft_timeout, hard_timeout=hard_timeout,
total_budget_limit=float(settings.get("TOTAL_BUDGET", 10.0)),
branch_dev=_workers_branch_dev, branch_stable=_workers_branch_stable,
)
from supervisor.events import dispatch_event
from supervisor.message_bus import send_with_budget
from ouroboros.consciousness import BackgroundConsciousness
import types
import queue as _queue_mod
restored_pending = restore_pending_from_snapshot()
kill_workers(preserve_pending=True)
spawn_workers(max_workers)
persist_queue_snapshot(reason="startup")
_resume_interrupted_project_deletions()
try:
from ouroboros.headless import prune_headless_task_drives, prune_task_drives, prune_task_trees
from ouroboros.utils import sweep_stale_temp_files
prune_report = prune_headless_task_drives(DATA_DIR)
task_drive_report = prune_task_drives(DATA_DIR)
# Ephemeral task-tree coordination ledgers age out with their terminal root.
prune_task_trees(DATA_DIR)
# Reap orphaned atomic-write temp files (.*.tmp.*) left by a hard kill.
sweep_stale_temp_files(DATA_DIR)
if (
prune_report.get("pruned")
or prune_report.get("errors")
or task_drive_report.get("pruned")
or task_drive_report.get("errors")
):
append_jsonl(DATA_DIR / "logs" / "events.jsonl", {
"ts": utc_now_iso(),
"type": "headless_task_drive_prune",
"report": prune_report,
"task_drives": task_drive_report,
})
except Exception:
log.debug("Headless task drive prune failed", exc_info=True)
try:
from ouroboros.process_custody import reap_orphaned_processes
reaped = reap_orphaned_processes(DATA_DIR, live_owner_skills=_installed_skill_names())
if reaped:
log.info("Process custody reaper killed %d orphaned process(es): %s", len(reaped), reaped)
except Exception:
log.debug("Process custody startup reap failed", exc_info=True)
try:
from ouroboros import subagent_worktrees
worktree_report = subagent_worktrees.prune_orphans()
if worktree_report.get("removed"):
append_jsonl(DATA_DIR / "logs" / "events.jsonl", {
"ts": utc_now_iso(),
"type": "subagent_worktree_prune",
"report": worktree_report,
})
except Exception:
log.debug("Subagent worktree prune failed", exc_info=True)
try:
from ouroboros.observability import prune_observability_blobs
from ouroboros.tools.services import prune_service_logs
observability_report = prune_observability_blobs(DATA_DIR)
service_report = prune_service_logs(DATA_DIR)
if (
observability_report.get("enabled")
or observability_report.get("manifest_count")
or observability_report.get("blob_count")
or observability_report.get("deleted_manifests")
or observability_report.get("deleted_blobs")
or observability_report.get("errors")
or service_report.get("deleted_dirs")
or service_report.get("deleted_files")
or service_report.get("errors")
):
append_jsonl(DATA_DIR / "logs" / "events.jsonl", {
"ts": utc_now_iso(),
"type": "runtime_artifact_prune",
"observability": observability_report,
"services": service_report,
})
except Exception:
log.debug("Runtime artifact prune failed", exc_info=True)
if restored_pending > 0:
st_boot = load_state()
if st_boot.get("owner_chat_id"):
send_with_budget(int(st_boot["owner_chat_id"]),
f"♻️ Restored pending queue from snapshot: {restored_pending} tasks.")
auto_resume_after_restart()
def _get_owner_chat_id() -> Optional[int]:
try:
st = load_state()
cid = st.get("owner_chat_id")
return int(cid) if cid else None
except Exception:
return None
_consciousness = BackgroundConsciousness(
drive_root=DATA_DIR, repo_dir=REPO_DIR,
event_queue=get_event_q(), owner_chat_id_fn=_get_owner_chat_id,
)
_bg_st = load_state()
if _bg_st.get("bg_consciousness_enabled"):
_consciousness.start()
log.info("Background consciousness auto-restored from saved state.")
branch_dev, branch_stable = _runtime_branch_defaults()
_event_ctx = types.SimpleNamespace(
DRIVE_ROOT=DATA_DIR, REPO_DIR=REPO_DIR,
BRANCH_DEV=branch_dev, BRANCH_STABLE=branch_stable,
bridge=bridge, WORKERS=WORKERS, PENDING=PENDING, RUNNING=RUNNING,
MAX_WORKERS=max_workers,
send_with_budget=send_with_budget, load_state=load_state, save_state=save_state,
update_state=update_state,
update_budget_from_usage=update_budget_from_usage, append_jsonl=append_jsonl,
enqueue_task=enqueue_task, cancel_task_by_id=cancel_task_by_id,
queue_deep_self_review_task=queue_deep_self_review_task, persist_queue_snapshot=persist_queue_snapshot,
safe_restart=safe_restart, kill_workers=kill_workers, spawn_workers=spawn_workers,
sort_pending=sort_pending, consciousness=_consciousness,
soft_timeout=soft_timeout, hard_timeout=hard_timeout,
get_chat_agent=_get_chat_agent, handle_chat_direct=handle_chat_direct,
handle_chat_ephemeral=handle_chat_ephemeral, request_restart=_request_restart_exit,
)
except Exception as exc:
_supervisor_error = f"Supervisor init failed: {exc}"
_consciousness = None
log.critical("Supervisor initialization failed", exc_info=True)
_supervisor_ready.set()
_supervisor_thread = None
return
_supervisor_ready.set()
log.info("Supervisor ready.")
offset = 0
crash_count = 0
_last_custody_reap = [time.time()]
_last_review_job_reconcile = [time.time()]
# WS3: a dedicated watchdog thread (outside this loop, so it fires even if the
# loop stalls) surfaces a wedge as an observable signal + owner alert instead
# of silent hours; the loop publishes a liveness tick each iteration.
_loop_liveness = [time.time()]
_watchdog_stop = threading.Event() # per-generation: stops the watchdog when THIS loop exits
_start_supervisor_liveness_watchdog(_loop_liveness, _watchdog_stop)
while not _restart_requested.is_set():
try:
_loop_liveness[0] = time.time()
rotate_chat_log_if_needed(DATA_DIR)
ensure_workers_healthy()
event_q = get_event_q()
while True:
try:
evt = event_q.get_nowait()
except _queue_mod.Empty:
break
if evt.get("type") == "restart_request":
_handle_restart_in_supervisor(evt, _event_ctx)
continue
dispatch_event(evt, _event_ctx)
if _restart_requested.is_set():
break
# WS3: intake new bridge messages EARLY — before the heavy steps
# (enforce_task_timeouts / assign_tasks / evolution) — so a later
# blocking step can never starve new-message intake (the wedge class
# where no task_received fired for hours until a full restart).
offset = _process_bridge_updates(bridge, offset, _event_ctx)
enforce_task_timeouts()
try:
from supervisor.queue import check_scheduled_tasks
check_scheduled_tasks()
except Exception:
log.warning("Scheduled task check failed", exc_info=True)
_periodic_supervisor_maintenance(_last_custody_reap, _last_review_job_reconcile)
# Loop-tick restart drain (no sleep, events keep flowing): while
# draining a deferred restart, skip starting new work the restart
# deadline would immediately chop (evolution / pending project tasks).
if not _check_pending_restart_drain(_event_ctx):
try:
from ouroboros.post_task_evolution import apply_pending_request
from supervisor import state as _pte_state
apply_pending_request(_pte_state.DRIVE_ROOT)
except Exception:
log.debug("Post-task evolution apply failed", exc_info=True)
enqueue_evolution_task_if_needed()
assign_tasks()
if _restart_requested.is_set():
break # restart just triggered (drain done) — exit without assigning new work (bridge intake already ran early this iteration)
persist_queue_snapshot(reason="main_loop")
crash_count = 0
time.sleep(0.5)
except Exception as exc:
crash_count += 1
log.error("Supervisor loop crash #%d: %s", crash_count, exc, exc_info=True)
if crash_count >= 3:
# Visible death: previously the loop returned with
# _supervisor_ready still set and no _supervisor_error, so
# tasks silently stopped being assigned with a healthy-looking
# /api/state. Record the failure and tell the owner.
_supervisor_error = f"Supervisor loop died after 3 consecutive crashes: {exc}"
_supervisor_ready.clear()
log.critical("Supervisor exceeded max retries: %s", _supervisor_error)
try:
st = load_state()
if st.get("owner_chat_id"):
send_with_budget(
int(st["owner_chat_id"]),
"🛑 Supervisor loop died after repeated crashes; tasks are no "
"longer being assigned. Saving settings or restarting the app "
f"will revive it. Last error: {exc}",
)
except Exception:
log.debug("Failed to notify owner about supervisor death", exc_info=True)
_watchdog_stop.set() # this generation is dead — stop its liveness watchdog
return
time.sleep(min(30, 2 ** crash_count))
_watchdog_stop.set() # loop exited (restart) — stop this generation's watchdog
_supervisor_thread = None
# Deferred restart-drain state (multi-project, v6.32.0). The drain MUST NOT
# sleep on the supervisor loop thread (it is the only thread that processes
# heartbeats / task_done and shrinks RUNNING). Instead a restart with live
# tasks is recorded here and re-checked every loop tick, so events keep
# flowing and the drain actually observes tasks finishing.
_pending_restart: Dict[str, Any] = {}
def _live_running_task_ids(ctx: Any) -> list:
"""RUNNING task ids with a fresh heartbeat — structured facts only.
Heartbeat staleness belongs to the generic supervisor queue, not to the
planning-scout wait policy. The latter intentionally waits until terminal
state or its shared cutoff even when a scout heartbeat is stale.
"""
from supervisor.queue import HEARTBEAT_STALE_SEC
now = time.time()
live = []
for tid, meta in dict(ctx.RUNNING or {}).items():
if not isinstance(meta, dict):
continue
try:
hb = float(meta.get("last_heartbeat_at") or 0.0)
except (TypeError, ValueError):
hb = 0.0
if hb and (now - hb) < HEARTBEAT_STALE_SEC:
live.append(str(tid))
return live
def _handle_restart_in_supervisor(evt: Dict[str, Any], ctx: Any) -> None:
"""Handle agent restart request: drain live tasks across loop ticks, then
graceful shutdown + exit(42). Never sleeps on the dispatch thread."""
st = ctx.load_state()
if st.get("owner_chat_id"):
ctx.send_with_budget(
int(st["owner_chat_id"]),
f"♻️ Restart requested by agent: {evt.get('reason')}",
)
from ouroboros.config import get_restart_drain_max_sec
max_wait = get_restart_drain_max_sec()
live = _live_running_task_ids(ctx) if max_wait > 0 else []
if live:
# Defer: re-checked each tick by _check_pending_restart_drain so the
# loop keeps draining events (heartbeats advance, RUNNING shrinks).
_pending_restart.clear()
_pending_restart.update({
"reason": str(evt.get("reason") or "agent_restart_request"),
"deadline": time.time() + min(max_wait, 1800),
"evolution_restart": bool(evt.get("evolution_restart")),
})
if st.get("owner_chat_id"):
ctx.send_with_budget(
int(st["owner_chat_id"]),
f"⏳ Restart drain: waiting up to {max_wait}s for running task(s) "
f"{', '.join(sorted(live))} to finish.",
)
return
_perform_supervisor_restart(
ctx, restart_reason=str(evt.get("reason") or "agent_restart_request"),
evolution_restart=bool(evt.get("evolution_restart")),
)
def _check_pending_restart_drain(ctx: Any) -> bool:
"""Loop-tick hook: complete a deferred restart once tasks drain or the
deadline passes (proceeds fail-closed). Returns True while STILL draining, so
the loop can skip starting new work that the restart would immediately chop."""
if not _pending_restart:
return False
live = _live_running_task_ids(ctx)
if live and time.time() < float(_pending_restart.get("deadline") or 0.0):
return True # keep draining — events still flow each tick
pending = dict(_pending_restart)
_pending_restart.clear()
_perform_supervisor_restart(
ctx, restart_reason=str(pending.get("reason") or "agent_restart_request"),
evolution_restart=bool(pending.get("evolution_restart")),
)
# Still "quiescing" this tick: _perform_supervisor_restart sets up the exit
# (or fail-closed pauses) and returns to the loop — the process exits on the
# next `while not _restart_requested` check. Returning True keeps the caller
# from starting new enqueue/assign work on this final pre-exit tick.
return True
def _perform_supervisor_restart(
ctx: Any, *, restart_reason: str = "agent_restart_request",
evolution_restart: bool = False,
) -> None:
"""Graceful shutdown + exit(42) (the post-drain tail; never sleeps)."""
st = ctx.load_state()
marker = read_json_dict(
pathlib.Path(ctx.DRIVE_ROOT) / "state" / "pending_restart_verify.json"
) or {}
claim = (
marker.get("evolution_claim")
if evolution_restart and marker.get("reason") == restart_reason
else {}
)
claim = claim if isinstance(claim, dict) else {}
if evolution_restart and not claim:
if st.get("owner_chat_id"):
ctx.send_with_budget(
int(st["owner_chat_id"]),
"🧬 Restart cancelled: the exact evolution restart receipt is missing.",
)
return
if claim:
from supervisor.evolution_lifecycle import check_evolution_authority
authority = check_evolution_authority(
str(claim.get("campaign_id") or ""),
str(claim.get("transaction_id") or ""),
str(claim.get("task_id") or ""),
commit_sha=str(claim.get("commit_sha") or ""),
)
if not authority.get("ok"):
if st.get("owner_chat_id"):
ctx.send_with_budget(
int(st["owner_chat_id"]),
"🧬 Restart cancelled: evolution authority changed "
f"({authority.get('reason') or 'unknown'}).",
)
return
expected_sha = str(claim.get("commit_sha") or "")
try:
head_proc = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=str(ctx.REPO_DIR),
check=False,
capture_output=True,
text=True,
)
status_proc = subprocess.run(
["git", "status", "--porcelain"],
cwd=str(ctx.REPO_DIR),
check=False,
capture_output=True,
text=True,
)
head = head_proc.stdout.strip() if head_proc.returncode == 0 else ""
clean = status_proc.returncode == 0 and not status_proc.stdout.strip()
except Exception:
head = ""
clean = False
if not expected_sha or head != expected_sha or not clean:
if st.get("owner_chat_id"):
ctx.send_with_budget(
int(st["owner_chat_id"]),
"🧬 Restart cancelled: the live checkout no longer matches "
"the exact reviewed evolution commit.",
)
return
ok, msg = _safe_restart_serialized(
ctx.safe_restart,
reason="agent_restart_request",
unsynced_policy="rescue_and_block",
)
if not ok:
try:
from supervisor.evolution_lifecycle import pause_evolution_campaign
st["evolution_mode_enabled"] = False
ctx.save_state(st)
pause_evolution_campaign(f"agent restart blocked to protect local changes: {msg}")
except Exception:
log.debug("Failed to pause evolution after blocked agent restart", exc_info=True)
if st.get("owner_chat_id"):
ctx.send_with_budget(int(st["owner_chat_id"]), f"⚠️ Restart skipped: {msg}")
return
cleanup_status, cleanup_reason = _shutdown_task_cleanup_args(restart_requested=True)
ctx.kill_workers(
force=True,
terminal_status=cleanup_status,
result_reason=cleanup_reason,
**_managed_update_pending_kwargs(),
)
st2 = ctx.load_state()
st2["session_id"] = uuid.uuid4().hex
ctx.save_state(st2)
ctx.persist_queue_snapshot(reason="pre_restart_exit")
_request_restart_exit()
def _request_restart_exit() -> None:
"""Signal server shutdown with restart exit code."""
_restart_requested.set()
def _managed_update_pending_kwargs() -> dict:
"""Preserve queued work while a durable tx or its pre-tx quiesce owns restart."""
try:
from supervisor.update_merge import active_update_tx
if active_update_tx():
return {"preserve_pending": True}
from supervisor.workers import repo_writer_admission_closed, worker_pool_admission_state
gate = repo_writer_admission_closed()
disabled = str(worker_pool_admission_state().get("disabled_reason") or "")
if gate.startswith("managed_update:") or disabled == "managed_update":
return {"preserve_pending": True}
return {}
except Exception:
return {"preserve_pending": True}
def _safe_restart_serialized(safe_restart_fn, *, reason: str, unsynced_policy: str):
"""Serialize checkout/reset with update apply; only a landed update may restart."""
from supervisor import git_ops
from supervisor.update_merge import (
acquire_update_lock,
read_update_tx_strict,
release_update_lock,
)
try:
lock_fh = acquire_update_lock()
except RuntimeError:
return False, "Managed update is changing the checkout; restart was deferred."
try:
status, tx = read_update_tx_strict()
if status == "corrupt":
return False, "Managed update state is unreadable; restart was deferred."
if status == "absent" and not git_ops._clear_update_intent():
return False, (
"An update intent marker with no update transaction could not be removed; "
"restart was deferred rather than applying an orphaned update."
)
allowed_phases = {"pending_boot_smoke", "applying_replace"}
if status == "valid" and str(tx.get("phase") or "") not in allowed_phases:
return False, "Managed update merge is still being resolved; restart was deferred."
return safe_restart_fn(reason=reason, unsynced_policy=unsynced_policy)
finally:
release_update_lock(lock_fh)
def _wait_for_supervisor_update_finalize() -> bool:
"""Wait for a real init outcome; slow dependency sync is not a failed boot."""
_supervisor_ready.wait()
return not bool(_supervisor_error)
def _boot_managed_update_tasks() -> None:
"""Finalize a pending update, restart after rollback, then refresh its feed."""
try:
from supervisor.git_ops import compute_managed_update_status
from supervisor.update_merge import finalize_managed_update_on_boot
result = finalize_managed_update_on_boot(
supervisor_ready=_wait_for_supervisor_update_finalize()
)
stash_note = str(result.get("stash_note") or "")
if stash_note:
# Q1=C disclosure contract: a stash restore that conflicted keeps the
# entry and the OWNER must see the exact recovery command, not only
# the supervisor log.
try:
from supervisor.message_bus import send_with_budget
from supervisor.state import load_state as _load_state
owner_chat = int((_load_state() or {}).get("owner_chat_id") or 0)
if owner_chat:
send_with_budget(owner_chat, f"📦 Managed update: {stash_note}")
except Exception:
log.debug("stash note owner notification failed", exc_info=True)
if result.get("rolled_back") is True:
# This generation imported the rejected candidate. Preserve queued roots
# through shutdown, then exec the restored code instead of limping on.
from supervisor.workers import close_repo_writer_admission
close_repo_writer_admission("managed_update:rollback_restart")
_request_restart_exit()
return
update_status = compute_managed_update_status(fetch=True)
broadcast_ws_sync({
"type": "update_status_ready",
"available": bool(update_status.get("available")),
"check_ok": update_status.get("check_ok"),
})
except Exception:
log.debug("boot managed-update tasks failed", exc_info=True)
def _shutdown_task_cleanup_args(restart_requested: bool) -> tuple[str, str]:
"""Return ``(terminal_status, result_reason)`` for tasks torn down by a
graceful server shutdown.
A graceful shutdown — a requested restart (exit 42) or an external
stop/restart signal (SIGTERM/SIGINT) — is not a worker crash storm, so a
still-running task is finalized as ``cancelled`` with an honest reason
instead of the default crash-storm text the supervisor uses for real
worker deaths.
"""
if restart_requested:
reason = (
"Server restarted before this task finished; the task was "
"interrupted by the restart, not a worker crash."
)
else:
reason = (
"Server shut down (external stop/restart signal) before this task "
"finished; the task was interrupted, not a worker crash."
)
return "cancelled", reason
def _shutdown_supervisor_event_bus() -> None:
try:
from supervisor.workers import shutdown_event_q
shutdown_event_q()
except Exception:
pass
def _execute_panic_stop(consciousness, kill_workers_fn) -> None:
_execute_panic_stop_impl(
consciousness,
kill_workers_fn,
data_dir=DATA_DIR,
panic_exit_code=PANIC_EXIT_CODE,
log=log,
)
APP_START = time.time()
def _sync_gateway_settings_module() -> None:
"""Keep legacy server.* monkeypatch tests wired to gateway.settings."""
_gateway_settings.load_settings = load_settings
_gateway_settings.save_settings = save_settings
_gateway_settings._apply_settings_to_env = _apply_settings_to_env
_gateway_settings.apply_runtime_provider_defaults = apply_runtime_provider_defaults
async def api_settings_get(request):
_sync_gateway_settings_module()
return await _gateway_settings.api_settings_get(request)
async def api_settings_post(request):
_sync_gateway_settings_module()
return await _gateway_settings.api_settings_post(request)
web_dir = resolve_web_dir(REPO_DIR)
web_dir.mkdir(parents=True, exist_ok=True)
index_page = make_index_page(web_dir)
routes = [
Route("/", endpoint=index_page),
*collect_routes(
data_dir=DATA_DIR,
settings_handlers={
"api_onboarding": _gateway_settings.api_onboarding,
"api_claude_code_status": _gateway_settings.api_claude_code_status,
"api_claude_code_install": _gateway_settings.api_claude_code_install,
"api_settings_get": api_settings_get,
"api_settings_post": api_settings_post,
},
),
Mount("/static", app=NoCacheStaticFiles(directory=str(web_dir)), name="static"),
]
from contextlib import asynccontextmanager, suppress
def _run_startup_task_recovery(
drive_root: pathlib.Path,
repo_dir: pathlib.Path,
*,
skip_live_data: bool,
) -> None:
"""Reconcile durable task phases once, after the prior process is gone."""
if skip_live_data:
return
try:
from ouroboros.task_status import reconcile_orphaned_running_tasks
reconcile_orphaned_running_tasks(drive_root)
except Exception:
log.warning("Orphaned running-task reconciliation at startup failed", exc_info=True)
try:
from ouroboros.agent_task_pipeline import recover_pending_root_post_task_synthesis
recover_pending_root_post_task_synthesis(drive_root, repo_dir)
except Exception:
log.warning("Root post-task synthesis recovery at startup failed", exc_info=True)
@asynccontextmanager
async def lifespan(app):
global _event_loop
_event_loop = asyncio.get_running_loop()
_set_ws_event_loop(_event_loop)
ws_heartbeat_task = asyncio.create_task(
ws_heartbeat_loop(_has_ws_clients, broadcast_ws),
name="ws-heartbeat",
)
settings, provider_defaults_changed, _provider_default_keys = apply_runtime_provider_defaults(load_settings())
if provider_defaults_changed:
save_settings(settings, allow_elevation=True)
_apply_settings_to_env(settings)
# Pin boot-time runtime-mode after env apply; save_settings compares to this owner baseline.
from ouroboros.config import initialize_runtime_mode_baseline
initialize_runtime_mode_baseline()
has_local = needs_local_model_autostart(settings)
lifespan_drive_root = pathlib.Path(
app.state.drive_root
if hasattr(app, "state") and hasattr(app.state, "drive_root")
else DATA_DIR
)
default_real_data_dir = pathlib.Path.home() / "Ouroboros" / "data"
pytest_default_real_data_dir = (
(bool(os.environ.get("PYTEST_CURRENT_TEST")) or "pytest" in sys.modules)
and lifespan_drive_root == default_real_data_dir
and not os.environ.get("OUROBOROS_DATA_DIR")
)
# Source-mode must seed native skills too, matching packaged launcher layout.
try:
if pytest_default_real_data_dir:
log.info("Skipping native skills bootstrap against real DATA_DIR during pytest")
else:
from ouroboros.launcher_bootstrap import ensure_data_skills_seeded
ensure_data_skills_seeded()
except Exception:
log.warning("Native skills bootstrap failed", exc_info=True)
# Boot-reconcile the project registry BEFORE /api/state and context-building
# can rely on registered_project_chat_ids (the multi-project isolation SSOT):
# register any pre-existing data/projects/<id>/ store whose row is missing, so
# an inherited project's raw chat is partitioned from turn one (not only after
# the 300s periodic tick). Idempotent and never prunes.
try:
if not pytest_default_real_data_dir:
from ouroboros.projects_registry import reconcile_projects
reconcile_projects(lifespan_drive_root)
except Exception:
log.warning("Project registry boot reconcile failed", exc_info=True)
if has_startup_ready_provider(settings):
_start_supervisor_if_needed(settings)
else:
_supervisor_ready.set()
log.info("No supported provider or local routing configured. Supervisor not started.")
# P2: finalize a pending managed merge update (post-boot smoke / boot-loop rollback)
# and run a one-shot boot-time update check (check-on-restart) so the main-screen
# Update badge reflects availability. Both run OFF the startup critical path and
# fail-soft — a missing managed remote / offline boot simply yields no badge.
threading.Thread(
target=_boot_managed_update_tasks, daemon=True, name="boot-managed-update",
).start()
if has_local and settings.get("LOCAL_MODEL_SOURCE"):
from ouroboros.local_model_autostart import auto_start_local_model
threading.Thread(
target=auto_start_local_model, args=(settings,),
daemon=True, name="local-model-autostart",
).start()
host_service_task = None
host_service_server = None
extension_reconcile_task = None
try:
from ouroboros.event_bus import init_global_event_bus
from ouroboros.extension_companion import init_global_supervisor
from ouroboros.gateway.host_service import (
DEFAULT_HOST_SERVICE_HOST,
create_host_service_app,
host_service_port,
)
init_global_event_bus().set_loop(_event_loop)
init_global_supervisor(lifespan_drive_root)
host_service_app = create_host_service_app(lifespan_drive_root)
host_port = host_service_port()
# Probe the port first: uvicorn's Server.startup() calls sys.exit(1) on a
# bind error, and SystemExit raised inside an asyncio task escapes
# run_forever and takes down the WHOLE main server (a stale prior
# instance still holding the port is exactly the realistic trigger).
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as _probe:
_probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
_probe.bind((DEFAULT_HOST_SERVICE_HOST, host_port))
except OSError as bind_exc:
raise RuntimeError(
f"Host Service port {host_port} is busy: {bind_exc}"
) from bind_exc
host_service_config = uvicorn.Config(
host_service_app,
host=DEFAULT_HOST_SERVICE_HOST,
port=host_port,
log_level="warning",
)
host_service_server = uvicorn.Server(host_service_config)
host_service_task = asyncio.create_task(
host_service_server.serve(),
name="host-service-api",
)
log.info("Host Service API listening on %s:%d", DEFAULT_HOST_SERVICE_HOST, host_port)
except Exception:
log.warning("Failed to start Host Service API", exc_info=True)
try:
from ouroboros.skill_review_runner import reconcile_stale_review_jobs
if pytest_default_real_data_dir:
log.info("Skipping stale skill-review reconciliation against real DATA_DIR during pytest")
else:
reconcile_stale_review_jobs(lifespan_drive_root)
except Exception:
log.warning("Stale skill-review reconciliation at startup failed", exc_info=True)
# Startup-only: after the prior process generation is gone, finalize orphaned
# RUNNING results and resolve an indeterminate post-task synthesis phase.
# The periodic zombie sweep intentionally does not perform this recovery.
_run_startup_task_recovery(
lifespan_drive_root,
REPO_DIR,
skip_live_data=pytest_default_real_data_dir,
)
# Reload enabled+reviewed extensions across restarts.
try:
from ouroboros.config import (
get_skills_repo_path,
load_settings as _load_settings,
)
from ouroboros.extension_loader import reload_all as _reload_extensions
from ouroboros.extension_loader import set_ws_broadcaster as _set_extension_ws_broadcaster
_set_extension_ws_broadcaster(broadcast_ws_sync)
repo_path = get_skills_repo_path()
if pytest_default_real_data_dir:
log.info("Skipping extension reload_all against real DATA_DIR during pytest")
else:
_reload_extensions(lifespan_drive_root, _load_settings, repo_path=repo_path or None)
except Exception:
log.error("Extension reload_all at startup failed", exc_info=True)
try:
from ouroboros.mcp_client import (
reconfigure_from_settings as _mcp_reconfigure_startup,
refresh_all_background as _mcp_refresh_background_startup,
)
_mcp_reconfigure_startup(settings)
_mcp_refresh_background_startup(reason="startup")
except Exception:
log.warning("MCP startup reconfigure failed", exc_info=True)
try:
from ouroboros.config import get_skills_repo_path
from ouroboros.config import load_settings as _load_settings
from ouroboros.extension_reconcile_queue import extension_reconcile_pickup_loop
if pytest_default_real_data_dir:
log.info("Skipping extension reconcile pickup against real DATA_DIR during pytest")
else:
extension_reconcile_task = asyncio.create_task(
extension_reconcile_pickup_loop(
lifespan_drive_root,
_load_settings,
repo_path_getter=lambda: get_skills_repo_path() or None,
),
name="extension-reconcile-pickup",
)
except Exception:
log.warning("Failed to start extension reconcile pickup task", exc_info=True)
try:
yield
finally:
if extension_reconcile_task is not None:
extension_reconcile_task.cancel()
with suppress(asyncio.CancelledError, asyncio.TimeoutError):
await asyncio.wait_for(extension_reconcile_task, timeout=30)
if host_service_server is not None:
try:
host_service_server.should_exit = True
except Exception:
pass
if host_service_task is not None:
with suppress(asyncio.CancelledError, asyncio.TimeoutError):
await asyncio.wait_for(host_service_task, timeout=5)
if not host_service_task.done():
host_service_task.cancel()
with suppress(asyncio.CancelledError, asyncio.TimeoutError):
await asyncio.wait_for(host_service_task, timeout=2)
ws_heartbeat_task.cancel()
with suppress(asyncio.CancelledError):
await ws_heartbeat_task
log.info("Server shutting down...")
try:
from ouroboros.local_model import get_manager
get_manager().stop_server()
except Exception:
pass
try:
from ouroboros.tools.shell import kill_all_tracked_subprocesses
kill_all_tracked_subprocesses()
except Exception:
pass
try:
from ouroboros.workspace_executor import kill_all_foreground
kill_all_foreground(lifespan_drive_root)
except Exception:
pass
try:
from ouroboros.tools.services import kill_all_services
kill_all_services(lifespan_drive_root)
except Exception:
pass
try:
from ouroboros.extension_companion import get_global_supervisor
supervisor = get_global_supervisor()
if supervisor is not None:
supervisor.stop_all()
except Exception:
pass
try:
restart_requested = _restart_requested.is_set()
# Record an explicit shutdown cause so a task interrupted by the
# shutdown is never later read as a worker crash storm.
try:
from ouroboros.utils import append_jsonl, utc_now_iso
append_jsonl(
lifespan_drive_root / "logs" / "supervisor.jsonl",
{
"ts": utc_now_iso(),
"type": "server_shutdown",
"cause": "restart_requested" if restart_requested else "external_signal",
"restart_exit": restart_requested,
},
)
except Exception:
log.debug("Failed to record server_shutdown event", exc_info=True)
from supervisor.workers import kill_workers
cleanup_status, cleanup_reason = _shutdown_task_cleanup_args(restart_requested)
kill_workers(
force=True,
terminal_status=cleanup_status,
result_reason=cleanup_reason,
**_managed_update_pending_kwargs(),
)
except Exception:
pass
try:
from supervisor.message_bus import get_bridge
get_bridge().shutdown()
except Exception:
pass
_shutdown_supervisor_event_bus()
app = NetworkAuthGate(Starlette(routes=routes, lifespan=lifespan))
app.app.state.drive_root = pathlib.Path(DATA_DIR) # type: ignore[attr-defined]
app.app.state.repo_dir = pathlib.Path(REPO_DIR) # type: ignore[attr-defined]
app.app.state.broadcast_ws_sync = broadcast_ws_sync # type: ignore[attr-defined]
app.app.state.app_start = APP_START # type: ignore[attr-defined]
app.app.state.supervisor_ready_event = _supervisor_ready # type: ignore[attr-defined]
app.app.state.get_supervisor_error = lambda: _supervisor_error # type: ignore[attr-defined]
app.app.state.describe_bg_consciousness_state = _describe_bg_consciousness_state # type: ignore[attr-defined]
app.app.state.request_restart = _request_restart_exit # type: ignore[attr-defined]
app.app.state.runtime_branch_defaults = _runtime_branch_defaults # type: ignore[attr-defined]
app.app.state.bind_host = _BIND_HOST # type: ignore[attr-defined]
app.app.state.port_file = PORT_FILE # type: ignore[attr-defined]
app.app.state.default_port = DEFAULT_PORT # type: ignore[attr-defined]
app.app.state.start_supervisor_if_needed = _start_supervisor_if_needed # type: ignore[attr-defined]
_ACTUAL_BOUND_PORT: Optional[int] = None
def _actual_bound_port() -> int:
"""Port the server actually bound (set in main(); DEFAULT_PORT before that)."""
return _ACTUAL_BOUND_PORT if _ACTUAL_BOUND_PORT else DEFAULT_PORT
def _emergency_process_cleanup(*, port_sweep: bool = True) -> None:
"""Kill child processes, workers, companions, and runtime port holders."""
try:
from ouroboros.tools.shell import kill_all_tracked_subprocesses
kill_all_tracked_subprocesses()
except Exception:
pass
try:
from ouroboros.workspace_executor import kill_all_foreground
kill_all_foreground(DATA_DIR, wait=False)
except Exception:
pass
try:
from ouroboros.tools.services import kill_all_services
kill_all_services(DATA_DIR, wait=False)
except Exception:
pass
try:
from supervisor.workers import kill_workers
if _restart_requested.is_set():
# A restart that hung past the uvicorn shutdown timeout still reaches
# here; finalize running tasks as an honest interrupted-by-restart,
# not a worker crash storm.
cleanup_status, cleanup_reason = _shutdown_task_cleanup_args(True)
kill_workers(
force=True,
archive_service_logs=False,
terminal_status=cleanup_status,
result_reason=cleanup_reason,
**_managed_update_pending_kwargs(),
)
else:
kill_workers(
force=True,
archive_service_logs=False,
**_managed_update_pending_kwargs(),
)
except Exception:
pass
import multiprocessing
from ouroboros.platform_layer import force_kill_pid, kill_process_on_port
for child in multiprocessing.active_children():
try:
force_kill_pid(child.pid)
except (ProcessLookupError, PermissionError):
pass
# Reap the Process object so it does not linger as a zombie / keep
# active_children non-empty if the main process exits before it dies.
try:
child.join(timeout=2)
except Exception:
pass
if port_sweep:
# Sweep the ACTUALLY bound port (find_free_port may have moved off
# DEFAULT_PORT); the old hardcoded 8765/8766 pair could kill an
# unrelated process on a custom-port install.
kill_process_on_port(_actual_bound_port())
try:
from ouroboros.extension_companion import panic_kill_all
from ouroboros.gateway.host_service import host_service_port
panic_kill_all()
if port_sweep:
kill_process_on_port(host_service_port())
except Exception:
pass
def main() -> int:
try:
saved_host = str(load_settings().get("OUROBOROS_SERVER_HOST") or "").strip()
except Exception:
saved_host = ""
default_host = os.environ.get("OUROBOROS_SERVER_HOST", "").strip() or saved_host or DEFAULT_HOST
args = parse_server_args(default_host, DEFAULT_PORT)
global _BIND_HOST
_BIND_HOST = args.host
app.app.state.bind_host = args.host # type: ignore[attr-defined]
auth_warning = get_network_auth_startup_warning(args.host)
if auth_warning:
log.warning(auth_warning)
auth_error = validate_network_auth_configuration(args.host)
if auth_error:
log.error(auth_error)
return 2
actual_port = find_free_port(args.host, args.port)
if actual_port != args.port:
log.info("Port %d busy on %s, using %d instead", args.port, args.host, actual_port)
global _ACTUAL_BOUND_PORT
_ACTUAL_BOUND_PORT = actual_port
write_port_file(PORT_FILE, actual_port)
log.info("Starting Ouroboros server on %s:%d", args.host, actual_port)
config = uvicorn.Config(
app,
host=args.host,
port=actual_port,
log_level="warning",
ws_ping_interval=20,
ws_ping_timeout=20,
)
server = uvicorn.Server(config)
_uvicorn_exited = threading.Event()
def _check_restart():
"""Monitor restart signal, then shut down uvicorn."""
while not _restart_requested.is_set():
time.sleep(0.5)
log.info("Restart requested — closing WebSocket clients and shutting down server.")
loop = _event_loop
if loop:
try:
future = asyncio.run_coroutine_threadsafe(close_all_ws(), loop)
future.result(timeout=3)
except Exception:
pass
server.should_exit = True
# Force-exit only if uvicorn never returns; direct-server mode needs cleanup/re-exec time.
force_exit_timeout_sec = 5 if _LAUNCHER_MANAGED else 30
if _uvicorn_exited.wait(timeout=force_exit_timeout_sec):
return
log.warning(
"Uvicorn did not exit within %ss — running emergency cleanup before os._exit(%d)",
force_exit_timeout_sec,
RESTART_EXIT_CODE,
)
_emergency_process_cleanup()
os._exit(RESTART_EXIT_CODE)
threading.Thread(target=_check_restart, daemon=True).start()
try:
server.run()
finally:
_uvicorn_exited.set()
if _restart_requested.is_set():
log.info("Exiting with code %d (restart signal).", RESTART_EXIT_CODE)
_emergency_process_cleanup(port_sweep=False)
if not _LAUNCHER_MANAGED:
_restart_current_process(args.host, actual_port)
os._exit(RESTART_EXIT_CODE)
return 0
if __name__ == "__main__":
sys.exit(main())