diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 193819ff2..a36438567 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -46,5 +46,5 @@ docker run --rm -p 8765:8765 \ ``` On a desktop install the same key lives in Settings → Advanced. It covers -Ouroboros's own HTTP clients only: `git`, `uv`, `pip`, the Claudexor engine and -the browsers keep their own trust stores. +Ouroboros's own HTTP clients only: `git`, `uv`, `pip`, the Claudexor engine, the +web-search scraper and the browsers keep their own trust stores. diff --git a/docs/PERSISTENCE.md b/docs/PERSISTENCE.md index 87941b801..2c8412010 100644 --- a/docs/PERSISTENCE.md +++ b/docs/PERSISTENCE.md @@ -59,7 +59,6 @@ scanned data-relative path to be covered by a row here (count-anchored both ways | `state/cancel_intents.json` | `ouroboros/cancel_intents.py` | own `schema_version: 1` | self-draining (settled rows leave) | in-flight cancels lost: cancelled-unsettled task revives as pending; forensics survive in supervisor.jsonl | | `state/update_letter.json` | `ouroboros/update_letter.py` (`refresh_after_check` — the ONE writer, synchronous inside a FETCHING update check: boot and the Updates panel's check button) | own record shape keyed by the update range (`base_sha`, `target_sha`, channel, target ref) — no `_schema_version` | overwrite per check; the letter is never deleted after the update lands (it outlives its range; `project_letter` reads `applied`/`other` by SHA equality + the recorded `target_in_head` fact) | absent = no letter: the Updates panel and the agent's Runtime context (`official_update_projection`) show the check's status alone; the next FETCHING check rewrites it | | `state/capability_evidence.json` | `ouroboros/capability_evidence.py` | none — accepted (self-healing cache; TTLs on read) | expired probe keys drop at the write seam: failed/unprobeable past their read TTL, confirmed past GC retention (blip-keep evidence survives inside retention); owner acks never expire | recreated; every reviewer window resolves `unknown` (sends are sized by the full-window assumption and re-probed), owner acks must be re-given; no review authority depends on it | -| `state/extra-ca-bundle.pem` | `ouroboros/net_transport.py` (`extra_ca_bundle`; tmp file + atomic replace) | none — derived (certifi followed by the owner's `OUROBOROS_EXTRA_CA_BUNDLE` PEM) | rewritten only when the merged content changes; never pruned | recreated on the next client construction; nothing beyond that process's TLS trust depends on it | | `state/evolution_campaign.json` | `supervisor/evolution_lifecycle.py` (CAS under state.lock) | own `schema_version: 1` (campaign) / 2 (active_transaction); the active transaction carries the pre-commit `commit_intent` (reviewed tree + parents) written BEFORE `git commit`, which boot recovery turns into the `commit_receipt` a crash never wrote | bounded histories (50) | in-flight self-modification transaction unabsorbable; anti-repeat fingerprints lost | | `state/evolution_metrics_cache.json` | `ouroboros/utils.py` | own `schema: 1` (strictly validated) | one point per git tag, no prune — accepted (derived cache) | pure cache; recomputed from git | | `state/projects.json`, `state/project_task_bindings.json` | `ouroboros/projects_registry.py` (sidecar locks) | `_schema_version: 2` / `1` (ABI-2) | never age-pruned (owner curates); deletes are durable tombstones | tombstones live here: losing it can resurrect deleted project rooms (marker unlink mitigates) | @@ -99,6 +98,7 @@ scanned data-relative path to be covered by a row here (count-anchored both ways | Path | Writer | schema_version | Retention | Reset | |---|---|---|---|---| +| `state/extra-ca-bundle/*.pem` | `ouroboros/net_transport.py` (`extra_ca_bundle`; tmp file + atomic replace) | none — derived (certifi followed by the owner's `OUROBOROS_EXTRA_CA_BUNDLE` PEM), each file named by the merged bytes' digest | a changed owner file writes a new file and removes the old ones | recreated on the next client construction; nothing beyond that process's TLS trust depends on it | | `state/skills//` owner state (`review.json`, `review_job.json`, `grants.json`, `enabled.json`, `deps.json`, `self_authored.json`, `owner_attestation.json`, `accepted_rebuttals.json`, `health.json`, `uninstalled.json`, provenance sidecars, `auto_repair.json`, `presence_profile_state.json`) | `ouroboros/skill_loader.py`, `skill_review_runner.py`, `skill_owner_attestation.py`, `skill_review_cycles.py`, `skill_uninstall_state.py`, `extension_health.py`, `marketplace/*`, `ouroboros/gateway/marketplace.py`; allowlist SSOT `contracts/skill_payload_policy.py` | `deps.json`/`self_authored.json`/provenance: `schema_version: 1`; `review.json`/`enabled.json`/`grants.json`/`review_job.json`/`owner_attestation.json`/`accepted_rebuttals.json`: `_schema_version: 1` (ABI-2, stamp-on-write — readers keep legacy-0 tolerance, unstamped files never retrofitted); verdict/grant staleness stays pinned by `content_hash` | no age GC; hub uninstalls write an `uninstalled.json` tombstone and the startup sweep clears the dead state BY that mark — `grants.json` survives as owner authority, a reinstall self-heals the tombstone; the gateway's local delete removes the whole state dir | absent state = disabled + pending review + grants revoked (fail-closed); `owner_attestation` absence invalidates its verdict | | `state/skills//review_history.jsonl` + `review_dispatch/` (legacy `review_dispatch.json`) | `ouroboros/skill_review_history.py` | rows carry `usage_attribution_schema: physical_attempt_v1`; no version key — accepted (derived-counter SSOT, P7) | history unbounded per skill — accepted with BOUNDED reads: every reader windows the 4 MB tail (`find_history_job_bounded` idiom); lifecycle terminal rows persist their ordinals so counters stay exact inside the window (a group aged past it restarts low — under-counts, never over-blocks); per-skill archive rotation declined (no per-skill archive plane; disclosed) | review-cycle ceiling resets to zero; paid dispatches become free again | | `state/delegate_project_retirements/.lock` | `ouroboros/delegate_custody_usage.py` (`project_retirement_lock`: exclusive file lock around one project's settlement/retirement decision; stale after 120 s, owner-aware) | none — not needed (lock file, no payload) | one file per project ever settled; reclaimed as stale by the next holder | delete freely; a live holder re-creates its lock | diff --git a/docs/architecture/01-high-level-architecture.md b/docs/architecture/01-high-level-architecture.md index 8879991e1..ea936056b 100644 --- a/docs/architecture/01-high-level-architecture.md +++ b/docs/architecture/01-high-level-architecture.md @@ -627,7 +627,7 @@ Bundled resources use the CLI / Headless Boundary lookup order rather than assum │ │ ├── usage_import_watermark.json ← resumable idempotent legacy-import watermark │ │ ├── request_wire_compatibility.json ← cross-process locked, schema-versioned 14-day exact-route wire evidence (request_wire_contract.py) │ │ ├── capability_evidence.json ← sourced model-capability evidence (capability_evidence.py) -│ │ ├── extra-ca-bundle.pem ← certifi plus the owner's `OUROBOROS_EXTRA_CA_BUNDLE` PEM, merged once and rewritten only on content change; the one path every first-party HTTP client verifies against (net_transport.py) +│ │ ├── extra-ca-bundle/.pem ← certifi plus the owner's `OUROBOROS_EXTRA_CA_BUNDLE` PEM, content-addressed so a changed owner file rotates every path-keyed cache (stale siblings removed); the one path every first-party HTTP client verifies against (net_transport.py) │ │ ├── process_ledger.jsonl ← durable process-custody ledger (process_custody.py; Runtime topology) │ │ ├── server_port ← active HTTP port for launcher/browser handoff │ │ ├── server_port.bindings.json ← informational endpoint snapshot owned by `server_process.py`: the main, Host Service and local-model owners publish their bound host/port with pid and process fingerprint while they hold it (`record_service_binding`/`clear_service_binding`, compare-and-remove); a browser identity fact, never a grant or a custody ledger (§6 MCP and browser-facing external tools) diff --git a/docs/architecture/06-agent-core.md b/docs/architecture/06-agent-core.md index ff35d0597..487b9e360 100644 --- a/docs/architecture/06-agent-core.md +++ b/docs/architecture/06-agent-core.md @@ -142,7 +142,7 @@ Retry rails nest, each with its own owner and bound; the table exists so the mul | served-model redo | `llm_substitution.py` inside one `chat_claudexor` | `OUROBOROS_SERVED_MODEL_REDOS` per call, and none under a pin, an admitted candidate, a spent send budget or a spent owner window | a round the engine says another model answered, as a new operation. It nests inside the rails above, so a repeat granted by one of them starts a call whose redo budget begins again | | review physical rail | `review_substrate.py`, `review_native_episode.py` | 2 sends per packet or session actor on the P3/acceptance surfaces; a native-retrieval slot carries no send count (its bounds are the episode's own, enumerated below); no rail elsewhere | a released review send, never an unknown one | -The cached OpenAI-compatible clients, the no-proxy per-call clients and the web-search clients share one transport factory (`net_transport.py`) that sets platform-guarded TCP keepalive socket options — on Linux and Darwin the idle threshold, probe interval and probe count from `config.py` (Darwin: `TCP_KEEPALIVE`, not XNU's 75 s × 8 default), on every other platform (Windows included) `SO_KEEPALIVE` alone — so a NAT/VPN mapping silently dropped during a long silent reasoning stretch is detected by kernel probes instead of hanging until the read timeout. When any proxy httpx would honor is configured, the cached and web-search clients skip the explicit transport (httpx env-proxy mounts require it absent). Disclosed residual: proxy-routed installs, the Anthropic-native `requests` lane, every non-Linux/non-Darwin platform and a handful of library clients run without keepalive tuning. Trust anchors ride the same seam: `net_transport.extra_ca_bundle` merges the owner's `OUROBOROS_EXTRA_CA_BUNDLE` PEM over certifi once (`state/extra-ca-bundle.pem`, rewritten only on content change) and every first-party client — the httpx transports, the proxy-routed Default client, the Anthropic `requests` lane, the GigaChat SDK, the catalog and probe clients — verifies against that one path, so a corporate or national CA is an owner setting rather than a system-store edit the no-proxy lanes would never consult; an unreadable file is a loud `ExtraCaBundleError`, never a silent fall-back. +The cached OpenAI-compatible clients, the no-proxy per-call clients and the web-search clients share one transport factory (`net_transport.py`) that sets platform-guarded TCP keepalive socket options — on Linux and Darwin the idle threshold, probe interval and probe count from `config.py` (Darwin: `TCP_KEEPALIVE`, not XNU's 75 s × 8 default), on every other platform (Windows included) `SO_KEEPALIVE` alone — so a NAT/VPN mapping silently dropped during a long silent reasoning stretch is detected by kernel probes instead of hanging until the read timeout. When any proxy httpx would honor is configured, the cached and web-search clients skip the explicit transport (httpx env-proxy mounts require it absent). Disclosed residual: proxy-routed installs, the Anthropic-native `requests` lane, every non-Linux/non-Darwin platform and a handful of library clients run without keepalive tuning. Trust anchors ride the same seam: `net_transport.extra_ca_bundle` merges the owner's `OUROBOROS_EXTRA_CA_BUNDLE` PEM over certifi into a content-addressed `state/extra-ca-bundle/.pem` (a changed owner file yields a new path, so the SSL context and the provider clients, all keyed on that path, rotate with it) and every first-party client — the httpx transports, the proxy-routed Default client, the Anthropic `requests` lane, the GigaChat SDK, the catalog and probe clients — verifies against that one path, so a corporate or national CA is an owner setting rather than a system-store edit the no-proxy lanes would never consult; an unreadable file is a loud `ExtraCaBundleError`, never a silent fall-back. Main-loop model cognition is a separate typed in-flight fact: immediately before each exact provider-call seam the worker sends a direct supervisor `started` event bound to the exact attempt, execution, round, call and retry, and every terminal sends the matching fact, so a stale terminal from an earlier retry or attempt cannot clear the current row. The supervisor keeps only that process-local active row, with no elapsed-time expiry, consulted only by the idle predicate. `OUROBOROS_LLM_TRANSPORT_READ_TIMEOUT_SEC` (default 2700 s) is a configurable dead-socket bound, not a cognition deadline; deadline, budget, cancellation and the absolute ceiling remain independent hard axes. diff --git a/docs/inventories/DATA_LAYOUT_INVENTORY.md b/docs/inventories/DATA_LAYOUT_INVENTORY.md index 3ebcf6671..936a38464 100644 --- a/docs/inventories/DATA_LAYOUT_INVENTORY.md +++ b/docs/inventories/DATA_LAYOUT_INVENTORY.md @@ -2,7 +2,7 @@ Machine extraction of the `docs/ARCHITECTURE.md` "Data layout (`~/Ouroboros/`)" tree — the durable-file orientation carrier (this tree's counterpart of the reference PERSISTENCE_OWNERS derivation checklist) — regenerated by `python scripts/regenerate_inventories.py`. Do not edit. Every entry is probed against reality: repo entries must exist as tracked paths; data-plane entries must appear as a literal in the runtime sources that construct them. A durable file renamed or removed in code while its tree row survives = red (`tests/test_generated_inventories.py`). -Source: `docs/architecture/01-high-level-architecture.md`, physical LF lines 603-693; UTF-8 SHA-256 `a4a7639c08410cae164bf3bb76883ceb86e8e8528f8313a740acbfcea194cb30`. +Source: `docs/architecture/01-high-level-architecture.md`, physical LF lines 603-693; UTF-8 SHA-256 `d276f4f125dcd4a470f0aa4fac1eef3495b3b213e4aa890612e4201bb4855f7e`. - entries: **80** (code-ref: 73, repo-dir: 6, repo-path: 1) @@ -29,7 +29,7 @@ Source: `docs/architecture/01-high-level-architecture.md`, physical LF lines 603 | `usage_import_watermark.json` | `usage_import_watermark.json` | code-ref | | `request_wire_compatibility.json` | `request_wire_compatibility.json` | code-ref | | `capability_evidence.json` | `capability_evidence.json` | code-ref | -| `extra-ca-bundle.pem` | `extra-ca-bundle.pem` | code-ref | +| `extra-ca-bundle/.pem` | `extra-ca-bundle` | code-ref | | `process_ledger.jsonl` | `process_ledger.jsonl` | code-ref | | `server_port` | `server_port` | code-ref | | `server_port.bindings.json` | `server_port.bindings.json` | code-ref | diff --git a/ouroboros/net_transport.py b/ouroboros/net_transport.py index f1b378fd5..7f5095df3 100644 --- a/ouroboros/net_transport.py +++ b/ouroboros/net_transport.py @@ -37,8 +37,11 @@ def extra_ca_bundle() -> Optional[str]: default bundle lacks (a TLS-inspecting corporate proxy, a national CA such as the one behind GigaChat). httpx, requests and the GigaChat SDK each take ONE bundle path and treat it as the whole trust list, so the owner's file is - merged with certifi into ``/state/extra-ca-bundle.pem`` — rewritten - only when its content would change — and that merged path is returned. + merged with certifi into a content-addressed + ``/state/extra-ca-bundle/.pem`` and that path is returned: a + changed owner file yields a new path, so every cache keyed on the path — the + SSL context below and the provider clients — rotates with it, and the stale + sibling files are removed. Unset returns None and every client is built exactly as before the setting existed. An unreadable or non-PEM file raises ``ExtraCaBundleError``: a silent fall-back to certifi would reproduce the very TLS failure the owner @@ -55,12 +58,12 @@ def extra_ca_bundle() -> Optional[str]: from ouroboros.config import DATA_DIR extra = pathlib.Path(raw).expanduser() - target = pathlib.Path(DATA_DIR) / "state" / "extra-ca-bundle.pem" + bundle_dir = pathlib.Path(DATA_DIR) / "state" / "extra-ca-bundle" try: stat = extra.stat() except OSError as exc: raise ExtraCaBundleError(f"{_EXTRA_CA_BUNDLE_KEY} is not readable: {extra} ({exc})") from exc - key = (str(extra), str(target), stat.st_mtime_ns, stat.st_size) + key = (str(extra), str(bundle_dir), stat.st_mtime_ns, stat.st_size) cached = _merged_bundle_cache.get(key) if cached and os.path.isfile(cached): return cached @@ -79,18 +82,27 @@ def extra_ca_bundle() -> Optional[str]: except (ssl.SSLError, ValueError) as exc: raise ExtraCaBundleError(f"{_EXTRA_CA_BUNDLE_KEY} holds no loadable PEM certificate: {extra} ({exc})") from exc + import hashlib + import threading + base = pathlib.Path(certifi.where()).read_bytes() merged = base.rstrip(b"\n") + b"\n" + extra_bytes.rstrip(b"\n") + b"\n" + digest = hashlib.sha256(merged).hexdigest()[:12] + target = bundle_dir / f"{digest}.pem" try: - if not (target.is_file() and target.read_bytes() == merged): - target.parent.mkdir(parents=True, exist_ok=True) - import threading - + if not target.is_file(): + bundle_dir.mkdir(parents=True, exist_ok=True) # Per-process AND per-thread temp name: two threads first building clients at once # must not share one temp file (Windows refuses to replace a file another thread holds open). tmp = target.with_name(f"{target.name}.{os.getpid()}-{threading.get_ident()}.tmp") tmp.write_bytes(merged) os.replace(tmp, target) + for stale in bundle_dir.glob("*.pem"): + if stale != target: + try: + stale.unlink() + except OSError: + pass # another process may be materializing; its own call re-resolves except OSError as exc: raise ExtraCaBundleError(f"cannot write the merged trust bundle {target}: {exc}") from exc _merged_bundle_cache[key] = str(target) diff --git a/tests/test_net_transport_extra_ca.py b/tests/test_net_transport_extra_ca.py index 59a529dec..dcd3ab043 100644 --- a/tests/test_net_transport_extra_ca.py +++ b/tests/test_net_transport_extra_ca.py @@ -16,6 +16,8 @@ import sys import threading import types +import re + import pytest from ouroboros import net_transport @@ -124,14 +126,15 @@ def test_unset_setting_leaves_every_client_as_before(monkeypatch): assert "verify" not in seen and seen["trust_env"] is False -def test_setting_merges_the_owner_file_over_certifi_and_rewrites_only_on_change(monkeypatch, tmp_path): +def test_setting_merges_the_owner_file_over_certifi_and_rotates_with_its_content(monkeypatch, tmp_path): import certifi extra, _cert, _key = _throwaway_ca(tmp_path) monkeypatch.setenv("OUROBOROS_EXTRA_CA_BUNDLE", str(extra)) merged = pathlib.Path(net_transport.extra_ca_bundle()) - assert merged == tmp_path / "data" / "state" / "extra-ca-bundle.pem" + assert merged.parent == tmp_path / "data" / "state" / "extra-ca-bundle" + assert re.fullmatch(r"[0-9a-f]{12}\.pem", merged.name), merged.name body = merged.read_bytes() assert body.startswith(pathlib.Path(certifi.where()).read_bytes().rstrip(b"\n")) assert body.endswith(extra.read_bytes().rstrip(b"\n") + b"\n") @@ -147,7 +150,11 @@ def test_setting_merges_the_owner_file_over_certifi_and_rewrites_only_on_change( second, _cert2, _key2 = _throwaway_ca(tmp_path / "second") extra.write_bytes(second.read_bytes()) - assert pathlib.Path(net_transport.extra_ca_bundle()).read_bytes().endswith(second.read_bytes().rstrip(b"\n") + b"\n") + rotated = pathlib.Path(net_transport.extra_ca_bundle()) + assert rotated != merged and rotated.read_bytes().endswith(second.read_bytes().rstrip(b"\n") + b"\n") + assert not merged.exists(), "the stale sibling is removed" + assert net_transport.trust_ssl_context() is not context, "a new owner file rotates the SSL context" + assert net_transport.verify_kwargs()["verify"] is net_transport.trust_ssl_context() @pytest.mark.parametrize("content", [None, b"not a certificate\n", b"-----BEGIN CERTIFICATE-----\nMIIBogus\n-----END CERTIFICATE-----\n"]) diff --git a/tests/test_persistence_inventory.py b/tests/test_persistence_inventory.py index c46a14a20..5800d9280 100644 --- a/tests/test_persistence_inventory.py +++ b/tests/test_persistence_inventory.py @@ -574,7 +574,7 @@ def scan_data_paths(root: pathlib.Path = REPO) -> frozenset[str]: # 295 -> 294: TZ-3 removed the destructive memory journal rewrite and its # ``.compact.tmp`` sibling path; PERSISTENCE.md keeps the journals, now # read-only observed and never age-digested. -EXPECTED_SCAN_PATHS = 295 +EXPECTED_SCAN_PATHS = 296 # Scanned paths that must always be present — guards the scanner itself # against a silent regression that would shrink coverage while keeping counts diff --git a/web/modules/settings_ui.js b/web/modules/settings_ui.js index 2e966dee3..cbb5ef085 100644 --- a/web/modules/settings_ui.js +++ b/web/modules/settings_ui.js @@ -851,7 +851,7 @@ export function renderSettingsPage() {
-
Absolute or ~-prefixed path to a PEM file on this machine. Leave empty unless a provider fails with a certificate error.
+
Absolute or ~-prefixed path to a PEM file on the machine that runs Ouroboros (its own filesystem, not the device showing this page). Leave empty unless a provider fails with a certificate error.