ouroboros/tests/test_skill_publish_github.py
Ouroboros becededcb7 Preserve typed failures across built-in plan, image and GitHub tools (#739)
A built-in tool that already knows its call failed must leave the registry
a typed result. The legacy adapter types a string by its first-line
`⚠️ IDENTIFIER` marker, so identifier-less prose and bare `ERROR:` strings
were recorded as successful calls in tools.jsonl, the outcome classifier
and the acceptance packet.

- plan_task: a schema-equivalent empty optional field (blank goal/plan,
  `{}` or a spec of declared keys holding [] / "") beside a real disposition
  no longer reads as a mixed envelope; a non-empty list, unknown key or
  wrong type still refuses typed. Every plan-review refusal publishes typed
  (argument vs state), and `_plan_unavailable` splits faults (error) from
  availability outcomes such as a budget-declined panel (unavailable).
- view_image / vlm_query: the local image loader publishes its own
  refusals as TOOL_ARG_ERROR; policy markers keep their owners' typing; the
  host's auto-attach stays non-fatal outside a registry call.
- GitHub: `_gh_run` returns a structured result (exit code, bounded and
  redacted stderr head, gh's own `(HTTP NNN)` marker read before bounding,
  failure class); `_gh_cmd` keeps the string ABI and its typed target
  refusals. Publication carries error_detail / github_status /
  github_operation beside the stage code and through the structured
  attempt projection; hints state producer facts and name Settings →
  Secrets; mandatory fork sync still stops before any mutation and PR
  settlement stays a read-only exact lookup.
- tests/test_typed_tool_refusals.py: shrink-only source lint over returned
  failure literals in ouroboros/tools (oracle: the real adapter); its
  allowlist is the disclosed residual for the other packages.
- docs: DEVELOPMENT (typed-refusal rule, plan closure), ARCHITECTURE
  (vision, skill publication, github.py row).
2026-09-08 16:16:24 +03:00

478 lines
17 KiB
Python

"""Hermetic tests for the one-shot GitHub publication transport."""
from __future__ import annotations
import base64
import json
import subprocess
import types
import pytest
from ouroboros import skill_publish_github as github
from ouroboros.tools import github as transport
from ouroboros.tools.github import GhResult
BASE_SHA = "1" * 40
COMMIT_SHA = "2" * 40
SNAPSHOT_SHA = "a" * 64
RULESET_SHA = "b" * 64
@pytest.fixture(autouse=True)
def synthetic_github_credentials(monkeypatch):
monkeypatch.setenv("GITHUB_TOKEN", "ghp_SYNTHETIC1234567890")
def _attempt():
facts = types.SimpleNamespace(
skill="demo",
snapshot_hash=SNAPSHOT_SHA,
scanner={"ruleset_sha256": RULESET_SHA},
marks=[],
)
def mark(stage, **values):
facts.marks.append((stage, values))
facts.mark = mark
return facts
def _pull_row(*, sha: str = COMMIT_SHA, owner: str = "alice", base: str = "main"):
return {
"number": 7,
"html_url": "https://github.com/hub/project/pull/7",
"head": {
"sha": sha,
"ref": "submit/demo-v1.0.0",
"repo": {"owner": {"login": owner}},
},
"base": {"ref": base},
}
def test_commit_payload_hardcodes_ouroboros_coauthor(monkeypatch):
captured = {}
def fake_json(_ctx, _args, **kwargs):
captured.update(json.loads(kwargs["input_data"]))
return {
"data": {
"createCommitOnBranch": {
"commit": {
"oid": COMMIT_SHA,
"url": "https://github.com/alice/project/commit/" + COMMIT_SHA,
}
}
}
}
monkeypatch.setattr(github, "_json_object", fake_json)
sha, url = github.commit_payload(
types.SimpleNamespace(),
"alice",
"project",
"submit/demo-v1.0.0",
BASE_SHA,
"Add skill: demo v1.0.0",
[{"path": "skills/demo/SKILL.md", "contents": "YQ=="}],
[{"path": "skills/demo/removed.py"}],
)
message = captured["variables"]["input"]["message"]
assert sha == COMMIT_SHA
assert url.endswith(COMMIT_SHA)
assert message == {
"headline": "Add skill: demo v1.0.0",
"body": ("Co-authored-by: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com>"),
}
assert captured["variables"]["input"]["fileChanges"] == {
"additions": [{"path": "skills/demo/SKILL.md", "contents": "YQ=="}],
"deletions": [{"path": "skills/demo/removed.py"}],
}
def test_upstream_catalog_is_read_from_the_exact_resolved_base_sha(monkeypatch):
calls = []
def fake_json(_ctx, args, **_kwargs):
calls.append(args)
if "/git/refs/heads/" in args[-1]:
return {"object": {"sha": BASE_SHA}}
return {
"content": base64.b64encode(b'{"skills":[]}').decode("ascii"),
}
monkeypatch.setattr(github, "_json_object", fake_json)
catalog, base_sha = github.fetch_upstream_catalog(
types.SimpleNamespace(),
"hub",
"project",
"main",
)
assert catalog == {"skills": []}
assert base_sha == BASE_SHA
assert calls[-1][-1] == f"/repos/hub/project/contents/catalog.json?ref={BASE_SHA}"
def test_owner_actor_skips_fork_and_sync(monkeypatch):
monkeypatch.setattr(
github,
"_gh_run",
lambda *_args, **_kwargs: pytest.fail("owner path must issue no fork command"),
)
attempt = _attempt()
github.prepare_publish_repository(
types.SimpleNamespace(),
attempt,
owner="HubOwner",
repo="project",
base_branch="main",
login="hubowner",
)
assert attempt.marks == [("fork_ready", {"repository": "hubowner/project", "actor": "hubowner"})]
def test_non_owner_sync_failure_is_typed(monkeypatch):
calls = []
def fake_gh(args, _ctx, **_kwargs):
calls.append(args)
if args[:2] == ["repo", "view"]:
return GhResult(True, '{"name":"project"}', 0, None, "")
return GhResult(False, "⚠️ GH_ERROR: synthetic", 1, None, "exit")
monkeypatch.setattr(github, "_gh_run", fake_gh)
with pytest.raises(github.SkillPublishGitHubError) as caught:
github.prepare_publish_repository(
types.SimpleNamespace(),
_attempt(),
owner="hub",
repo="project",
base_branch="main",
login="alice",
)
assert caught.value.reason_code == "fork_sync_failed"
assert [call[0] for call in calls] == ["repo", "api"]
def test_direct_pr_url_yields_validated_receipt_without_lookup(monkeypatch):
calls = []
def fake_gh(args, _ctx, **_kwargs):
calls.append(args)
return GhResult(True, "https://github.com/hub/project/pull/7", 0, None, "")
monkeypatch.setattr(github, "_gh_run", fake_gh)
monkeypatch.setattr(
github,
"_json_value",
lambda *_args, **_kwargs: pytest.fail("valid direct output must not settle"),
)
attempt = _attempt()
receipt = github.create_pr_receipt(
types.SimpleNamespace(),
attempt,
owner="hub",
repo="project",
base_branch="main",
login="alice",
branch="submit/demo-v1.0.0",
title="Add demo",
body="body",
commit_sha=COMMIT_SHA,
)
assert receipt and receipt["number"] == 7
assert len(calls) == 1
assert [stage for stage, _facts in attempt.marks] == ["pr_create_attempted"]
def test_ambiguous_create_uses_one_exact_read_only_settlement(monkeypatch):
create_calls = []
lookup_calls = []
def fake_gh(args, _ctx, **_kwargs):
create_calls.append(args)
return GhResult(False, "⚠️ GH_TIMEOUT: synthetic", None, None, "timeout")
def fake_json(_ctx, args, **_kwargs):
lookup_calls.append(args)
return [_pull_row()]
monkeypatch.setattr(github, "_gh_run", fake_gh)
monkeypatch.setattr(github, "_json_value", fake_json)
receipt = github.create_pr_receipt(
types.SimpleNamespace(),
_attempt(),
owner="hub",
repo="project",
base_branch="main",
login="alice",
branch="submit/demo-v1.0.0",
title="Add demo",
body="body",
commit_sha=COMMIT_SHA,
)
assert receipt and receipt["url"].endswith("/pull/7")
assert len(create_calls) == 1
assert len(lookup_calls) == 1
assert "--method" in lookup_calls[0]
assert "GET" in lookup_calls[0]
@pytest.mark.parametrize(
"rows",
[
[],
[_pull_row(), _pull_row()],
[_pull_row(sha="3" * 40)],
[_pull_row(owner="other")],
[_pull_row(base="wrong")],
[_pull_row() | {"number": 8}],
],
)
def test_ambiguous_settlement_never_claims_wrong_or_nonunique_pr(monkeypatch, rows):
monkeypatch.setattr(github, "_gh_run", lambda *_args, **_kwargs: GhResult(True, "garbage", 0, None, ""))
monkeypatch.setattr(github, "_json_value", lambda *_args, **_kwargs: rows)
receipt = github.create_pr_receipt(
types.SimpleNamespace(),
_attempt(),
owner="hub",
repo="project",
base_branch="main",
login="alice",
branch="submit/demo-v1.0.0",
title="Add demo",
body="body",
commit_sha=COMMIT_SHA,
)
assert receipt is None
def test_existing_branch_is_never_overwritten(monkeypatch):
monkeypatch.setattr(github, "_gh_run", lambda *_args, **_kwargs: GhResult(True, '{"ref":"exists"}', 0, None, ""))
with pytest.raises(github.SkillPublishGitHubError) as caught:
github.ensure_branch(
types.SimpleNamespace(),
"alice",
"project",
"submit/demo-v1.0.0",
BASE_SHA,
)
assert caught.value.reason_code == "submission_branch_exists"
@pytest.mark.parametrize(
"failure,stderr,http_status,hint",
[
("exit", "gh: Resource not accessible by personal access token (HTTP 403)", 403, "Settings → Secrets"),
("exit", "gh: Conflict (HTTP 409)", 409, "conflict (HTTP 409)"),
("timeout", "", None, "the outcome may be unknown"),
("cli_missing", "", None, "Install the GitHub CLI (gh)"),
("exit", "gh: ghp_SYNTHETIC1234567890 refused (HTTP 403)", 403, "Settings → Secrets"),
],
)
def test_sync_failure_preserves_process_evidence_and_stops(
monkeypatch, tmp_path, failure, stderr, http_status, hint,
):
calls = []
def run(cmd, *, cwd, capture_output, text, timeout, input, env):
calls.append(cmd)
assert cwd == str(tmp_path)
assert capture_output is True and text is True and input is None
assert env["GH_TOKEN"] == "ghp_SYNTHETIC1234567890"
if cmd[1:3] == ["repo", "view"]:
return subprocess.CompletedProcess(cmd, 0, '{"name":"project"}', "")
assert cmd == ["gh", "api", "-X", "POST", "/repos/alice/project/merge-upstream", "-f", "branch=main"]
assert timeout == 45
if failure == "timeout":
raise subprocess.TimeoutExpired(cmd, timeout)
if failure == "cli_missing":
raise FileNotFoundError("synthetic missing CLI")
return subprocess.CompletedProcess(cmd, 1, "", stderr)
monkeypatch.setattr(subprocess, "run", run)
attempt = _attempt()
with pytest.raises(github.SkillPublishGitHubError) as caught:
github.prepare_publish_repository(
types.SimpleNamespace(repo_dir=tmp_path), attempt,
owner="hub", repo="project", base_branch="main", login="alice",
)
error = caught.value
assert error.reason_code == "fork_sync_failed"
assert error.http_status == http_status
assert error.operation == "merge-upstream"
assert hint in error.repair_hint
assert [stage for stage, _ in attempt.marks] == ["fork_ready"]
assert len(calls) == 2 # Exactly one merge-upstream; never retry a mutation.
assert "ghp_SYNTHETIC1234567890" not in error.detail
if "ghp_SYNTHETIC" in stderr:
assert "***" in error.detail
if http_status:
assert str(http_status) in error.detail
if failure == "cli_missing":
assert "token" not in error.repair_hint.lower()
if failure == "timeout":
assert error.detail == "⚠️ GH_TIMEOUT: exceeded 45s."
def test_sync_success_marks_fork_synced(monkeypatch, tmp_path):
calls = []
def run(cmd, **kwargs):
calls.append(cmd)
return subprocess.CompletedProcess(cmd, 0, '{}', "")
monkeypatch.setattr(subprocess, "run", run)
attempt = _attempt()
github.prepare_publish_repository(
types.SimpleNamespace(repo_dir=tmp_path), attempt,
owner="hub", repo="project", base_branch="main", login="alice",
)
assert [stage for stage, _ in attempt.marks] == ["fork_ready", "fork_synced"]
assert len(calls) == 2
@pytest.mark.parametrize(
"stderr,expected_status,expected_text",
[
# the FIRST gh marker wins; the bounded head keeps three non-empty lines
("\n first\n\n second (HTTP 401)\n third\n ignored (HTTP 403)", 401,
"⚠️ GH_ERROR: first | second (HTTP 401) | third"),
# a bare number is prose, never a status
("permission denied, status 403", None, "⚠️ GH_ERROR: permission denied, status 403"),
# the status is read from the WHOLE redacted stderr, before the head is cut
("first\nsecond\nthird\nlater line (HTTP 403)", 403, "⚠️ GH_ERROR: first | second | third"),
("x" * 700 + " (HTTP 403)", 403, None),
("", None, "⚠️ GH_ERROR: "),
],
)
def test_transport_bounds_stderr_and_only_reads_gh_http_marker(
monkeypatch, tmp_path, stderr, expected_status, expected_text,
):
monkeypatch.setattr(
subprocess, "run", lambda cmd, **kwargs: subprocess.CompletedProcess(cmd, 7, "ignored stdout", stderr),
)
result = transport._gh_run(["api", "/user"], types.SimpleNamespace(repo_dir=tmp_path))
assert (result.ok, result.exit_code, result.http_status, result.failure) == (False, 7, expected_status, "exit")
assert result.text.startswith("⚠️ GH_ERROR: ")
assert len(result.text.removeprefix("⚠️ GH_ERROR: ")) <= 600
assert "ignored" not in result.text
if expected_text is not None:
assert result.text == expected_text
@pytest.mark.parametrize("failure", ["cli_missing", "timeout", "exit", "exception"])
def test_transport_never_publishes_a_sidecar(monkeypatch, tmp_path, failure):
sentinel = object()
ctx = types.SimpleNamespace(repo_dir=tmp_path, _active_builtin_tool_result=sentinel)
def run(cmd, **kwargs):
if failure == "cli_missing":
raise FileNotFoundError()
if failure == "timeout":
raise subprocess.TimeoutExpired(cmd, kwargs["timeout"])
if failure == "exception":
raise RuntimeError("synthetic ghp_SYNTHETIC1234567890")
return subprocess.CompletedProcess(cmd, 1, "", "synthetic")
monkeypatch.setattr(subprocess, "run", run)
result = transport._gh_run(["api", "/user"], ctx)
assert result.failure == failure
assert ctx._active_builtin_tool_result is sentinel
assert transport._gh_cmd(["api", "/user"], ctx) == result.text
assert ctx._active_builtin_tool_result is sentinel
assert "ghp_SYNTHETIC1234567890" not in result.text
def test_unconfigured_credentials_hint_uses_settings(monkeypatch):
monkeypatch.setattr(github, "github_cli_configured", lambda: False)
result = GhResult(False, "⚠️ GH_ERROR: synthetic", 1, None, "exit")
assert github.github_repair_hint(result, operation="repo view", repository="alice/project", default="default") == (
"No GitHub credential is configured; add GITHUB_TOKEN in Settings → Secrets, then retry."
)
@pytest.mark.parametrize("handler,kwargs", [
(transport._get_issue, {}),
(transport._comment_on_issue, {"body": "comment"}),
(transport._close_issue, {}),
])
def test_nonpositive_issue_number_publishes_typed_argument_error(handler, kwargs):
ctx = types.SimpleNamespace(_active_builtin_tool_result=None)
text = handler(ctx, number=0, **kwargs)
result = ctx._active_builtin_tool_result
assert (result.status, result.code) == ("error", "TOOL_ARG_ERROR")
assert result.text == text == "⚠️ TOOL_ARG_ERROR: issue number must be positive"
@pytest.mark.parametrize("create_failed", [True, False])
def test_failed_pr_settlement_keeps_producer_evidence_without_retry(monkeypatch, tmp_path, create_failed):
calls = []
def run(cmd, **kwargs):
calls.append(cmd)
if cmd[1:3] == ["pr", "create"]:
if create_failed:
raise subprocess.TimeoutExpired(cmd, kwargs["timeout"])
return subprocess.CompletedProcess(cmd, 0, "malformed URL", "")
assert cmd[1:4] == ["api", "--method", "GET"]
return subprocess.CompletedProcess(cmd, 1, "", "gh: read refused (HTTP 403)")
monkeypatch.setattr(subprocess, "run", run)
attempt = _attempt()
with pytest.raises(github.SkillPublishGitHubError) as caught:
github.create_pr_receipt(
types.SimpleNamespace(repo_dir=tmp_path), attempt,
owner="hub", repo="project", base_branch="main", login="alice",
branch="submit/demo-v1.0.0", title="Add demo", body="body", commit_sha=COMMIT_SHA,
)
error = caught.value
assert error.reason_code == "pr_open_indeterminate"
assert len(calls) == 2
assert [stage for stage, _ in attempt.marks] == ["pr_create_attempted"]
assert error.operation == ("pr create" if create_failed else "pulls")
assert error.http_status == (None if create_failed else 403)
assert ("GH_TIMEOUT" if create_failed else "403") in error.detail
@pytest.mark.parametrize("stdout", ['["wrong shape"]', 'invalid ghp_SYNTHETIC1234567890 ' + 'x' * 1000])
def test_malformed_json_keeps_bounded_redacted_detail(monkeypatch, tmp_path, stdout):
monkeypatch.setattr(
subprocess, "run", lambda cmd, **kwargs: subprocess.CompletedProcess(cmd, 0, stdout, ""),
)
with pytest.raises(github.SkillPublishGitHubError) as caught:
github.fetch_upstream_catalog(types.SimpleNamespace(repo_dir=tmp_path), "hub", "project", "main")
error = caught.value
assert error.reason_code == "upstream_read_failed"
assert error.http_status is None
assert error.operation == "git/refs"
assert error.detail
assert len(error.detail) <= 640
assert "ghp_SYNTHETIC1234567890" not in error.detail
@pytest.mark.parametrize("returncode,stdout,stderr", [
(1, "", "gh: mutation refused (HTTP 403)"),
(0, '{"errors":[{"message":"synthetic GraphQL rejection"}]}', ""),
])
def test_commit_failure_keeps_graphql_cause(monkeypatch, tmp_path, returncode, stdout, stderr):
calls = []
def run(cmd, **kwargs):
calls.append(cmd)
return subprocess.CompletedProcess(cmd, returncode, stdout, stderr)
monkeypatch.setattr(subprocess, "run", run)
with pytest.raises(github.SkillPublishGitHubError) as caught:
github.commit_payload(
types.SimpleNamespace(repo_dir=tmp_path), "alice", "project", "submit/demo-v1.0.0",
BASE_SHA, "Add demo", [],
)
error = caught.value
assert error.reason_code == "commit_create_failed"
assert error.operation == "graphql"
assert error.http_status == (403 if returncode else None)
assert ("403" if returncode else "synthetic GraphQL rejection") in error.detail
assert len(calls) == 1