mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
feat(nodes): add computer use (computer.act) on macOS nodes via Peekaboo (#102776)
* feat(nodes): add computer use (computer.act) on macOS nodes via Peekaboo * fix(nodes): release held computer.act input on node disconnect/stop/disable * fix(nodes): release held computer.act input armed after a lifecycle release * fix(nodes): scope computer.act lifecycle catch-up release to the arming action * chore(nodes): register computer tool display metadata, sync native i18n, doc-comment constants * fix(nodes): preserve caught-error cause in computer tool resolver; regen docs map
This commit is contained in:
Peter Steinberger
GitHub
parent
d9f8fc6322
commit
f8dbd34ab8
35 changed files with 2632 additions and 127 deletions
|
|
@ -16171,7 +16171,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 68,
|
||||
"line": 69,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "General",
|
||||
"surface": "apple",
|
||||
|
|
@ -16179,7 +16179,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 69,
|
||||
"line": 70,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Everyday OpenClaw app behavior.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16187,7 +16187,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 73,
|
||||
"line": 74,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "App",
|
||||
"surface": "apple",
|
||||
|
|
@ -16195,7 +16195,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 75,
|
||||
"line": 76,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Launch at login",
|
||||
"surface": "apple",
|
||||
|
|
@ -16203,7 +16203,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 76,
|
||||
"line": 77,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Automatically start OpenClaw after you sign in.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16211,7 +16211,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 80,
|
||||
"line": 81,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Show Dock icon",
|
||||
"surface": "apple",
|
||||
|
|
@ -16219,7 +16219,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 81,
|
||||
"line": 82,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Keep OpenClaw visible in the Dock. When off, windows still show the Dock icon while open.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16227,7 +16227,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 85,
|
||||
"line": 86,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Play menu bar icon animations",
|
||||
"surface": "apple",
|
||||
|
|
@ -16235,7 +16235,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 86,
|
||||
"line": 87,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Enable idle blinks and wiggles on the status icon.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16243,7 +16243,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 91,
|
||||
"line": 92,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Capabilities",
|
||||
"surface": "apple",
|
||||
|
|
@ -16251,7 +16251,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 93,
|
||||
"line": 94,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Allow Canvas",
|
||||
"surface": "apple",
|
||||
|
|
@ -16259,7 +16259,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 94,
|
||||
"line": 95,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Allow the agent to show and control the Canvas panel.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16267,7 +16267,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 98,
|
||||
"line": 99,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Allow Camera",
|
||||
"surface": "apple",
|
||||
|
|
@ -16275,7 +16275,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 99,
|
||||
"line": 100,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Allow the agent to capture a photo or short video via the built-in camera.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16283,7 +16283,23 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 103,
|
||||
"line": 104,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Allow Computer Control",
|
||||
"surface": "apple",
|
||||
"id": "native.apple.6db2112acf394e43"
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 105,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Let the agent move the pointer, click, and type on this Mac. Requires Accessibility permission and stays disarmed until an operator arms it. High risk.",
|
||||
"surface": "apple",
|
||||
"id": "native.apple.b59b6b6769b0ad2f"
|
||||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 109,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Enable Peekaboo Bridge",
|
||||
"surface": "apple",
|
||||
|
|
@ -16291,7 +16307,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 104,
|
||||
"line": 110,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Allow signed tools (e.g. `peekaboo`) to drive UI automation via PeekabooBridge.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16299,7 +16315,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 109,
|
||||
"line": 115,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Developer",
|
||||
"surface": "apple",
|
||||
|
|
@ -16307,7 +16323,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 111,
|
||||
"line": 117,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Enable debug tools",
|
||||
"surface": "apple",
|
||||
|
|
@ -16315,7 +16331,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 112,
|
||||
"line": 118,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Show the Debug page with development utilities.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16323,7 +16339,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 119,
|
||||
"line": 125,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "App session",
|
||||
"surface": "apple",
|
||||
|
|
@ -16331,7 +16347,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 121,
|
||||
"line": 127,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Quit only when you want to stop the menu bar app completely.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16339,7 +16355,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 126,
|
||||
"line": 132,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Quit",
|
||||
"surface": "apple",
|
||||
|
|
@ -16347,7 +16363,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 146,
|
||||
"line": 152,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "OpenClaw paused",
|
||||
"surface": "apple",
|
||||
|
|
@ -16355,7 +16371,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 156,
|
||||
"line": 162,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "OpenClaw active",
|
||||
"surface": "apple",
|
||||
|
|
@ -16363,7 +16379,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 174,
|
||||
"line": 180,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Processing messages through the local Gateway on this Mac.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16371,7 +16387,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 176,
|
||||
"line": 182,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Connected to a remote Gateway configuration.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16379,7 +16395,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 178,
|
||||
"line": 184,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Ready to run after you choose a Gateway connection.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16387,7 +16403,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 186,
|
||||
"line": 192,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Connection",
|
||||
"surface": "apple",
|
||||
|
|
@ -16395,7 +16411,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 187,
|
||||
"line": 193,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Choose where the Gateway runs and how this Mac app reaches it.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16403,7 +16419,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 244,
|
||||
"line": 250,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "\\(Int(ping)) ms",
|
||||
"surface": "apple",
|
||||
|
|
@ -16411,7 +16427,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 278,
|
||||
"line": 284,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Local Gateway",
|
||||
"surface": "apple",
|
||||
|
|
@ -16419,7 +16435,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 279,
|
||||
"line": 285,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Remote Gateway direct",
|
||||
"surface": "apple",
|
||||
|
|
@ -16427,7 +16443,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 279,
|
||||
"line": 285,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Remote Gateway via SSH",
|
||||
"surface": "apple",
|
||||
|
|
@ -16435,7 +16451,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 280,
|
||||
"line": 286,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Gateway not configured",
|
||||
"surface": "apple",
|
||||
|
|
@ -16443,7 +16459,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 286,
|
||||
"line": 292,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "OpenClaw starts and monitors the Gateway on this Mac.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16451,7 +16467,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 295,
|
||||
"line": 301,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Choose local or remote before the app can attach to a Gateway.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16459,7 +16475,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 301,
|
||||
"line": 307,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Gateway",
|
||||
"surface": "apple",
|
||||
|
|
@ -16467,7 +16483,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 303,
|
||||
"line": 309,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "OpenClaw runs",
|
||||
"surface": "apple",
|
||||
|
|
@ -16475,7 +16491,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 304,
|
||||
"line": 310,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Pick whether this app owns a local Gateway or attaches to another host.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16483,7 +16499,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 307,
|
||||
"line": 313,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Gateway location",
|
||||
"surface": "apple",
|
||||
|
|
@ -16491,7 +16507,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 308,
|
||||
"line": 314,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Not configured",
|
||||
"surface": "apple",
|
||||
|
|
@ -16499,7 +16515,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 309,
|
||||
"line": 315,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Local (this Mac)",
|
||||
"surface": "apple",
|
||||
|
|
@ -16507,7 +16523,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 310,
|
||||
"line": 316,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Remote (another host)",
|
||||
"surface": "apple",
|
||||
|
|
@ -16515,7 +16531,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 319,
|
||||
"line": 325,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Setup needed",
|
||||
"surface": "apple",
|
||||
|
|
@ -16523,7 +16539,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 320,
|
||||
"line": 326,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Local is best for this Mac. Remote is best when the Gateway already runs on a Mac Studio or server.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16531,7 +16547,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 349,
|
||||
"line": 355,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Remote Access",
|
||||
"surface": "apple",
|
||||
|
|
@ -16539,7 +16555,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 360,
|
||||
"line": 366,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Discovery & Status",
|
||||
"surface": "apple",
|
||||
|
|
@ -16547,7 +16563,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 376,
|
||||
"line": 382,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Nearby gateways",
|
||||
"surface": "apple",
|
||||
|
|
@ -16555,7 +16571,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 399,
|
||||
"line": 405,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Remote test",
|
||||
"surface": "apple",
|
||||
|
|
@ -16563,7 +16579,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 406,
|
||||
"line": 412,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Control channel",
|
||||
"surface": "apple",
|
||||
|
|
@ -16571,7 +16587,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 433,
|
||||
"line": 439,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Recommended setup",
|
||||
"surface": "apple",
|
||||
|
|
@ -16579,7 +16595,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 435,
|
||||
"line": 441,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Use Tailscale Serve so the gateway has a valid HTTPS certificate.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16587,7 +16603,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "conditional-branch",
|
||||
"line": 435,
|
||||
"line": 441,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Use Tailscale plus an SSH tunnel for stable private access.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16595,7 +16611,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 445,
|
||||
"line": 451,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Advanced",
|
||||
"surface": "apple",
|
||||
|
|
@ -16603,7 +16619,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 448,
|
||||
"line": 454,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Identity file",
|
||||
"surface": "apple",
|
||||
|
|
@ -16611,7 +16627,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 450,
|
||||
"line": 456,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "/Users/you/.ssh/id_ed25519",
|
||||
"surface": "apple",
|
||||
|
|
@ -16619,7 +16635,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 452,
|
||||
"line": 458,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Project root",
|
||||
"surface": "apple",
|
||||
|
|
@ -16627,7 +16643,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 454,
|
||||
"line": 460,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "/home/you/Projects/openclaw",
|
||||
"surface": "apple",
|
||||
|
|
@ -16635,7 +16651,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 456,
|
||||
"line": 462,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "CLI path",
|
||||
"surface": "apple",
|
||||
|
|
@ -16643,7 +16659,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 458,
|
||||
"line": 464,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "/Applications/OpenClaw.app/.../openclaw",
|
||||
"surface": "apple",
|
||||
|
|
@ -16651,7 +16667,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 463,
|
||||
"line": 469,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "SSH command details",
|
||||
"surface": "apple",
|
||||
|
|
@ -16659,7 +16675,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 483,
|
||||
"line": 489,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Transport",
|
||||
"surface": "apple",
|
||||
|
|
@ -16667,7 +16683,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 484,
|
||||
"line": 490,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "SSH keeps the Gateway private; direct is best for HTTPS or Tailscale Serve.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16675,7 +16691,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 487,
|
||||
"line": 493,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "SSH tunnel",
|
||||
"surface": "apple",
|
||||
|
|
@ -16683,7 +16699,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 488,
|
||||
"line": 494,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Direct (ws/wss)",
|
||||
"surface": "apple",
|
||||
|
|
@ -16691,7 +16707,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 501,
|
||||
"line": 507,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "SSH target",
|
||||
"surface": "apple",
|
||||
|
|
@ -16699,7 +16715,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 501,
|
||||
"line": 507,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "User and host for the remote Gateway machine.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16707,7 +16723,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 518,
|
||||
"line": 524,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Gateway URL",
|
||||
"surface": "apple",
|
||||
|
|
@ -16715,7 +16731,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 518,
|
||||
"line": 524,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "The WebSocket URL exposed by the remote Gateway.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16723,7 +16739,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 521,
|
||||
"line": 527,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "wss://gateway.example.ts.net",
|
||||
"surface": "apple",
|
||||
|
|
@ -16731,7 +16747,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 527,
|
||||
"line": 533,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Use wss:// for public hosts. ws:// is allowed for localhost, LAN, .local, and Tailnet hosts.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16739,7 +16755,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 538,
|
||||
"line": 544,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Gateway token",
|
||||
"surface": "apple",
|
||||
|
|
@ -16747,7 +16763,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-named-argument",
|
||||
"line": 539,
|
||||
"line": 545,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Used when the remote gateway requires token auth.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16755,7 +16771,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 542,
|
||||
"line": 548,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "remote gateway auth token (gateway.remote.token)",
|
||||
"surface": "apple",
|
||||
|
|
@ -16763,7 +16779,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call-concatenated",
|
||||
"line": 547,
|
||||
"line": 553,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "The current gateway.remote.token value is not plain text. OpenClaw for macOS cannot use it directly; enter a plaintext token here to replace it.",
|
||||
"surface": "apple",
|
||||
|
|
@ -16771,7 +16787,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 566,
|
||||
"line": 572,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Test remote",
|
||||
"surface": "apple",
|
||||
|
|
@ -16779,7 +16795,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 589,
|
||||
"line": 595,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Testing…",
|
||||
"surface": "apple",
|
||||
|
|
@ -16787,7 +16803,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 627,
|
||||
"line": 633,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Installed: \\(gatewayVersion) · Required: \\(required)",
|
||||
"surface": "apple",
|
||||
|
|
@ -16795,7 +16811,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 631,
|
||||
"line": 637,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Gateway \\(gatewayVersion) detected",
|
||||
"surface": "apple",
|
||||
|
|
@ -16803,7 +16819,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 637,
|
||||
"line": 643,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Node \\(node)",
|
||||
"surface": "apple",
|
||||
|
|
@ -16811,7 +16827,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 649,
|
||||
"line": 655,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Last failure: \\(failure)",
|
||||
"surface": "apple",
|
||||
|
|
@ -16819,7 +16835,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 654,
|
||||
"line": 660,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Recheck",
|
||||
"surface": "apple",
|
||||
|
|
@ -16827,7 +16843,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 657,
|
||||
"line": 663,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Gateway auto-starts in local mode via launchd (\\(gatewayLaunchdLabel)).",
|
||||
"surface": "apple",
|
||||
|
|
@ -16835,7 +16851,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 712,
|
||||
"line": 718,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Retry now",
|
||||
"surface": "apple",
|
||||
|
|
@ -16843,7 +16859,7 @@
|
|||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 717,
|
||||
"line": 723,
|
||||
"path": "apps/macos/Sources/OpenClaw/GeneralSettings.swift",
|
||||
"source": "Open logs",
|
||||
"surface": "apple",
|
||||
|
|
|
|||
555
apps/macos/Sources/OpenClaw/ComputerActionService.swift
Normal file
555
apps/macos/Sources/OpenClaw/ComputerActionService.swift
Normal file
|
|
@ -0,0 +1,555 @@
|
|||
import AppKit
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
import OpenClawKit
|
||||
import PeekabooAutomationKit
|
||||
import PeekabooFoundation
|
||||
@preconcurrency import ScreenCaptureKit
|
||||
|
||||
/// Fulfills `computer.act` on this Mac by driving the embedded Peekaboo
|
||||
/// automation engine in-process. Peekaboo covers single/right/double click,
|
||||
/// move, drag, scroll, type, and key/hold. A narrow CoreGraphics path handles
|
||||
/// the computer_20251124 primitives Peekaboo cannot express: middle click,
|
||||
/// triple click, separate mouse down/up, and modifier-held clicks/scroll.
|
||||
@MainActor
|
||||
final class ComputerActionService {
|
||||
enum ComputerActionError: LocalizedError {
|
||||
case accessibilityNotTrusted
|
||||
case noDisplays
|
||||
case invalidScreenIndex(Int)
|
||||
case missingCoordinate
|
||||
case coordinateOutOfBounds
|
||||
case invalidReferenceWidth
|
||||
case missingKeys
|
||||
case emptyText
|
||||
case invalidScroll
|
||||
case invalidModifier(String)
|
||||
case eventCreationFailed
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .accessibilityNotTrusted:
|
||||
"Accessibility permission is required for computer control"
|
||||
case .noDisplays:
|
||||
"No displays available for computer control"
|
||||
case let .invalidScreenIndex(idx):
|
||||
"Invalid screen index \(idx)"
|
||||
case .missingCoordinate:
|
||||
"coordinate is required for this action"
|
||||
case .coordinateOutOfBounds:
|
||||
"coordinate is outside the captured screen"
|
||||
case .invalidReferenceWidth:
|
||||
"refWidth must be a positive integer"
|
||||
case .missingKeys:
|
||||
"keys are required for this action"
|
||||
case .emptyText:
|
||||
"text is required for this action"
|
||||
case .invalidScroll:
|
||||
"scrollDirection is required for scroll"
|
||||
case let .invalidModifier(token):
|
||||
"unsupported modifier '\(token)'"
|
||||
case .eventCreationFailed:
|
||||
"Failed to synthesize input event"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private let automation: UIAutomationService
|
||||
private let permissions: PermissionsService
|
||||
/// Tracks whether a left_mouse_down is outstanding so mouse_move emits
|
||||
/// drag events (state persists across invokes on the shared instance).
|
||||
private var leftButtonDown = false
|
||||
/// Bounded watchdog that releases a stuck left button if the matching
|
||||
/// left_mouse_up never arrives (arm expiry, disconnect, or a failed turn).
|
||||
private var buttonReleaseTask: Task<Void, Never>?
|
||||
/// Modifier flags held since the outstanding left_mouse_down, reapplied to
|
||||
/// drag and release events so a modifier-held split drag keeps Cmd/Opt/Shift
|
||||
/// for the whole gesture even when later turns omit the modifier.
|
||||
private var heldButtonFlags: CGEventFlags = []
|
||||
/// Bumped on every lifecycle release request. `perform` captures this before
|
||||
/// it suspends and re-checks after dispatch: a disconnect/stop/disable release
|
||||
/// can run during an await while `leftButtonDown` is still false, see nothing
|
||||
/// held, and no-op, after which dispatch arms the button. The post-dispatch
|
||||
/// re-check releases that just-armed button so a lifecycle release can never
|
||||
/// be defeated by actor reentrancy at an await before the button is set.
|
||||
private var releaseGeneration: UInt64 = 0
|
||||
|
||||
// Drag pacing: fast enough to feel responsive, slow enough that dropped
|
||||
// targets (AppKit hit-testing mid-drag) do not misfire.
|
||||
private static let dragDurationMs = 400
|
||||
private static let dragSteps = 24
|
||||
private static let clickInterEventDelay: useconds_t = 12000
|
||||
/// Cap wheel ticks at the node so a direct armed caller cannot overflow the
|
||||
/// Int32 wheel delta (line count = ticks * 5) and crash the app.
|
||||
private static let maxScrollTicks = 100
|
||||
/// Cap hold_key at the node: computer.act is directly invocable once armed,
|
||||
/// so an unbounded durationMs must not pin a key down for minutes.
|
||||
private static let maxHoldMs = 10000
|
||||
/// Allow slightly-past-edge coordinates so clicks on the last row/column of
|
||||
/// the reported frame still land instead of erroring on rounding.
|
||||
private static let coordinateBoundsEpsilon: Double = 2
|
||||
/// Idle timeout for an outstanding left button. Refreshed by each drag move,
|
||||
/// so a legitimate multi-turn drag (every turn adds a screenshot plus a model
|
||||
/// inference) is not force-released mid-gesture. Only a truly abandoned button
|
||||
/// (arm expiry, disconnect, or a failed turn with no further activity) hits
|
||||
/// this bounded cleanup.
|
||||
private static let buttonHoldIdleTimeoutNanoseconds: UInt64 = 120 * 1_000_000_000
|
||||
|
||||
init() {
|
||||
self.automation = UIAutomationService()
|
||||
self.permissions = PermissionsService()
|
||||
}
|
||||
|
||||
func perform(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult {
|
||||
guard self.permissions.checkAccessibilityPermission() else {
|
||||
throw ComputerActionError.accessibilityNotTrusted
|
||||
}
|
||||
// Capture the release generation before the first suspension. If a
|
||||
// lifecycle release runs while this action is awaiting below (when
|
||||
// leftButtonDown is still false), it no-ops; the check after dispatch
|
||||
// then releases any button this action armed so the release wins.
|
||||
let releaseGenerationAtStart = self.releaseGeneration
|
||||
let display = try await self.resolveDisplay(screenIndex: params.screenIndex)
|
||||
try await self.dispatch(params, display: display)
|
||||
// Catch-up release scoped to the left_mouse_down that armed the button:
|
||||
// that branch is synchronous with no await before the check, so
|
||||
// leftButtonDown here reflects THIS action's own arm, not a concurrent
|
||||
// invoke's. Restricting to leftMouseDown keeps a stale non-arming invoke
|
||||
// (scroll/type/move) that predates a lifecycle release from releasing a
|
||||
// button a newer action legitimately holds. If the generation moved while
|
||||
// this action was suspended, a lifecycle release ran before it armed, so
|
||||
// the release must win and the just-armed button is released.
|
||||
if params.action == .leftMouseDown,
|
||||
self.leftButtonDown,
|
||||
self.releaseGeneration != releaseGenerationAtStart
|
||||
{
|
||||
self.releaseHeldInput()
|
||||
}
|
||||
let cursor = self.automation.currentMouseLocation() ?? CGPoint.zero
|
||||
return OpenClawComputerActResult(ok: true, cursorX: cursor.x, cursorY: cursor.y)
|
||||
}
|
||||
|
||||
// MARK: - Dispatch
|
||||
|
||||
private func dispatch(
|
||||
_ params: OpenClawComputerActParams,
|
||||
display: ResolvedDisplay) async throws
|
||||
{
|
||||
let modifiers = try ComputerModifiers.parse(params.modifiers)
|
||||
switch params.action {
|
||||
case .leftClick, .rightClick, .doubleClick:
|
||||
let point = try self.requiredPoint(params, display: display)
|
||||
let button: ComputerMouseButton = params.action == .rightClick ? .right : .left
|
||||
let count = params.action == .doubleClick ? 2 : 1
|
||||
if modifiers.isEmpty {
|
||||
try await self.peekabooClick(at: point, action: params.action)
|
||||
} else {
|
||||
try self.rawClick(at: point, button: button, count: count, flags: modifiers.flags)
|
||||
}
|
||||
case .middleClick:
|
||||
let point = try self.requiredPoint(params, display: display)
|
||||
try self.rawClick(at: point, button: .middle, count: 1, flags: modifiers.flags)
|
||||
case .tripleClick:
|
||||
let point = try self.requiredPoint(params, display: display)
|
||||
try self.rawClick(at: point, button: .left, count: 3, flags: modifiers.flags)
|
||||
case .mouseMove:
|
||||
let point = try self.requiredPoint(params, display: display)
|
||||
if self.leftButtonDown {
|
||||
// A drag is in progress; ordinary moveMouse would post
|
||||
// mouseMoved and break drag targets, so emit dragged events
|
||||
// carrying the modifiers held since left_mouse_down.
|
||||
try self.postMouseEvent(
|
||||
.leftMouseDragged,
|
||||
at: point,
|
||||
button: .left,
|
||||
clickState: 1,
|
||||
flags: self.heldButtonFlags)
|
||||
// Refresh the release watchdog: an active drag must not be
|
||||
// auto-released mid-gesture during normal tool-loop latency.
|
||||
self.armButtonWatchdog()
|
||||
} else {
|
||||
try await self.automation.moveMouse(to: point, duration: 0, steps: 1, profile: .linear)
|
||||
}
|
||||
case .leftClickDrag:
|
||||
let to = try self.requiredPoint(params, display: display)
|
||||
let from = try self.point(params.fromX, params.fromY, params: params, display: display)
|
||||
?? to
|
||||
try await self.automation.drag(DragOperationRequest(
|
||||
from: from,
|
||||
to: to,
|
||||
duration: Self.dragDurationMs,
|
||||
steps: Self.dragSteps,
|
||||
modifiers: modifiers.peekabooString,
|
||||
profile: .linear))
|
||||
case .leftMouseDown, .leftMouseUp:
|
||||
// Coordinate is optional: press/release at the current cursor when omitted.
|
||||
let point = try self.point(params.x, params.y, params: params, display: display)
|
||||
?? (self.automation.currentMouseLocation() ?? CGPoint.zero)
|
||||
if params.action == .leftMouseDown {
|
||||
try self.rawMouseButton(down: true, at: point, flags: modifiers.flags)
|
||||
self.setLeftButtonDown(true, flags: modifiers.flags)
|
||||
} else {
|
||||
// Release with the modifiers held since left_mouse_down (unioned
|
||||
// with any the release turn resends) so modifier-held drops keep
|
||||
// their copy/move semantics.
|
||||
let releaseFlags = self.heldButtonFlags.union(modifiers.flags)
|
||||
try self.rawMouseButton(down: false, at: point, flags: releaseFlags)
|
||||
self.setLeftButtonDown(false)
|
||||
}
|
||||
case .scroll:
|
||||
try await self.performScroll(params, display: display, modifiers: modifiers)
|
||||
case .type:
|
||||
guard let text = params.text, !text.isEmpty else { throw ComputerActionError.emptyText }
|
||||
try await self.automation.type(
|
||||
text: text,
|
||||
target: nil,
|
||||
clearExisting: false,
|
||||
typingDelay: 0,
|
||||
snapshotId: nil)
|
||||
case .key:
|
||||
let keys = try self.requireKeys(params.keys)
|
||||
try await self.automation.hotkey(keys: keys, holdDuration: 0)
|
||||
case .holdKey:
|
||||
let keys = try self.requireKeys(params.keys)
|
||||
let holdMs = min(Self.maxHoldMs, max(0, params.durationMs ?? 1000))
|
||||
try await self.automation.hotkey(keys: keys, holdDuration: holdMs)
|
||||
}
|
||||
}
|
||||
|
||||
private func peekabooClick(at point: CGPoint, action: OpenClawComputerAction) async throws {
|
||||
let clickType: ClickType = switch action {
|
||||
case .rightClick: .right
|
||||
case .doubleClick: .double
|
||||
default: .single
|
||||
}
|
||||
try await self.automation.click(target: .coordinates(point), clickType: clickType, snapshotId: nil)
|
||||
}
|
||||
|
||||
private func performScroll(
|
||||
_ params: OpenClawComputerActParams,
|
||||
display: ResolvedDisplay,
|
||||
modifiers: ComputerModifiers) async throws
|
||||
{
|
||||
guard let direction = params.scrollDirection else { throw ComputerActionError.invalidScroll }
|
||||
let amount = min(Self.maxScrollTicks, max(1, params.scrollAmount ?? 3))
|
||||
// Position the pointer over the requested region first; both Peekaboo
|
||||
// and the raw wheel event scroll at the current mouse location.
|
||||
if let point = try self.point(params.x, params.y, params: params, display: display) {
|
||||
try await self.automation.moveMouse(to: point, duration: 0, steps: 1, profile: .linear)
|
||||
}
|
||||
if modifiers.isEmpty {
|
||||
try await self.automation.scroll(ScrollRequest(
|
||||
direction: Self.scrollDirection(direction),
|
||||
amount: amount))
|
||||
} else {
|
||||
try self.rawScroll(direction: direction, amount: amount, flags: modifiers.flags)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Coordinate mapping
|
||||
|
||||
/// The target display in global points plus the capture source width used to
|
||||
/// derive the captured screenshot pixel width for coordinate scaling.
|
||||
private struct ResolvedDisplay {
|
||||
var geometry: OpenClawComputerDisplayGeometry
|
||||
var sourceWidth: Double
|
||||
var sourceHeight: Double
|
||||
}
|
||||
|
||||
private func requiredPoint(
|
||||
_ params: OpenClawComputerActParams,
|
||||
display: ResolvedDisplay) throws -> CGPoint
|
||||
{
|
||||
guard let point = try self.point(params.x, params.y, params: params, display: display) else {
|
||||
throw ComputerActionError.missingCoordinate
|
||||
}
|
||||
return point
|
||||
}
|
||||
|
||||
private func point(
|
||||
_ x: Double?,
|
||||
_ y: Double?,
|
||||
params: OpenClawComputerActParams,
|
||||
display: ResolvedDisplay) throws -> CGPoint?
|
||||
{
|
||||
if x == nil, y == nil {
|
||||
return nil
|
||||
}
|
||||
// A partial coordinate (only x or only y) is malformed: optional-coordinate
|
||||
// actions (scroll, mouse down/up) must fail rather than silently acting at
|
||||
// the current cursor, and a partial drag origin must not fall back to the
|
||||
// destination.
|
||||
guard let x, let y else { throw ComputerActionError.missingCoordinate }
|
||||
// A malformed direct computer.act request could carry refWidth <= 0, which
|
||||
// would make capturedWidth non-positive and silently map every coordinate
|
||||
// to the display origin. Reject it as an invalid request before clicking.
|
||||
if let refWidth = params.refWidth, refWidth <= 0 {
|
||||
throw ComputerActionError.invalidReferenceWidth
|
||||
}
|
||||
let capturedWidth = OpenClawComputerInputGeometry.capturedWidth(
|
||||
refWidth: params.refWidth,
|
||||
sourceWidth: display.sourceWidth,
|
||||
sourceHeight: display.sourceHeight)
|
||||
let mapped = OpenClawComputerInputGeometry.mapReferencePointToGlobal(
|
||||
x: x,
|
||||
y: y,
|
||||
capturedWidthPixels: capturedWidth,
|
||||
display: display.geometry)
|
||||
// Reject coordinates well outside the captured display: on a multi-display
|
||||
// Mac an out-of-frame coordinate could otherwise map onto an adjacent
|
||||
// screen and click content the model never saw.
|
||||
let geometry = display.geometry
|
||||
let epsilon = Self.coordinateBoundsEpsilon
|
||||
let withinX = mapped.x >= geometry.originX - epsilon
|
||||
&& mapped.x <= geometry.originX + geometry.widthPoints + epsilon
|
||||
let withinY = mapped.y >= geometry.originY - epsilon
|
||||
&& mapped.y <= geometry.originY + geometry.heightPoints + epsilon
|
||||
guard withinX, withinY else { throw ComputerActionError.coordinateOutOfBounds }
|
||||
// Clamp the epsilon-tolerated rounding to strictly inside the selected
|
||||
// display so a far-edge coordinate cannot post onto an adjacent screen.
|
||||
let clamped = OpenClawComputerInputGeometry.clampToDisplay(
|
||||
x: mapped.x,
|
||||
y: mapped.y,
|
||||
display: geometry)
|
||||
return CGPoint(x: clamped.x, y: clamped.y)
|
||||
}
|
||||
|
||||
// MARK: - Button-hold watchdog
|
||||
|
||||
private func setLeftButtonDown(_ down: Bool, flags: CGEventFlags = []) {
|
||||
self.buttonReleaseTask?.cancel()
|
||||
self.buttonReleaseTask = nil
|
||||
self.leftButtonDown = down
|
||||
self.heldButtonFlags = down ? flags : []
|
||||
guard down else { return }
|
||||
self.armButtonWatchdog()
|
||||
}
|
||||
|
||||
/// Arms or re-arms the bounded idle watchdog for an outstanding left button.
|
||||
/// Re-armed on each drag move so a live multi-turn gesture is never cut off,
|
||||
/// while an abandoned button still gets released after the idle timeout.
|
||||
private func armButtonWatchdog() {
|
||||
self.buttonReleaseTask?.cancel()
|
||||
self.buttonReleaseTask = Task { [weak self] in
|
||||
try? await Task.sleep(nanoseconds: Self.buttonHoldIdleTimeoutNanoseconds)
|
||||
guard !Task.isCancelled else { return }
|
||||
self?.autoReleaseLeftButton()
|
||||
}
|
||||
}
|
||||
|
||||
private func autoReleaseLeftButton() {
|
||||
guard self.leftButtonDown else { return }
|
||||
let point = self.automation.currentMouseLocation() ?? CGPoint.zero
|
||||
try? self.rawMouseButton(down: false, at: point, flags: self.heldButtonFlags)
|
||||
self.leftButtonDown = false
|
||||
self.heldButtonFlags = []
|
||||
self.buttonReleaseTask = nil
|
||||
}
|
||||
|
||||
/// Releases any outstanding synthetic left button immediately. Called on
|
||||
/// lifecycle transitions (node disconnect, node stop, Computer Control
|
||||
/// disabled) so a stranded left_mouse_down is not held until the idle
|
||||
/// watchdog fires. Idempotent when nothing is held.
|
||||
func releaseHeldInput() {
|
||||
// Bump first so a computer.act action suspended at an await before it
|
||||
// arms the button observes the changed generation after it resumes and
|
||||
// releases itself (see perform); otherwise this no-ops for a not-yet-held
|
||||
// button and the action would leave it stuck until the idle watchdog.
|
||||
self.releaseGeneration &+= 1
|
||||
self.buttonReleaseTask?.cancel()
|
||||
self.buttonReleaseTask = nil
|
||||
guard self.leftButtonDown else { return }
|
||||
let point = self.automation.currentMouseLocation() ?? CGPoint.zero
|
||||
try? self.rawMouseButton(down: false, at: point, flags: self.heldButtonFlags)
|
||||
self.leftButtonDown = false
|
||||
self.heldButtonFlags = []
|
||||
}
|
||||
|
||||
private func resolveDisplay(screenIndex: Int?) async throws -> ResolvedDisplay {
|
||||
// Match ScreenSnapshotService display ordering so a computer.act
|
||||
// screenIndex targets the same display the model saw in screen.snapshot.
|
||||
let content = try await SCShareableContent.current
|
||||
let displays = content.displays.sorted { $0.displayID < $1.displayID }
|
||||
guard !displays.isEmpty else { throw ComputerActionError.noDisplays }
|
||||
let idx = screenIndex ?? 0
|
||||
guard idx >= 0, idx < displays.count else { throw ComputerActionError.invalidScreenIndex(idx) }
|
||||
// CGDisplayBounds is the global top-left point space CGEvent uses;
|
||||
// SCDisplay.width/height is the capture source size ScreenSnapshotService
|
||||
// caps to refWidth, so together they recover the captured pixel scale and
|
||||
// the source aspect ratio needed for portrait longest-edge scaling.
|
||||
let bounds = CGDisplayBounds(displays[idx].displayID)
|
||||
return ResolvedDisplay(
|
||||
geometry: OpenClawComputerDisplayGeometry(
|
||||
originX: bounds.origin.x,
|
||||
originY: bounds.origin.y,
|
||||
widthPoints: bounds.width,
|
||||
heightPoints: bounds.height),
|
||||
sourceWidth: Double(displays[idx].width),
|
||||
sourceHeight: Double(displays[idx].height))
|
||||
}
|
||||
|
||||
private func requireKeys(_ keys: String?) throws -> String {
|
||||
guard let keys, !keys.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
|
||||
throw ComputerActionError.missingKeys
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
private static func scrollDirection(
|
||||
_ direction: OpenClawComputerScrollDirection) -> PeekabooFoundation.ScrollDirection
|
||||
{
|
||||
switch direction {
|
||||
case .up: .up
|
||||
case .down: .down
|
||||
case .left: .left
|
||||
case .right: .right
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Raw CoreGraphics primitives
|
||||
|
||||
private func rawClick(at point: CGPoint, button: ComputerMouseButton, count: Int, flags: CGEventFlags) throws {
|
||||
for click in 1...max(1, count) {
|
||||
try self.postMouseEvent(
|
||||
button.downType,
|
||||
at: point,
|
||||
button: button.cgButton,
|
||||
clickState: click,
|
||||
flags: flags)
|
||||
try self.postMouseEvent(button.upType, at: point, button: button.cgButton, clickState: click, flags: flags)
|
||||
usleep(Self.clickInterEventDelay)
|
||||
}
|
||||
}
|
||||
|
||||
private func rawMouseButton(down: Bool, at point: CGPoint, flags: CGEventFlags) throws {
|
||||
let type: CGEventType = down ? .leftMouseDown : .leftMouseUp
|
||||
try self.postMouseEvent(type, at: point, button: .left, clickState: 1, flags: flags)
|
||||
}
|
||||
|
||||
private func postMouseEvent(
|
||||
_ type: CGEventType,
|
||||
at point: CGPoint,
|
||||
button: CGMouseButton,
|
||||
clickState: Int,
|
||||
flags: CGEventFlags) throws
|
||||
{
|
||||
guard let event = CGEvent(
|
||||
mouseEventSource: nil,
|
||||
mouseType: type,
|
||||
mouseCursorPosition: point,
|
||||
mouseButton: button)
|
||||
else {
|
||||
throw ComputerActionError.eventCreationFailed
|
||||
}
|
||||
if clickState > 1 {
|
||||
event.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))
|
||||
}
|
||||
if !flags.isEmpty {
|
||||
event.flags = flags
|
||||
}
|
||||
event.post(tap: .cghidEventTap)
|
||||
}
|
||||
|
||||
private func rawScroll(direction: OpenClawComputerScrollDirection, amount: Int, flags: CGEventFlags) throws {
|
||||
// Line units per tick match Peekaboo's non-smooth scroll (~5 lines).
|
||||
let lines = Int32(amount * 5)
|
||||
let (wheel1, wheel2): (Int32, Int32) = switch direction {
|
||||
case .up: (lines, 0)
|
||||
case .down: (-lines, 0)
|
||||
case .left: (0, lines)
|
||||
case .right: (0, -lines)
|
||||
}
|
||||
guard let event = CGEvent(
|
||||
scrollWheelEvent2Source: nil,
|
||||
units: .line,
|
||||
wheelCount: 2,
|
||||
wheel1: wheel1,
|
||||
wheel2: wheel2,
|
||||
wheel3: 0)
|
||||
else {
|
||||
throw ComputerActionError.eventCreationFailed
|
||||
}
|
||||
if !flags.isEmpty {
|
||||
event.flags = flags
|
||||
}
|
||||
event.post(tap: .cghidEventTap)
|
||||
}
|
||||
}
|
||||
|
||||
/// Mouse button plus the CoreGraphics event types for the raw click path.
|
||||
private enum ComputerMouseButton {
|
||||
case left
|
||||
case right
|
||||
case middle
|
||||
|
||||
var cgButton: CGMouseButton {
|
||||
switch self {
|
||||
case .left: .left
|
||||
case .right: .right
|
||||
case .middle: .center
|
||||
}
|
||||
}
|
||||
|
||||
var downType: CGEventType {
|
||||
switch self {
|
||||
case .left: .leftMouseDown
|
||||
case .right: .rightMouseDown
|
||||
case .middle: .otherMouseDown
|
||||
}
|
||||
}
|
||||
|
||||
var upType: CGEventType {
|
||||
switch self {
|
||||
case .left: .leftMouseUp
|
||||
case .right: .rightMouseUp
|
||||
case .middle: .otherMouseUp
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a portable modifier string ("shift", "cmd+alt") into CGEvent flags and
|
||||
/// the comma-separated form Peekaboo's drag request expects.
|
||||
struct ComputerModifiers {
|
||||
var flags: CGEventFlags
|
||||
var peekabooTokens: [String]
|
||||
|
||||
var isEmpty: Bool {
|
||||
self.flags.isEmpty
|
||||
}
|
||||
|
||||
var peekabooString: String? {
|
||||
self.peekabooTokens.isEmpty ? nil : self.peekabooTokens.joined(separator: ",")
|
||||
}
|
||||
|
||||
static func parse(_ raw: String?) throws -> ComputerModifiers {
|
||||
guard let raw, !raw.isEmpty else { return ComputerModifiers(flags: [], peekabooTokens: []) }
|
||||
var flags: CGEventFlags = []
|
||||
var tokens: [String] = []
|
||||
for piece in raw.split(whereSeparator: { $0 == "+" || $0 == "," || $0 == " " }) {
|
||||
let key = piece.lowercased()
|
||||
switch key {
|
||||
case "cmd", "command", "meta", "super", "win", "windows":
|
||||
flags.insert(.maskCommand)
|
||||
tokens.append("cmd")
|
||||
case "shift":
|
||||
flags.insert(.maskShift)
|
||||
tokens.append("shift")
|
||||
case "ctrl", "control":
|
||||
flags.insert(.maskControl)
|
||||
tokens.append("ctrl")
|
||||
case "alt", "opt", "option":
|
||||
flags.insert(.maskAlternate)
|
||||
tokens.append("alt")
|
||||
case "fn", "function":
|
||||
flags.insert(.maskSecondaryFn)
|
||||
tokens.append("fn")
|
||||
default:
|
||||
// A typo like "shfit" would otherwise silently drop the modifier
|
||||
// and perform a materially different high-risk gesture (a plain
|
||||
// click instead of a modifier-click); reject it instead.
|
||||
throw ComputerActionService.ComputerActionError.invalidModifier(key)
|
||||
}
|
||||
}
|
||||
return ComputerModifiers(flags: flags, peekabooTokens: tokens)
|
||||
}
|
||||
}
|
||||
|
|
@ -34,6 +34,7 @@ let remoteProjectRootKey = "openclaw.remoteProjectRoot"
|
|||
let remoteCliPathKey = "openclaw.remoteCliPath"
|
||||
let canvasEnabledKey = "openclaw.canvasEnabled"
|
||||
let cameraEnabledKey = "openclaw.cameraEnabled"
|
||||
let computerControlEnabledKey = "openclaw.computerControlEnabled"
|
||||
let systemRunPolicyKey = "openclaw.systemRunPolicy"
|
||||
let systemRunAllowlistKey = "openclaw.systemRunAllowlist"
|
||||
let systemRunEnabledKey = "openclaw.systemRunEnabled"
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ struct GeneralSettings: View {
|
|||
|
||||
@Bindable var state: AppState
|
||||
@AppStorage(cameraEnabledKey) private var cameraEnabled: Bool = false
|
||||
@AppStorage(computerControlEnabledKey) private var computerControlEnabled: Bool = false
|
||||
let page: Page
|
||||
let isActive: Bool
|
||||
private let healthStore = HealthStore.shared
|
||||
|
|
@ -99,6 +100,11 @@ struct GeneralSettings: View {
|
|||
subtitle: "Allow the agent to capture a photo or short video via the built-in camera.",
|
||||
binding: self.$cameraEnabled)
|
||||
|
||||
SettingsCardToggleRow(
|
||||
title: "Allow Computer Control",
|
||||
subtitle: "Let the agent move the pointer, click, and type on this Mac. Requires Accessibility permission and stays disarmed until an operator arms it. High risk.",
|
||||
binding: self.$computerControlEnabled)
|
||||
|
||||
SettingsCardToggleRow(
|
||||
title: "Enable Peekaboo Bridge",
|
||||
subtitle: "Allow signed tools (e.g. `peekaboo`) to drive UI automation via PeekabooBridge.",
|
||||
|
|
|
|||
|
|
@ -140,7 +140,10 @@ final class MacNodeModeCoordinator: NSObject {
|
|||
self.endpointRefreshTask = nil
|
||||
self.reconnectProbeTask?.cancel()
|
||||
self.reconnectProbeTask = nil
|
||||
Task { await self.session.disconnect() }
|
||||
Task {
|
||||
await self.runtime.releaseHeldComputerInput()
|
||||
await self.session.disconnect()
|
||||
}
|
||||
}
|
||||
|
||||
func setPreferredGatewayStableID(_ stableID: String?) {
|
||||
|
|
@ -179,6 +182,13 @@ final class MacNodeModeCoordinator: NSObject {
|
|||
browserControlEnabled: browserControlEnabled,
|
||||
cameraEnabled: cameraEnabled,
|
||||
codexThreadCatalogEnabled: codexThreadCatalogEnabled)
|
||||
// If Computer Control was turned off, release any button the
|
||||
// computer.act service is still holding rather than waiting for
|
||||
// the idle watchdog. This refresh loop re-runs on the settings
|
||||
// change that drops the cap.
|
||||
if !caps.contains(OpenClawCapability.computer.rawValue) {
|
||||
await self.runtime.releaseHeldComputerInput()
|
||||
}
|
||||
let commands = self.currentCommands(caps: caps)
|
||||
let permissions = await self.currentPermissions()
|
||||
let connectOptions = GatewayConnectOptions(
|
||||
|
|
@ -216,6 +226,7 @@ final class MacNodeModeCoordinator: NSObject {
|
|||
onDisconnected: { [weak self] reason in
|
||||
guard let self else { return }
|
||||
await self.runtime.setEventSender(nil)
|
||||
await self.runtime.releaseHeldComputerInput()
|
||||
await self.scheduleReconnectProbe()
|
||||
self.logger.error("mac node disconnected: \(reason, privacy: .public)")
|
||||
},
|
||||
|
|
@ -270,6 +281,7 @@ final class MacNodeModeCoordinator: NSObject {
|
|||
nonisolated static func resolvedCaps(
|
||||
browserControlEnabled: Bool,
|
||||
cameraEnabled: Bool,
|
||||
computerControlEnabled: Bool,
|
||||
locationMode: OpenClawLocationMode,
|
||||
connectionMode: AppState.ConnectionMode,
|
||||
codexThreadCatalogEnabled: Bool = false) -> [String]
|
||||
|
|
@ -284,6 +296,11 @@ final class MacNodeModeCoordinator: NSObject {
|
|||
if cameraEnabled {
|
||||
caps.append(OpenClawCapability.camera.rawValue)
|
||||
}
|
||||
// Advertised only when the operator has enabled Computer Control; the
|
||||
// command is dangerous and stays disarmed until allowlisted on the gateway.
|
||||
if computerControlEnabled {
|
||||
caps.append(OpenClawCapability.computer.rawValue)
|
||||
}
|
||||
if locationMode != .off {
|
||||
caps.append(OpenClawCapability.location.rawValue)
|
||||
}
|
||||
|
|
@ -299,9 +316,12 @@ final class MacNodeModeCoordinator: NSObject {
|
|||
codexThreadCatalogEnabled: Bool) -> [String]
|
||||
{
|
||||
let rawLocationMode = UserDefaults.standard.string(forKey: locationModeKey) ?? "off"
|
||||
let computerControlEnabled =
|
||||
UserDefaults.standard.object(forKey: computerControlEnabledKey) as? Bool ?? false
|
||||
return Self.resolvedCaps(
|
||||
browserControlEnabled: browserControlEnabled,
|
||||
cameraEnabled: cameraEnabled,
|
||||
computerControlEnabled: computerControlEnabled,
|
||||
locationMode: OpenClawLocationMode(rawValue: rawLocationMode) ?? .off,
|
||||
connectionMode: AppStateStore.shared.connectionMode,
|
||||
codexThreadCatalogEnabled: codexThreadCatalogEnabled)
|
||||
|
|
@ -346,6 +366,9 @@ final class MacNodeModeCoordinator: NSObject {
|
|||
if capsSet.contains(MacNodeCodexThreadCatalogContract.capability) {
|
||||
commands.append(MacNodeCodexThreadCatalogContract.listCommand)
|
||||
}
|
||||
if capsSet.contains(OpenClawCapability.computer.rawValue) {
|
||||
commands.append(OpenClawComputerCommand.act.rawValue)
|
||||
}
|
||||
|
||||
return commands
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,9 +9,11 @@ actor MacNodeRuntime {
|
|||
private let cameraCapture = CameraCaptureService()
|
||||
private let makeMainActorServices: () async -> any MacNodeRuntimeMainActorServices
|
||||
private let browserProxyRequest: @Sendable (String?) async throws -> String
|
||||
// Injectable so tests can pin the gate instead of racing on process-global
|
||||
// OPENCLAW_CONFIG_PATH; config parsing is covered by OpenClawConfigFileTests.
|
||||
/// Injectable so tests can pin the gate instead of racing on process-global
|
||||
/// OPENCLAW_CONFIG_PATH; config parsing is covered by OpenClawConfigFileTests.
|
||||
private let browserControlEnabled: @Sendable () -> Bool
|
||||
// Injectable so tests pin the gate instead of racing on process-global UserDefaults.
|
||||
private let computerControlEnabled: @Sendable () -> Bool
|
||||
private let canvasSurfaceUrl: @Sendable () async -> String?
|
||||
private let refreshCanvasSurfaceUrl: @Sendable () async -> String?
|
||||
private let codexThreadCatalogEnabled: @Sendable () -> Bool
|
||||
|
|
@ -30,6 +32,9 @@ actor MacNodeRuntime {
|
|||
browserControlEnabled: @escaping @Sendable () -> Bool = {
|
||||
OpenClawConfigFile.browserControlEnabled()
|
||||
},
|
||||
computerControlEnabled: @escaping @Sendable () -> Bool = {
|
||||
MacNodeRuntime.computerControlEnabledDefault()
|
||||
},
|
||||
canvasSurfaceUrl: @escaping @Sendable () async -> String? = {
|
||||
await GatewayConnection.shared.canvasPluginSurfaceUrl()
|
||||
},
|
||||
|
|
@ -45,6 +50,7 @@ actor MacNodeRuntime {
|
|||
self.makeMainActorServices = makeMainActorServices
|
||||
self.browserProxyRequest = browserProxyRequest
|
||||
self.browserControlEnabled = browserControlEnabled
|
||||
self.computerControlEnabled = computerControlEnabled
|
||||
self.canvasSurfaceUrl = canvasSurfaceUrl
|
||||
self.refreshCanvasSurfaceUrl = refreshCanvasSurfaceUrl
|
||||
self.codexThreadCatalogEnabled = codexThreadCatalogEnabled
|
||||
|
|
@ -95,6 +101,8 @@ actor MacNodeRuntime {
|
|||
return try await self.handleScreenSnapshotInvoke(req)
|
||||
case MacNodeScreenCommand.record.rawValue:
|
||||
return try await self.handleScreenRecordInvoke(req)
|
||||
case OpenClawComputerCommand.act.rawValue:
|
||||
return try await self.handleComputerActInvoke(req)
|
||||
case OpenClawSystemCommand.run.rawValue:
|
||||
return try await self.handleSystemRun(req)
|
||||
case OpenClawSystemCommand.which.rawValue:
|
||||
|
|
@ -174,7 +182,9 @@ actor MacNodeRuntime {
|
|||
let params = try? Self.decodeParams(OpenClawCanvasSnapshotParams.self, from: req.paramsJSON)
|
||||
let format = params?.format ?? .jpeg
|
||||
let maxWidth: Int? = {
|
||||
if let raw = params?.maxWidth, raw > 0 { return raw }
|
||||
if let raw = params?.maxWidth, raw > 0 {
|
||||
return raw
|
||||
}
|
||||
return switch format {
|
||||
case .png: 900
|
||||
case .jpeg: 1600
|
||||
|
|
@ -360,6 +370,51 @@ actor MacNodeRuntime {
|
|||
}
|
||||
}
|
||||
|
||||
private func handleComputerActInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
guard self.computerControlEnabled() else {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: OpenClawNodeError(
|
||||
code: .unavailable,
|
||||
message: "COMPUTER_DISABLED: enable Computer Control in Settings"))
|
||||
}
|
||||
let params: OpenClawComputerActParams
|
||||
do {
|
||||
params = try Self.decodeParams(OpenClawComputerActParams.self, from: req.paramsJSON)
|
||||
} catch {
|
||||
return Self.errorResponse(
|
||||
req,
|
||||
code: .invalidRequest,
|
||||
message: "INVALID_REQUEST: invalid computer.act params")
|
||||
}
|
||||
let services = await self.mainActorServices()
|
||||
do {
|
||||
let result = try await services.performComputerAct(params)
|
||||
let payload = try Self.encodePayload(result)
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
} catch let error as ComputerActionService.ComputerActionError {
|
||||
switch error {
|
||||
case .accessibilityNotTrusted:
|
||||
return Self.errorResponse(
|
||||
req,
|
||||
code: .unavailable,
|
||||
message: "ACCESSIBILITY_REQUIRED: grant Accessibility permission to OpenClaw")
|
||||
case .noDisplays, .invalidScreenIndex, .missingCoordinate, .coordinateOutOfBounds,
|
||||
.invalidReferenceWidth, .missingKeys, .emptyText, .invalidScroll, .invalidModifier:
|
||||
return Self.errorResponse(
|
||||
req,
|
||||
code: .invalidRequest,
|
||||
message: "INVALID_REQUEST: \(error.localizedDescription)")
|
||||
case .eventCreationFailed:
|
||||
return Self.errorResponse(
|
||||
req,
|
||||
code: .unavailable,
|
||||
message: "UNAVAILABLE: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleScreenRecordInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
let params = (try? Self.decodeParams(MacNodeScreenRecordParams.self, from: req.paramsJSON)) ??
|
||||
MacNodeScreenRecordParams()
|
||||
|
|
@ -472,12 +527,22 @@ actor MacNodeRuntime {
|
|||
}
|
||||
|
||||
private func mainActorServices() async -> any MacNodeRuntimeMainActorServices {
|
||||
if let cachedMainActorServices { return cachedMainActorServices }
|
||||
if let cachedMainActorServices {
|
||||
return cachedMainActorServices
|
||||
}
|
||||
let services = await self.makeMainActorServices()
|
||||
self.cachedMainActorServices = services
|
||||
return services
|
||||
}
|
||||
|
||||
/// Releases any synthetic input the computer.act service is still holding
|
||||
/// (a left_mouse_down without its matching up) on lifecycle transitions:
|
||||
/// node disconnect, node stop, or Computer Control disabled. Uses the cached
|
||||
/// services directly so it never spins up services just to release nothing.
|
||||
func releaseHeldComputerInput() async {
|
||||
await self.cachedMainActorServices?.releaseHeldInput()
|
||||
}
|
||||
|
||||
private func handleA2UIReset(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
try await self.ensureA2UIHost()
|
||||
|
||||
|
|
@ -529,7 +594,9 @@ actor MacNodeRuntime {
|
|||
}
|
||||
|
||||
private func ensureA2UIHost() async throws {
|
||||
if await self.isA2UIReady() { return }
|
||||
if await self.isA2UIReady() {
|
||||
return
|
||||
}
|
||||
guard let a2uiUrl = await self.resolveA2UIHostUrlWithCapabilityRefresh() else {
|
||||
throw NSError(domain: "Canvas", code: 30, userInfo: [
|
||||
NSLocalizedDescriptionKey: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
|
||||
|
|
@ -539,12 +606,16 @@ actor MacNodeRuntime {
|
|||
_ = try await MainActor.run {
|
||||
try CanvasManager.shared.show(sessionKey: sessionKey, path: a2uiUrl)
|
||||
}
|
||||
if await self.isA2UIReady(poll: true) { return }
|
||||
if await self.isA2UIReady(poll: true) {
|
||||
return
|
||||
}
|
||||
if let refreshedUrl = await self.resolveA2UIHostUrlWithCapabilityRefresh(forceRefresh: true) {
|
||||
_ = try await MainActor.run {
|
||||
try CanvasManager.shared.show(sessionKey: sessionKey, path: refreshedUrl)
|
||||
}
|
||||
if await self.isA2UIReady(poll: true) { return }
|
||||
if await self.isA2UIReady(poll: true) {
|
||||
return
|
||||
}
|
||||
}
|
||||
throw NSError(domain: "Canvas", code: 31, userInfo: [
|
||||
NSLocalizedDescriptionKey: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable",
|
||||
|
|
@ -583,7 +654,9 @@ actor MacNodeRuntime {
|
|||
})()
|
||||
""")
|
||||
let trimmed = ready.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if trimmed == "true" { return true }
|
||||
if trimmed == "true" {
|
||||
return true
|
||||
}
|
||||
} catch {
|
||||
// Ignore transient eval failures while the page is loading.
|
||||
}
|
||||
|
|
@ -656,7 +729,9 @@ actor MacNodeRuntime {
|
|||
skillAllow: evaluation.skillAllow,
|
||||
sessionKey: sessionKey,
|
||||
runId: runId))
|
||||
if let response = approval.response { return response }
|
||||
if let response = approval.response {
|
||||
return response
|
||||
}
|
||||
let approvedByAsk = approval.approvedByAsk
|
||||
let persistAllowlist = approval.persistAllowlist
|
||||
self.persistAllowlistPatterns(
|
||||
|
|
@ -1146,13 +1221,19 @@ extension MacNodeRuntime {
|
|||
UserDefaults.standard.object(forKey: cameraEnabledKey) as? Bool ?? false
|
||||
}
|
||||
|
||||
nonisolated static func computerControlEnabledDefault() -> Bool {
|
||||
UserDefaults.standard.object(forKey: computerControlEnabledKey) as? Bool ?? false
|
||||
}
|
||||
|
||||
private nonisolated static func locationMode() -> OpenClawLocationMode {
|
||||
let raw = UserDefaults.standard.string(forKey: locationModeKey) ?? "off"
|
||||
return OpenClawLocationMode(rawValue: raw) ?? .off
|
||||
}
|
||||
|
||||
private nonisolated static func locationPreciseEnabled() -> Bool {
|
||||
if UserDefaults.standard.object(forKey: locationPreciseKey) == nil { return true }
|
||||
if UserDefaults.standard.object(forKey: locationPreciseKey) == nil {
|
||||
return true
|
||||
}
|
||||
return UserDefaults.standard.bool(forKey: locationPreciseKey)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,9 @@ protocol MacNodeRuntimeMainActorServices: Sendable {
|
|||
desiredAccuracy: OpenClawLocationAccuracy,
|
||||
maxAgeMs: Int?,
|
||||
timeoutMs: Int?) async throws -> CLLocation
|
||||
|
||||
func performComputerAct(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult
|
||||
func releaseHeldInput()
|
||||
}
|
||||
|
||||
@MainActor
|
||||
|
|
@ -31,6 +34,7 @@ final class LiveMacNodeRuntimeMainActorServices: MacNodeRuntimeMainActorServices
|
|||
private let screenSnapshotter = ScreenSnapshotService()
|
||||
private let screenRecorder = ScreenRecordService()
|
||||
private let locationService = MacNodeLocationService()
|
||||
private let computerAction = ComputerActionService()
|
||||
|
||||
func snapshotScreen(
|
||||
screenIndex: Int?,
|
||||
|
|
@ -79,4 +83,12 @@ final class LiveMacNodeRuntimeMainActorServices: MacNodeRuntimeMainActorServices
|
|||
maxAgeMs: maxAgeMs,
|
||||
timeoutMs: timeoutMs)
|
||||
}
|
||||
|
||||
func performComputerAct(_ params: OpenClawComputerActParams) async throws -> OpenClawComputerActResult {
|
||||
try await self.computerAction.perform(params)
|
||||
}
|
||||
|
||||
func releaseHeldInput() {
|
||||
self.computerAction.releaseHeldInput()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ struct MacNodeModeCoordinatorTests {
|
|||
let caps = MacNodeModeCoordinator.resolvedCaps(
|
||||
browserControlEnabled: true,
|
||||
cameraEnabled: false,
|
||||
computerControlEnabled: false,
|
||||
locationMode: .off,
|
||||
connectionMode: .remote)
|
||||
let commands = MacNodeModeCoordinator.resolvedCommands(caps: caps)
|
||||
|
|
@ -44,6 +45,7 @@ struct MacNodeModeCoordinatorTests {
|
|||
let caps = MacNodeModeCoordinator.resolvedCaps(
|
||||
browserControlEnabled: true,
|
||||
cameraEnabled: false,
|
||||
computerControlEnabled: false,
|
||||
locationMode: .off,
|
||||
connectionMode: .local)
|
||||
let commands = MacNodeModeCoordinator.resolvedCommands(caps: caps)
|
||||
|
|
@ -56,6 +58,7 @@ struct MacNodeModeCoordinatorTests {
|
|||
let caps = MacNodeModeCoordinator.resolvedCaps(
|
||||
browserControlEnabled: false,
|
||||
cameraEnabled: false,
|
||||
computerControlEnabled: false,
|
||||
locationMode: .off,
|
||||
connectionMode: .remote,
|
||||
codexThreadCatalogEnabled: true)
|
||||
|
|
@ -112,6 +115,28 @@ struct MacNodeModeCoordinatorTests {
|
|||
root: paddedDeny))
|
||||
}
|
||||
|
||||
@Test func `computer control cap gates the computer.act command`() {
|
||||
let enabledCaps = MacNodeModeCoordinator.resolvedCaps(
|
||||
browserControlEnabled: false,
|
||||
cameraEnabled: false,
|
||||
computerControlEnabled: true,
|
||||
locationMode: .off,
|
||||
connectionMode: .local)
|
||||
let enabledCommands = MacNodeModeCoordinator.resolvedCommands(caps: enabledCaps)
|
||||
#expect(enabledCaps.contains(OpenClawCapability.computer.rawValue))
|
||||
#expect(enabledCommands.contains(OpenClawComputerCommand.act.rawValue))
|
||||
|
||||
let disabledCaps = MacNodeModeCoordinator.resolvedCaps(
|
||||
browserControlEnabled: false,
|
||||
cameraEnabled: false,
|
||||
computerControlEnabled: false,
|
||||
locationMode: .off,
|
||||
connectionMode: .local)
|
||||
let disabledCommands = MacNodeModeCoordinator.resolvedCommands(caps: disabledCaps)
|
||||
#expect(!disabledCaps.contains(OpenClawCapability.computer.rawValue))
|
||||
#expect(!disabledCommands.contains(OpenClawComputerCommand.act.rawValue))
|
||||
}
|
||||
|
||||
@Test func `tls pin store key uses default wss port`() throws {
|
||||
let url = try #require(URL(string: "wss://gateway.example.ts.net"))
|
||||
#expect(MacNodeModeCoordinator.tlsPinStoreKey(for: url) == "gateway.example.ts.net:443")
|
||||
|
|
|
|||
|
|
@ -96,6 +96,15 @@ struct MacNodeRuntimeTests {
|
|||
_ = timeoutMs
|
||||
return CLLocation(latitude: 0, longitude: 0)
|
||||
}
|
||||
|
||||
func performComputerAct(_ params: OpenClawComputerActParams) async throws
|
||||
-> OpenClawComputerActResult
|
||||
{
|
||||
_ = params
|
||||
return OpenClawComputerActResult(ok: true, cursorX: 0, cursorY: 0)
|
||||
}
|
||||
|
||||
func releaseHeldInput() {}
|
||||
}
|
||||
|
||||
@Test func `handle invoke rejects unknown command`() async {
|
||||
|
|
@ -105,7 +114,7 @@ struct MacNodeRuntimeTests {
|
|||
#expect(response.ok == false)
|
||||
}
|
||||
|
||||
@Test func `handle invoke returns injected Codex thread catalog`() async throws {
|
||||
@Test func `handle invoke returns injected Codex thread catalog`() async {
|
||||
let payload = #"{"sessions":[]}"#
|
||||
let runtime = MacNodeRuntime(
|
||||
codexThreadCatalogEnabled: { true },
|
||||
|
|
@ -185,7 +194,7 @@ struct MacNodeRuntimeTests {
|
|||
paramsJSON: json))
|
||||
|
||||
#expect(response.ok == false)
|
||||
let denied = try #require((await probe.events()).first { $0.event == "exec.denied" })
|
||||
let denied = try #require(await (probe.events()).first { $0.event == "exec.denied" })
|
||||
struct Payload: Decodable {
|
||||
var sessionKey: String
|
||||
var runId: String
|
||||
|
|
@ -294,6 +303,15 @@ struct MacNodeRuntimeTests {
|
|||
{
|
||||
CLLocation(latitude: 0, longitude: 0)
|
||||
}
|
||||
|
||||
func performComputerAct(_ params: OpenClawComputerActParams) async throws
|
||||
-> OpenClawComputerActResult
|
||||
{
|
||||
_ = params
|
||||
return OpenClawComputerActResult(ok: true, cursorX: 0, cursorY: 0)
|
||||
}
|
||||
|
||||
func releaseHeldInput() {}
|
||||
}
|
||||
|
||||
let services = await MainActor.run { FakeMainActorServices() }
|
||||
|
|
@ -365,6 +383,15 @@ struct MacNodeRuntimeTests {
|
|||
_ = timeoutMs
|
||||
return CLLocation(latitude: 0, longitude: 0)
|
||||
}
|
||||
|
||||
func performComputerAct(_ params: OpenClawComputerActParams) async throws
|
||||
-> OpenClawComputerActResult
|
||||
{
|
||||
_ = params
|
||||
return OpenClawComputerActResult(ok: true, cursorX: 0, cursorY: 0)
|
||||
}
|
||||
|
||||
func releaseHeldInput() {}
|
||||
}
|
||||
|
||||
let services = await MainActor.run { FakeMainActorServices() }
|
||||
|
|
@ -403,7 +430,7 @@ struct MacNodeRuntimeTests {
|
|||
#expect(payload.capturedAtMs <= snapshotCalledAtMs!)
|
||||
}
|
||||
|
||||
@Test func `handle invoke screen snapshot rejects malformed params before capture`() async throws {
|
||||
@Test func `handle invoke screen snapshot rejects malformed params before capture`() async {
|
||||
let services = await MainActor.run { ScreenSnapshotProbeServices() }
|
||||
let runtime = MacNodeRuntime(makeMainActorServices: { services })
|
||||
|
||||
|
|
@ -420,7 +447,7 @@ struct MacNodeRuntimeTests {
|
|||
#expect(snapshotCallCount == 0)
|
||||
}
|
||||
|
||||
@Test func `handle invoke screen snapshot keeps nil params as defaults`() async throws {
|
||||
@Test func `handle invoke screen snapshot keeps nil params as defaults`() async {
|
||||
let services = await MainActor.run { ScreenSnapshotProbeServices() }
|
||||
let runtime = MacNodeRuntime(makeMainActorServices: { services })
|
||||
|
||||
|
|
@ -434,10 +461,153 @@ struct MacNodeRuntimeTests {
|
|||
#expect(received == MacNodeScreenSnapshotParams())
|
||||
}
|
||||
|
||||
@Test func `handle invoke screen snapshot sanitizes capture failures`() async throws {
|
||||
@MainActor
|
||||
final class ComputerActProbeServices: MacNodeRuntimeMainActorServices, @unchecked Sendable {
|
||||
var receivedParams: OpenClawComputerActParams?
|
||||
var actError: Error?
|
||||
|
||||
init(actError: Error? = nil) {
|
||||
self.actError = actError
|
||||
}
|
||||
|
||||
func snapshotScreen(
|
||||
screenIndex: Int?,
|
||||
maxWidth: Int?,
|
||||
quality: Double?,
|
||||
format: OpenClawScreenSnapshotFormat?) async throws
|
||||
-> (data: Data, format: OpenClawScreenSnapshotFormat, width: Int, height: Int)
|
||||
{
|
||||
(Data("ok".utf8), format ?? .jpeg, 10, 10)
|
||||
}
|
||||
|
||||
func recordScreen(
|
||||
screenIndex: Int?,
|
||||
durationMs: Int?,
|
||||
fps: Double?,
|
||||
includeAudio: Bool?,
|
||||
outPath: String?) async throws -> (path: String, hasAudio: Bool)
|
||||
{
|
||||
(path: "/tmp/none", hasAudio: false)
|
||||
}
|
||||
|
||||
func locationAuthorizationStatus() -> CLAuthorizationStatus {
|
||||
.authorizedAlways
|
||||
}
|
||||
|
||||
func locationAccuracyAuthorization() -> CLAccuracyAuthorization {
|
||||
.fullAccuracy
|
||||
}
|
||||
|
||||
func currentLocation(
|
||||
desiredAccuracy: OpenClawLocationAccuracy,
|
||||
maxAgeMs: Int?,
|
||||
timeoutMs: Int?) async throws -> CLLocation
|
||||
{
|
||||
CLLocation(latitude: 0, longitude: 0)
|
||||
}
|
||||
|
||||
func performComputerAct(_ params: OpenClawComputerActParams) async throws
|
||||
-> OpenClawComputerActResult
|
||||
{
|
||||
self.receivedParams = params
|
||||
if let actError {
|
||||
throw actError
|
||||
}
|
||||
return OpenClawComputerActResult(ok: true, cursorX: params.x ?? 0, cursorY: params.y ?? 0)
|
||||
}
|
||||
|
||||
func releaseHeldInput() {}
|
||||
}
|
||||
|
||||
@Test func `handle invoke rejects computer act when control disabled`() async throws {
|
||||
let services = await MainActor.run { ComputerActProbeServices() }
|
||||
let runtime = MacNodeRuntime(
|
||||
makeMainActorServices: { services },
|
||||
computerControlEnabled: { false })
|
||||
|
||||
let params = OpenClawComputerActParams(action: .leftClick, x: 5, y: 6, refWidth: 1280)
|
||||
let json = try String(data: JSONEncoder().encode(params), encoding: .utf8)
|
||||
let response = await runtime.handleInvoke(
|
||||
BridgeInvokeRequest(
|
||||
id: "req-computer-disabled",
|
||||
command: OpenClawComputerCommand.act.rawValue,
|
||||
paramsJSON: json))
|
||||
|
||||
#expect(response.ok == false)
|
||||
#expect(response.error?.code == .unavailable)
|
||||
#expect(response.error?.message == "COMPUTER_DISABLED: enable Computer Control in Settings")
|
||||
let received = await MainActor.run { services.receivedParams }
|
||||
#expect(received == nil)
|
||||
}
|
||||
|
||||
@Test func `handle invoke routes computer act to the injected services when enabled`() async throws {
|
||||
let services = await MainActor.run { ComputerActProbeServices() }
|
||||
let runtime = MacNodeRuntime(
|
||||
makeMainActorServices: { services },
|
||||
computerControlEnabled: { true })
|
||||
|
||||
let params = OpenClawComputerActParams(action: .leftClick, x: 12, y: 34, refWidth: 1280)
|
||||
let json = try String(data: JSONEncoder().encode(params), encoding: .utf8)
|
||||
let response = await runtime.handleInvoke(
|
||||
BridgeInvokeRequest(
|
||||
id: "req-computer-ok",
|
||||
command: OpenClawComputerCommand.act.rawValue,
|
||||
paramsJSON: json))
|
||||
|
||||
#expect(response.ok == true)
|
||||
let received = await MainActor.run { services.receivedParams }
|
||||
#expect(received?.action == .leftClick)
|
||||
#expect(received?.x == 12)
|
||||
let payloadJSON = try #require(response.payloadJSON)
|
||||
let result = try JSONDecoder().decode(OpenClawComputerActResult.self, from: Data(payloadJSON.utf8))
|
||||
#expect(result.ok == true)
|
||||
#expect(result.cursorX == 12)
|
||||
}
|
||||
|
||||
@Test func `handle invoke maps accessibility denial to unavailable`() async throws {
|
||||
let services = await MainActor.run {
|
||||
ComputerActProbeServices(actError: ComputerActionService.ComputerActionError.accessibilityNotTrusted)
|
||||
}
|
||||
let runtime = MacNodeRuntime(
|
||||
makeMainActorServices: { services },
|
||||
computerControlEnabled: { true })
|
||||
|
||||
let params = OpenClawComputerActParams(action: .leftClick, x: 1, y: 1, refWidth: 1280)
|
||||
let json = try String(data: JSONEncoder().encode(params), encoding: .utf8)
|
||||
let response = await runtime.handleInvoke(
|
||||
BridgeInvokeRequest(
|
||||
id: "req-computer-ax",
|
||||
command: OpenClawComputerCommand.act.rawValue,
|
||||
paramsJSON: json))
|
||||
|
||||
#expect(response.ok == false)
|
||||
#expect(response.error?.code == .unavailable)
|
||||
#expect(response.error?.message == "ACCESSIBILITY_REQUIRED: grant Accessibility permission to OpenClaw")
|
||||
}
|
||||
|
||||
@Test func `handle invoke rejects malformed computer act params`() async {
|
||||
let services = await MainActor.run { ComputerActProbeServices() }
|
||||
let runtime = MacNodeRuntime(
|
||||
makeMainActorServices: { services },
|
||||
computerControlEnabled: { true })
|
||||
|
||||
let response = await runtime.handleInvoke(
|
||||
BridgeInvokeRequest(
|
||||
id: "req-computer-bad",
|
||||
command: OpenClawComputerCommand.act.rawValue,
|
||||
paramsJSON: #"{"action":"#))
|
||||
|
||||
#expect(response.ok == false)
|
||||
#expect(response.error?.code == .invalidRequest)
|
||||
#expect(response.error?.message == "INVALID_REQUEST: invalid computer.act params")
|
||||
}
|
||||
|
||||
@Test func `handle invoke screen snapshot sanitizes capture failures`() async {
|
||||
struct SensitiveError: LocalizedError {
|
||||
let detail: String
|
||||
var errorDescription: String? { detail }
|
||||
var errorDescription: String? {
|
||||
self.detail
|
||||
}
|
||||
}
|
||||
|
||||
let services = await MainActor.run {
|
||||
|
|
@ -455,7 +625,7 @@ struct MacNodeRuntimeTests {
|
|||
#expect(response.error?.message == "UNAVAILABLE: screen snapshot failed")
|
||||
}
|
||||
|
||||
@Test func `handle invoke screen snapshot reports validation failures as invalid request`() async throws {
|
||||
@Test func `handle invoke screen snapshot reports validation failures as invalid request`() async {
|
||||
let invalidIndexServices = await MainActor.run {
|
||||
ScreenSnapshotProbeServices(
|
||||
snapshotError: ScreenSnapshotService.ScreenSnapshotError.invalidScreenIndex(4))
|
||||
|
|
@ -486,7 +656,7 @@ struct MacNodeRuntimeTests {
|
|||
"INVALID_REQUEST: no displays available for screen snapshot")
|
||||
}
|
||||
|
||||
@Test func `handle invoke screen snapshot rejects raw payloads above base64 ceiling`() async throws {
|
||||
@Test func `handle invoke screen snapshot rejects raw payloads above base64 ceiling`() async {
|
||||
let payloadSize = 19_660_801
|
||||
let services = await MainActor.run {
|
||||
ScreenSnapshotProbeServices(snapshotResult: (
|
||||
|
|
@ -510,7 +680,7 @@ struct MacNodeRuntimeTests {
|
|||
"UNAVAILABLE: screen snapshot payload too large; reduce maxWidth or use jpeg")
|
||||
}
|
||||
|
||||
@Test func `handle invoke screen snapshot rejects escaped oversized outer frames`() async throws {
|
||||
@Test func `handle invoke screen snapshot rejects escaped oversized outer frames`() async {
|
||||
let payloadSize = 12 * 1024 * 1024
|
||||
let services = await MainActor.run {
|
||||
ScreenSnapshotProbeServices(snapshotResult: (
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ public enum OpenClawCapability: String, Codable, Sendable {
|
|||
case browser
|
||||
case camera
|
||||
case screen
|
||||
case computer
|
||||
case voiceWake
|
||||
case talk
|
||||
case location
|
||||
|
|
|
|||
|
|
@ -0,0 +1,98 @@
|
|||
import Foundation
|
||||
|
||||
/// Node command that mirrors the Anthropic `computer_20251124` action set. One
|
||||
/// action per invoke; pointer coordinates are in reference-screenshot pixels
|
||||
/// (the `screen.snapshot` frame captured at `maxWidth == refWidth`), which the
|
||||
/// fulfilling node maps back to display points.
|
||||
public enum OpenClawComputerCommand: String, Codable, Sendable {
|
||||
case act = "computer.act"
|
||||
}
|
||||
|
||||
/// Discriminates the requested computer action. The macOS node maps each case
|
||||
/// onto the embedded Peekaboo automation engine plus a narrow CoreGraphics
|
||||
/// path for primitives Peekaboo does not express (middle/triple click,
|
||||
/// separate mouse down/up, modifier-held clicks/scroll).
|
||||
public enum OpenClawComputerAction: String, Codable, Sendable {
|
||||
case leftClick = "left_click"
|
||||
case rightClick = "right_click"
|
||||
case middleClick = "middle_click"
|
||||
case doubleClick = "double_click"
|
||||
case tripleClick = "triple_click"
|
||||
case mouseMove = "mouse_move"
|
||||
case leftClickDrag = "left_click_drag"
|
||||
case leftMouseDown = "left_mouse_down"
|
||||
case leftMouseUp = "left_mouse_up"
|
||||
case scroll
|
||||
case type
|
||||
case key
|
||||
case holdKey = "hold_key"
|
||||
}
|
||||
|
||||
public enum OpenClawComputerScrollDirection: String, Codable, Sendable {
|
||||
case up
|
||||
case down
|
||||
case left
|
||||
case right
|
||||
}
|
||||
|
||||
/// Wire params for `computer.act`. All coordinate fields are reference-screenshot
|
||||
/// pixels at `refWidth`; `keys` is a chord for key/hold_key; `modifiers` are
|
||||
/// modifier keys held during pointer actions; `scrollAmount` is wheel ticks.
|
||||
public struct OpenClawComputerActParams: Codable, Sendable, Equatable {
|
||||
public var action: OpenClawComputerAction
|
||||
public var x: Double?
|
||||
public var y: Double?
|
||||
public var fromX: Double?
|
||||
public var fromY: Double?
|
||||
public var text: String?
|
||||
public var keys: String?
|
||||
public var modifiers: String?
|
||||
public var scrollDirection: OpenClawComputerScrollDirection?
|
||||
public var scrollAmount: Int?
|
||||
public var durationMs: Int?
|
||||
public var screenIndex: Int?
|
||||
public var refWidth: Int?
|
||||
|
||||
public init(
|
||||
action: OpenClawComputerAction,
|
||||
x: Double? = nil,
|
||||
y: Double? = nil,
|
||||
fromX: Double? = nil,
|
||||
fromY: Double? = nil,
|
||||
text: String? = nil,
|
||||
keys: String? = nil,
|
||||
modifiers: String? = nil,
|
||||
scrollDirection: OpenClawComputerScrollDirection? = nil,
|
||||
scrollAmount: Int? = nil,
|
||||
durationMs: Int? = nil,
|
||||
screenIndex: Int? = nil,
|
||||
refWidth: Int? = nil)
|
||||
{
|
||||
self.action = action
|
||||
self.x = x
|
||||
self.y = y
|
||||
self.fromX = fromX
|
||||
self.fromY = fromY
|
||||
self.text = text
|
||||
self.keys = keys
|
||||
self.modifiers = modifiers
|
||||
self.scrollDirection = scrollDirection
|
||||
self.scrollAmount = scrollAmount
|
||||
self.durationMs = durationMs
|
||||
self.screenIndex = screenIndex
|
||||
self.refWidth = refWidth
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of a `computer.act` input action.
|
||||
public struct OpenClawComputerActResult: Codable, Sendable, Equatable {
|
||||
public var ok: Bool
|
||||
public var cursorX: Double
|
||||
public var cursorY: Double
|
||||
|
||||
public init(ok: Bool, cursorX: Double, cursorY: Double) {
|
||||
self.ok = ok
|
||||
self.cursorX = cursorX
|
||||
self.cursorY = cursorY
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
import Foundation
|
||||
|
||||
/// A target display expressed in the global CoreGraphics point space (top-left
|
||||
/// origin), which is exactly the space CGEvent mouse coordinates use.
|
||||
public struct OpenClawComputerDisplayGeometry: Sendable, Equatable {
|
||||
public var originX: Double
|
||||
public var originY: Double
|
||||
public var widthPoints: Double
|
||||
public var heightPoints: Double
|
||||
|
||||
public init(originX: Double, originY: Double, widthPoints: Double, heightPoints: Double) {
|
||||
self.originX = originX
|
||||
self.originY = originY
|
||||
self.widthPoints = widthPoints
|
||||
self.heightPoints = heightPoints
|
||||
}
|
||||
}
|
||||
|
||||
/// Maps reference-screenshot pixel coordinates to global display points.
|
||||
///
|
||||
/// The model reasons over the `screen.snapshot` image whose pixel width equals
|
||||
/// the captured width (ScreenSnapshotService downscales the capture source to
|
||||
/// `min(refWidth, sourceWidth)`), so model coordinates live in captured-pixel
|
||||
/// space. A single uniform factor (display point width / captured pixel width)
|
||||
/// recovers global display points; aspect ratio is preserved so it also applies
|
||||
/// to the vertical axis. Deriving the factor from the captured pixel width
|
||||
/// rather than the display point width keeps clicks aligned on Retina modes
|
||||
/// where physical pixels and logical points differ. Retina backing scale never
|
||||
/// enters CGEvent coordinates, which are always points.
|
||||
public enum OpenClawComputerInputGeometry {
|
||||
/// The delivered screenshot pixel width for a reference width and the capture
|
||||
/// source dimensions. `sourceWidth`/`sourceHeight` are the capture source
|
||||
/// dimensions in the same units ScreenSnapshotService reads from the display.
|
||||
public static func capturedWidth(
|
||||
refWidth: Int?,
|
||||
sourceWidth: Double,
|
||||
sourceHeight: Double) -> Double
|
||||
{
|
||||
guard sourceWidth > 0 else { return 0 }
|
||||
// Node width cap: ScreenSnapshotService downscales the source to
|
||||
// min(refWidth, sourceWidth) and never upscales.
|
||||
let widthCap = refWidth.map { min(Double($0), sourceWidth) } ?? sourceWidth
|
||||
guard let refWidth, refWidth > 0, sourceHeight > 0 else { return widthCap }
|
||||
// The agent additionally caps the delivered screenshot's LONGEST edge to
|
||||
// the reference width (this turn and on later replay-sanitization), so a
|
||||
// portrait capture whose height exceeds the reference width is scaled down
|
||||
// uniformly. Mirror that scaling here so coordinates map against the same
|
||||
// pixel width the model actually saw. Landscape frames, whose longest edge
|
||||
// is the already-capped width, are unaffected.
|
||||
let cappedHeight = widthCap * sourceHeight / sourceWidth
|
||||
let longestEdge = max(widthCap, cappedHeight)
|
||||
let referenceWidth = Double(refWidth)
|
||||
guard longestEdge > referenceWidth else { return widthCap }
|
||||
return widthCap * referenceWidth / longestEdge
|
||||
}
|
||||
|
||||
/// Converts a captured-pixel-space point to a global display point.
|
||||
/// `capturedWidthPixels` is the actual pixel width of the screenshot the
|
||||
/// model saw (see `capturedWidth`).
|
||||
public static func mapReferencePointToGlobal(
|
||||
x: Double,
|
||||
y: Double,
|
||||
capturedWidthPixels: Double,
|
||||
display: OpenClawComputerDisplayGeometry) -> (x: Double, y: Double)
|
||||
{
|
||||
guard capturedWidthPixels > 0, display.widthPoints > 0 else {
|
||||
return (x: display.originX, y: display.originY)
|
||||
}
|
||||
let scale = display.widthPoints / capturedWidthPixels
|
||||
return (
|
||||
x: display.originX + x * scale,
|
||||
y: display.originY + y * scale)
|
||||
}
|
||||
|
||||
/// Clamps a global point to strictly inside the display. Coordinate mapping
|
||||
/// tolerates a small rounding epsilon at the edges, but the posted event must
|
||||
/// stay on the selected display: a far-edge point (e.g. x == captured width,
|
||||
/// which maps to originX + widthPoints) would otherwise fall on the adjacent
|
||||
/// screen the model never saw.
|
||||
public static func clampToDisplay(
|
||||
x: Double,
|
||||
y: Double,
|
||||
display: OpenClawComputerDisplayGeometry) -> (x: Double, y: Double)
|
||||
{
|
||||
let maxX = display.originX + max(0, display.widthPoints - 1)
|
||||
let maxY = display.originY + max(0, display.heightPoints - 1)
|
||||
return (
|
||||
x: min(max(x, display.originX), maxX),
|
||||
y: min(max(y, display.originY), maxY))
|
||||
}
|
||||
}
|
||||
|
|
@ -32,6 +32,18 @@
|
|||
"command"
|
||||
]
|
||||
},
|
||||
"computer": {
|
||||
"emoji": "🖱️",
|
||||
"title": "Computer",
|
||||
"detailKeys": [
|
||||
"action",
|
||||
"coordinate",
|
||||
"text",
|
||||
"node",
|
||||
"nodeId",
|
||||
"screenIndex"
|
||||
]
|
||||
},
|
||||
"process": {
|
||||
"emoji": "🧰",
|
||||
"title": "Process",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,184 @@
|
|||
import Foundation
|
||||
import OpenClawKit
|
||||
import Testing
|
||||
|
||||
struct ComputerInputGeometryTests {
|
||||
private func display(
|
||||
originX: Double = 0,
|
||||
originY: Double = 0,
|
||||
width: Double,
|
||||
height: Double) -> OpenClawComputerDisplayGeometry
|
||||
{
|
||||
OpenClawComputerDisplayGeometry(
|
||||
originX: originX,
|
||||
originY: originY,
|
||||
widthPoints: width,
|
||||
heightPoints: height)
|
||||
}
|
||||
|
||||
private func capturedWidth(refWidth: Int?, sourceWidth: Double, sourceHeight: Double = 0) -> Double {
|
||||
OpenClawComputerInputGeometry.capturedWidth(
|
||||
refWidth: refWidth,
|
||||
sourceWidth: sourceWidth,
|
||||
sourceHeight: sourceHeight)
|
||||
}
|
||||
|
||||
@Test func `maps one to one when capture matches display`() {
|
||||
// Source width equals display points and refWidth: captured 1280px over
|
||||
// a 1280pt display → 1:1.
|
||||
let captured = self.capturedWidth(refWidth: 1280, sourceWidth: 1280)
|
||||
#expect(captured == 1280)
|
||||
let mapped = OpenClawComputerInputGeometry.mapReferencePointToGlobal(
|
||||
x: 100,
|
||||
y: 200,
|
||||
capturedWidthPixels: captured,
|
||||
display: self.display(width: 1280, height: 800))
|
||||
#expect(mapped.x == 100)
|
||||
#expect(mapped.y == 200)
|
||||
}
|
||||
|
||||
@Test func `never upscales when ref width exceeds source`() {
|
||||
// refWidth larger than the capture source means the screenshot is the
|
||||
// full source width, so mapping over a matching-point display stays 1:1.
|
||||
let captured = self.capturedWidth(refWidth: 4000, sourceWidth: 1512)
|
||||
#expect(captured == 1512)
|
||||
let mapped = OpenClawComputerInputGeometry.mapReferencePointToGlobal(
|
||||
x: 300,
|
||||
y: 150,
|
||||
capturedWidthPixels: captured,
|
||||
display: self.display(width: 1512, height: 982))
|
||||
#expect(mapped.x == 300)
|
||||
#expect(mapped.y == 150)
|
||||
}
|
||||
|
||||
@Test func `scales up from downscaled screenshot`() {
|
||||
// Display is 2560pt wide, capture source 2560; screenshot captured at
|
||||
// refWidth 1280 → captured 1280px, scale 2x.
|
||||
let captured = self.capturedWidth(refWidth: 1280, sourceWidth: 2560)
|
||||
#expect(captured == 1280)
|
||||
let mapped = OpenClawComputerInputGeometry.mapReferencePointToGlobal(
|
||||
x: 100,
|
||||
y: 50,
|
||||
capturedWidthPixels: captured,
|
||||
display: self.display(width: 2560, height: 1600))
|
||||
#expect(mapped.x == 200)
|
||||
#expect(mapped.y == 100)
|
||||
}
|
||||
|
||||
@Test func `scales down on retina when pixels exceed points`() {
|
||||
// Retina display set to 1024 logical points but 2048 physical pixels:
|
||||
// capture source (pixels) 2048 downscales to refWidth 1280, and the
|
||||
// 1280px screenshot must map back onto the 1024pt display (scale 0.8).
|
||||
let captured = self.capturedWidth(refWidth: 1280, sourceWidth: 2048)
|
||||
#expect(captured == 1280)
|
||||
let mapped = OpenClawComputerInputGeometry.mapReferencePointToGlobal(
|
||||
x: 1280,
|
||||
y: 0,
|
||||
capturedWidthPixels: captured,
|
||||
display: self.display(width: 1024, height: 640))
|
||||
#expect(mapped.x == 1024)
|
||||
#expect(mapped.y == 0)
|
||||
}
|
||||
|
||||
@Test func `adds display origin after scaling`() {
|
||||
let captured = self.capturedWidth(refWidth: 1280, sourceWidth: 2560)
|
||||
let mapped = OpenClawComputerInputGeometry.mapReferencePointToGlobal(
|
||||
x: 100,
|
||||
y: 0,
|
||||
capturedWidthPixels: captured,
|
||||
display: self.display(originX: 1512, originY: 0, width: 2560, height: 1600))
|
||||
// 100 * (2560/1280) + 1512 origin.
|
||||
#expect(mapped.x == 1712)
|
||||
#expect(mapped.y == 0)
|
||||
}
|
||||
|
||||
@Test func `scales down portrait capture whose height exceeds reference width`() {
|
||||
// Portrait source 1080x1920: width cap keeps 1080, but the capture's
|
||||
// longest edge (1920) exceeds the 1280 reference width, so the agent
|
||||
// scales the whole frame down. Delivered width = 1080 * 1280 / 1920 = 720.
|
||||
let captured = self.capturedWidth(refWidth: 1280, sourceWidth: 1080, sourceHeight: 1920)
|
||||
#expect(captured == 720)
|
||||
// The 720px-wide delivered frame maps back onto the 1080pt portrait
|
||||
// display (scale 1.5), so replayed coordinates land on target.
|
||||
let mapped = OpenClawComputerInputGeometry.mapReferencePointToGlobal(
|
||||
x: 360,
|
||||
y: 0,
|
||||
capturedWidthPixels: captured,
|
||||
display: self.display(width: 1080, height: 1920))
|
||||
#expect(mapped.x == 540)
|
||||
#expect(mapped.y == 0)
|
||||
}
|
||||
|
||||
@Test func `keeps landscape width when the width is the longest edge`() {
|
||||
// Landscape source 2560x1440: the width cap (1280) is already the longest
|
||||
// edge, so longest-edge scaling is a no-op and the width cap stands.
|
||||
let captured = self.capturedWidth(refWidth: 1280, sourceWidth: 2560, sourceHeight: 1440)
|
||||
#expect(captured == 1280)
|
||||
}
|
||||
|
||||
@Test func `clamps far-edge and out-of-bounds points strictly inside the display`() {
|
||||
let display = self.display(originX: 100, originY: 200, width: 1280, height: 800)
|
||||
// The far right/bottom edge maps to origin + size, which belongs to the
|
||||
// adjacent display; clamp keeps it on the last in-bounds point.
|
||||
let farEdge = OpenClawComputerInputGeometry.clampToDisplay(
|
||||
x: 100 + 1280,
|
||||
y: 200 + 800,
|
||||
display: display)
|
||||
#expect(farEdge.x == 100 + 1279)
|
||||
#expect(farEdge.y == 200 + 799)
|
||||
// A slightly-negative epsilon overrun clamps back to the origin.
|
||||
let negative = OpenClawComputerInputGeometry.clampToDisplay(x: 99, y: 199, display: display)
|
||||
#expect(negative.x == 100)
|
||||
#expect(negative.y == 200)
|
||||
// A point already inside is unchanged.
|
||||
let inside = OpenClawComputerInputGeometry.clampToDisplay(x: 640, y: 400, display: display)
|
||||
#expect(inside.x == 640)
|
||||
#expect(inside.y == 400)
|
||||
}
|
||||
|
||||
@Test func `captured width is non-positive for a non-positive reference width`() {
|
||||
// Justifies the executor rejecting refWidth <= 0 before mapping: geometry
|
||||
// would otherwise yield a non-positive width that collapses every
|
||||
// coordinate onto the display origin.
|
||||
#expect(self.capturedWidth(refWidth: 0, sourceWidth: 1280, sourceHeight: 800) <= 0)
|
||||
#expect(self.capturedWidth(refWidth: -5, sourceWidth: 1280, sourceHeight: 800) <= 0)
|
||||
}
|
||||
|
||||
@Test func `falls back to origin for degenerate display`() {
|
||||
let mapped = OpenClawComputerInputGeometry.mapReferencePointToGlobal(
|
||||
x: 42,
|
||||
y: 42,
|
||||
capturedWidthPixels: 0,
|
||||
display: self.display(originX: 10, originY: 20, width: 0, height: 0))
|
||||
#expect(mapped.x == 10)
|
||||
#expect(mapped.y == 20)
|
||||
}
|
||||
|
||||
@Test func `decodes partial act params`() throws {
|
||||
let json = """
|
||||
{"action":"left_click","x":12,"y":34,"modifiers":"shift","screenIndex":0,"refWidth":1280}
|
||||
"""
|
||||
let data = try #require(json.data(using: .utf8))
|
||||
let params = try JSONDecoder().decode(OpenClawComputerActParams.self, from: data)
|
||||
#expect(params.action == .leftClick)
|
||||
#expect(params.x == 12)
|
||||
#expect(params.modifiers == "shift")
|
||||
#expect(params.refWidth == 1280)
|
||||
}
|
||||
|
||||
@Test func `decodes scroll and hold actions`() throws {
|
||||
let scroll = try #require(
|
||||
"{\"action\":\"scroll\",\"scrollDirection\":\"down\",\"scrollAmount\":3}".data(using: .utf8))
|
||||
let scrollParams = try JSONDecoder().decode(OpenClawComputerActParams.self, from: scroll)
|
||||
#expect(scrollParams.action == .scroll)
|
||||
#expect(scrollParams.scrollDirection == .down)
|
||||
#expect(scrollParams.scrollAmount == 3)
|
||||
|
||||
let hold = try #require(
|
||||
"{\"action\":\"hold_key\",\"keys\":\"space\",\"durationMs\":2000}".data(using: .utf8))
|
||||
let holdParams = try JSONDecoder().decode(OpenClawComputerActParams.self, from: hold)
|
||||
#expect(holdParams.action == .holdKey)
|
||||
#expect(holdParams.keys == "space")
|
||||
#expect(holdParams.durationMs == 2000)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue