mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
feat(browser): extend --labels overlay to full-page and element captures (#92834)
Summary: - The replacement PR extends Browser plugin labeled screenshots to honor Playwright full-page/ref/element scope, returns annotation bounding boxes, and updates docs, tests, and skill guidance. - PR surface: Source +415, Tests +550, Docs +24. Total +989 across 12 files. - Reproducibility: yes. Current main source shows the labeled Playwright helper ignores fullPage/ref/element and omits annotations, and the source PR supplies live before/after commands for the Browser plugin path. Automerge notes: - PR branch already contained follow-up commit before automerge: docs(browser): correct raw-CDP labels caveat in automation skill - PR branch already contained follow-up commit before automerge: fix(browser): preserve labelsSkipped semantics for off-viewport refs - PR branch already contained follow-up commit before automerge: docs(browser): scope labels docs by driver - PR branch already contained follow-up commit before automerge: docs(browser): fix labels annotation indent and document scope fix - PR branch already contained follow-up commit before automerge: docs(browser): indent annotations box schema under --labels bullet - PR branch already contained follow-up commit before automerge: docs(browser): indent labels annotation schema Validation: - ClawSweeper review passed for head70aca6c506. - Required merge gates passed before the squash merge. Prepared head SHA:70aca6c506Review: https://github.com/openclaw/openclaw/pull/92834#issuecomment-4700431344 Co-authored-by: FMLS <kfliuyang@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Mason Huang <masonxhuang@tencent.com> Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com> Approved-by: hxy91819 Co-authored-by: hxy91819 <8814856+hxy91819@users.noreply.github.com>
This commit is contained in:
clawsweeper[bot]
GitHub
parent
aef670cf0c
commit
24e729fc4e
12 changed files with 1093 additions and 104 deletions
|
|
@ -174,7 +174,22 @@ Notes:
|
|||
or `--element`.
|
||||
- `existing-session` / `user` profiles support page screenshots and `--ref`
|
||||
screenshots from snapshot output, but not CSS `--element` screenshots.
|
||||
- `--labels` overlays current snapshot refs on the screenshot.
|
||||
- `--labels` overlays current snapshot refs on the screenshot. On
|
||||
Playwright-backed profiles, it works with `--full-page` (full-page label
|
||||
overlay), `--ref` (element-clip label overlay by ARIA ref), and `--element`
|
||||
(element-clip label overlay by CSS selector); in element-clip modes, labels
|
||||
are projected relative to the element. The response also includes an
|
||||
`annotations` array with each ref's bounding box. Each item has `ref`,
|
||||
`number`, `role`, optional `name`, and `box: {x, y, width, height}`;
|
||||
coordinates are in the captured image's space (viewport / fullpage /
|
||||
element-relative). The field is omitted when empty.
|
||||
`existing-session` profiles render a chrome-mcp overlay on page screenshots
|
||||
but do not use the Playwright projection helper and do not include
|
||||
`annotations`; CSS `--element` screenshots are unsupported there. Without
|
||||
Playwright or chrome-mcp, labeled screenshots are not available. Prior
|
||||
releases ignored `--full-page`, `--ref`, and `--element` on labeled
|
||||
Playwright screenshots and always returned a viewport capture; labeled
|
||||
screenshots now honor those scopes.
|
||||
- `snapshot --urls` appends discovered link destinations to AI snapshots so
|
||||
agents can choose direct navigation targets instead of guessing from link
|
||||
text alone.
|
||||
|
|
|
|||
|
|
@ -258,7 +258,14 @@ Snapshot flags at a glance:
|
|||
- `--format aria`: accessibility tree with `axN` refs. When Playwright is available, OpenClaw binds refs with backend DOM ids to the live page so follow-up actions can use them; otherwise treat the output as inspection-only.
|
||||
- `--efficient` (or `--mode efficient`): compact role snapshot preset. Set `browser.snapshotDefaults.mode: "efficient"` to make this the default (see [Gateway configuration](/gateway/configuration-reference#browser)).
|
||||
- `--interactive`, `--compact`, `--depth`, `--selector` force a role snapshot with `ref=e12` refs. `--frame "<iframe>"` scopes role snapshots to an iframe.
|
||||
- `--labels` adds a viewport-only screenshot with overlayed ref labels and prints the saved path.
|
||||
- With Playwright, `--labels` adds a screenshot with overlayed ref labels
|
||||
(prints `MEDIA:<path>`) plus an `annotations` array with each ref's bounding
|
||||
box. On `screenshot`, Playwright-backed labels work with `--full-page`,
|
||||
`--ref`, and `--element`; on `snapshot`, the accompanying screenshot remains
|
||||
viewport-only. Existing-session/chrome-mcp profiles render overlay labels on
|
||||
page screenshots but do not return `annotations` or use the Playwright
|
||||
full-page/ref/element projection helper. Without Playwright or chrome-mcp,
|
||||
labeled screenshots are not available.
|
||||
- `--urls` appends discovered link destinations to AI snapshots.
|
||||
|
||||
## Snapshots and refs
|
||||
|
|
@ -274,7 +281,9 @@ OpenClaw supports two "snapshot" styles:
|
|||
- Output: a role-based list/tree with `[ref=e12]` (and optional `[nth=1]`).
|
||||
- Actions: `openclaw browser click e12`, `openclaw browser highlight e12`.
|
||||
- Internally, the ref is resolved via `getByRole(...)` (plus `nth()` for duplicates).
|
||||
- Add `--labels` to include a viewport screenshot with overlayed `e12` labels.
|
||||
- Add `--labels` to include a screenshot with overlayed `e12` labels. On
|
||||
Playwright-backed profiles this also returns per-ref bounding-box metadata
|
||||
(`annotations[]`).
|
||||
- Add `--urls` when link text is ambiguous and the agent needs concrete
|
||||
navigation targets.
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ Use this skill when you need the `browser` tool for anything beyond a single pag
|
|||
- Use the same `targetId` for follow-up actions so refs stay on the same tab.
|
||||
- For durable Playwright refs, request `refs="aria"` when supported. If you receive `axN` refs from `snapshotFormat="aria"`, use them only after that same snapshot call; stale or unbound `axN` refs fail fast and need a fresh snapshot.
|
||||
- Use `urls=true` when link text is ambiguous or a direct navigation target would avoid brittle clicks.
|
||||
- Use `labels=true` on snapshot or screenshot when visual position matters.
|
||||
- Use `labels=true` on snapshot or screenshot when visual position matters. On Playwright-backed profiles, the response includes an `annotations` array (`{ref, number, role, name?, box}`) with each ref's bounding box in the captured image's coordinate space, so you can reason about position without re-snapshotting; screenshot labels can also combine with `fullPage=true` (CLI: `--full-page`) to label the whole document, or `ref` / `element` to clip to one element. `profile="user"` and other existing-session (chrome-mcp) profiles render an overlay into page screenshots but do not attach `annotations` or use the Playwright full-page/ref/element projection helper, so read positions from the labeled image itself on those profiles. The raw-CDP fallback (no Playwright) does not support labeled screenshots at all and returns a 501, so only request `labels` when Playwright is available.
|
||||
4. Act narrowly:
|
||||
- Prefer `action="act"` with a ref from the latest snapshot.
|
||||
- After navigation, modal changes, or form submission, snapshot again before the next action.
|
||||
|
|
|
|||
|
|
@ -486,6 +486,7 @@ export async function executeSnapshotAction(params: {
|
|||
labels: snapshot.labels,
|
||||
labelsCount: snapshot.labelsCount,
|
||||
labelsSkipped: snapshot.labelsSkipped,
|
||||
annotations: snapshot.annotations,
|
||||
imagePath: snapshot.imagePath,
|
||||
imageType: snapshot.imageType,
|
||||
refsFallback,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
/**
|
||||
* Shared result types for browser client action helpers.
|
||||
*/
|
||||
import type { AnnotationItem } from "./screenshot-annotate.js";
|
||||
|
||||
/** Generic success result for action endpoints. */
|
||||
export type BrowserActionOk = { ok: true };
|
||||
|
||||
|
|
@ -20,4 +22,10 @@ export type BrowserActionPathResult = {
|
|||
labels?: boolean;
|
||||
labelsCount?: number;
|
||||
labelsSkipped?: number;
|
||||
/**
|
||||
* Per-ref bounding boxes when labels=true. Coordinates are in the
|
||||
* captured image's space (viewport / fullpage / element-relative).
|
||||
* Omitted when empty.
|
||||
*/
|
||||
annotations?: AnnotationItem[];
|
||||
};
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import type {
|
|||
} from "./client.types.js";
|
||||
import { DEFAULT_BROWSER_SNAPSHOT_TIMEOUT_MS } from "./constants.js";
|
||||
import type { BrowserDoctorReport } from "./doctor.js";
|
||||
import type { AnnotationItem } from "./screenshot-annotate.js";
|
||||
|
||||
export type { BrowserStatus, BrowserTab, BrowserTransport } from "./client.types.js";
|
||||
export type { BrowserDoctorCheck, BrowserDoctorReport } from "./doctor.js";
|
||||
|
|
@ -124,6 +125,11 @@ export type SnapshotResult =
|
|||
labels?: boolean;
|
||||
labelsCount?: number;
|
||||
labelsSkipped?: number;
|
||||
/**
|
||||
* Per-ref bounding boxes when labels=true. Coordinates are in the
|
||||
* captured image's space. Omitted when empty.
|
||||
*/
|
||||
annotations?: AnnotationItem[];
|
||||
imagePath?: string;
|
||||
imageType?: "png" | "jpeg";
|
||||
blockedByDialog?: boolean;
|
||||
|
|
|
|||
205
extensions/browser/src/browser/pw-tools-core.annotate.test.ts
Normal file
205
extensions/browser/src/browser/pw-tools-core.annotate.test.ts
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
installPwToolsCoreTestHooks,
|
||||
setPwToolsCoreCurrentPage,
|
||||
setPwToolsCoreCurrentRefLocator,
|
||||
} from "./pw-tools-core.test-harness.js";
|
||||
|
||||
installPwToolsCoreTestHooks();
|
||||
const mod = await import("./pw-tools-core.js");
|
||||
|
||||
type EvaluateArg = unknown;
|
||||
|
||||
function evaluateMockReturning(view: { x: number; y: number; width?: number; height?: number }) {
|
||||
// Caller reads { x, y, width, height } in one evaluate; default to a normal
|
||||
// desktop viewport so refs near the top stay in-viewport unless a test puts
|
||||
// them out of range explicitly.
|
||||
const result = { width: 1280, height: 720, ...view };
|
||||
return vi.fn(async (arg: EvaluateArg) => {
|
||||
if (typeof arg === "function") {
|
||||
return result;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
describe("screenshotWithLabelsViaPlaywright (viewport)", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("calls page.screenshot without fullPage and returns annotations", async () => {
|
||||
const evaluate = evaluateMockReturning({ x: 0, y: 100 });
|
||||
const screenshot = vi.fn(async () => Buffer.from("PNG"));
|
||||
setPwToolsCoreCurrentPage({ evaluate, screenshot, url: () => "https://example.com" });
|
||||
setPwToolsCoreCurrentRefLocator({
|
||||
boundingBox: async () => ({ x: 10, y: 200, width: 50, height: 20 }),
|
||||
});
|
||||
|
||||
const result = await mod.screenshotWithLabelsViaPlaywright({
|
||||
cdpUrl: "http://127.0.0.1:18792",
|
||||
targetId: "T1",
|
||||
refs: { e1: { role: "button", name: "Submit" } },
|
||||
type: "png",
|
||||
});
|
||||
|
||||
expect(screenshot).toHaveBeenCalledWith(expect.objectContaining({ type: "png" }));
|
||||
expect(screenshot).not.toHaveBeenCalledWith(expect.objectContaining({ fullPage: true }));
|
||||
|
||||
expect(result.annotations).toHaveLength(1);
|
||||
expect(result.annotations[0]).toMatchObject({
|
||||
ref: "e1",
|
||||
number: 1,
|
||||
role: "button",
|
||||
name: "Submit",
|
||||
});
|
||||
// viewport-mode box = doc(box.x + scroll.x, box.y + scroll.y) - scroll = bbox
|
||||
expect(result.annotations[0]?.box).toEqual({ x: 10, y: 200, width: 50, height: 20 });
|
||||
expect(result.skipped).toBe(0);
|
||||
});
|
||||
|
||||
it("runs the clear script even when screenshot throws", async () => {
|
||||
const evaluate = evaluateMockReturning({ x: 0, y: 0 });
|
||||
const screenshot = vi.fn(async () => {
|
||||
throw new Error("boom");
|
||||
});
|
||||
setPwToolsCoreCurrentPage({ evaluate, screenshot });
|
||||
setPwToolsCoreCurrentRefLocator({
|
||||
boundingBox: async () => ({ x: 0, y: 0, width: 1, height: 1 }),
|
||||
});
|
||||
|
||||
await expect(
|
||||
mod.screenshotWithLabelsViaPlaywright({
|
||||
cdpUrl: "http://127.0.0.1:18792",
|
||||
targetId: "T1",
|
||||
refs: { e1: { role: "button" } },
|
||||
}),
|
||||
).rejects.toThrow(/boom/);
|
||||
|
||||
// The clear script must have run (string evaluate calls include the overlay attr)
|
||||
const clearCalls = evaluate.mock.calls.filter(
|
||||
([arg]) => typeof arg === "string" && arg.includes("data-openclaw-labels"),
|
||||
);
|
||||
// inject + clear = at least 2 string evaluations
|
||||
expect(clearCalls.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("counts off-viewport refs as skipped but still surfaces them in annotations", async () => {
|
||||
const evaluate = evaluateMockReturning({ x: 0, y: 0, width: 1280, height: 720 });
|
||||
const screenshot = vi.fn(async () => Buffer.from("PNG"));
|
||||
setPwToolsCoreCurrentPage({ evaluate, screenshot });
|
||||
// bbox is far below the viewport (y: 5000): not drawn, but still reported
|
||||
// so callers keep the position and a non-zero skipped count.
|
||||
setPwToolsCoreCurrentRefLocator({
|
||||
boundingBox: async () => ({ x: 0, y: 5000, width: 50, height: 20 }),
|
||||
});
|
||||
|
||||
const result = await mod.screenshotWithLabelsViaPlaywright({
|
||||
cdpUrl: "http://127.0.0.1:18792",
|
||||
targetId: "T1",
|
||||
refs: { e1: { role: "button" } },
|
||||
});
|
||||
|
||||
expect(result.skipped).toBe(1);
|
||||
expect(result.labels).toBe(0);
|
||||
expect(result.annotations).toHaveLength(1);
|
||||
expect(result.annotations[0]?.ref).toBe("e1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("screenshotWithLabelsViaPlaywright (fullpage)", () => {
|
||||
beforeEach(() => vi.clearAllMocks());
|
||||
|
||||
it("forwards fullPage:true to page.screenshot and uses doc-space annotations", async () => {
|
||||
const evaluate = evaluateMockReturning({ x: 0, y: 1000 });
|
||||
const screenshot = vi.fn(async () => Buffer.from("FULL"));
|
||||
setPwToolsCoreCurrentPage({ evaluate, screenshot });
|
||||
setPwToolsCoreCurrentRefLocator({
|
||||
boundingBox: async () => ({ x: 10, y: 200, width: 50, height: 20 }),
|
||||
});
|
||||
|
||||
const result = await mod.screenshotWithLabelsViaPlaywright({
|
||||
cdpUrl: "http://127.0.0.1:18792",
|
||||
targetId: "T1",
|
||||
refs: { e1: { role: "button" } },
|
||||
fullPage: true,
|
||||
});
|
||||
|
||||
expect(screenshot).toHaveBeenCalledWith(expect.objectContaining({ fullPage: true }));
|
||||
// doc-space: scroll y=1000 + bbox y=200 = 1200
|
||||
expect(result.annotations[0]?.box.y).toBe(1200);
|
||||
expect(result.annotations[0]?.box.x).toBe(10);
|
||||
});
|
||||
});
|
||||
|
||||
describe("screenshotWithLabelsViaPlaywright (element/ref)", () => {
|
||||
beforeEach(() => vi.clearAllMocks());
|
||||
|
||||
it("uses refLocator.screenshot for ref mode and projects relative to element", async () => {
|
||||
const evaluate = evaluateMockReturning({ x: 0, y: 0 });
|
||||
// First call resolves the element rect (container), second resolves e1 annotation bbox.
|
||||
const boundingBox = vi
|
||||
.fn<() => Promise<{ x: number; y: number; width: number; height: number } | null>>()
|
||||
.mockResolvedValueOnce({ x: 50, y: 100, width: 200, height: 300 })
|
||||
.mockResolvedValueOnce({ x: 60, y: 110, width: 30, height: 20 });
|
||||
const elementScreenshot = vi.fn(async () => Buffer.from("ELEM"));
|
||||
setPwToolsCoreCurrentPage({ evaluate, screenshot: vi.fn() });
|
||||
setPwToolsCoreCurrentRefLocator({ boundingBox, screenshot: elementScreenshot });
|
||||
|
||||
const result = await mod.screenshotWithLabelsViaPlaywright({
|
||||
cdpUrl: "http://127.0.0.1:18792",
|
||||
targetId: "T1",
|
||||
refs: { e1: { role: "button" } },
|
||||
ref: "container",
|
||||
});
|
||||
|
||||
expect(elementScreenshot).toHaveBeenCalledTimes(1);
|
||||
// Element-relative: doc(60,110) - elementRect(50,100) = (10,10)
|
||||
expect(result.annotations).toHaveLength(1);
|
||||
expect(result.annotations[0]?.box).toEqual({ x: 10, y: 10, width: 30, height: 20 });
|
||||
});
|
||||
|
||||
it("throws when ref/element cannot be resolved", async () => {
|
||||
const evaluate = evaluateMockReturning({ x: 0, y: 0 });
|
||||
setPwToolsCoreCurrentPage({ evaluate, screenshot: vi.fn() });
|
||||
setPwToolsCoreCurrentRefLocator({
|
||||
boundingBox: async () => null,
|
||||
screenshot: vi.fn(),
|
||||
});
|
||||
|
||||
await expect(
|
||||
mod.screenshotWithLabelsViaPlaywright({
|
||||
cdpUrl: "http://127.0.0.1:18792",
|
||||
targetId: "T1",
|
||||
refs: { e1: { role: "button" } },
|
||||
ref: "missing",
|
||||
}),
|
||||
).rejects.toThrow(/element not found/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe("screenshotWithLabelsViaPlaywright (skipped accounting)", () => {
|
||||
beforeEach(() => vi.clearAllMocks());
|
||||
|
||||
it("counts refs whose boundingBox is null toward skipped", async () => {
|
||||
const evaluate = evaluateMockReturning({ x: 0, y: 0 });
|
||||
const screenshot = vi.fn(async () => Buffer.from("PNG"));
|
||||
setPwToolsCoreCurrentPage({ evaluate, screenshot });
|
||||
// Two refs: first returns a box, second returns null (e.g. element detached).
|
||||
const boundingBox = vi
|
||||
.fn<() => Promise<{ x: number; y: number; width: number; height: number } | null>>()
|
||||
.mockResolvedValueOnce({ x: 10, y: 20, width: 30, height: 40 })
|
||||
.mockResolvedValueOnce(null);
|
||||
setPwToolsCoreCurrentRefLocator({ boundingBox });
|
||||
|
||||
const result = await mod.screenshotWithLabelsViaPlaywright({
|
||||
cdpUrl: "http://127.0.0.1:18792",
|
||||
targetId: "T1",
|
||||
refs: { e1: { role: "button" }, e2: { role: "link" } },
|
||||
});
|
||||
|
||||
expect(result.annotations).toHaveLength(1);
|
||||
expect(result.annotations[0]?.ref).toBe("e1");
|
||||
expect(result.skipped).toBe(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -41,6 +41,15 @@ import {
|
|||
toAIFriendlyError,
|
||||
} from "./pw-tools-core.shared.js";
|
||||
import { closePageViaPlaywright, resizeViewportViaPlaywright } from "./pw-tools-core.snapshot.js";
|
||||
import {
|
||||
ANNOTATION_MAX_LABELS_DEFAULT,
|
||||
type AnnotationItem,
|
||||
buildOverlayClearScript,
|
||||
buildOverlayInjectionScript,
|
||||
type CoordinateSpace,
|
||||
planAnnotations,
|
||||
type RawAnnotationInput,
|
||||
} from "./screenshot-annotate.js";
|
||||
|
||||
type TargetOpts = {
|
||||
cdpUrl: string;
|
||||
|
|
@ -1287,7 +1296,15 @@ export async function screenshotWithLabelsViaPlaywright(opts: {
|
|||
maxLabels?: number;
|
||||
type?: "png" | "jpeg";
|
||||
timeoutMs?: number;
|
||||
}): Promise<{ buffer: Buffer; labels: number; skipped: number }> {
|
||||
fullPage?: boolean;
|
||||
ref?: string;
|
||||
element?: string;
|
||||
}): Promise<{
|
||||
buffer: Buffer;
|
||||
labels: number;
|
||||
skipped: number;
|
||||
annotations: AnnotationItem[];
|
||||
}> {
|
||||
const page = await getPageForTargetId(opts);
|
||||
ensurePageState(page);
|
||||
restoreRoleRefsForTarget({ cdpUrl: opts.cdpUrl, targetId: opts.targetId, page });
|
||||
|
|
@ -1295,119 +1312,151 @@ export async function screenshotWithLabelsViaPlaywright(opts: {
|
|||
const maxLabels =
|
||||
typeof opts.maxLabels === "number" && Number.isFinite(opts.maxLabels)
|
||||
? Math.max(1, Math.floor(opts.maxLabels))
|
||||
: 150;
|
||||
: ANNOTATION_MAX_LABELS_DEFAULT;
|
||||
|
||||
const viewport = await page.evaluate(() => ({
|
||||
scrollX: window.scrollX || 0,
|
||||
scrollY: window.scrollY || 0,
|
||||
const refKey = normalizeOptionalString(opts.ref) ?? undefined;
|
||||
const elementSelector = normalizeOptionalString(opts.element) ?? undefined;
|
||||
const space: CoordinateSpace = opts.fullPage
|
||||
? "fullpage"
|
||||
: refKey || elementSelector
|
||||
? "element"
|
||||
: "viewport";
|
||||
|
||||
// Read scroll + viewport size. Scroll converts Playwright's viewport-space
|
||||
// boundingBoxes into document-space inputs; the viewport size lets the helper
|
||||
// restore the shipped `labelsSkipped` semantics by counting off-viewport refs
|
||||
// as skipped (in viewport capture mode).
|
||||
const view = await page.evaluate(() => ({
|
||||
x: window.scrollX || 0,
|
||||
y: window.scrollY || 0,
|
||||
width: window.innerWidth || 0,
|
||||
height: window.innerHeight || 0,
|
||||
}));
|
||||
const scroll = { x: view.x, y: view.y };
|
||||
|
||||
const refs = Object.keys(opts.refs ?? {});
|
||||
const boxes: Array<{ ref: string; x: number; y: number; w: number; h: number }> = [];
|
||||
let skipped = 0;
|
||||
let elementRect: { x: number; y: number; width: number; height: number } | undefined;
|
||||
if (space === "element") {
|
||||
const box = await resolveElementBoundingBoxForLabels(page, refKey, elementSelector);
|
||||
if (!box) {
|
||||
throw new Error(
|
||||
`screenshotWithLabelsViaPlaywright: element not found for ${
|
||||
refKey ? `ref="${refKey}"` : `selector="${elementSelector ?? ""}"`
|
||||
}`,
|
||||
);
|
||||
}
|
||||
// Convert viewport-space bbox to document space.
|
||||
elementRect = {
|
||||
x: box.x + scroll.x,
|
||||
y: box.y + scroll.y,
|
||||
width: box.width,
|
||||
height: box.height,
|
||||
};
|
||||
}
|
||||
|
||||
for (const ref of refs) {
|
||||
if (boxes.length >= maxLabels) {
|
||||
skipped += 1;
|
||||
const refKeys = Object.keys(opts.refs ?? {});
|
||||
const inputs: RawAnnotationInput[] = [];
|
||||
let bboxFailures = 0;
|
||||
for (const ref of refKeys) {
|
||||
const box = await refLocator(page, ref)
|
||||
.boundingBox()
|
||||
.catch(() => null);
|
||||
if (!box) {
|
||||
bboxFailures += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const box = await refLocator(page, ref).boundingBox();
|
||||
if (!box) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
const x0 = box.x;
|
||||
const y0 = box.y;
|
||||
const x1 = box.x + box.width;
|
||||
const y1 = box.y + box.height;
|
||||
const vx0 = viewport.scrollX;
|
||||
const vy0 = viewport.scrollY;
|
||||
const vx1 = viewport.scrollX + viewport.width;
|
||||
const vy1 = viewport.scrollY + viewport.height;
|
||||
if (x1 < vx0 || x0 > vx1 || y1 < vy0 || y0 > vy1) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
boxes.push({
|
||||
ref,
|
||||
x: x0 - viewport.scrollX,
|
||||
y: y0 - viewport.scrollY,
|
||||
w: Math.max(1, box.width),
|
||||
h: Math.max(1, box.height),
|
||||
});
|
||||
} catch {
|
||||
skipped += 1;
|
||||
}
|
||||
inputs.push({
|
||||
ref,
|
||||
role: opts.refs[ref].role,
|
||||
name: opts.refs[ref].name,
|
||||
doc: {
|
||||
x: box.x + scroll.x,
|
||||
y: box.y + scroll.y,
|
||||
width: box.width,
|
||||
height: box.height,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const plan = planAnnotations({
|
||||
inputs,
|
||||
space,
|
||||
scroll,
|
||||
viewport: { width: view.width, height: view.height },
|
||||
elementRect,
|
||||
maxLabels,
|
||||
});
|
||||
|
||||
try {
|
||||
if (boxes.length > 0) {
|
||||
await page.evaluate((labels) => {
|
||||
const existing = document.querySelectorAll("[data-openclaw-labels]");
|
||||
existing.forEach((el) => el.remove());
|
||||
|
||||
const root = document.createElement("div");
|
||||
root.setAttribute("data-openclaw-labels", "1");
|
||||
root.style.position = "fixed";
|
||||
root.style.left = "0";
|
||||
root.style.top = "0";
|
||||
root.style.zIndex = "2147483647";
|
||||
root.style.pointerEvents = "none";
|
||||
root.style.fontFamily =
|
||||
'"SF Mono","SFMono-Regular",Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace';
|
||||
|
||||
const clamp = (value: number, min: number, max: number) =>
|
||||
Math.min(max, Math.max(min, value));
|
||||
|
||||
for (const label of labels) {
|
||||
const box = document.createElement("div");
|
||||
box.setAttribute("data-openclaw-labels", "1");
|
||||
box.style.position = "absolute";
|
||||
box.style.left = `${label.x}px`;
|
||||
box.style.top = `${label.y}px`;
|
||||
box.style.width = `${label.w}px`;
|
||||
box.style.height = `${label.h}px`;
|
||||
box.style.border = "2px solid #ffb020";
|
||||
box.style.boxSizing = "border-box";
|
||||
|
||||
const tag = document.createElement("div");
|
||||
tag.setAttribute("data-openclaw-labels", "1");
|
||||
tag.textContent = label.ref;
|
||||
tag.style.position = "absolute";
|
||||
tag.style.left = `${label.x}px`;
|
||||
tag.style.top = `${clamp(label.y - 18, 0, 20000)}px`;
|
||||
tag.style.background = "#ffb020";
|
||||
tag.style.color = "#1a1a1a";
|
||||
tag.style.fontSize = "12px";
|
||||
tag.style.lineHeight = "14px";
|
||||
tag.style.padding = "1px 4px";
|
||||
tag.style.borderRadius = "3px";
|
||||
tag.style.boxShadow = "0 1px 2px rgba(0,0,0,0.35)";
|
||||
tag.style.whiteSpace = "nowrap";
|
||||
|
||||
root.appendChild(box);
|
||||
root.appendChild(tag);
|
||||
}
|
||||
|
||||
document.documentElement.appendChild(root);
|
||||
}, boxes);
|
||||
if (plan.overlayItems.length > 0) {
|
||||
const captureY = space === "element" ? elementRect?.y : space === "viewport" ? scroll.y : 0;
|
||||
await page.evaluate(buildOverlayInjectionScript({ items: plan.overlayItems, captureY }));
|
||||
}
|
||||
|
||||
const buffer = await page.screenshot({ type, timeout: opts.timeoutMs });
|
||||
return { buffer, labels: boxes.length, skipped };
|
||||
const buffer =
|
||||
space === "element"
|
||||
? await captureElementScreenshotForLabels(
|
||||
page,
|
||||
refKey,
|
||||
elementSelector,
|
||||
type,
|
||||
opts.timeoutMs,
|
||||
)
|
||||
: await page.screenshot({
|
||||
type,
|
||||
fullPage: Boolean(opts.fullPage),
|
||||
timeout: opts.timeoutMs,
|
||||
});
|
||||
return {
|
||||
// `labels` reports overlay boxes actually drawn on the captured image
|
||||
// (in-viewport, within budget); off-viewport refs are surfaced via
|
||||
// `annotations` but not drawn, and are reflected in `skipped`.
|
||||
buffer,
|
||||
labels: plan.overlayItems.length,
|
||||
skipped: plan.skipped + bboxFailures,
|
||||
annotations: plan.annotations,
|
||||
};
|
||||
} finally {
|
||||
await page
|
||||
.evaluate(() => {
|
||||
const existing = document.querySelectorAll("[data-openclaw-labels]");
|
||||
existing.forEach((el) => el.remove());
|
||||
})
|
||||
.catch(() => {});
|
||||
await page.evaluate(buildOverlayClearScript()).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveElementBoundingBoxForLabels(
|
||||
page: Page,
|
||||
refKey: string | undefined,
|
||||
cssSelector: string | undefined,
|
||||
): Promise<{ x: number; y: number; width: number; height: number } | null> {
|
||||
if (refKey) {
|
||||
try {
|
||||
return await refLocator(page, refKey).boundingBox();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
if (cssSelector) {
|
||||
try {
|
||||
return await page.locator(cssSelector).first().boundingBox();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function captureElementScreenshotForLabels(
|
||||
page: Page,
|
||||
refKey: string | undefined,
|
||||
cssSelector: string | undefined,
|
||||
type: "png" | "jpeg",
|
||||
timeoutMs: number | undefined,
|
||||
): Promise<Buffer> {
|
||||
if (refKey) {
|
||||
return await refLocator(page, refKey).screenshot({ type, timeout: timeoutMs });
|
||||
}
|
||||
if (cssSelector) {
|
||||
return await page.locator(cssSelector).first().screenshot({ type, timeout: timeoutMs });
|
||||
}
|
||||
throw new Error("captureElementScreenshotForLabels: requires refKey or cssSelector");
|
||||
}
|
||||
|
||||
/** Sets file inputs for a role ref or selector with strict existing-path checks. */
|
||||
export async function setInputFilesViaPlaywright(opts: {
|
||||
cdpUrl: string;
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
* navigation policy checks, media storage, and screenshot normalization.
|
||||
*/
|
||||
import path from "node:path";
|
||||
import { getImageMetadata } from "../../media/media-services.js";
|
||||
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
|
||||
import { captureScreenshot, snapshotAria, snapshotRoleViaCdp } from "../cdp.js";
|
||||
import {
|
||||
|
|
@ -24,6 +25,8 @@ import {
|
|||
assertBrowserNavigationResultAllowed,
|
||||
} from "../navigation-guard.js";
|
||||
import { getBrowserProfileCapabilities } from "../profile-capabilities.js";
|
||||
import type { AnnotationItem } from "../screenshot-annotate.js";
|
||||
import { scaleAnnotations } from "../screenshot-annotate.js";
|
||||
import {
|
||||
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
||||
|
|
@ -192,11 +195,24 @@ async function saveNormalizedScreenshotResponse(params: {
|
|||
labels?: boolean;
|
||||
labelsCount?: number;
|
||||
labelsSkipped?: number;
|
||||
annotations?: AnnotationItem[];
|
||||
}) {
|
||||
// Measure original dimensions BEFORE normalization so we can rescale
|
||||
// annotation coordinates if the response pipeline shrinks the image
|
||||
// (longest-side or byte-budget cap). Annotation boxes are in the captured
|
||||
// image's pixel space, so they would otherwise drift from the saved media.
|
||||
const originalMeta = params.annotations?.length
|
||||
? ((await getImageMetadata(params.buffer)) ?? undefined)
|
||||
: undefined;
|
||||
const normalized = await normalizeBrowserScreenshot(params.buffer, {
|
||||
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
||||
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
});
|
||||
const annotations = await rescaleAnnotationsForNormalization({
|
||||
annotations: params.annotations,
|
||||
originalMeta,
|
||||
normalizedBuffer: normalized.buffer,
|
||||
});
|
||||
await saveBrowserMediaResponse({
|
||||
res: params.res,
|
||||
buffer: normalized.buffer,
|
||||
|
|
@ -207,9 +223,39 @@ async function saveNormalizedScreenshotResponse(params: {
|
|||
labels: params.labels,
|
||||
labelsCount: params.labelsCount,
|
||||
labelsSkipped: params.labelsSkipped,
|
||||
annotations,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep annotation coordinates aligned with the saved media after
|
||||
* normalizeBrowserScreenshot. Returns the original annotations unchanged
|
||||
* when normalization did not change the image dimensions, or when image
|
||||
* metadata is unavailable (best-effort: better to ship pre-resize coords
|
||||
* than to drop the field entirely).
|
||||
*/
|
||||
async function rescaleAnnotationsForNormalization(params: {
|
||||
annotations?: AnnotationItem[];
|
||||
originalMeta?: { width?: number; height?: number };
|
||||
normalizedBuffer: Buffer;
|
||||
}): Promise<AnnotationItem[] | undefined> {
|
||||
if (!params.annotations || params.annotations.length === 0) {
|
||||
return params.annotations;
|
||||
}
|
||||
const orig = params.originalMeta;
|
||||
if (!orig?.width || !orig?.height) {
|
||||
return params.annotations;
|
||||
}
|
||||
const next = await getImageMetadata(params.normalizedBuffer);
|
||||
if (!next?.width || !next?.height) {
|
||||
return params.annotations;
|
||||
}
|
||||
if (next.width === orig.width && next.height === orig.height) {
|
||||
return params.annotations;
|
||||
}
|
||||
return scaleAnnotations(params.annotations, next.width / orig.width, next.height / orig.height);
|
||||
}
|
||||
|
||||
async function saveBrowserMediaResponse(params: {
|
||||
res: BrowserResponse;
|
||||
buffer: Buffer;
|
||||
|
|
@ -220,6 +266,7 @@ async function saveBrowserMediaResponse(params: {
|
|||
labels?: boolean;
|
||||
labelsCount?: number;
|
||||
labelsSkipped?: number;
|
||||
annotations?: AnnotationItem[];
|
||||
}) {
|
||||
await ensureMediaDir();
|
||||
const saved = await saveMediaBuffer(
|
||||
|
|
@ -236,6 +283,9 @@ async function saveBrowserMediaResponse(params: {
|
|||
...(params.labels ? { labels: true } : {}),
|
||||
...(typeof params.labelsCount === "number" ? { labelsCount: params.labelsCount } : {}),
|
||||
...(typeof params.labelsSkipped === "number" ? { labelsSkipped: params.labelsSkipped } : {}),
|
||||
...(params.annotations && params.annotations.length > 0
|
||||
? { annotations: params.annotations }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -478,6 +528,9 @@ export function registerBrowserAgentSnapshotRoutes(
|
|||
refs: snap.refs,
|
||||
type,
|
||||
timeoutMs,
|
||||
fullPage,
|
||||
ref,
|
||||
element,
|
||||
});
|
||||
await saveNormalizedScreenshotResponse({
|
||||
res,
|
||||
|
|
@ -488,6 +541,7 @@ export function registerBrowserAgentSnapshotRoutes(
|
|||
labels: true,
|
||||
labelsCount: labeled.labels,
|
||||
labelsSkipped: labeled.skipped,
|
||||
annotations: labeled.annotations,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
|
@ -743,10 +797,18 @@ export function registerBrowserAgentSnapshotRoutes(
|
|||
type: "png",
|
||||
timeoutMs: plan.timeoutMs,
|
||||
});
|
||||
const originalMeta = labeled.annotations.length
|
||||
? ((await getImageMetadata(labeled.buffer)) ?? undefined)
|
||||
: undefined;
|
||||
const normalized = await normalizeBrowserScreenshot(labeled.buffer, {
|
||||
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
||||
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
});
|
||||
const scaledAnnotations = await rescaleAnnotationsForNormalization({
|
||||
annotations: labeled.annotations,
|
||||
originalMeta,
|
||||
normalizedBuffer: normalized.buffer,
|
||||
});
|
||||
await ensureMediaDir();
|
||||
const saved = await saveMediaBuffer(
|
||||
normalized.buffer,
|
||||
|
|
@ -764,6 +826,9 @@ export function registerBrowserAgentSnapshotRoutes(
|
|||
labels: true,
|
||||
labelsCount: labeled.labels,
|
||||
labelsSkipped: labeled.skipped,
|
||||
...(scaledAnnotations && scaledAnnotations.length > 0
|
||||
? { annotations: scaledAnnotations }
|
||||
: {}),
|
||||
imagePath: path.resolve(saved.path),
|
||||
imageType,
|
||||
...snap,
|
||||
|
|
|
|||
345
extensions/browser/src/browser/screenshot-annotate.test.ts
Normal file
345
extensions/browser/src/browser/screenshot-annotate.test.ts
Normal file
|
|
@ -0,0 +1,345 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
ANNOTATION_OVERLAY_ATTR,
|
||||
type AnnotationItem,
|
||||
buildOverlayClearScript,
|
||||
buildOverlayInjectionScript,
|
||||
planAnnotations,
|
||||
type RawAnnotationInput,
|
||||
refToNumber,
|
||||
scaleAnnotations,
|
||||
} from "./screenshot-annotate.js";
|
||||
|
||||
const sampleInputs: RawAnnotationInput[] = [
|
||||
{
|
||||
ref: "e1",
|
||||
role: "button",
|
||||
name: "Submit",
|
||||
doc: { x: 100, y: 200, width: 50, height: 20 },
|
||||
},
|
||||
{
|
||||
ref: "e2",
|
||||
role: "link",
|
||||
doc: { x: 300, y: 1500, width: 80, height: 18 },
|
||||
},
|
||||
];
|
||||
|
||||
describe("refToNumber", () => {
|
||||
it("extracts number from `e<N>` form", () => {
|
||||
expect(refToNumber("e12")).toBe(12);
|
||||
expect(refToNumber("e0")).toBe(0);
|
||||
});
|
||||
|
||||
it("extracts number from `ax<N>` form", () => {
|
||||
expect(refToNumber("ax12")).toBe(12);
|
||||
});
|
||||
|
||||
it("extracts number from bare numeric form", () => {
|
||||
expect(refToNumber("12")).toBe(12);
|
||||
});
|
||||
|
||||
it("returns 0 for non-numeric refs", () => {
|
||||
expect(refToNumber("foo")).toBe(0);
|
||||
expect(refToNumber("")).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("planAnnotations - viewport mode", () => {
|
||||
it("subtracts scroll from doc coords", () => {
|
||||
const plan = planAnnotations({
|
||||
inputs: sampleInputs,
|
||||
space: "viewport",
|
||||
scroll: { x: 0, y: 1000 },
|
||||
});
|
||||
|
||||
expect(plan.annotations).toHaveLength(2);
|
||||
expect(plan.annotations[0]).toEqual({
|
||||
ref: "e1",
|
||||
number: 1,
|
||||
role: "button",
|
||||
name: "Submit",
|
||||
box: { x: 100, y: -800, width: 50, height: 20 },
|
||||
});
|
||||
expect(plan.annotations[1]).toEqual({
|
||||
ref: "e2",
|
||||
number: 2,
|
||||
role: "link",
|
||||
box: { x: 300, y: 500, width: 80, height: 18 },
|
||||
});
|
||||
expect(plan.skipped).toBe(0);
|
||||
});
|
||||
|
||||
it("keeps overlay items in document space regardless of mode", () => {
|
||||
const plan = planAnnotations({
|
||||
inputs: sampleInputs,
|
||||
space: "viewport",
|
||||
scroll: { x: 0, y: 1000 },
|
||||
});
|
||||
expect(plan.overlayItems).toEqual([
|
||||
{ ref: "e1", x: 100, y: 200, w: 50, h: 20 },
|
||||
{ ref: "e2", x: 300, y: 1500, w: 80, h: 18 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("omits empty name field", () => {
|
||||
const plan = planAnnotations({
|
||||
inputs: [{ ref: "e1", role: "button", name: "", doc: { x: 0, y: 0, width: 1, height: 1 } }],
|
||||
space: "viewport",
|
||||
scroll: { x: 0, y: 0 },
|
||||
});
|
||||
expect(plan.annotations[0]).not.toHaveProperty("name");
|
||||
});
|
||||
|
||||
it("throws when scroll missing in viewport mode", () => {
|
||||
expect(() => planAnnotations({ inputs: sampleInputs, space: "viewport" })).toThrow(/scroll/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("planAnnotations - viewport off-screen accounting", () => {
|
||||
it("counts off-viewport refs as skipped but keeps them in annotations when viewport size is given", () => {
|
||||
const plan = planAnnotations({
|
||||
inputs: [
|
||||
{ ref: "e1", role: "button", doc: { x: 10, y: 50, width: 40, height: 20 } }, // in viewport
|
||||
{ ref: "e2", role: "link", doc: { x: 10, y: 5000, width: 40, height: 20 } }, // below viewport
|
||||
],
|
||||
space: "viewport",
|
||||
scroll: { x: 0, y: 0 },
|
||||
viewport: { width: 1280, height: 720 },
|
||||
});
|
||||
|
||||
// Only the in-viewport ref is drawn.
|
||||
expect(plan.overlayItems.map((o) => o.ref)).toEqual(["e1"]);
|
||||
// Both refs are surfaced for callers (off-viewport box can be out of image).
|
||||
expect(plan.annotations.map((a) => a.ref)).toEqual(["e1", "e2"]);
|
||||
// The off-viewport ref raises skipped, preserving the shipped contract.
|
||||
expect(plan.skipped).toBe(1);
|
||||
});
|
||||
|
||||
it("does not count off-viewport refs when viewport size is omitted", () => {
|
||||
const plan = planAnnotations({
|
||||
inputs: [{ ref: "e2", role: "link", doc: { x: 10, y: 5000, width: 40, height: 20 } }],
|
||||
space: "viewport",
|
||||
scroll: { x: 0, y: 0 },
|
||||
});
|
||||
|
||||
expect(plan.skipped).toBe(0);
|
||||
expect(plan.overlayItems).toHaveLength(1);
|
||||
expect(plan.annotations).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("planAnnotations - fullpage mode", () => {
|
||||
it("returns box equal to doc (document coordinates)", () => {
|
||||
const plan = planAnnotations({ inputs: sampleInputs, space: "fullpage" });
|
||||
expect(plan.annotations[0].box).toEqual({ x: 100, y: 200, width: 50, height: 20 });
|
||||
expect(plan.annotations[1].box).toEqual({ x: 300, y: 1500, width: 80, height: 18 });
|
||||
});
|
||||
|
||||
it("does not require scroll", () => {
|
||||
expect(() => planAnnotations({ inputs: sampleInputs, space: "fullpage" })).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("planAnnotations - element mode", () => {
|
||||
const elementRect = { x: 50, y: 100, width: 200, height: 300 };
|
||||
|
||||
it("projects box relative to element top-left", () => {
|
||||
const plan = planAnnotations({
|
||||
inputs: [{ ref: "e1", role: "button", doc: { x: 60, y: 110, width: 40, height: 20 } }],
|
||||
space: "element",
|
||||
elementRect,
|
||||
});
|
||||
expect(plan.annotations[0].box).toEqual({ x: 10, y: 10, width: 40, height: 20 });
|
||||
});
|
||||
|
||||
it("filters out inputs that do not overlap element rect", () => {
|
||||
const plan = planAnnotations({
|
||||
inputs: [
|
||||
{ ref: "e1", role: "button", doc: { x: 60, y: 110, width: 40, height: 20 } }, // inside
|
||||
{ ref: "e2", role: "link", doc: { x: 500, y: 500, width: 40, height: 20 } }, // outside
|
||||
],
|
||||
space: "element",
|
||||
elementRect,
|
||||
});
|
||||
expect(plan.annotations).toHaveLength(1);
|
||||
expect(plan.annotations[0].ref).toBe("e1");
|
||||
expect(plan.overlayItems).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("throws when elementRect missing", () => {
|
||||
expect(() => planAnnotations({ inputs: [], space: "element" })).toThrow(/elementRect/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("planAnnotations - maxLabels", () => {
|
||||
it("truncates to maxLabels and reports skipped", () => {
|
||||
const inputs = Array.from({ length: 5 }, (_, i) => ({
|
||||
ref: `e${i + 1}`,
|
||||
role: "button",
|
||||
doc: { x: 0, y: i * 10, width: 5, height: 5 },
|
||||
}));
|
||||
const plan = planAnnotations({ inputs, space: "fullpage", maxLabels: 2 });
|
||||
expect(plan.annotations).toHaveLength(2);
|
||||
expect(plan.overlayItems).toHaveLength(2);
|
||||
expect(plan.skipped).toBe(3);
|
||||
});
|
||||
|
||||
it("uses ANNOTATION_MAX_LABELS_DEFAULT when not specified", () => {
|
||||
const inputs = Array.from({ length: 200 }, (_, i) => ({
|
||||
ref: `e${i + 1}`,
|
||||
role: "button",
|
||||
doc: { x: 0, y: i, width: 5, height: 5 },
|
||||
}));
|
||||
const plan = planAnnotations({ inputs, space: "fullpage" });
|
||||
expect(plan.annotations).toHaveLength(150);
|
||||
expect(plan.skipped).toBe(50);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildOverlayInjectionScript", () => {
|
||||
it("returns a self-contained IIFE", () => {
|
||||
const script = buildOverlayInjectionScript({
|
||||
items: [{ ref: "e1", x: 100, y: 200, w: 50, h: 20 }],
|
||||
});
|
||||
expect(script).toMatch(/^\(\s*\(\s*\)\s*=>\s*\{/);
|
||||
expect(script).toMatch(/\}\s*\)\s*\(\s*\)\s*;?\s*$/);
|
||||
});
|
||||
|
||||
it("embeds the overlay attr", () => {
|
||||
const script = buildOverlayInjectionScript({ items: [] });
|
||||
expect(script).toContain(ANNOTATION_OVERLAY_ATTR);
|
||||
});
|
||||
|
||||
it("embeds each item's ref text and coordinates", () => {
|
||||
const script = buildOverlayInjectionScript({
|
||||
items: [
|
||||
{ ref: "e1", x: 100, y: 200, w: 50, h: 20 },
|
||||
{ ref: "ax42", x: 999, y: 1500, w: 80, h: 18 },
|
||||
],
|
||||
});
|
||||
expect(script).toMatch(/"ref":\s*"e1"/);
|
||||
expect(script).toMatch(/"ref":\s*"ax42"/);
|
||||
expect(script).toMatch(/"x":\s*100/);
|
||||
expect(script).toMatch(/"x":\s*999/);
|
||||
});
|
||||
|
||||
it("handles empty items without throwing", () => {
|
||||
expect(() => buildOverlayInjectionScript({ items: [] })).not.toThrow();
|
||||
});
|
||||
|
||||
it("rounds coordinates to integers", () => {
|
||||
const script = buildOverlayInjectionScript({
|
||||
items: [{ ref: "e1", x: 100.7, y: 200.4, w: 50.6, h: 20.1 }],
|
||||
});
|
||||
expect(script).toMatch(/"x":\s*101/); // 100.7 -> 101
|
||||
expect(script).toMatch(/"y":\s*200/); // 200.4 -> 200
|
||||
});
|
||||
|
||||
it("clamps zero/negative-size boxes to 1px so they remain visible", () => {
|
||||
const script = buildOverlayInjectionScript({
|
||||
items: [{ ref: "e1", x: 10, y: 10, w: 0, h: 0 }],
|
||||
});
|
||||
expect(script).toMatch(/"w":\s*1/);
|
||||
expect(script).toMatch(/"h":\s*1/);
|
||||
});
|
||||
|
||||
it("escapes hostile ref characters via JSON.stringify (no breakout)", () => {
|
||||
const hostile = 'e1");alert(1);//';
|
||||
const script = buildOverlayInjectionScript({
|
||||
items: [{ ref: hostile, x: 0, y: 0, w: 1, h: 1 }],
|
||||
});
|
||||
// The hostile `"` MUST be escaped as `\"` inside the JSON literal.
|
||||
expect(script).toContain('"e1\\");alert(1);//"');
|
||||
// The unescaped breakout MUST NOT appear anywhere in the script as a
|
||||
// bare statement that would terminate the JSON literal early.
|
||||
expect(script).not.toContain('e1");alert(1);');
|
||||
});
|
||||
|
||||
it("flips label below the box when y < 14 (no headroom)", () => {
|
||||
const script = buildOverlayInjectionScript({
|
||||
items: [{ ref: "e1", x: 0, y: 5, w: 10, h: 10 }],
|
||||
});
|
||||
// labelTop = relativeY < 14 ? it.y + 2 : it.y - 14
|
||||
// The expression literal `relativeY < 14 ? (it.y + 2) : (it.y - 14)` is in the script.
|
||||
expect(script).toContain("relativeY < 14 ? (it.y + 2) : (it.y - 14)");
|
||||
});
|
||||
|
||||
it("uses capture-relative y when deciding whether to flip labels below boxes", () => {
|
||||
const script = buildOverlayInjectionScript({
|
||||
items: [{ ref: "e1", x: 0, y: 1005, w: 10, h: 10 }],
|
||||
captureY: 1000,
|
||||
});
|
||||
|
||||
expect(script).toContain("var captureY = 1000;");
|
||||
expect(script).toContain("var relativeY = it.y - captureY;");
|
||||
expect(script).toContain("relativeY < 14 ? (it.y + 2) : (it.y - 14)");
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildOverlayClearScript", () => {
|
||||
it("returns an IIFE selecting overlay attr", () => {
|
||||
const script = buildOverlayClearScript();
|
||||
expect(script).toContain(`[${ANNOTATION_OVERLAY_ATTR}]`);
|
||||
expect(script).toMatch(/^\(\s*\(\s*\)\s*=>\s*\{/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("scaleAnnotations", () => {
|
||||
const sample: AnnotationItem[] = [
|
||||
{
|
||||
ref: "e1",
|
||||
number: 1,
|
||||
role: "button",
|
||||
name: "Submit",
|
||||
box: { x: 100, y: 200, width: 50, height: 20 },
|
||||
},
|
||||
];
|
||||
|
||||
it("returns identity (structural copy) when both factors are 1", () => {
|
||||
const out = scaleAnnotations(sample, 1, 1);
|
||||
expect(out[0]).toEqual(sample[0]);
|
||||
expect(out[0]).not.toBe(sample[0]);
|
||||
expect(out[0]?.box).not.toBe(sample[0]?.box);
|
||||
});
|
||||
|
||||
it("scales box dimensions by independent x/y factors", () => {
|
||||
const out = scaleAnnotations(sample, 0.5, 0.485);
|
||||
expect(out[0]?.box).toEqual({
|
||||
x: 50,
|
||||
y: 97,
|
||||
width: 25,
|
||||
height: 10,
|
||||
});
|
||||
});
|
||||
|
||||
it("clamps width/height to a minimum of 1 to avoid disappearing labels", () => {
|
||||
const tiny: AnnotationItem[] = [
|
||||
{
|
||||
ref: "e1",
|
||||
number: 1,
|
||||
role: "button",
|
||||
box: { x: 0, y: 0, width: 1, height: 1 },
|
||||
},
|
||||
];
|
||||
const out = scaleAnnotations(tiny, 0.1, 0.1);
|
||||
expect(out[0]?.box.width).toBeGreaterThanOrEqual(1);
|
||||
expect(out[0]?.box.height).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("returns identity (structural copy) for invalid factors", () => {
|
||||
const out = scaleAnnotations(sample, Number.NaN, 0.5);
|
||||
expect(out[0]?.box).toEqual(sample[0]?.box);
|
||||
const out2 = scaleAnnotations(sample, 0, 0.5);
|
||||
expect(out2[0]?.box).toEqual(sample[0]?.box);
|
||||
const out3 = scaleAnnotations(sample, -1, 1);
|
||||
expect(out3[0]?.box).toEqual(sample[0]?.box);
|
||||
});
|
||||
|
||||
it("preserves ref/number/role/name fields verbatim", () => {
|
||||
const out = scaleAnnotations(sample, 0.5, 0.5);
|
||||
expect(out[0]?.ref).toBe("e1");
|
||||
expect(out[0]?.number).toBe(1);
|
||||
expect(out[0]?.role).toBe("button");
|
||||
expect(out[0]?.name).toBe("Submit");
|
||||
});
|
||||
});
|
||||
282
extensions/browser/src/browser/screenshot-annotate.ts
Normal file
282
extensions/browser/src/browser/screenshot-annotate.ts
Normal file
|
|
@ -0,0 +1,282 @@
|
|||
// extensions/browser/src/browser/screenshot-annotate.ts
|
||||
//
|
||||
// Pure helper module for screenshot label annotations.
|
||||
// Has no Playwright / CDP / page dependency: takes document-space inputs,
|
||||
// returns coordinate-projected annotations + IIFE strings the caller can
|
||||
// hand to page.evaluate / Runtime.evaluate.
|
||||
//
|
||||
// Used by:
|
||||
// - pw-tools-core.interactions.ts (Playwright path, M1.2-a)
|
||||
// - planned: raw-CDP path in M1.2-b
|
||||
//
|
||||
// chrome-mcp path keeps its own inline overlay (renderChromeMcpLabels) for now.
|
||||
|
||||
export const ANNOTATION_OVERLAY_ATTR = "data-openclaw-labels";
|
||||
export const ANNOTATION_OVERLAY_ROOT_ID = "__openclaw-annotations__";
|
||||
export const ANNOTATION_MAX_LABELS_DEFAULT = 150;
|
||||
|
||||
export type CoordinateSpace = "viewport" | "fullpage" | "element";
|
||||
|
||||
export interface RawAnnotationInput {
|
||||
ref: string;
|
||||
role: string;
|
||||
name?: string;
|
||||
/** Bounding box in document coordinates (viewport top-left + scroll). */
|
||||
doc: { x: number; y: number; width: number; height: number };
|
||||
}
|
||||
|
||||
export interface AnnotationBox {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface AnnotationItem {
|
||||
ref: string;
|
||||
number: number;
|
||||
role: string;
|
||||
name?: string;
|
||||
box: AnnotationBox;
|
||||
}
|
||||
|
||||
export interface OverlayItem {
|
||||
ref: string;
|
||||
x: number;
|
||||
y: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
export interface AnnotationPlan {
|
||||
/** Always document-space items, fed to buildOverlayInjectionScript. */
|
||||
overlayItems: OverlayItem[];
|
||||
/** Items projected into the capture mode's image-space coordinates. */
|
||||
annotations: AnnotationItem[];
|
||||
/** Refs dropped because of maxLabels truncation. */
|
||||
skipped: number;
|
||||
}
|
||||
|
||||
export interface PlanAnnotationsParams {
|
||||
inputs: RawAnnotationInput[];
|
||||
space: CoordinateSpace;
|
||||
/** Required when space === "viewport". */
|
||||
scroll?: { x: number; y: number };
|
||||
/**
|
||||
* Viewport size (CSS px). Only meaningful when space === "viewport". When
|
||||
* provided, refs whose document box falls outside the current viewport rect
|
||||
* (`scroll` + this size) are counted as skipped instead of drawn, preserving
|
||||
* the shipped `labelsSkipped` contract. Omit it to disable that accounting.
|
||||
*/
|
||||
viewport?: { width: number; height: number };
|
||||
/** Required when space === "element". */
|
||||
elementRect?: { x: number; y: number; width: number; height: number };
|
||||
maxLabels?: number;
|
||||
}
|
||||
|
||||
export function refToNumber(ref: string): number {
|
||||
const match = ref.match(/(\d+)/);
|
||||
if (!match) {
|
||||
return 0;
|
||||
}
|
||||
const n = Number(match[1]);
|
||||
return Number.isFinite(n) ? n : 0;
|
||||
}
|
||||
|
||||
export function planAnnotations(params: PlanAnnotationsParams): AnnotationPlan {
|
||||
const maxLabels = params.maxLabels ?? ANNOTATION_MAX_LABELS_DEFAULT;
|
||||
|
||||
if (params.space === "viewport" && !params.scroll) {
|
||||
throw new Error("planAnnotations: scroll is required when space is 'viewport'");
|
||||
}
|
||||
if (params.space === "element" && !params.elementRect) {
|
||||
throw new Error("planAnnotations: elementRect is required when space is 'element'");
|
||||
}
|
||||
|
||||
// Element-mode filter: discard inputs that do not overlap the element rect.
|
||||
let kept = params.inputs;
|
||||
if (params.space === "element" && params.elementRect) {
|
||||
const er = params.elementRect;
|
||||
kept = params.inputs.filter((input) => rectsOverlap(input.doc, er));
|
||||
}
|
||||
|
||||
// Viewport capture only shows refs inside the current viewport rect. An
|
||||
// off-viewport ref is still surfaced in `annotations` (with its real,
|
||||
// possibly out-of-image box) so callers can locate it, but it is not drawn
|
||||
// and is counted as skipped. This keeps the shipped `labelsSkipped` meaning
|
||||
// ("refs not present in the captured viewport image") instead of silently
|
||||
// narrowing it. Only applied when the caller supplies the viewport size;
|
||||
// without it we cannot decide off-screen state and skip nothing.
|
||||
const viewportRect =
|
||||
params.space === "viewport" && params.scroll && params.viewport
|
||||
? {
|
||||
x: params.scroll.x,
|
||||
y: params.scroll.y,
|
||||
width: params.viewport.width,
|
||||
height: params.viewport.height,
|
||||
}
|
||||
: undefined;
|
||||
|
||||
const overlayItems: OverlayItem[] = [];
|
||||
const annotations: AnnotationItem[] = [];
|
||||
let skipped = 0;
|
||||
|
||||
for (const input of kept) {
|
||||
if (viewportRect && !rectsOverlap(input.doc, viewportRect)) {
|
||||
// Outside the captured viewport: count as skipped (compat) but still
|
||||
// report the annotation; do not draw it or consume the label budget.
|
||||
skipped += 1;
|
||||
annotations.push(toAnnotation(input, params));
|
||||
continue;
|
||||
}
|
||||
if (overlayItems.length >= maxLabels) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
overlayItems.push({
|
||||
ref: input.ref,
|
||||
x: input.doc.x,
|
||||
y: input.doc.y,
|
||||
w: input.doc.width,
|
||||
h: input.doc.height,
|
||||
});
|
||||
annotations.push(toAnnotation(input, params));
|
||||
}
|
||||
|
||||
return { overlayItems, annotations, skipped };
|
||||
}
|
||||
|
||||
function toAnnotation(input: RawAnnotationInput, params: PlanAnnotationsParams): AnnotationItem {
|
||||
return {
|
||||
ref: input.ref,
|
||||
number: refToNumber(input.ref),
|
||||
role: input.role,
|
||||
...(input.name ? { name: input.name } : {}),
|
||||
box: projectBox(input.doc, params),
|
||||
};
|
||||
}
|
||||
|
||||
function projectBox(
|
||||
doc: { x: number; y: number; width: number; height: number },
|
||||
params: PlanAnnotationsParams,
|
||||
): AnnotationBox {
|
||||
if (params.space === "viewport") {
|
||||
const scroll = params.scroll!;
|
||||
return {
|
||||
x: doc.x - scroll.x,
|
||||
y: doc.y - scroll.y,
|
||||
width: doc.width,
|
||||
height: doc.height,
|
||||
};
|
||||
}
|
||||
if (params.space === "element") {
|
||||
const er = params.elementRect!;
|
||||
// NOTE: width/height pass through unchanged even when the input rect
|
||||
// partially extends past the element. The capture backend (e.g.
|
||||
// locator.screenshot) is responsible for clipping; the box may have
|
||||
// negative x/y or extend past elementRect width/height for partial overlaps.
|
||||
return {
|
||||
x: doc.x - er.x,
|
||||
y: doc.y - er.y,
|
||||
width: doc.width,
|
||||
height: doc.height,
|
||||
};
|
||||
}
|
||||
// fullpage: document coordinates as-is
|
||||
return { x: doc.x, y: doc.y, width: doc.width, height: doc.height };
|
||||
}
|
||||
|
||||
function rectsOverlap(
|
||||
a: { x: number; y: number; width: number; height: number },
|
||||
b: { x: number; y: number; width: number; height: number },
|
||||
): boolean {
|
||||
return a.x < b.x + b.width && a.x + a.width > b.x && a.y < b.y + b.height && a.y + a.height > b.y;
|
||||
}
|
||||
|
||||
export function buildOverlayInjectionScript(params: {
|
||||
items: OverlayItem[];
|
||||
captureY?: number;
|
||||
}): string {
|
||||
const itemsJson = JSON.stringify(
|
||||
params.items.map((it) => ({
|
||||
ref: it.ref,
|
||||
x: round(it.x),
|
||||
y: round(it.y),
|
||||
w: Math.max(1, round(it.w)),
|
||||
h: Math.max(1, round(it.h)),
|
||||
})),
|
||||
);
|
||||
const attr = ANNOTATION_OVERLAY_ATTR;
|
||||
const rootId = ANNOTATION_OVERLAY_ROOT_ID;
|
||||
const captureY = Number.isFinite(params.captureY) ? round(params.captureY ?? 0) : 0;
|
||||
return `(() => {
|
||||
var items = ${itemsJson};
|
||||
var captureY = ${captureY};
|
||||
var existing = document.querySelectorAll("[${attr}]");
|
||||
for (var k = 0; k < existing.length; k++) existing[k].remove();
|
||||
var root = document.createElement("div");
|
||||
root.id = ${JSON.stringify(rootId)};
|
||||
root.setAttribute("${attr}", "1");
|
||||
root.style.cssText = "position:absolute;top:0;left:0;width:0;height:0;pointer-events:none;z-index:2147483647;font-family:'SF Mono','SFMono-Regular',Menlo,Monaco,Consolas,'Liberation Mono','Courier New',monospace;";
|
||||
for (var i = 0; i < items.length; i++) {
|
||||
var it = items[i];
|
||||
var box = document.createElement("div");
|
||||
box.setAttribute("${attr}", "1");
|
||||
box.style.cssText = "position:absolute;left:" + it.x + "px;top:" + it.y + "px;width:" + it.w + "px;height:" + it.h + "px;border:2px solid #ffb020;box-sizing:border-box;pointer-events:none;";
|
||||
var tag = document.createElement("div");
|
||||
tag.setAttribute("${attr}", "1");
|
||||
tag.textContent = String(it.ref);
|
||||
var relativeY = it.y - captureY;
|
||||
var labelTop = relativeY < 14 ? (it.y + 2) : (it.y - 14);
|
||||
tag.style.cssText = "position:absolute;left:" + it.x + "px;top:" + labelTop + "px;background:#ffb020;color:#1a1a1a;font:bold 11px/14px monospace;padding:0 4px;border-radius:2px;white-space:nowrap;pointer-events:none;";
|
||||
root.appendChild(box);
|
||||
root.appendChild(tag);
|
||||
}
|
||||
document.documentElement.appendChild(root);
|
||||
return true;
|
||||
})();`;
|
||||
}
|
||||
|
||||
export function buildOverlayClearScript(): string {
|
||||
const attr = ANNOTATION_OVERLAY_ATTR;
|
||||
return `(() => {
|
||||
var existing = document.querySelectorAll("[${attr}]");
|
||||
for (var k = 0; k < existing.length; k++) existing[k].remove();
|
||||
return true;
|
||||
})();`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scale annotation boxes by independent x/y factors. Used to keep annotation
|
||||
* coordinates aligned with the saved image after the response pipeline
|
||||
* resizes the screenshot (e.g. via normalizeBrowserScreenshot capping the
|
||||
* longest side or the byte budget). Returns a new array; inputs are not
|
||||
* mutated. When both factors are 1 the boxes are returned unchanged (modulo
|
||||
* structural copy) so callers can share the same code path for resized and
|
||||
* non-resized captures.
|
||||
*/
|
||||
export function scaleAnnotations(
|
||||
items: AnnotationItem[],
|
||||
scaleX: number,
|
||||
scaleY: number,
|
||||
): AnnotationItem[] {
|
||||
if (!Number.isFinite(scaleX) || !Number.isFinite(scaleY) || scaleX <= 0 || scaleY <= 0) {
|
||||
return items.map((it) => ({ ...it, box: { ...it.box } }));
|
||||
}
|
||||
if (scaleX === 1 && scaleY === 1) {
|
||||
return items.map((it) => ({ ...it, box: { ...it.box } }));
|
||||
}
|
||||
return items.map((it) => ({
|
||||
...it,
|
||||
box: {
|
||||
x: round(it.box.x * scaleX),
|
||||
y: round(it.box.y * scaleY),
|
||||
width: Math.max(1, round(it.box.width * scaleX)),
|
||||
height: Math.max(1, round(it.box.height * scaleY)),
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
function round(v: number): number {
|
||||
return Math.round(v);
|
||||
}
|
||||
|
|
@ -51,7 +51,11 @@ export function registerBrowserInspectCommands(
|
|||
.option("--full-page", "Capture full scrollable page", false)
|
||||
.option("--ref <ref>", "ARIA ref from ai snapshot")
|
||||
.option("--element <selector>", "CSS selector for element screenshot")
|
||||
.option("--labels", "Overlay role refs on the screenshot", false)
|
||||
.option(
|
||||
"--labels",
|
||||
"Overlay role refs on the screenshot (works with --full-page, --ref, and --element)",
|
||||
false,
|
||||
)
|
||||
.option("--type <png|jpeg>", "Output type (default: png)", "png")
|
||||
.action(async (targetId: string | undefined, opts, cmd) => {
|
||||
const parent = parentOpts(cmd);
|
||||
|
|
@ -98,7 +102,7 @@ export function registerBrowserInspectCommands(
|
|||
.option("--depth <n>", "Role snapshot: max depth")
|
||||
.option("--selector <sel>", "Role snapshot: scope to CSS selector")
|
||||
.option("--frame <sel>", "Role snapshot: scope to an iframe selector")
|
||||
.option("--labels", "Include viewport label overlay screenshot", false)
|
||||
.option("--labels", "Include label overlay screenshot with annotations", false)
|
||||
.option("--urls", "Append discovered link URLs to AI snapshots", false)
|
||||
.option("--out <path>", "Write snapshot to a file")
|
||||
.action(async (opts, cmd) => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue