From 24e729fc4e077d0ea23a14c4497d343c5f9569c3 Mon Sep 17 00:00:00 2001
From: "clawsweeper[bot]" <274271284+clawsweeper[bot]@users.noreply.github.com>
Date: Sun, 14 Jun 2026 02:21:23 +0000
Subject: [PATCH] feat(browser): extend --labels overlay to full-page and
 element captures (#92834)

Summary:
- The replacement PR extends Browser plugin labeled screenshots to honor Playwright full-page/ref/element scope, returns annotation bounding boxes, and updates docs, tests, and skill guidance.
- PR surface: Source +415, Tests +550, Docs +24. Total +989 across 12 files.
- Reproducibility: yes. Current main source shows the labeled Playwright helper ignores fullPage/ref/element and omits annotations, and the source PR supplies live before/after commands for the Browser plugin path.

Automerge notes:
- PR branch already contained follow-up commit before automerge: docs(browser): correct raw-CDP labels caveat in automation skill
- PR branch already contained follow-up commit before automerge: fix(browser): preserve labelsSkipped semantics for off-viewport refs
- PR branch already contained follow-up commit before automerge: docs(browser): scope labels docs by driver
- PR branch already contained follow-up commit before automerge: docs(browser): fix labels annotation indent and document scope fix
- PR branch already contained follow-up commit before automerge: docs(browser): indent annotations box schema under --labels bullet
- PR branch already contained follow-up commit before automerge: docs(browser): indent labels annotation schema

Validation:
- ClawSweeper review passed for head 70aca6c5065ae68bbf1037949e28045f479c0355.
- Required merge gates passed before the squash merge.

Prepared head SHA: 70aca6c5065ae68bbf1037949e28045f479c0355
Review: https://github.com/openclaw/openclaw/pull/92834#issuecomment-4700431344

Co-authored-by: FMLS <kfliuyang@gmail.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Mason Huang <masonxhuang@tencent.com>
Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com>
Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com>
Approved-by: hxy91819
Co-authored-by: hxy91819 <8814856+hxy91819@users.noreply.github.com>
---
 docs/cli/browser.md                           |  17 +-
 docs/tools/browser-control.md                 |  13 +-
 .../skills/browser-automation/SKILL.md        |   2 +-
 .../browser/src/browser-tool.actions.ts       |   1 +
 .../src/browser/client-actions-types.ts       |   8 +
 extensions/browser/src/browser/client.ts      |   6 +
 .../browser/pw-tools-core.annotate.test.ts    | 205 +++++++++++
 .../src/browser/pw-tools-core.interactions.ts | 245 ++++++++-----
 .../src/browser/routes/agent.snapshot.ts      |  65 ++++
 .../src/browser/screenshot-annotate.test.ts   | 345 ++++++++++++++++++
 .../src/browser/screenshot-annotate.ts        | 282 ++++++++++++++
 .../browser/src/cli/browser-cli-inspect.ts    |   8 +-
 12 files changed, 1093 insertions(+), 104 deletions(-)
 create mode 100644 extensions/browser/src/browser/pw-tools-core.annotate.test.ts
 create mode 100644 extensions/browser/src/browser/screenshot-annotate.test.ts
 create mode 100644 extensions/browser/src/browser/screenshot-annotate.ts

diff --git a/docs/cli/browser.md b/docs/cli/browser.md
index 0925016a159..10cd0e4e798 100644
--- a/docs/cli/browser.md
+++ b/docs/cli/browser.md
@@ -174,7 +174,22 @@ Notes:
   or `--element`.
 - `existing-session` / `user` profiles support page screenshots and `--ref`
   screenshots from snapshot output, but not CSS `--element` screenshots.
-- `--labels` overlays current snapshot refs on the screenshot.
+- `--labels` overlays current snapshot refs on the screenshot. On
+  Playwright-backed profiles, it works with `--full-page` (full-page label
+  overlay), `--ref` (element-clip label overlay by ARIA ref), and `--element`
+  (element-clip label overlay by CSS selector); in element-clip modes, labels
+  are projected relative to the element. The response also includes an
+  `annotations` array with each ref's bounding box. Each item has `ref`,
+  `number`, `role`, optional `name`, and `box: {x, y, width, height}`;
+  coordinates are in the captured image's space (viewport / fullpage /
+  element-relative). The field is omitted when empty.
+  `existing-session` profiles render a chrome-mcp overlay on page screenshots
+  but do not use the Playwright projection helper and do not include
+  `annotations`; CSS `--element` screenshots are unsupported there. Without
+  Playwright or chrome-mcp, labeled screenshots are not available. Prior
+  releases ignored `--full-page`, `--ref`, and `--element` on labeled
+  Playwright screenshots and always returned a viewport capture; labeled
+  screenshots now honor those scopes.
 - `snapshot --urls` appends discovered link destinations to AI snapshots so
   agents can choose direct navigation targets instead of guessing from link
   text alone.
diff --git a/docs/tools/browser-control.md b/docs/tools/browser-control.md
index 9ab421417c9..afb319f7a68 100644
--- a/docs/tools/browser-control.md
+++ b/docs/tools/browser-control.md
@@ -258,7 +258,14 @@ Snapshot flags at a glance:
 - `--format aria`: accessibility tree with `axN` refs. When Playwright is available, OpenClaw binds refs with backend DOM ids to the live page so follow-up actions can use them; otherwise treat the output as inspection-only.
 - `--efficient` (or `--mode efficient`): compact role snapshot preset. Set `browser.snapshotDefaults.mode: "efficient"` to make this the default (see [Gateway configuration](/gateway/configuration-reference#browser)).
 - `--interactive`, `--compact`, `--depth`, `--selector` force a role snapshot with `ref=e12` refs. `--frame "<iframe>"` scopes role snapshots to an iframe.
-- `--labels` adds a viewport-only screenshot with overlayed ref labels and prints the saved path.
+- With Playwright, `--labels` adds a screenshot with overlayed ref labels
+  (prints `MEDIA:<path>`) plus an `annotations` array with each ref's bounding
+  box. On `screenshot`, Playwright-backed labels work with `--full-page`,
+  `--ref`, and `--element`; on `snapshot`, the accompanying screenshot remains
+  viewport-only. Existing-session/chrome-mcp profiles render overlay labels on
+  page screenshots but do not return `annotations` or use the Playwright
+  full-page/ref/element projection helper. Without Playwright or chrome-mcp,
+  labeled screenshots are not available.
 - `--urls` appends discovered link destinations to AI snapshots.
 
 ## Snapshots and refs
@@ -274,7 +281,9 @@ OpenClaw supports two "snapshot" styles:
   - Output: a role-based list/tree with `[ref=e12]` (and optional `[nth=1]`).
   - Actions: `openclaw browser click e12`, `openclaw browser highlight e12`.
   - Internally, the ref is resolved via `getByRole(...)` (plus `nth()` for duplicates).
-  - Add `--labels` to include a viewport screenshot with overlayed `e12` labels.
+  - Add `--labels` to include a screenshot with overlayed `e12` labels. On
+    Playwright-backed profiles this also returns per-ref bounding-box metadata
+    (`annotations[]`).
   - Add `--urls` when link text is ambiguous and the agent needs concrete
     navigation targets.
 
diff --git a/extensions/browser/skills/browser-automation/SKILL.md b/extensions/browser/skills/browser-automation/SKILL.md
index b0516b4e333..83a8aa2adfc 100644
--- a/extensions/browser/skills/browser-automation/SKILL.md
+++ b/extensions/browser/skills/browser-automation/SKILL.md
@@ -25,7 +25,7 @@ Use this skill when you need the `browser` tool for anything beyond a single pag
    - Use the same `targetId` for follow-up actions so refs stay on the same tab.
    - For durable Playwright refs, request `refs="aria"` when supported. If you receive `axN` refs from `snapshotFormat="aria"`, use them only after that same snapshot call; stale or unbound `axN` refs fail fast and need a fresh snapshot.
    - Use `urls=true` when link text is ambiguous or a direct navigation target would avoid brittle clicks.
-   - Use `labels=true` on snapshot or screenshot when visual position matters.
+   - Use `labels=true` on snapshot or screenshot when visual position matters. On Playwright-backed profiles, the response includes an `annotations` array (`{ref, number, role, name?, box}`) with each ref's bounding box in the captured image's coordinate space, so you can reason about position without re-snapshotting; screenshot labels can also combine with `fullPage=true` (CLI: `--full-page`) to label the whole document, or `ref` / `element` to clip to one element. `profile="user"` and other existing-session (chrome-mcp) profiles render an overlay into page screenshots but do not attach `annotations` or use the Playwright full-page/ref/element projection helper, so read positions from the labeled image itself on those profiles. The raw-CDP fallback (no Playwright) does not support labeled screenshots at all and returns a 501, so only request `labels` when Playwright is available.
 4. Act narrowly:
    - Prefer `action="act"` with a ref from the latest snapshot.
    - After navigation, modal changes, or form submission, snapshot again before the next action.
diff --git a/extensions/browser/src/browser-tool.actions.ts b/extensions/browser/src/browser-tool.actions.ts
index bb6a3fd5dbb..1c86a145829 100644
--- a/extensions/browser/src/browser-tool.actions.ts
+++ b/extensions/browser/src/browser-tool.actions.ts
@@ -486,6 +486,7 @@ export async function executeSnapshotAction(params: {
       labels: snapshot.labels,
       labelsCount: snapshot.labelsCount,
       labelsSkipped: snapshot.labelsSkipped,
+      annotations: snapshot.annotations,
       imagePath: snapshot.imagePath,
       imageType: snapshot.imageType,
       refsFallback,
diff --git a/extensions/browser/src/browser/client-actions-types.ts b/extensions/browser/src/browser/client-actions-types.ts
index 37142b4aebf..1953158334d 100644
--- a/extensions/browser/src/browser/client-actions-types.ts
+++ b/extensions/browser/src/browser/client-actions-types.ts
@@ -1,6 +1,8 @@
 /**
  * Shared result types for browser client action helpers.
  */
+import type { AnnotationItem } from "./screenshot-annotate.js";
+
 /** Generic success result for action endpoints. */
 export type BrowserActionOk = { ok: true };
 
@@ -20,4 +22,10 @@ export type BrowserActionPathResult = {
   labels?: boolean;
   labelsCount?: number;
   labelsSkipped?: number;
+  /**
+   * Per-ref bounding boxes when labels=true. Coordinates are in the
+   * captured image's space (viewport / fullpage / element-relative).
+   * Omitted when empty.
+   */
+  annotations?: AnnotationItem[];
 };
diff --git a/extensions/browser/src/browser/client.ts b/extensions/browser/src/browser/client.ts
index 111adc6948f..13a802d0f59 100644
--- a/extensions/browser/src/browser/client.ts
+++ b/extensions/browser/src/browser/client.ts
@@ -18,6 +18,7 @@ import type {
 } from "./client.types.js";
 import { DEFAULT_BROWSER_SNAPSHOT_TIMEOUT_MS } from "./constants.js";
 import type { BrowserDoctorReport } from "./doctor.js";
+import type { AnnotationItem } from "./screenshot-annotate.js";
 
 export type { BrowserStatus, BrowserTab, BrowserTransport } from "./client.types.js";
 export type { BrowserDoctorCheck, BrowserDoctorReport } from "./doctor.js";
@@ -124,6 +125,11 @@ export type SnapshotResult =
       labels?: boolean;
       labelsCount?: number;
       labelsSkipped?: number;
+      /**
+       * Per-ref bounding boxes when labels=true. Coordinates are in the
+       * captured image's space. Omitted when empty.
+       */
+      annotations?: AnnotationItem[];
       imagePath?: string;
       imageType?: "png" | "jpeg";
       blockedByDialog?: boolean;
diff --git a/extensions/browser/src/browser/pw-tools-core.annotate.test.ts b/extensions/browser/src/browser/pw-tools-core.annotate.test.ts
new file mode 100644
index 00000000000..7a7fbc271fc
--- /dev/null
+++ b/extensions/browser/src/browser/pw-tools-core.annotate.test.ts
@@ -0,0 +1,205 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  installPwToolsCoreTestHooks,
+  setPwToolsCoreCurrentPage,
+  setPwToolsCoreCurrentRefLocator,
+} from "./pw-tools-core.test-harness.js";
+
+installPwToolsCoreTestHooks();
+const mod = await import("./pw-tools-core.js");
+
+type EvaluateArg = unknown;
+
+function evaluateMockReturning(view: { x: number; y: number; width?: number; height?: number }) {
+  // Caller reads { x, y, width, height } in one evaluate; default to a normal
+  // desktop viewport so refs near the top stay in-viewport unless a test puts
+  // them out of range explicitly.
+  const result = { width: 1280, height: 720, ...view };
+  return vi.fn(async (arg: EvaluateArg) => {
+    if (typeof arg === "function") {
+      return result;
+    }
+    return true;
+  });
+}
+
+describe("screenshotWithLabelsViaPlaywright (viewport)", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("calls page.screenshot without fullPage and returns annotations", async () => {
+    const evaluate = evaluateMockReturning({ x: 0, y: 100 });
+    const screenshot = vi.fn(async () => Buffer.from("PNG"));
+    setPwToolsCoreCurrentPage({ evaluate, screenshot, url: () => "https://example.com" });
+    setPwToolsCoreCurrentRefLocator({
+      boundingBox: async () => ({ x: 10, y: 200, width: 50, height: 20 }),
+    });
+
+    const result = await mod.screenshotWithLabelsViaPlaywright({
+      cdpUrl: "http://127.0.0.1:18792",
+      targetId: "T1",
+      refs: { e1: { role: "button", name: "Submit" } },
+      type: "png",
+    });
+
+    expect(screenshot).toHaveBeenCalledWith(expect.objectContaining({ type: "png" }));
+    expect(screenshot).not.toHaveBeenCalledWith(expect.objectContaining({ fullPage: true }));
+
+    expect(result.annotations).toHaveLength(1);
+    expect(result.annotations[0]).toMatchObject({
+      ref: "e1",
+      number: 1,
+      role: "button",
+      name: "Submit",
+    });
+    // viewport-mode box = doc(box.x + scroll.x, box.y + scroll.y) - scroll = bbox
+    expect(result.annotations[0]?.box).toEqual({ x: 10, y: 200, width: 50, height: 20 });
+    expect(result.skipped).toBe(0);
+  });
+
+  it("runs the clear script even when screenshot throws", async () => {
+    const evaluate = evaluateMockReturning({ x: 0, y: 0 });
+    const screenshot = vi.fn(async () => {
+      throw new Error("boom");
+    });
+    setPwToolsCoreCurrentPage({ evaluate, screenshot });
+    setPwToolsCoreCurrentRefLocator({
+      boundingBox: async () => ({ x: 0, y: 0, width: 1, height: 1 }),
+    });
+
+    await expect(
+      mod.screenshotWithLabelsViaPlaywright({
+        cdpUrl: "http://127.0.0.1:18792",
+        targetId: "T1",
+        refs: { e1: { role: "button" } },
+      }),
+    ).rejects.toThrow(/boom/);
+
+    // The clear script must have run (string evaluate calls include the overlay attr)
+    const clearCalls = evaluate.mock.calls.filter(
+      ([arg]) => typeof arg === "string" && arg.includes("data-openclaw-labels"),
+    );
+    // inject + clear = at least 2 string evaluations
+    expect(clearCalls.length).toBeGreaterThanOrEqual(2);
+  });
+
+  it("counts off-viewport refs as skipped but still surfaces them in annotations", async () => {
+    const evaluate = evaluateMockReturning({ x: 0, y: 0, width: 1280, height: 720 });
+    const screenshot = vi.fn(async () => Buffer.from("PNG"));
+    setPwToolsCoreCurrentPage({ evaluate, screenshot });
+    // bbox is far below the viewport (y: 5000): not drawn, but still reported
+    // so callers keep the position and a non-zero skipped count.
+    setPwToolsCoreCurrentRefLocator({
+      boundingBox: async () => ({ x: 0, y: 5000, width: 50, height: 20 }),
+    });
+
+    const result = await mod.screenshotWithLabelsViaPlaywright({
+      cdpUrl: "http://127.0.0.1:18792",
+      targetId: "T1",
+      refs: { e1: { role: "button" } },
+    });
+
+    expect(result.skipped).toBe(1);
+    expect(result.labels).toBe(0);
+    expect(result.annotations).toHaveLength(1);
+    expect(result.annotations[0]?.ref).toBe("e1");
+  });
+});
+
+describe("screenshotWithLabelsViaPlaywright (fullpage)", () => {
+  beforeEach(() => vi.clearAllMocks());
+
+  it("forwards fullPage:true to page.screenshot and uses doc-space annotations", async () => {
+    const evaluate = evaluateMockReturning({ x: 0, y: 1000 });
+    const screenshot = vi.fn(async () => Buffer.from("FULL"));
+    setPwToolsCoreCurrentPage({ evaluate, screenshot });
+    setPwToolsCoreCurrentRefLocator({
+      boundingBox: async () => ({ x: 10, y: 200, width: 50, height: 20 }),
+    });
+
+    const result = await mod.screenshotWithLabelsViaPlaywright({
+      cdpUrl: "http://127.0.0.1:18792",
+      targetId: "T1",
+      refs: { e1: { role: "button" } },
+      fullPage: true,
+    });
+
+    expect(screenshot).toHaveBeenCalledWith(expect.objectContaining({ fullPage: true }));
+    // doc-space: scroll y=1000 + bbox y=200 = 1200
+    expect(result.annotations[0]?.box.y).toBe(1200);
+    expect(result.annotations[0]?.box.x).toBe(10);
+  });
+});
+
+describe("screenshotWithLabelsViaPlaywright (element/ref)", () => {
+  beforeEach(() => vi.clearAllMocks());
+
+  it("uses refLocator.screenshot for ref mode and projects relative to element", async () => {
+    const evaluate = evaluateMockReturning({ x: 0, y: 0 });
+    // First call resolves the element rect (container), second resolves e1 annotation bbox.
+    const boundingBox = vi
+      .fn<() => Promise<{ x: number; y: number; width: number; height: number } | null>>()
+      .mockResolvedValueOnce({ x: 50, y: 100, width: 200, height: 300 })
+      .mockResolvedValueOnce({ x: 60, y: 110, width: 30, height: 20 });
+    const elementScreenshot = vi.fn(async () => Buffer.from("ELEM"));
+    setPwToolsCoreCurrentPage({ evaluate, screenshot: vi.fn() });
+    setPwToolsCoreCurrentRefLocator({ boundingBox, screenshot: elementScreenshot });
+
+    const result = await mod.screenshotWithLabelsViaPlaywright({
+      cdpUrl: "http://127.0.0.1:18792",
+      targetId: "T1",
+      refs: { e1: { role: "button" } },
+      ref: "container",
+    });
+
+    expect(elementScreenshot).toHaveBeenCalledTimes(1);
+    // Element-relative: doc(60,110) - elementRect(50,100) = (10,10)
+    expect(result.annotations).toHaveLength(1);
+    expect(result.annotations[0]?.box).toEqual({ x: 10, y: 10, width: 30, height: 20 });
+  });
+
+  it("throws when ref/element cannot be resolved", async () => {
+    const evaluate = evaluateMockReturning({ x: 0, y: 0 });
+    setPwToolsCoreCurrentPage({ evaluate, screenshot: vi.fn() });
+    setPwToolsCoreCurrentRefLocator({
+      boundingBox: async () => null,
+      screenshot: vi.fn(),
+    });
+
+    await expect(
+      mod.screenshotWithLabelsViaPlaywright({
+        cdpUrl: "http://127.0.0.1:18792",
+        targetId: "T1",
+        refs: { e1: { role: "button" } },
+        ref: "missing",
+      }),
+    ).rejects.toThrow(/element not found/i);
+  });
+});
+
+describe("screenshotWithLabelsViaPlaywright (skipped accounting)", () => {
+  beforeEach(() => vi.clearAllMocks());
+
+  it("counts refs whose boundingBox is null toward skipped", async () => {
+    const evaluate = evaluateMockReturning({ x: 0, y: 0 });
+    const screenshot = vi.fn(async () => Buffer.from("PNG"));
+    setPwToolsCoreCurrentPage({ evaluate, screenshot });
+    // Two refs: first returns a box, second returns null (e.g. element detached).
+    const boundingBox = vi
+      .fn<() => Promise<{ x: number; y: number; width: number; height: number } | null>>()
+      .mockResolvedValueOnce({ x: 10, y: 20, width: 30, height: 40 })
+      .mockResolvedValueOnce(null);
+    setPwToolsCoreCurrentRefLocator({ boundingBox });
+
+    const result = await mod.screenshotWithLabelsViaPlaywright({
+      cdpUrl: "http://127.0.0.1:18792",
+      targetId: "T1",
+      refs: { e1: { role: "button" }, e2: { role: "link" } },
+    });
+
+    expect(result.annotations).toHaveLength(1);
+    expect(result.annotations[0]?.ref).toBe("e1");
+    expect(result.skipped).toBe(1);
+  });
+});
diff --git a/extensions/browser/src/browser/pw-tools-core.interactions.ts b/extensions/browser/src/browser/pw-tools-core.interactions.ts
index b7119754bad..4856737469e 100644
--- a/extensions/browser/src/browser/pw-tools-core.interactions.ts
+++ b/extensions/browser/src/browser/pw-tools-core.interactions.ts
@@ -41,6 +41,15 @@ import {
   toAIFriendlyError,
 } from "./pw-tools-core.shared.js";
 import { closePageViaPlaywright, resizeViewportViaPlaywright } from "./pw-tools-core.snapshot.js";
+import {
+  ANNOTATION_MAX_LABELS_DEFAULT,
+  type AnnotationItem,
+  buildOverlayClearScript,
+  buildOverlayInjectionScript,
+  type CoordinateSpace,
+  planAnnotations,
+  type RawAnnotationInput,
+} from "./screenshot-annotate.js";
 
 type TargetOpts = {
   cdpUrl: string;
@@ -1287,7 +1296,15 @@ export async function screenshotWithLabelsViaPlaywright(opts: {
   maxLabels?: number;
   type?: "png" | "jpeg";
   timeoutMs?: number;
-}): Promise<{ buffer: Buffer; labels: number; skipped: number }> {
+  fullPage?: boolean;
+  ref?: string;
+  element?: string;
+}): Promise<{
+  buffer: Buffer;
+  labels: number;
+  skipped: number;
+  annotations: AnnotationItem[];
+}> {
   const page = await getPageForTargetId(opts);
   ensurePageState(page);
   restoreRoleRefsForTarget({ cdpUrl: opts.cdpUrl, targetId: opts.targetId, page });
@@ -1295,119 +1312,151 @@ export async function screenshotWithLabelsViaPlaywright(opts: {
   const maxLabels =
     typeof opts.maxLabels === "number" && Number.isFinite(opts.maxLabels)
       ? Math.max(1, Math.floor(opts.maxLabels))
-      : 150;
+      : ANNOTATION_MAX_LABELS_DEFAULT;
 
-  const viewport = await page.evaluate(() => ({
-    scrollX: window.scrollX || 0,
-    scrollY: window.scrollY || 0,
+  const refKey = normalizeOptionalString(opts.ref) ?? undefined;
+  const elementSelector = normalizeOptionalString(opts.element) ?? undefined;
+  const space: CoordinateSpace = opts.fullPage
+    ? "fullpage"
+    : refKey || elementSelector
+      ? "element"
+      : "viewport";
+
+  // Read scroll + viewport size. Scroll converts Playwright's viewport-space
+  // boundingBoxes into document-space inputs; the viewport size lets the helper
+  // restore the shipped `labelsSkipped` semantics by counting off-viewport refs
+  // as skipped (in viewport capture mode).
+  const view = await page.evaluate(() => ({
+    x: window.scrollX || 0,
+    y: window.scrollY || 0,
     width: window.innerWidth || 0,
     height: window.innerHeight || 0,
   }));
+  const scroll = { x: view.x, y: view.y };
 
-  const refs = Object.keys(opts.refs ?? {});
-  const boxes: Array<{ ref: string; x: number; y: number; w: number; h: number }> = [];
-  let skipped = 0;
+  let elementRect: { x: number; y: number; width: number; height: number } | undefined;
+  if (space === "element") {
+    const box = await resolveElementBoundingBoxForLabels(page, refKey, elementSelector);
+    if (!box) {
+      throw new Error(
+        `screenshotWithLabelsViaPlaywright: element not found for ${
+          refKey ? `ref="${refKey}"` : `selector="${elementSelector ?? ""}"`
+        }`,
+      );
+    }
+    // Convert viewport-space bbox to document space.
+    elementRect = {
+      x: box.x + scroll.x,
+      y: box.y + scroll.y,
+      width: box.width,
+      height: box.height,
+    };
+  }
 
-  for (const ref of refs) {
-    if (boxes.length >= maxLabels) {
-      skipped += 1;
+  const refKeys = Object.keys(opts.refs ?? {});
+  const inputs: RawAnnotationInput[] = [];
+  let bboxFailures = 0;
+  for (const ref of refKeys) {
+    const box = await refLocator(page, ref)
+      .boundingBox()
+      .catch(() => null);
+    if (!box) {
+      bboxFailures += 1;
       continue;
     }
-    try {
-      const box = await refLocator(page, ref).boundingBox();
-      if (!box) {
-        skipped += 1;
-        continue;
-      }
-      const x0 = box.x;
-      const y0 = box.y;
-      const x1 = box.x + box.width;
-      const y1 = box.y + box.height;
-      const vx0 = viewport.scrollX;
-      const vy0 = viewport.scrollY;
-      const vx1 = viewport.scrollX + viewport.width;
-      const vy1 = viewport.scrollY + viewport.height;
-      if (x1 < vx0 || x0 > vx1 || y1 < vy0 || y0 > vy1) {
-        skipped += 1;
-        continue;
-      }
-      boxes.push({
-        ref,
-        x: x0 - viewport.scrollX,
-        y: y0 - viewport.scrollY,
-        w: Math.max(1, box.width),
-        h: Math.max(1, box.height),
-      });
-    } catch {
-      skipped += 1;
-    }
+    inputs.push({
+      ref,
+      role: opts.refs[ref].role,
+      name: opts.refs[ref].name,
+      doc: {
+        x: box.x + scroll.x,
+        y: box.y + scroll.y,
+        width: box.width,
+        height: box.height,
+      },
+    });
   }
 
+  const plan = planAnnotations({
+    inputs,
+    space,
+    scroll,
+    viewport: { width: view.width, height: view.height },
+    elementRect,
+    maxLabels,
+  });
+
   try {
-    if (boxes.length > 0) {
-      await page.evaluate((labels) => {
-        const existing = document.querySelectorAll("[data-openclaw-labels]");
-        existing.forEach((el) => el.remove());
-
-        const root = document.createElement("div");
-        root.setAttribute("data-openclaw-labels", "1");
-        root.style.position = "fixed";
-        root.style.left = "0";
-        root.style.top = "0";
-        root.style.zIndex = "2147483647";
-        root.style.pointerEvents = "none";
-        root.style.fontFamily =
-          '"SF Mono","SFMono-Regular",Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace';
-
-        const clamp = (value: number, min: number, max: number) =>
-          Math.min(max, Math.max(min, value));
-
-        for (const label of labels) {
-          const box = document.createElement("div");
-          box.setAttribute("data-openclaw-labels", "1");
-          box.style.position = "absolute";
-          box.style.left = `${label.x}px`;
-          box.style.top = `${label.y}px`;
-          box.style.width = `${label.w}px`;
-          box.style.height = `${label.h}px`;
-          box.style.border = "2px solid #ffb020";
-          box.style.boxSizing = "border-box";
-
-          const tag = document.createElement("div");
-          tag.setAttribute("data-openclaw-labels", "1");
-          tag.textContent = label.ref;
-          tag.style.position = "absolute";
-          tag.style.left = `${label.x}px`;
-          tag.style.top = `${clamp(label.y - 18, 0, 20000)}px`;
-          tag.style.background = "#ffb020";
-          tag.style.color = "#1a1a1a";
-          tag.style.fontSize = "12px";
-          tag.style.lineHeight = "14px";
-          tag.style.padding = "1px 4px";
-          tag.style.borderRadius = "3px";
-          tag.style.boxShadow = "0 1px 2px rgba(0,0,0,0.35)";
-          tag.style.whiteSpace = "nowrap";
-
-          root.appendChild(box);
-          root.appendChild(tag);
-        }
-
-        document.documentElement.appendChild(root);
-      }, boxes);
+    if (plan.overlayItems.length > 0) {
+      const captureY = space === "element" ? elementRect?.y : space === "viewport" ? scroll.y : 0;
+      await page.evaluate(buildOverlayInjectionScript({ items: plan.overlayItems, captureY }));
     }
-
-    const buffer = await page.screenshot({ type, timeout: opts.timeoutMs });
-    return { buffer, labels: boxes.length, skipped };
+    const buffer =
+      space === "element"
+        ? await captureElementScreenshotForLabels(
+            page,
+            refKey,
+            elementSelector,
+            type,
+            opts.timeoutMs,
+          )
+        : await page.screenshot({
+            type,
+            fullPage: Boolean(opts.fullPage),
+            timeout: opts.timeoutMs,
+          });
+    return {
+      // `labels` reports overlay boxes actually drawn on the captured image
+      // (in-viewport, within budget); off-viewport refs are surfaced via
+      // `annotations` but not drawn, and are reflected in `skipped`.
+      buffer,
+      labels: plan.overlayItems.length,
+      skipped: plan.skipped + bboxFailures,
+      annotations: plan.annotations,
+    };
   } finally {
-    await page
-      .evaluate(() => {
-        const existing = document.querySelectorAll("[data-openclaw-labels]");
-        existing.forEach((el) => el.remove());
-      })
-      .catch(() => {});
+    await page.evaluate(buildOverlayClearScript()).catch(() => {});
   }
 }
 
+async function resolveElementBoundingBoxForLabels(
+  page: Page,
+  refKey: string | undefined,
+  cssSelector: string | undefined,
+): Promise<{ x: number; y: number; width: number; height: number } | null> {
+  if (refKey) {
+    try {
+      return await refLocator(page, refKey).boundingBox();
+    } catch {
+      return null;
+    }
+  }
+  if (cssSelector) {
+    try {
+      return await page.locator(cssSelector).first().boundingBox();
+    } catch {
+      return null;
+    }
+  }
+  return null;
+}
+
+async function captureElementScreenshotForLabels(
+  page: Page,
+  refKey: string | undefined,
+  cssSelector: string | undefined,
+  type: "png" | "jpeg",
+  timeoutMs: number | undefined,
+): Promise<Buffer> {
+  if (refKey) {
+    return await refLocator(page, refKey).screenshot({ type, timeout: timeoutMs });
+  }
+  if (cssSelector) {
+    return await page.locator(cssSelector).first().screenshot({ type, timeout: timeoutMs });
+  }
+  throw new Error("captureElementScreenshotForLabels: requires refKey or cssSelector");
+}
+
 /** Sets file inputs for a role ref or selector with strict existing-path checks. */
 export async function setInputFilesViaPlaywright(opts: {
   cdpUrl: string;
diff --git a/extensions/browser/src/browser/routes/agent.snapshot.ts b/extensions/browser/src/browser/routes/agent.snapshot.ts
index 267cd31a890..17a6ead989d 100644
--- a/extensions/browser/src/browser/routes/agent.snapshot.ts
+++ b/extensions/browser/src/browser/routes/agent.snapshot.ts
@@ -5,6 +5,7 @@
  * navigation policy checks, media storage, and screenshot normalization.
  */
 import path from "node:path";
+import { getImageMetadata } from "../../media/media-services.js";
 import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
 import { captureScreenshot, snapshotAria, snapshotRoleViaCdp } from "../cdp.js";
 import {
@@ -24,6 +25,8 @@ import {
   assertBrowserNavigationResultAllowed,
 } from "../navigation-guard.js";
 import { getBrowserProfileCapabilities } from "../profile-capabilities.js";
+import type { AnnotationItem } from "../screenshot-annotate.js";
+import { scaleAnnotations } from "../screenshot-annotate.js";
 import {
   DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
   DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
@@ -192,11 +195,24 @@ async function saveNormalizedScreenshotResponse(params: {
   labels?: boolean;
   labelsCount?: number;
   labelsSkipped?: number;
+  annotations?: AnnotationItem[];
 }) {
+  // Measure original dimensions BEFORE normalization so we can rescale
+  // annotation coordinates if the response pipeline shrinks the image
+  // (longest-side or byte-budget cap). Annotation boxes are in the captured
+  // image's pixel space, so they would otherwise drift from the saved media.
+  const originalMeta = params.annotations?.length
+    ? ((await getImageMetadata(params.buffer)) ?? undefined)
+    : undefined;
   const normalized = await normalizeBrowserScreenshot(params.buffer, {
     maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
     maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
   });
+  const annotations = await rescaleAnnotationsForNormalization({
+    annotations: params.annotations,
+    originalMeta,
+    normalizedBuffer: normalized.buffer,
+  });
   await saveBrowserMediaResponse({
     res: params.res,
     buffer: normalized.buffer,
@@ -207,9 +223,39 @@ async function saveNormalizedScreenshotResponse(params: {
     labels: params.labels,
     labelsCount: params.labelsCount,
     labelsSkipped: params.labelsSkipped,
+    annotations,
   });
 }
 
+/**
+ * Keep annotation coordinates aligned with the saved media after
+ * normalizeBrowserScreenshot. Returns the original annotations unchanged
+ * when normalization did not change the image dimensions, or when image
+ * metadata is unavailable (best-effort: better to ship pre-resize coords
+ * than to drop the field entirely).
+ */
+async function rescaleAnnotationsForNormalization(params: {
+  annotations?: AnnotationItem[];
+  originalMeta?: { width?: number; height?: number };
+  normalizedBuffer: Buffer;
+}): Promise<AnnotationItem[] | undefined> {
+  if (!params.annotations || params.annotations.length === 0) {
+    return params.annotations;
+  }
+  const orig = params.originalMeta;
+  if (!orig?.width || !orig?.height) {
+    return params.annotations;
+  }
+  const next = await getImageMetadata(params.normalizedBuffer);
+  if (!next?.width || !next?.height) {
+    return params.annotations;
+  }
+  if (next.width === orig.width && next.height === orig.height) {
+    return params.annotations;
+  }
+  return scaleAnnotations(params.annotations, next.width / orig.width, next.height / orig.height);
+}
+
 async function saveBrowserMediaResponse(params: {
   res: BrowserResponse;
   buffer: Buffer;
@@ -220,6 +266,7 @@ async function saveBrowserMediaResponse(params: {
   labels?: boolean;
   labelsCount?: number;
   labelsSkipped?: number;
+  annotations?: AnnotationItem[];
 }) {
   await ensureMediaDir();
   const saved = await saveMediaBuffer(
@@ -236,6 +283,9 @@ async function saveBrowserMediaResponse(params: {
     ...(params.labels ? { labels: true } : {}),
     ...(typeof params.labelsCount === "number" ? { labelsCount: params.labelsCount } : {}),
     ...(typeof params.labelsSkipped === "number" ? { labelsSkipped: params.labelsSkipped } : {}),
+    ...(params.annotations && params.annotations.length > 0
+      ? { annotations: params.annotations }
+      : {}),
   });
 }
 
@@ -478,6 +528,9 @@ export function registerBrowserAgentSnapshotRoutes(
                 refs: snap.refs,
                 type,
                 timeoutMs,
+                fullPage,
+                ref,
+                element,
               });
               await saveNormalizedScreenshotResponse({
                 res,
@@ -488,6 +541,7 @@ export function registerBrowserAgentSnapshotRoutes(
                 labels: true,
                 labelsCount: labeled.labels,
                 labelsSkipped: labeled.skipped,
+                annotations: labeled.annotations,
               });
               return;
             }
@@ -743,10 +797,18 @@ export function registerBrowserAgentSnapshotRoutes(
               type: "png",
               timeoutMs: plan.timeoutMs,
             });
+            const originalMeta = labeled.annotations.length
+              ? ((await getImageMetadata(labeled.buffer)) ?? undefined)
+              : undefined;
             const normalized = await normalizeBrowserScreenshot(labeled.buffer, {
               maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
               maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
             });
+            const scaledAnnotations = await rescaleAnnotationsForNormalization({
+              annotations: labeled.annotations,
+              originalMeta,
+              normalizedBuffer: normalized.buffer,
+            });
             await ensureMediaDir();
             const saved = await saveMediaBuffer(
               normalized.buffer,
@@ -764,6 +826,9 @@ export function registerBrowserAgentSnapshotRoutes(
               labels: true,
               labelsCount: labeled.labels,
               labelsSkipped: labeled.skipped,
+              ...(scaledAnnotations && scaledAnnotations.length > 0
+                ? { annotations: scaledAnnotations }
+                : {}),
               imagePath: path.resolve(saved.path),
               imageType,
               ...snap,
diff --git a/extensions/browser/src/browser/screenshot-annotate.test.ts b/extensions/browser/src/browser/screenshot-annotate.test.ts
new file mode 100644
index 00000000000..bce33211cde
--- /dev/null
+++ b/extensions/browser/src/browser/screenshot-annotate.test.ts
@@ -0,0 +1,345 @@
+import { describe, expect, it } from "vitest";
+import {
+  ANNOTATION_OVERLAY_ATTR,
+  type AnnotationItem,
+  buildOverlayClearScript,
+  buildOverlayInjectionScript,
+  planAnnotations,
+  type RawAnnotationInput,
+  refToNumber,
+  scaleAnnotations,
+} from "./screenshot-annotate.js";
+
+const sampleInputs: RawAnnotationInput[] = [
+  {
+    ref: "e1",
+    role: "button",
+    name: "Submit",
+    doc: { x: 100, y: 200, width: 50, height: 20 },
+  },
+  {
+    ref: "e2",
+    role: "link",
+    doc: { x: 300, y: 1500, width: 80, height: 18 },
+  },
+];
+
+describe("refToNumber", () => {
+  it("extracts number from `e<N>` form", () => {
+    expect(refToNumber("e12")).toBe(12);
+    expect(refToNumber("e0")).toBe(0);
+  });
+
+  it("extracts number from `ax<N>` form", () => {
+    expect(refToNumber("ax12")).toBe(12);
+  });
+
+  it("extracts number from bare numeric form", () => {
+    expect(refToNumber("12")).toBe(12);
+  });
+
+  it("returns 0 for non-numeric refs", () => {
+    expect(refToNumber("foo")).toBe(0);
+    expect(refToNumber("")).toBe(0);
+  });
+});
+
+describe("planAnnotations - viewport mode", () => {
+  it("subtracts scroll from doc coords", () => {
+    const plan = planAnnotations({
+      inputs: sampleInputs,
+      space: "viewport",
+      scroll: { x: 0, y: 1000 },
+    });
+
+    expect(plan.annotations).toHaveLength(2);
+    expect(plan.annotations[0]).toEqual({
+      ref: "e1",
+      number: 1,
+      role: "button",
+      name: "Submit",
+      box: { x: 100, y: -800, width: 50, height: 20 },
+    });
+    expect(plan.annotations[1]).toEqual({
+      ref: "e2",
+      number: 2,
+      role: "link",
+      box: { x: 300, y: 500, width: 80, height: 18 },
+    });
+    expect(plan.skipped).toBe(0);
+  });
+
+  it("keeps overlay items in document space regardless of mode", () => {
+    const plan = planAnnotations({
+      inputs: sampleInputs,
+      space: "viewport",
+      scroll: { x: 0, y: 1000 },
+    });
+    expect(plan.overlayItems).toEqual([
+      { ref: "e1", x: 100, y: 200, w: 50, h: 20 },
+      { ref: "e2", x: 300, y: 1500, w: 80, h: 18 },
+    ]);
+  });
+
+  it("omits empty name field", () => {
+    const plan = planAnnotations({
+      inputs: [{ ref: "e1", role: "button", name: "", doc: { x: 0, y: 0, width: 1, height: 1 } }],
+      space: "viewport",
+      scroll: { x: 0, y: 0 },
+    });
+    expect(plan.annotations[0]).not.toHaveProperty("name");
+  });
+
+  it("throws when scroll missing in viewport mode", () => {
+    expect(() => planAnnotations({ inputs: sampleInputs, space: "viewport" })).toThrow(/scroll/);
+  });
+});
+
+describe("planAnnotations - viewport off-screen accounting", () => {
+  it("counts off-viewport refs as skipped but keeps them in annotations when viewport size is given", () => {
+    const plan = planAnnotations({
+      inputs: [
+        { ref: "e1", role: "button", doc: { x: 10, y: 50, width: 40, height: 20 } }, // in viewport
+        { ref: "e2", role: "link", doc: { x: 10, y: 5000, width: 40, height: 20 } }, // below viewport
+      ],
+      space: "viewport",
+      scroll: { x: 0, y: 0 },
+      viewport: { width: 1280, height: 720 },
+    });
+
+    // Only the in-viewport ref is drawn.
+    expect(plan.overlayItems.map((o) => o.ref)).toEqual(["e1"]);
+    // Both refs are surfaced for callers (off-viewport box can be out of image).
+    expect(plan.annotations.map((a) => a.ref)).toEqual(["e1", "e2"]);
+    // The off-viewport ref raises skipped, preserving the shipped contract.
+    expect(plan.skipped).toBe(1);
+  });
+
+  it("does not count off-viewport refs when viewport size is omitted", () => {
+    const plan = planAnnotations({
+      inputs: [{ ref: "e2", role: "link", doc: { x: 10, y: 5000, width: 40, height: 20 } }],
+      space: "viewport",
+      scroll: { x: 0, y: 0 },
+    });
+
+    expect(plan.skipped).toBe(0);
+    expect(plan.overlayItems).toHaveLength(1);
+    expect(plan.annotations).toHaveLength(1);
+  });
+});
+
+describe("planAnnotations - fullpage mode", () => {
+  it("returns box equal to doc (document coordinates)", () => {
+    const plan = planAnnotations({ inputs: sampleInputs, space: "fullpage" });
+    expect(plan.annotations[0].box).toEqual({ x: 100, y: 200, width: 50, height: 20 });
+    expect(plan.annotations[1].box).toEqual({ x: 300, y: 1500, width: 80, height: 18 });
+  });
+
+  it("does not require scroll", () => {
+    expect(() => planAnnotations({ inputs: sampleInputs, space: "fullpage" })).not.toThrow();
+  });
+});
+
+describe("planAnnotations - element mode", () => {
+  const elementRect = { x: 50, y: 100, width: 200, height: 300 };
+
+  it("projects box relative to element top-left", () => {
+    const plan = planAnnotations({
+      inputs: [{ ref: "e1", role: "button", doc: { x: 60, y: 110, width: 40, height: 20 } }],
+      space: "element",
+      elementRect,
+    });
+    expect(plan.annotations[0].box).toEqual({ x: 10, y: 10, width: 40, height: 20 });
+  });
+
+  it("filters out inputs that do not overlap element rect", () => {
+    const plan = planAnnotations({
+      inputs: [
+        { ref: "e1", role: "button", doc: { x: 60, y: 110, width: 40, height: 20 } }, // inside
+        { ref: "e2", role: "link", doc: { x: 500, y: 500, width: 40, height: 20 } }, // outside
+      ],
+      space: "element",
+      elementRect,
+    });
+    expect(plan.annotations).toHaveLength(1);
+    expect(plan.annotations[0].ref).toBe("e1");
+    expect(plan.overlayItems).toHaveLength(1);
+  });
+
+  it("throws when elementRect missing", () => {
+    expect(() => planAnnotations({ inputs: [], space: "element" })).toThrow(/elementRect/);
+  });
+});
+
+describe("planAnnotations - maxLabels", () => {
+  it("truncates to maxLabels and reports skipped", () => {
+    const inputs = Array.from({ length: 5 }, (_, i) => ({
+      ref: `e${i + 1}`,
+      role: "button",
+      doc: { x: 0, y: i * 10, width: 5, height: 5 },
+    }));
+    const plan = planAnnotations({ inputs, space: "fullpage", maxLabels: 2 });
+    expect(plan.annotations).toHaveLength(2);
+    expect(plan.overlayItems).toHaveLength(2);
+    expect(plan.skipped).toBe(3);
+  });
+
+  it("uses ANNOTATION_MAX_LABELS_DEFAULT when not specified", () => {
+    const inputs = Array.from({ length: 200 }, (_, i) => ({
+      ref: `e${i + 1}`,
+      role: "button",
+      doc: { x: 0, y: i, width: 5, height: 5 },
+    }));
+    const plan = planAnnotations({ inputs, space: "fullpage" });
+    expect(plan.annotations).toHaveLength(150);
+    expect(plan.skipped).toBe(50);
+  });
+});
+
+describe("buildOverlayInjectionScript", () => {
+  it("returns a self-contained IIFE", () => {
+    const script = buildOverlayInjectionScript({
+      items: [{ ref: "e1", x: 100, y: 200, w: 50, h: 20 }],
+    });
+    expect(script).toMatch(/^\(\s*\(\s*\)\s*=>\s*\{/);
+    expect(script).toMatch(/\}\s*\)\s*\(\s*\)\s*;?\s*$/);
+  });
+
+  it("embeds the overlay attr", () => {
+    const script = buildOverlayInjectionScript({ items: [] });
+    expect(script).toContain(ANNOTATION_OVERLAY_ATTR);
+  });
+
+  it("embeds each item's ref text and coordinates", () => {
+    const script = buildOverlayInjectionScript({
+      items: [
+        { ref: "e1", x: 100, y: 200, w: 50, h: 20 },
+        { ref: "ax42", x: 999, y: 1500, w: 80, h: 18 },
+      ],
+    });
+    expect(script).toMatch(/"ref":\s*"e1"/);
+    expect(script).toMatch(/"ref":\s*"ax42"/);
+    expect(script).toMatch(/"x":\s*100/);
+    expect(script).toMatch(/"x":\s*999/);
+  });
+
+  it("handles empty items without throwing", () => {
+    expect(() => buildOverlayInjectionScript({ items: [] })).not.toThrow();
+  });
+
+  it("rounds coordinates to integers", () => {
+    const script = buildOverlayInjectionScript({
+      items: [{ ref: "e1", x: 100.7, y: 200.4, w: 50.6, h: 20.1 }],
+    });
+    expect(script).toMatch(/"x":\s*101/); // 100.7 -> 101
+    expect(script).toMatch(/"y":\s*200/); // 200.4 -> 200
+  });
+
+  it("clamps zero/negative-size boxes to 1px so they remain visible", () => {
+    const script = buildOverlayInjectionScript({
+      items: [{ ref: "e1", x: 10, y: 10, w: 0, h: 0 }],
+    });
+    expect(script).toMatch(/"w":\s*1/);
+    expect(script).toMatch(/"h":\s*1/);
+  });
+
+  it("escapes hostile ref characters via JSON.stringify (no breakout)", () => {
+    const hostile = 'e1");alert(1);//';
+    const script = buildOverlayInjectionScript({
+      items: [{ ref: hostile, x: 0, y: 0, w: 1, h: 1 }],
+    });
+    // The hostile `"` MUST be escaped as `\"` inside the JSON literal.
+    expect(script).toContain('"e1\\");alert(1);//"');
+    // The unescaped breakout MUST NOT appear anywhere in the script as a
+    // bare statement that would terminate the JSON literal early.
+    expect(script).not.toContain('e1");alert(1);');
+  });
+
+  it("flips label below the box when y < 14 (no headroom)", () => {
+    const script = buildOverlayInjectionScript({
+      items: [{ ref: "e1", x: 0, y: 5, w: 10, h: 10 }],
+    });
+    // labelTop = relativeY < 14 ? it.y + 2 : it.y - 14
+    // The expression literal `relativeY < 14 ? (it.y + 2) : (it.y - 14)` is in the script.
+    expect(script).toContain("relativeY < 14 ? (it.y + 2) : (it.y - 14)");
+  });
+
+  it("uses capture-relative y when deciding whether to flip labels below boxes", () => {
+    const script = buildOverlayInjectionScript({
+      items: [{ ref: "e1", x: 0, y: 1005, w: 10, h: 10 }],
+      captureY: 1000,
+    });
+
+    expect(script).toContain("var captureY = 1000;");
+    expect(script).toContain("var relativeY = it.y - captureY;");
+    expect(script).toContain("relativeY < 14 ? (it.y + 2) : (it.y - 14)");
+  });
+});
+
+describe("buildOverlayClearScript", () => {
+  it("returns an IIFE selecting overlay attr", () => {
+    const script = buildOverlayClearScript();
+    expect(script).toContain(`[${ANNOTATION_OVERLAY_ATTR}]`);
+    expect(script).toMatch(/^\(\s*\(\s*\)\s*=>\s*\{/);
+  });
+});
+
+describe("scaleAnnotations", () => {
+  const sample: AnnotationItem[] = [
+    {
+      ref: "e1",
+      number: 1,
+      role: "button",
+      name: "Submit",
+      box: { x: 100, y: 200, width: 50, height: 20 },
+    },
+  ];
+
+  it("returns identity (structural copy) when both factors are 1", () => {
+    const out = scaleAnnotations(sample, 1, 1);
+    expect(out[0]).toEqual(sample[0]);
+    expect(out[0]).not.toBe(sample[0]);
+    expect(out[0]?.box).not.toBe(sample[0]?.box);
+  });
+
+  it("scales box dimensions by independent x/y factors", () => {
+    const out = scaleAnnotations(sample, 0.5, 0.485);
+    expect(out[0]?.box).toEqual({
+      x: 50,
+      y: 97,
+      width: 25,
+      height: 10,
+    });
+  });
+
+  it("clamps width/height to a minimum of 1 to avoid disappearing labels", () => {
+    const tiny: AnnotationItem[] = [
+      {
+        ref: "e1",
+        number: 1,
+        role: "button",
+        box: { x: 0, y: 0, width: 1, height: 1 },
+      },
+    ];
+    const out = scaleAnnotations(tiny, 0.1, 0.1);
+    expect(out[0]?.box.width).toBeGreaterThanOrEqual(1);
+    expect(out[0]?.box.height).toBeGreaterThanOrEqual(1);
+  });
+
+  it("returns identity (structural copy) for invalid factors", () => {
+    const out = scaleAnnotations(sample, Number.NaN, 0.5);
+    expect(out[0]?.box).toEqual(sample[0]?.box);
+    const out2 = scaleAnnotations(sample, 0, 0.5);
+    expect(out2[0]?.box).toEqual(sample[0]?.box);
+    const out3 = scaleAnnotations(sample, -1, 1);
+    expect(out3[0]?.box).toEqual(sample[0]?.box);
+  });
+
+  it("preserves ref/number/role/name fields verbatim", () => {
+    const out = scaleAnnotations(sample, 0.5, 0.5);
+    expect(out[0]?.ref).toBe("e1");
+    expect(out[0]?.number).toBe(1);
+    expect(out[0]?.role).toBe("button");
+    expect(out[0]?.name).toBe("Submit");
+  });
+});
diff --git a/extensions/browser/src/browser/screenshot-annotate.ts b/extensions/browser/src/browser/screenshot-annotate.ts
new file mode 100644
index 00000000000..c030effc21f
--- /dev/null
+++ b/extensions/browser/src/browser/screenshot-annotate.ts
@@ -0,0 +1,282 @@
+// extensions/browser/src/browser/screenshot-annotate.ts
+//
+// Pure helper module for screenshot label annotations.
+// Has no Playwright / CDP / page dependency: takes document-space inputs,
+// returns coordinate-projected annotations + IIFE strings the caller can
+// hand to page.evaluate / Runtime.evaluate.
+//
+// Used by:
+//   - pw-tools-core.interactions.ts (Playwright path, M1.2-a)
+//   - planned: raw-CDP path in M1.2-b
+//
+// chrome-mcp path keeps its own inline overlay (renderChromeMcpLabels) for now.
+
+export const ANNOTATION_OVERLAY_ATTR = "data-openclaw-labels";
+export const ANNOTATION_OVERLAY_ROOT_ID = "__openclaw-annotations__";
+export const ANNOTATION_MAX_LABELS_DEFAULT = 150;
+
+export type CoordinateSpace = "viewport" | "fullpage" | "element";
+
+export interface RawAnnotationInput {
+  ref: string;
+  role: string;
+  name?: string;
+  /** Bounding box in document coordinates (viewport top-left + scroll). */
+  doc: { x: number; y: number; width: number; height: number };
+}
+
+export interface AnnotationBox {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+export interface AnnotationItem {
+  ref: string;
+  number: number;
+  role: string;
+  name?: string;
+  box: AnnotationBox;
+}
+
+export interface OverlayItem {
+  ref: string;
+  x: number;
+  y: number;
+  w: number;
+  h: number;
+}
+
+export interface AnnotationPlan {
+  /** Always document-space items, fed to buildOverlayInjectionScript. */
+  overlayItems: OverlayItem[];
+  /** Items projected into the capture mode's image-space coordinates. */
+  annotations: AnnotationItem[];
+  /** Refs dropped because of maxLabels truncation. */
+  skipped: number;
+}
+
+export interface PlanAnnotationsParams {
+  inputs: RawAnnotationInput[];
+  space: CoordinateSpace;
+  /** Required when space === "viewport". */
+  scroll?: { x: number; y: number };
+  /**
+   * Viewport size (CSS px). Only meaningful when space === "viewport". When
+   * provided, refs whose document box falls outside the current viewport rect
+   * (`scroll` + this size) are counted as skipped instead of drawn, preserving
+   * the shipped `labelsSkipped` contract. Omit it to disable that accounting.
+   */
+  viewport?: { width: number; height: number };
+  /** Required when space === "element". */
+  elementRect?: { x: number; y: number; width: number; height: number };
+  maxLabels?: number;
+}
+
+export function refToNumber(ref: string): number {
+  const match = ref.match(/(\d+)/);
+  if (!match) {
+    return 0;
+  }
+  const n = Number(match[1]);
+  return Number.isFinite(n) ? n : 0;
+}
+
+export function planAnnotations(params: PlanAnnotationsParams): AnnotationPlan {
+  const maxLabels = params.maxLabels ?? ANNOTATION_MAX_LABELS_DEFAULT;
+
+  if (params.space === "viewport" && !params.scroll) {
+    throw new Error("planAnnotations: scroll is required when space is 'viewport'");
+  }
+  if (params.space === "element" && !params.elementRect) {
+    throw new Error("planAnnotations: elementRect is required when space is 'element'");
+  }
+
+  // Element-mode filter: discard inputs that do not overlap the element rect.
+  let kept = params.inputs;
+  if (params.space === "element" && params.elementRect) {
+    const er = params.elementRect;
+    kept = params.inputs.filter((input) => rectsOverlap(input.doc, er));
+  }
+
+  // Viewport capture only shows refs inside the current viewport rect. An
+  // off-viewport ref is still surfaced in `annotations` (with its real,
+  // possibly out-of-image box) so callers can locate it, but it is not drawn
+  // and is counted as skipped. This keeps the shipped `labelsSkipped` meaning
+  // ("refs not present in the captured viewport image") instead of silently
+  // narrowing it. Only applied when the caller supplies the viewport size;
+  // without it we cannot decide off-screen state and skip nothing.
+  const viewportRect =
+    params.space === "viewport" && params.scroll && params.viewport
+      ? {
+          x: params.scroll.x,
+          y: params.scroll.y,
+          width: params.viewport.width,
+          height: params.viewport.height,
+        }
+      : undefined;
+
+  const overlayItems: OverlayItem[] = [];
+  const annotations: AnnotationItem[] = [];
+  let skipped = 0;
+
+  for (const input of kept) {
+    if (viewportRect && !rectsOverlap(input.doc, viewportRect)) {
+      // Outside the captured viewport: count as skipped (compat) but still
+      // report the annotation; do not draw it or consume the label budget.
+      skipped += 1;
+      annotations.push(toAnnotation(input, params));
+      continue;
+    }
+    if (overlayItems.length >= maxLabels) {
+      skipped += 1;
+      continue;
+    }
+    overlayItems.push({
+      ref: input.ref,
+      x: input.doc.x,
+      y: input.doc.y,
+      w: input.doc.width,
+      h: input.doc.height,
+    });
+    annotations.push(toAnnotation(input, params));
+  }
+
+  return { overlayItems, annotations, skipped };
+}
+
+function toAnnotation(input: RawAnnotationInput, params: PlanAnnotationsParams): AnnotationItem {
+  return {
+    ref: input.ref,
+    number: refToNumber(input.ref),
+    role: input.role,
+    ...(input.name ? { name: input.name } : {}),
+    box: projectBox(input.doc, params),
+  };
+}
+
+function projectBox(
+  doc: { x: number; y: number; width: number; height: number },
+  params: PlanAnnotationsParams,
+): AnnotationBox {
+  if (params.space === "viewport") {
+    const scroll = params.scroll!;
+    return {
+      x: doc.x - scroll.x,
+      y: doc.y - scroll.y,
+      width: doc.width,
+      height: doc.height,
+    };
+  }
+  if (params.space === "element") {
+    const er = params.elementRect!;
+    // NOTE: width/height pass through unchanged even when the input rect
+    // partially extends past the element. The capture backend (e.g.
+    // locator.screenshot) is responsible for clipping; the box may have
+    // negative x/y or extend past elementRect width/height for partial overlaps.
+    return {
+      x: doc.x - er.x,
+      y: doc.y - er.y,
+      width: doc.width,
+      height: doc.height,
+    };
+  }
+  // fullpage: document coordinates as-is
+  return { x: doc.x, y: doc.y, width: doc.width, height: doc.height };
+}
+
+function rectsOverlap(
+  a: { x: number; y: number; width: number; height: number },
+  b: { x: number; y: number; width: number; height: number },
+): boolean {
+  return a.x < b.x + b.width && a.x + a.width > b.x && a.y < b.y + b.height && a.y + a.height > b.y;
+}
+
+export function buildOverlayInjectionScript(params: {
+  items: OverlayItem[];
+  captureY?: number;
+}): string {
+  const itemsJson = JSON.stringify(
+    params.items.map((it) => ({
+      ref: it.ref,
+      x: round(it.x),
+      y: round(it.y),
+      w: Math.max(1, round(it.w)),
+      h: Math.max(1, round(it.h)),
+    })),
+  );
+  const attr = ANNOTATION_OVERLAY_ATTR;
+  const rootId = ANNOTATION_OVERLAY_ROOT_ID;
+  const captureY = Number.isFinite(params.captureY) ? round(params.captureY ?? 0) : 0;
+  return `(() => {
+  var items = ${itemsJson};
+  var captureY = ${captureY};
+  var existing = document.querySelectorAll("[${attr}]");
+  for (var k = 0; k < existing.length; k++) existing[k].remove();
+  var root = document.createElement("div");
+  root.id = ${JSON.stringify(rootId)};
+  root.setAttribute("${attr}", "1");
+  root.style.cssText = "position:absolute;top:0;left:0;width:0;height:0;pointer-events:none;z-index:2147483647;font-family:'SF Mono','SFMono-Regular',Menlo,Monaco,Consolas,'Liberation Mono','Courier New',monospace;";
+  for (var i = 0; i < items.length; i++) {
+    var it = items[i];
+    var box = document.createElement("div");
+    box.setAttribute("${attr}", "1");
+    box.style.cssText = "position:absolute;left:" + it.x + "px;top:" + it.y + "px;width:" + it.w + "px;height:" + it.h + "px;border:2px solid #ffb020;box-sizing:border-box;pointer-events:none;";
+    var tag = document.createElement("div");
+    tag.setAttribute("${attr}", "1");
+    tag.textContent = String(it.ref);
+    var relativeY = it.y - captureY;
+    var labelTop = relativeY < 14 ? (it.y + 2) : (it.y - 14);
+    tag.style.cssText = "position:absolute;left:" + it.x + "px;top:" + labelTop + "px;background:#ffb020;color:#1a1a1a;font:bold 11px/14px monospace;padding:0 4px;border-radius:2px;white-space:nowrap;pointer-events:none;";
+    root.appendChild(box);
+    root.appendChild(tag);
+  }
+  document.documentElement.appendChild(root);
+  return true;
+})();`;
+}
+
+export function buildOverlayClearScript(): string {
+  const attr = ANNOTATION_OVERLAY_ATTR;
+  return `(() => {
+  var existing = document.querySelectorAll("[${attr}]");
+  for (var k = 0; k < existing.length; k++) existing[k].remove();
+  return true;
+})();`;
+}
+
+/**
+ * Scale annotation boxes by independent x/y factors. Used to keep annotation
+ * coordinates aligned with the saved image after the response pipeline
+ * resizes the screenshot (e.g. via normalizeBrowserScreenshot capping the
+ * longest side or the byte budget). Returns a new array; inputs are not
+ * mutated. When both factors are 1 the boxes are returned unchanged (modulo
+ * structural copy) so callers can share the same code path for resized and
+ * non-resized captures.
+ */
+export function scaleAnnotations(
+  items: AnnotationItem[],
+  scaleX: number,
+  scaleY: number,
+): AnnotationItem[] {
+  if (!Number.isFinite(scaleX) || !Number.isFinite(scaleY) || scaleX <= 0 || scaleY <= 0) {
+    return items.map((it) => ({ ...it, box: { ...it.box } }));
+  }
+  if (scaleX === 1 && scaleY === 1) {
+    return items.map((it) => ({ ...it, box: { ...it.box } }));
+  }
+  return items.map((it) => ({
+    ...it,
+    box: {
+      x: round(it.box.x * scaleX),
+      y: round(it.box.y * scaleY),
+      width: Math.max(1, round(it.box.width * scaleX)),
+      height: Math.max(1, round(it.box.height * scaleY)),
+    },
+  }));
+}
+
+function round(v: number): number {
+  return Math.round(v);
+}
diff --git a/extensions/browser/src/cli/browser-cli-inspect.ts b/extensions/browser/src/cli/browser-cli-inspect.ts
index ef460414e33..020154957ee 100644
--- a/extensions/browser/src/cli/browser-cli-inspect.ts
+++ b/extensions/browser/src/cli/browser-cli-inspect.ts
@@ -51,7 +51,11 @@ export function registerBrowserInspectCommands(
     .option("--full-page", "Capture full scrollable page", false)
     .option("--ref <ref>", "ARIA ref from ai snapshot")
     .option("--element <selector>", "CSS selector for element screenshot")
-    .option("--labels", "Overlay role refs on the screenshot", false)
+    .option(
+      "--labels",
+      "Overlay role refs on the screenshot (works with --full-page, --ref, and --element)",
+      false,
+    )
     .option("--type <png|jpeg>", "Output type (default: png)", "png")
     .action(async (targetId: string | undefined, opts, cmd) => {
       const parent = parentOpts(cmd);
@@ -98,7 +102,7 @@ export function registerBrowserInspectCommands(
     .option("--depth <n>", "Role snapshot: max depth")
     .option("--selector <sel>", "Role snapshot: scope to CSS selector")
     .option("--frame <sel>", "Role snapshot: scope to an iframe selector")
-    .option("--labels", "Include viewport label overlay screenshot", false)
+    .option("--labels", "Include label overlay screenshot with annotations", false)
     .option("--urls", "Append discovered link URLs to AI snapshots", false)
     .option("--out <path>", "Write snapshot to a file")
     .action(async (opts, cmd) => {