mirror of
https://github.com/agent0ai/agent-zero.git
synced 2026-05-16 19:50:43 +00:00
Add browser annotate mode
Add Codex-inspired annotation UI to the built-in Browser surfaces, including the Annotate toggle, Cmd/Ctrl+. shortcut, selection overlay, inline comments, and batch Draft to chat / Send now actions. Wire browser_viewer_annotation through the WebSocket and runtime layers, and expose safe DOM metadata extraction for clicked elements and selected areas without leaking password/value data. Expand regression coverage for the Browser UI, annotation dispatch, runtime helper exposure, prompt formatting, and WebUI extension surface harness behavior.
This commit is contained in:
Alessandro
parent
58a5f8276b
commit
4ff3244ce6
7 changed files with 1382 additions and 35 deletions
|
|
@ -43,6 +43,8 @@ class WsBrowser(WsHandler):
|
|||
return await self._command(data, sid)
|
||||
if event == "browser_viewer_input":
|
||||
return await self._input(data, sid)
|
||||
if event == "browser_viewer_annotation":
|
||||
return await self._annotation(data, sid)
|
||||
|
||||
return WsResult.error(
|
||||
code="UNKNOWN_BROWSER_EVENT",
|
||||
|
|
@ -215,6 +217,29 @@ class WsBrowser(WsHandler):
|
|||
else None,
|
||||
}
|
||||
|
||||
async def _annotation(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:
|
||||
context_id = self._context_id(data)
|
||||
if not context_id:
|
||||
return self._error("MISSING_CONTEXT", "context_id is required", data)
|
||||
runtime = await get_runtime(context_id, create=False)
|
||||
if not runtime:
|
||||
return self._error("NO_BROWSER_RUNTIME", "No browser runtime exists for this context", data)
|
||||
|
||||
browser_id = data.get("browser_id")
|
||||
viewer_id = str(data.get("viewer_id") or "")
|
||||
payload = data.get("payload") if isinstance(data.get("payload"), dict) else {}
|
||||
try:
|
||||
annotation = await runtime.call("annotation_target", browser_id, payload)
|
||||
except Exception as exc:
|
||||
return self._error("ANNOTATION_FAILED", str(exc), data)
|
||||
|
||||
return {
|
||||
"annotation": annotation,
|
||||
"context_id": context_id,
|
||||
"browser_id": browser_id,
|
||||
"viewer_id": viewer_id,
|
||||
}
|
||||
|
||||
async def _snapshot_for_result(
|
||||
self,
|
||||
runtime: Any,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
(() => {
|
||||
const GLOBAL_KEY = "__spaceBrowserPageContent__";
|
||||
const DOM_HELPER_KEY = "__spaceBrowserDomHelper__";
|
||||
const VERSION = "6";
|
||||
const VERSION = "7";
|
||||
const BLOCK_TAGS = new Set([
|
||||
"ADDRESS",
|
||||
"ARTICLE",
|
||||
|
|
@ -2842,10 +2842,377 @@
|
|||
});
|
||||
}
|
||||
|
||||
function cssEscape(value) {
|
||||
const rawValue = String(value || "");
|
||||
if (!rawValue) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (typeof globalThis.CSS?.escape === "function") {
|
||||
return globalThis.CSS.escape(rawValue);
|
||||
}
|
||||
|
||||
return rawValue.replace(/[^a-zA-Z0-9_-]/gu, (character) => `\\${character}`);
|
||||
}
|
||||
|
||||
function getClassSummary(element) {
|
||||
try {
|
||||
return [...(element?.classList || [])]
|
||||
.map((className) => normalizeAttributeText(className))
|
||||
.filter(Boolean)
|
||||
.slice(0, 4)
|
||||
.join(" ");
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function buildCssSelector(element) {
|
||||
if (!isElementNode(element)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const id = normalizeAttributeText(element.getAttribute?.("id"));
|
||||
if (id) {
|
||||
return `#${cssEscape(id)}`;
|
||||
}
|
||||
|
||||
const parts = [];
|
||||
let current = element;
|
||||
while (isElementNode(current) && current !== globalThis.document?.documentElement && parts.length < 6) {
|
||||
const tagName = getTagName(current).toLowerCase();
|
||||
if (!tagName) {
|
||||
break;
|
||||
}
|
||||
|
||||
let part = tagName;
|
||||
const classes = getClassSummary(current)
|
||||
.split(/\s+/u)
|
||||
.filter(Boolean)
|
||||
.slice(0, 2);
|
||||
if (classes.length && !["body", "html"].includes(tagName)) {
|
||||
part += classes.map((className) => `.${cssEscape(className)}`).join("");
|
||||
}
|
||||
|
||||
const parent = current.parentElement;
|
||||
if (parent) {
|
||||
const siblings = [...parent.children].filter((sibling) => getTagName(sibling) === getTagName(current));
|
||||
if (siblings.length > 1) {
|
||||
part += `:nth-of-type(${siblings.indexOf(current) + 1})`;
|
||||
}
|
||||
}
|
||||
|
||||
parts.unshift(part);
|
||||
if (tagName === "body") {
|
||||
break;
|
||||
}
|
||||
current = parent;
|
||||
}
|
||||
|
||||
return parts.join(" > ");
|
||||
}
|
||||
|
||||
function sanitizeAnnotationDom(value) {
|
||||
return truncateText(
|
||||
String(value || "")
|
||||
.replace(/(<input\b(?=[^>]*\btype\s*=\s*(["'])?password\2?)[^>]*?)\s+value\s*=\s*(["'])[\s\S]*?\3/giu, "$1 value=\"[redacted]\"")
|
||||
.replace(/\svalue\s*=\s*(["'])[\s\S]{0,600}?\1/giu, " value=\"[redacted]\"")
|
||||
.replace(/\sdata-space-browser-live-value\s*=\s*(["'])[\s\S]{0,600}?\1/giu, "")
|
||||
.replace(/\sdata-space-browser-selected-text\s*=\s*(["'])[\s\S]{0,600}?\1/giu, ""),
|
||||
1200
|
||||
);
|
||||
}
|
||||
|
||||
function summarizeAnnotationElement(element) {
|
||||
if (!isElementNode(element)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const summaryData = collectReferenceSummaryData(element, {
|
||||
includeLabelQuotes: false,
|
||||
includeLinkUrls: true,
|
||||
includeSemanticTags: true,
|
||||
includeStateTags: true
|
||||
});
|
||||
const rawDom = serializeElementSnapshot(element);
|
||||
return {
|
||||
classes: getClassSummary(element),
|
||||
dom: sanitizeAnnotationDom(rawDom),
|
||||
id: normalizeAttributeText(element.getAttribute?.("id")),
|
||||
kind: summaryData.kind,
|
||||
name: normalizeAttributeText(element.getAttribute?.("name")),
|
||||
rect: getElementRectSafe(element),
|
||||
role: normalizeAttributeText(element.getAttribute?.("role")).toLowerCase(),
|
||||
selector: buildCssSelector(element),
|
||||
semanticTags: Array.isArray(summaryData.semanticTags) ? summaryData.semanticTags.slice(0, 4) : [],
|
||||
stateTags: Array.isArray(summaryData.state?.stateTags) ? summaryData.state.stateTags.slice(0, 8) : [],
|
||||
summary: truncateText(summaryData.summary || getLabelText(element, {
|
||||
includeAlt: true,
|
||||
includeDescendantImageAlt: true,
|
||||
includePlaceholder: true,
|
||||
includeText: true
|
||||
}), 240),
|
||||
tagName: getTagName(element)
|
||||
};
|
||||
}
|
||||
|
||||
function annotationViewport() {
|
||||
return {
|
||||
height: Math.max(0, Number(globalThis.innerHeight || globalThis.document?.documentElement?.clientHeight || 0)),
|
||||
scrollX: Number(globalThis.scrollX || globalThis.pageXOffset || 0),
|
||||
scrollY: Number(globalThis.scrollY || globalThis.pageYOffset || 0),
|
||||
width: Math.max(0, Number(globalThis.innerWidth || globalThis.document?.documentElement?.clientWidth || 0))
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeAnnotationPoint(payload = {}, viewport = annotationViewport()) {
|
||||
const source = payload?.point && typeof payload.point === "object" ? payload.point : payload;
|
||||
const width = Math.max(1, Number(viewport.width || 1));
|
||||
const height = Math.max(1, Number(viewport.height || 1));
|
||||
return {
|
||||
x: Math.max(0, Math.min(width, Number(source?.x || 0))),
|
||||
y: Math.max(0, Math.min(height, Number(source?.y || 0)))
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeAnnotationRectPayload(payload = {}, viewport = annotationViewport()) {
|
||||
const source = payload?.rect && typeof payload.rect === "object" ? payload.rect : payload;
|
||||
const width = Math.max(1, Number(viewport.width || 1));
|
||||
const height = Math.max(1, Number(viewport.height || 1));
|
||||
const x = Math.max(0, Math.min(width, Number(source?.x || 0)));
|
||||
const y = Math.max(0, Math.min(height, Number(source?.y || 0)));
|
||||
return {
|
||||
height: Math.max(1, Math.min(height - y, Number(source?.height || source?.h || 1))),
|
||||
width: Math.max(1, Math.min(width - x, Number(source?.width || source?.w || 1))),
|
||||
x,
|
||||
y
|
||||
};
|
||||
}
|
||||
|
||||
function intersectRects(leftRect, rightRect) {
|
||||
if (!leftRect || !rightRect) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const x = Math.max(Number(leftRect.x || 0), Number(rightRect.x || 0));
|
||||
const y = Math.max(Number(leftRect.y || 0), Number(rightRect.y || 0));
|
||||
const right = Math.min(
|
||||
Number(leftRect.x || 0) + Number(leftRect.width || 0),
|
||||
Number(rightRect.x || 0) + Number(rightRect.width || 0)
|
||||
);
|
||||
const bottom = Math.min(
|
||||
Number(leftRect.y || 0) + Number(leftRect.height || 0),
|
||||
Number(rightRect.y || 0) + Number(rightRect.height || 0)
|
||||
);
|
||||
const width = right - x;
|
||||
const height = bottom - y;
|
||||
if (width <= 0 || height <= 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
area: width * height,
|
||||
height,
|
||||
width,
|
||||
x,
|
||||
y
|
||||
};
|
||||
}
|
||||
|
||||
function deepElementFromPoint(x, y) {
|
||||
let element = null;
|
||||
try {
|
||||
element = globalThis.document?.elementFromPoint?.(x, y) || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
let guard = 0;
|
||||
while (isElementNode(element) && element.shadowRoot && guard < 8) {
|
||||
guard += 1;
|
||||
try {
|
||||
const nestedElement = element.shadowRoot.elementFromPoint?.(x, y);
|
||||
if (!nestedElement || nestedElement === element) {
|
||||
break;
|
||||
}
|
||||
element = nestedElement;
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return element;
|
||||
}
|
||||
|
||||
function findAnnotationTarget(element) {
|
||||
if (!isElementNode(element)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const selector = [
|
||||
"a[href]",
|
||||
"button",
|
||||
"input",
|
||||
"textarea",
|
||||
"select",
|
||||
"summary",
|
||||
"[role]",
|
||||
"img",
|
||||
"label",
|
||||
"form",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"p",
|
||||
"li",
|
||||
"td",
|
||||
"th",
|
||||
"article",
|
||||
"section",
|
||||
"nav",
|
||||
"header",
|
||||
"main",
|
||||
"footer"
|
||||
].join(",");
|
||||
const target = element.closest?.(selector) || element;
|
||||
return isElementNode(target) && !isHiddenElement(target) ? target : element;
|
||||
}
|
||||
|
||||
function isMeaningfulAnnotationElement(element) {
|
||||
if (!isElementNode(element) || isHiddenElement(element)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isInteractiveElement(element) || getTagName(element) === "IMG") {
|
||||
return true;
|
||||
}
|
||||
|
||||
const tagName = getTagName(element);
|
||||
const role = normalizeAttributeText(element.getAttribute?.("role")).toLowerCase();
|
||||
return Boolean(
|
||||
role
|
||||
|| /^H[1-6]$/u.test(tagName)
|
||||
|| ["ARTICLE", "SECTION", "MAIN", "NAV", "HEADER", "FOOTER", "FORM", "LABEL", "P", "LI", "TD", "TH"].includes(tagName)
|
||||
);
|
||||
}
|
||||
|
||||
function collectIntersectingAnnotationElements(rect) {
|
||||
const selector = [
|
||||
"a[href]",
|
||||
"button",
|
||||
"input",
|
||||
"textarea",
|
||||
"select",
|
||||
"summary",
|
||||
"[role]",
|
||||
"img",
|
||||
"label",
|
||||
"form",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"p",
|
||||
"li",
|
||||
"td",
|
||||
"th",
|
||||
"article",
|
||||
"section",
|
||||
"main",
|
||||
"nav",
|
||||
"header",
|
||||
"footer"
|
||||
].join(",");
|
||||
let candidates = [];
|
||||
try {
|
||||
candidates = [...(globalThis.document?.querySelectorAll?.(selector) || [])];
|
||||
} catch {
|
||||
candidates = [];
|
||||
}
|
||||
|
||||
const seen = new Set();
|
||||
return candidates
|
||||
.map((element) => {
|
||||
if (!isMeaningfulAnnotationElement(element) || seen.has(element)) {
|
||||
return null;
|
||||
}
|
||||
seen.add(element);
|
||||
const elementRect = getElementRectSafe(element);
|
||||
const intersection = intersectRects(rect, elementRect);
|
||||
if (!intersection || intersection.area < 48) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
element,
|
||||
elementArea: Math.max(1, Number(elementRect.width || 0) * Number(elementRect.height || 0)),
|
||||
intersection
|
||||
};
|
||||
})
|
||||
.filter(Boolean)
|
||||
.sort((left, right) => {
|
||||
if (right.intersection.area !== left.intersection.area) {
|
||||
return right.intersection.area - left.intersection.area;
|
||||
}
|
||||
return left.elementArea - right.elementArea;
|
||||
})
|
||||
.slice(0, 12)
|
||||
.map((entry) => summarizeAnnotationElement(entry.element))
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function annotate(payload = null) {
|
||||
const request = payload && typeof payload === "object" ? payload : {};
|
||||
const viewport = annotationViewport();
|
||||
const kind = request.kind === "area" || request.rect ? "area" : "element";
|
||||
|
||||
if (kind === "area") {
|
||||
const rect = normalizeAnnotationRectPayload(request, viewport);
|
||||
const point = {
|
||||
x: rect.x + rect.width / 2,
|
||||
y: rect.y + rect.height / 2
|
||||
};
|
||||
const elements = collectIntersectingAnnotationElements(rect);
|
||||
const fallbackElement = findAnnotationTarget(deepElementFromPoint(point.x, point.y));
|
||||
const fallbackTarget = fallbackElement ? summarizeAnnotationElement(fallbackElement) : null;
|
||||
return {
|
||||
elements,
|
||||
kind,
|
||||
point,
|
||||
rect,
|
||||
status: elements.length || fallbackTarget ? "ok" : "empty",
|
||||
target: elements[0] || fallbackTarget,
|
||||
viewport
|
||||
};
|
||||
}
|
||||
|
||||
const point = normalizeAnnotationPoint(request, viewport);
|
||||
const rawElement = deepElementFromPoint(point.x, point.y);
|
||||
const targetElement = findAnnotationTarget(rawElement);
|
||||
const target = targetElement ? summarizeAnnotationElement(targetElement) : null;
|
||||
return {
|
||||
kind,
|
||||
point,
|
||||
rect: target?.rect || {
|
||||
height: 1,
|
||||
width: 1,
|
||||
x: point.x,
|
||||
y: point.y
|
||||
},
|
||||
status: target ? "ok" : "empty",
|
||||
target,
|
||||
viewport
|
||||
};
|
||||
}
|
||||
|
||||
globalThis[GLOBAL_KEY] = {
|
||||
click(referenceId) {
|
||||
return activateElement(referenceId);
|
||||
},
|
||||
annotate,
|
||||
capture,
|
||||
clear() {
|
||||
state.captureId = 0;
|
||||
|
|
|
|||
|
|
@ -557,6 +557,22 @@ class _BrowserRuntimeCore:
|
|||
self.last_interacted_browser_id = resolved_id
|
||||
return result or {}
|
||||
|
||||
async def annotation_target(
|
||||
self,
|
||||
browser_id: int | str | None,
|
||||
payload: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
await self.ensure_started()
|
||||
resolved_id = self._resolve_browser_id(browser_id)
|
||||
page = self._page(resolved_id)
|
||||
await self._ensure_content_helper(page)
|
||||
result = await page.evaluate(
|
||||
"(payload) => globalThis.__spaceBrowserPageContent__.annotate(payload || null)",
|
||||
payload or None,
|
||||
)
|
||||
self.last_interacted_browser_id = resolved_id
|
||||
return result or {}
|
||||
|
||||
async def evaluate(self, browser_id: int | str | None, script: str) -> dict[str, Any]:
|
||||
await self.ensure_started()
|
||||
resolved_id = self._resolve_browser_id(browser_id)
|
||||
|
|
@ -918,7 +934,7 @@ class _BrowserRuntimeCore:
|
|||
|
||||
async def _ensure_content_helper(self, page: Any) -> None:
|
||||
has_helper = await page.evaluate(
|
||||
"() => Boolean(globalThis.__spaceBrowserPageContent__?.capture)"
|
||||
"() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate)"
|
||||
)
|
||||
if has_helper:
|
||||
return
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
<div x-data>
|
||||
<template x-if="$store.browserPage">
|
||||
<div class="browser-panel" x-create="$store.browserPage.onOpen($el, xAttrs($el) || {})" x-destroy="$store.browserPage.cleanup()"
|
||||
@keydown.window="$store.browserPage.sendKey($event)">
|
||||
@keydown.window="$store.browserPage.handleKeydown($event)">
|
||||
<div class="browser-meta">
|
||||
<div class="browser-meta-top">
|
||||
<div class="browser-session-tabs" role="tablist" aria-label="Browser sessions">
|
||||
|
|
@ -41,6 +41,15 @@
|
|||
</div>
|
||||
|
||||
<div class="browser-session-controls">
|
||||
<button type="button" class="btn browser-annotate-toggle" title="Annotate" aria-label="Annotate"
|
||||
:aria-pressed="$store.browserPage.annotating.toString()"
|
||||
:class="{ 'is-active': $store.browserPage.annotating }"
|
||||
:disabled="!$store.browserPage.canAnnotate() && !$store.browserPage.annotating"
|
||||
@click="$store.browserPage.toggleAnnotationMode()">
|
||||
<span class="material-symbols-outlined" aria-hidden="true"
|
||||
x-text="$store.browserPage.annotating ? 'rate_review' : 'edit_note'"></span>
|
||||
<span x-text="$store.browserPage.annotating ? 'Annotating' : 'Annotate'"></span>
|
||||
</button>
|
||||
<div class="browser-extension-menu" @click.outside="$store.browserPage.closeExtensionsMenu()"
|
||||
@keydown.escape.window="$store.browserPage.closeExtensionsMenu()">
|
||||
<button type="button" class="btn btn-icon-action browser-extensions" title="Browser settings"
|
||||
|
|
@ -161,12 +170,82 @@
|
|||
</div>
|
||||
|
||||
<div class="browser-stage" tabindex="0" @click="$el.focus()"
|
||||
@wheel.prevent="$store.browserPage.sendWheel($event)">
|
||||
:class="{ 'is-annotating': $store.browserPage.annotating }"
|
||||
@wheel.prevent="$store.browserPage.handleStageWheel($event)">
|
||||
<template x-if="$store.browserPage.frameSrc">
|
||||
<img class="browser-frame" :src="$store.browserPage.frameSrc"
|
||||
@click="$store.browserPage.sendMouse('click', $event)"
|
||||
@mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)" draggable="false" />
|
||||
</template>
|
||||
<template x-if="$store.browserPage.annotating && $store.browserPage.frameSrc">
|
||||
<div class="browser-annotation-layer"
|
||||
:class="{ 'is-busy': $store.browserPage.annotationBusy }"
|
||||
@pointerdown.stop.prevent="$store.browserPage.startAnnotationSelection($event)"
|
||||
@pointermove.stop.prevent="$store.browserPage.moveAnnotationSelection($event)"
|
||||
@pointerup.stop.prevent="$store.browserPage.finishAnnotationSelection($event)"
|
||||
@pointercancel.stop.prevent="$store.browserPage.cancelAnnotationSelection($event)">
|
||||
<template x-for="annotation in $store.browserPage.visibleAnnotations()" :key="annotation.id">
|
||||
<div class="browser-annotation-box is-saved"
|
||||
:style="$store.browserPage.annotationBoxStyle(annotation.rect)">
|
||||
<span class="browser-annotation-number" x-text="annotation.index"></span>
|
||||
</div>
|
||||
</template>
|
||||
<template x-if="$store.browserPage.annotationDragRect">
|
||||
<div class="browser-annotation-box is-draft"
|
||||
:style="$store.browserPage.annotationBoxStyle($store.browserPage.annotationDragRect)">
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
</template>
|
||||
<template x-if="$store.browserPage.annotationDraft">
|
||||
<div class="browser-annotation-popover"
|
||||
:style="$store.browserPage.annotationPopoverStyle()"
|
||||
@click.stop @pointerdown.stop @keydown.stop>
|
||||
<div class="browser-annotation-popover-title">
|
||||
<span class="browser-annotation-number" x-text="$store.browserPage.nextAnnotationIndex()"></span>
|
||||
<span x-text="$store.browserPage.annotationDraftTitle()"></span>
|
||||
</div>
|
||||
<textarea x-model="$store.browserPage.annotationDraftText" placeholder="Comment"
|
||||
maxlength="1200"></textarea>
|
||||
<div class="browser-annotation-actions">
|
||||
<button type="button" class="btn btn-field"
|
||||
@click="$store.browserPage.cancelAnnotationDraft()">Cancel</button>
|
||||
<button type="button" class="btn btn-ok"
|
||||
:disabled="!String($store.browserPage.annotationDraftText || '').trim()"
|
||||
@click="$store.browserPage.addAnnotationComment()">Add</button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
<div class="browser-annotation-tray"
|
||||
x-show="$store.browserPage.visibleAnnotations().length"
|
||||
x-transition style="display: none;"
|
||||
@click.stop @pointerdown.stop @keydown.stop>
|
||||
<div class="browser-annotation-tray-header">
|
||||
<span>Annotations</span>
|
||||
<button type="button" class="browser-annotation-clear" title="Clear annotations"
|
||||
aria-label="Clear annotations" @click="$store.browserPage.clearVisibleAnnotations()">
|
||||
<span class="material-symbols-outlined">delete</span>
|
||||
</button>
|
||||
</div>
|
||||
<div class="browser-annotation-chips">
|
||||
<template x-for="annotation in $store.browserPage.visibleAnnotations()" :key="annotation.id">
|
||||
<div class="browser-annotation-chip">
|
||||
<span class="browser-annotation-number" x-text="annotation.index"></span>
|
||||
<span class="browser-annotation-chip-text" x-text="annotation.comment"></span>
|
||||
<button type="button" title="Remove annotation" aria-label="Remove annotation"
|
||||
@click="$store.browserPage.removeAnnotationComment(annotation.id)">
|
||||
<span class="material-symbols-outlined">close</span>
|
||||
</button>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
<div class="browser-annotation-tray-actions">
|
||||
<button type="button" class="btn btn-field"
|
||||
@click="$store.browserPage.draftAnnotationsToChat()">Draft to chat</button>
|
||||
<button type="button" class="btn btn-ok"
|
||||
@click="$store.browserPage.sendAnnotationsToChat()">Send now</button>
|
||||
</div>
|
||||
</div>
|
||||
<template x-if="!$store.browserPage.frameSrc && !$store.browserPage.isBusy()">
|
||||
<div class="browser-empty">
|
||||
<span class="material-symbols-outlined">captive_portal</span>
|
||||
|
|
@ -549,6 +628,34 @@
|
|||
padding-bottom: 1px;
|
||||
}
|
||||
|
||||
.browser-annotate-toggle {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 6px;
|
||||
min-width: 0;
|
||||
min-height: var(--browser-control-size);
|
||||
padding: 0 10px;
|
||||
border: 1px solid var(--browser-chrome-border);
|
||||
border-radius: var(--browser-control-radius);
|
||||
background: color-mix(in srgb, var(--color-background) 26%, transparent);
|
||||
color: color-mix(in srgb, var(--color-text) 74%, var(--color-primary) 26%);
|
||||
font-size: 0.82rem;
|
||||
font-weight: 650;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.browser-annotate-toggle:hover:not(:disabled),
|
||||
.browser-annotate-toggle.is-active {
|
||||
border-color: color-mix(in srgb, var(--color-primary) 48%, var(--browser-chrome-border));
|
||||
background: color-mix(in srgb, var(--color-primary) 16%, var(--color-background));
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.browser-annotate-toggle .material-symbols-outlined {
|
||||
font-size: 1.05rem;
|
||||
}
|
||||
|
||||
.browser-session-controls .browser-extensions.is-active {
|
||||
color: #2e7d32;
|
||||
}
|
||||
|
|
@ -813,6 +920,10 @@
|
|||
outline: none;
|
||||
}
|
||||
|
||||
.browser-stage.is-annotating {
|
||||
cursor: crosshair;
|
||||
}
|
||||
|
||||
.browser-frame {
|
||||
display: block;
|
||||
position: absolute;
|
||||
|
|
@ -827,6 +938,194 @@
|
|||
background: #fff;
|
||||
}
|
||||
|
||||
.browser-annotation-layer {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
z-index: 12;
|
||||
cursor: crosshair;
|
||||
touch-action: none;
|
||||
background: rgba(37, 99, 235, 0.035);
|
||||
}
|
||||
|
||||
.browser-annotation-layer.is-busy {
|
||||
cursor: progress;
|
||||
}
|
||||
|
||||
.browser-annotation-box {
|
||||
position: absolute;
|
||||
box-sizing: border-box;
|
||||
min-width: 8px;
|
||||
min-height: 8px;
|
||||
border: 2px solid #3399ff;
|
||||
background: rgba(51, 153, 255, 0.16);
|
||||
box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.52), 0 8px 22px rgba(0, 0, 0, 0.18);
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.browser-annotation-box.is-draft {
|
||||
border-style: dashed;
|
||||
background: rgba(51, 153, 255, 0.1);
|
||||
}
|
||||
|
||||
.browser-annotation-number {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 22px;
|
||||
min-width: 22px;
|
||||
height: 22px;
|
||||
min-height: 22px;
|
||||
border-radius: 50%;
|
||||
background: #3399ff;
|
||||
color: #fff;
|
||||
font-size: 0.74rem;
|
||||
font-weight: 800;
|
||||
line-height: 1;
|
||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.28);
|
||||
}
|
||||
|
||||
.browser-annotation-box .browser-annotation-number {
|
||||
position: absolute;
|
||||
top: -12px;
|
||||
left: -12px;
|
||||
}
|
||||
|
||||
.browser-annotation-popover,
|
||||
.browser-annotation-tray {
|
||||
position: absolute;
|
||||
z-index: 18;
|
||||
border: 1px solid color-mix(in srgb, var(--color-border) 76%, transparent);
|
||||
border-radius: 7px;
|
||||
background: color-mix(in srgb, var(--color-background) 96%, #000 4%);
|
||||
color: var(--color-text);
|
||||
box-shadow: 0 16px 38px rgba(0, 0, 0, 0.28);
|
||||
}
|
||||
|
||||
.browser-annotation-popover {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
.browser-annotation-popover-title,
|
||||
.browser-annotation-tray-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
min-width: 0;
|
||||
font-size: 0.82rem;
|
||||
font-weight: 750;
|
||||
}
|
||||
|
||||
.browser-annotation-popover textarea {
|
||||
width: 100%;
|
||||
min-height: 82px;
|
||||
max-height: 160px;
|
||||
resize: vertical;
|
||||
padding: 8px;
|
||||
border: 1px solid color-mix(in srgb, var(--color-border) 74%, transparent);
|
||||
border-radius: 6px;
|
||||
background: var(--color-input);
|
||||
color: var(--color-text);
|
||||
font: inherit;
|
||||
line-height: 1.35;
|
||||
}
|
||||
|
||||
.browser-annotation-actions,
|
||||
.browser-annotation-tray-actions {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
gap: 7px;
|
||||
}
|
||||
|
||||
.browser-annotation-actions .btn,
|
||||
.browser-annotation-tray-actions .btn {
|
||||
min-height: 30px;
|
||||
padding: 0 10px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.browser-annotation-tray {
|
||||
right: 10px;
|
||||
bottom: 10px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 9px;
|
||||
width: min(360px, calc(100% - 20px));
|
||||
max-height: min(48%, 310px);
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
.browser-annotation-tray-header {
|
||||
justify-content: space-between;
|
||||
}
|
||||
|
||||
.browser-annotation-clear,
|
||||
.browser-annotation-chip button {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
min-width: 24px;
|
||||
min-height: 24px;
|
||||
padding: 0;
|
||||
border: 0;
|
||||
border-radius: 6px;
|
||||
background: transparent;
|
||||
color: color-mix(in srgb, var(--color-text) 64%, transparent);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.browser-annotation-clear:hover,
|
||||
.browser-annotation-chip button:hover {
|
||||
background: color-mix(in srgb, var(--color-panel) 82%, transparent);
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.browser-annotation-clear .material-symbols-outlined,
|
||||
.browser-annotation-chip button .material-symbols-outlined {
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
.browser-annotation-chips {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
min-height: 0;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
.browser-annotation-chip {
|
||||
display: grid;
|
||||
grid-template-columns: auto minmax(0, 1fr) auto;
|
||||
align-items: center;
|
||||
gap: 7px;
|
||||
min-height: 32px;
|
||||
padding: 5px 6px;
|
||||
border: 1px solid color-mix(in srgb, var(--color-border) 54%, transparent);
|
||||
border-radius: 7px;
|
||||
background: color-mix(in srgb, var(--color-panel) 74%, transparent);
|
||||
}
|
||||
|
||||
.browser-annotation-chip .browser-annotation-number {
|
||||
width: 20px;
|
||||
min-width: 20px;
|
||||
height: 20px;
|
||||
min-height: 20px;
|
||||
font-size: 0.68rem;
|
||||
}
|
||||
|
||||
.browser-annotation-chip-text {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
font-size: 0.78rem;
|
||||
line-height: 1.25;
|
||||
}
|
||||
|
||||
.browser-empty {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
|
@ -928,6 +1227,16 @@
|
|||
height: var(--browser-control-size);
|
||||
}
|
||||
|
||||
.browser-annotate-toggle {
|
||||
width: var(--browser-control-size);
|
||||
min-width: var(--browser-control-size);
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.browser-annotate-toggle span:not(.material-symbols-outlined) {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.browser-extension-dropdown {
|
||||
right: 0;
|
||||
left: auto;
|
||||
|
|
|
|||
|
|
@ -13,6 +13,9 @@ const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;
|
|||
const BROWSER_CONFIG_REFRESH_MS = 15000;
|
||||
const VIEWPORT_SYNC_DEBOUNCE_MS = 220;
|
||||
const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;
|
||||
const ANNOTATION_DRAG_THRESHOLD = 6;
|
||||
const ANNOTATION_MAX_COMMENTS = 24;
|
||||
const ANNOTATION_DOM_LIMIT = 1200;
|
||||
|
||||
function makeViewerToken() {
|
||||
return globalThis.crypto?.randomUUID?.()
|
||||
|
|
@ -60,6 +63,13 @@ const model = {
|
|||
address: "",
|
||||
frameSrc: "",
|
||||
frameState: null,
|
||||
annotating: false,
|
||||
annotationComments: [],
|
||||
annotationDraft: null,
|
||||
annotationDraftText: "",
|
||||
annotationDragRect: null,
|
||||
annotationBusy: false,
|
||||
annotationError: "",
|
||||
connected: false,
|
||||
switchingBrowserId: null,
|
||||
commandInFlight: false,
|
||||
|
|
@ -76,6 +86,8 @@ const model = {
|
|||
_viewportSyncTimer: null,
|
||||
_lastViewportKey: "",
|
||||
_lastViewport: null,
|
||||
_annotationPointer: null,
|
||||
_annotationSequence: 0,
|
||||
_mode: "",
|
||||
_surfaceMounted: false,
|
||||
_surfaceSwitching: false,
|
||||
|
|
@ -613,6 +625,7 @@ const model = {
|
|||
|
||||
async command(command, extra = {}) {
|
||||
this.error = "";
|
||||
this.annotationError = "";
|
||||
this.commandInFlight = true;
|
||||
const previousActiveBrowserId = this.activeBrowserId;
|
||||
try {
|
||||
|
|
@ -642,13 +655,17 @@ const model = {
|
|||
this.frameState = null;
|
||||
this.frameSrc = "";
|
||||
}
|
||||
if (result.state?.currentUrl || result.currentUrl) {
|
||||
this.address = result.state?.currentUrl || result.currentUrl;
|
||||
}
|
||||
this.applySnapshot(data.snapshot);
|
||||
const activeChanged = this.activeBrowserId && this.activeBrowserId !== previousActiveBrowserId;
|
||||
if ((command === "open" || command === "close" || activeChanged) && this.contextId && this.activeBrowserId) {
|
||||
await this.connectViewer({ browserId: this.activeBrowserId });
|
||||
if (result.state?.currentUrl || result.currentUrl) {
|
||||
this.address = result.state?.currentUrl || result.currentUrl;
|
||||
}
|
||||
this.applySnapshot(data.snapshot);
|
||||
if (["navigate", "back", "forward", "reload", "close"].includes(String(command || "").toLowerCase())) {
|
||||
this.clearAnnotationsForBrowser(previousActiveBrowserId);
|
||||
this.cancelAnnotationDraft();
|
||||
}
|
||||
const activeChanged = this.activeBrowserId && this.activeBrowserId !== previousActiveBrowserId;
|
||||
if ((command === "open" || command === "close" || activeChanged) && this.contextId && this.activeBrowserId) {
|
||||
await this.connectViewer({ browserId: this.activeBrowserId });
|
||||
}
|
||||
} catch (error) {
|
||||
this.error = error instanceof Error ? error.message : String(error);
|
||||
|
|
@ -761,17 +778,22 @@ const model = {
|
|||
return null;
|
||||
},
|
||||
|
||||
applyActiveFrameState(nextState = null) {
|
||||
if (!nextState) return;
|
||||
const stateId = this.normalizeBrowserId(nextState.id);
|
||||
if (stateId && this.activeBrowserId && !this.sameBrowserId(stateId, this.activeBrowserId)) {
|
||||
return;
|
||||
applyActiveFrameState(nextState = null) {
|
||||
if (!nextState) return;
|
||||
const stateId = this.normalizeBrowserId(nextState.id);
|
||||
if (stateId && this.activeBrowserId && !this.sameBrowserId(stateId, this.activeBrowserId)) {
|
||||
return;
|
||||
}
|
||||
const previousUrl = String(this.frameState?.currentUrl || "");
|
||||
const nextUrl = String(nextState.currentUrl || "");
|
||||
this.frameState = nextState;
|
||||
if (previousUrl && nextUrl && previousUrl !== nextUrl) {
|
||||
this.cancelAnnotationDraft();
|
||||
}
|
||||
if (!this.addressFocused && nextState.currentUrl) {
|
||||
this.address = nextState.currentUrl;
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
applySnapshot(snapshot = null) {
|
||||
if (!snapshot?.image) return;
|
||||
|
|
@ -805,6 +827,7 @@ const model = {
|
|||
if (this.activeBrowserId !== previous) {
|
||||
this._lastViewportKey = "";
|
||||
this._lastViewport = null;
|
||||
this.cancelAnnotationDraft();
|
||||
}
|
||||
},
|
||||
|
||||
|
|
@ -849,6 +872,419 @@ const model = {
|
|||
};
|
||||
},
|
||||
|
||||
handleKeydown(event) {
|
||||
const annotateShortcut = event?.key === "." && (event.metaKey || event.ctrlKey) && !event.altKey;
|
||||
if (annotateShortcut && this._surfaceMounted) {
|
||||
event.preventDefault();
|
||||
event.stopPropagation?.();
|
||||
this.toggleAnnotationMode();
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.annotating) {
|
||||
if (event?.key === "Escape") {
|
||||
event.preventDefault();
|
||||
if (this.annotationDraft || this.annotationDragRect) {
|
||||
this.cancelAnnotationDraft();
|
||||
} else {
|
||||
this.toggleAnnotationMode(false);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
void this.sendKey(event);
|
||||
},
|
||||
|
||||
handleStageWheel(event) {
|
||||
if (this.annotating) return;
|
||||
void this.sendWheel(event);
|
||||
},
|
||||
|
||||
toggleAnnotationMode(force = null) {
|
||||
const nextValue = force === null ? !this.annotating : Boolean(force);
|
||||
if (nextValue && !this.canAnnotate()) return;
|
||||
|
||||
this.annotating = nextValue;
|
||||
this.annotationError = "";
|
||||
this.closeExtensionsMenu();
|
||||
if (!nextValue) {
|
||||
this.cancelAnnotationDraft();
|
||||
this.annotationDragRect = null;
|
||||
this._annotationPointer = null;
|
||||
} else {
|
||||
this._stageElement?.focus?.({ preventScroll: true });
|
||||
}
|
||||
},
|
||||
|
||||
canAnnotate() {
|
||||
return Boolean(this.activeBrowserId && this.frameSrc && !this.isBusy());
|
||||
},
|
||||
|
||||
activeAnnotationUrl() {
|
||||
return String(this.frameState?.currentUrl || this.address || "about:blank");
|
||||
},
|
||||
|
||||
visibleAnnotations() {
|
||||
const browserId = this.normalizeBrowserId(this.activeBrowserId);
|
||||
const url = this.activeAnnotationUrl();
|
||||
return this.annotationComments.filter((annotation) => (
|
||||
this.sameBrowserId(annotation.browserId, browserId)
|
||||
&& String(annotation.url || "") === url
|
||||
));
|
||||
},
|
||||
|
||||
nextAnnotationIndex() {
|
||||
return this.visibleAnnotations().length + 1;
|
||||
},
|
||||
|
||||
clearVisibleAnnotations() {
|
||||
this.clearAnnotationsForBrowser(this.activeBrowserId, this.activeAnnotationUrl());
|
||||
},
|
||||
|
||||
clearAnnotationsForBrowser(browserId, url = null) {
|
||||
const numericBrowserId = this.normalizeBrowserId(browserId);
|
||||
if (!numericBrowserId) return;
|
||||
this.annotationComments = this.annotationComments.filter((annotation) => {
|
||||
if (!this.sameBrowserId(annotation.browserId, numericBrowserId)) return true;
|
||||
return url ? String(annotation.url || "") !== String(url) : false;
|
||||
});
|
||||
},
|
||||
|
||||
annotationBoxStyle(rect = {}) {
|
||||
const viewport = this.currentViewportSize() || this._lastViewport || {};
|
||||
const width = Math.max(1, Number(viewport.width || rect.width || 1));
|
||||
const height = Math.max(1, Number(viewport.height || rect.height || 1));
|
||||
const normalized = this.clampAnnotationRect(rect);
|
||||
return [
|
||||
`left: ${(normalized.x / width) * 100}%`,
|
||||
`top: ${(normalized.y / height) * 100}%`,
|
||||
`width: ${(Math.max(1, normalized.width) / width) * 100}%`,
|
||||
`height: ${(Math.max(1, normalized.height) / height) * 100}%`,
|
||||
].join("; ");
|
||||
},
|
||||
|
||||
annotationPopoverStyle() {
|
||||
const rect = this.annotationDraft?.rect || this.annotationDragRect || {};
|
||||
const viewport = this.currentViewportSize() || this._lastViewport || {};
|
||||
const width = Math.max(1, Number(viewport.width || 1));
|
||||
const height = Math.max(1, Number(viewport.height || 1));
|
||||
const popoverWidth = Math.min(320, Math.max(240, width - 20));
|
||||
const popoverHeight = 190;
|
||||
const nextLeft = Math.min(
|
||||
Math.max(10, Number(rect.x || 0) + Number(rect.width || 0) + 10),
|
||||
Math.max(10, width - popoverWidth - 10),
|
||||
);
|
||||
const nextTop = Math.min(
|
||||
Math.max(10, Number(rect.y || 0) + Number(rect.height || 0) + 10),
|
||||
Math.max(10, height - popoverHeight - 10),
|
||||
);
|
||||
return [
|
||||
`left: ${(nextLeft / width) * 100}%`,
|
||||
`top: ${(nextTop / height) * 100}%`,
|
||||
`width: min(${popoverWidth}px, calc(100% - 20px))`,
|
||||
].join("; ");
|
||||
},
|
||||
|
||||
annotationDraftTitle() {
|
||||
if (!this.annotationDraft) return "Annotation";
|
||||
return this.annotationDraft.kind === "area" ? "Area annotation" : "Element annotation";
|
||||
},
|
||||
|
||||
stagePointForEvent(event) {
|
||||
const image = this._stageElement?.querySelector?.(".browser-frame") || null;
|
||||
return this.pointerCoordinatesFor(event, image);
|
||||
},
|
||||
|
||||
normalizeAnnotationRect(start = {}, end = {}) {
|
||||
const x1 = Number(start.x || 0);
|
||||
const y1 = Number(start.y || 0);
|
||||
const x2 = Number(end.x || x1);
|
||||
const y2 = Number(end.y || y1);
|
||||
return this.clampAnnotationRect({
|
||||
x: Math.min(x1, x2),
|
||||
y: Math.min(y1, y2),
|
||||
width: Math.abs(x2 - x1),
|
||||
height: Math.abs(y2 - y1),
|
||||
});
|
||||
},
|
||||
|
||||
clampAnnotationRect(rect = {}) {
|
||||
const viewport = this.currentViewportSize() || this._lastViewport || {};
|
||||
const viewportWidth = Math.max(1, Number(viewport.width || rect.x + rect.width || 1));
|
||||
const viewportHeight = Math.max(1, Number(viewport.height || rect.y + rect.height || 1));
|
||||
const x = Math.max(0, Math.min(viewportWidth, Number(rect.x || 0)));
|
||||
const y = Math.max(0, Math.min(viewportHeight, Number(rect.y || 0)));
|
||||
const width = Math.max(1, Math.min(viewportWidth - x, Number(rect.width || 1)));
|
||||
const height = Math.max(1, Math.min(viewportHeight - y, Number(rect.height || 1)));
|
||||
return {
|
||||
x: Math.round(x),
|
||||
y: Math.round(y),
|
||||
width: Math.round(width),
|
||||
height: Math.round(height),
|
||||
};
|
||||
},
|
||||
|
||||
startAnnotationSelection(event) {
|
||||
if (!this.annotating || this.annotationBusy || !this.canAnnotate()) return;
|
||||
const point = this.stagePointForEvent(event);
|
||||
if (!point) return;
|
||||
this.cancelAnnotationDraft();
|
||||
this.annotationError = "";
|
||||
this._annotationPointer = {
|
||||
id: event.pointerId,
|
||||
start: point,
|
||||
last: point,
|
||||
};
|
||||
this.annotationDragRect = this.clampAnnotationRect({
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
width: 1,
|
||||
height: 1,
|
||||
});
|
||||
event.currentTarget?.setPointerCapture?.(event.pointerId);
|
||||
},
|
||||
|
||||
moveAnnotationSelection(event) {
|
||||
if (!this.annotating || !this._annotationPointer) return;
|
||||
if (event.pointerId !== this._annotationPointer.id) return;
|
||||
const point = this.stagePointForEvent(event);
|
||||
if (!point) return;
|
||||
this._annotationPointer.last = point;
|
||||
this.annotationDragRect = this.normalizeAnnotationRect(this._annotationPointer.start, point);
|
||||
},
|
||||
|
||||
async finishAnnotationSelection(event) {
|
||||
if (!this.annotating || !this._annotationPointer) return;
|
||||
if (event.pointerId !== this._annotationPointer.id) return;
|
||||
const pointer = this._annotationPointer;
|
||||
this._annotationPointer = null;
|
||||
event.currentTarget?.releasePointerCapture?.(event.pointerId);
|
||||
const endPoint = this.stagePointForEvent(event) || pointer.last || pointer.start;
|
||||
const rect = this.normalizeAnnotationRect(pointer.start, endPoint);
|
||||
this.annotationDragRect = null;
|
||||
const isDrag = rect.width >= ANNOTATION_DRAG_THRESHOLD || rect.height >= ANNOTATION_DRAG_THRESHOLD;
|
||||
const point = {
|
||||
x: Math.round(endPoint.x),
|
||||
y: Math.round(endPoint.y),
|
||||
};
|
||||
const payload = {
|
||||
kind: isDrag ? "area" : "element",
|
||||
point,
|
||||
rect: isDrag ? rect : null,
|
||||
viewport: this.currentViewportSize(),
|
||||
url: this.activeAnnotationUrl(),
|
||||
title: this.activeTitle,
|
||||
};
|
||||
await this.createAnnotationDraft(payload, isDrag ? rect : {
|
||||
x: point.x - 10,
|
||||
y: point.y - 10,
|
||||
width: 20,
|
||||
height: 20,
|
||||
});
|
||||
},
|
||||
|
||||
cancelAnnotationSelection(event = null) {
|
||||
if (event && this._annotationPointer?.id === event.pointerId) {
|
||||
event.currentTarget?.releasePointerCapture?.(event.pointerId);
|
||||
}
|
||||
this._annotationPointer = null;
|
||||
this.annotationDragRect = null;
|
||||
},
|
||||
|
||||
cancelAnnotationDraft() {
|
||||
this.annotationDraft = null;
|
||||
this.annotationDraftText = "";
|
||||
this.annotationDragRect = null;
|
||||
},
|
||||
|
||||
async createAnnotationDraft(payload, fallbackRect) {
|
||||
if (!this.activeBrowserId || !this.contextId) return;
|
||||
const sequence = this._annotationSequence + 1;
|
||||
const browserId = this.activeBrowserId;
|
||||
const url = this.activeAnnotationUrl();
|
||||
const title = this.activeTitle;
|
||||
this._annotationSequence = sequence;
|
||||
this.annotationBusy = true;
|
||||
this.annotationError = "";
|
||||
try {
|
||||
const response = await websocket.request(
|
||||
"browser_viewer_annotation",
|
||||
{
|
||||
context_id: this.contextId,
|
||||
browser_id: browserId,
|
||||
viewer_id: this._viewerToken,
|
||||
payload,
|
||||
},
|
||||
{ timeoutMs: 10000 },
|
||||
);
|
||||
if (sequence !== this._annotationSequence) return;
|
||||
const data = firstOk(response);
|
||||
const metadata = data.annotation || {};
|
||||
this.annotationDraft = {
|
||||
id: makeViewerToken(),
|
||||
browserId,
|
||||
url,
|
||||
title,
|
||||
kind: metadata.kind || payload.kind,
|
||||
rect: this.annotationRectFromMetadata(metadata, fallbackRect),
|
||||
metadata,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
this.annotationDraftText = "";
|
||||
} catch (error) {
|
||||
this.annotationError = error instanceof Error ? error.message : String(error);
|
||||
this.error = this.annotationError;
|
||||
} finally {
|
||||
if (sequence === this._annotationSequence) {
|
||||
this.annotationBusy = false;
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
annotationRectFromMetadata(metadata = {}, fallbackRect = {}) {
|
||||
const targetRect = metadata?.target?.rect || metadata?.rect || null;
|
||||
return this.clampAnnotationRect(targetRect || fallbackRect);
|
||||
},
|
||||
|
||||
addAnnotationComment() {
|
||||
const comment = String(this.annotationDraftText || "").trim();
|
||||
if (!this.annotationDraft || !comment) return;
|
||||
if (this.visibleAnnotations().length >= ANNOTATION_MAX_COMMENTS) {
|
||||
this.annotationError = `Keep each batch to ${ANNOTATION_MAX_COMMENTS} annotations or fewer.`;
|
||||
this.error = this.annotationError;
|
||||
return;
|
||||
}
|
||||
this.annotationComments = [
|
||||
...this.annotationComments,
|
||||
{
|
||||
...this.annotationDraft,
|
||||
comment,
|
||||
index: this.nextAnnotationIndex(),
|
||||
},
|
||||
];
|
||||
this.cancelAnnotationDraft();
|
||||
},
|
||||
|
||||
removeAnnotationComment(annotationId) {
|
||||
this.annotationComments = this.annotationComments.filter((annotation) => annotation.id !== annotationId);
|
||||
},
|
||||
|
||||
annotationChipLabel(annotation) {
|
||||
const prefix = annotation?.kind === "area" ? "Area" : "Element";
|
||||
return `${prefix} ${annotation?.index || ""}`.trim();
|
||||
},
|
||||
|
||||
formatAnnotationRect(rect = {}) {
|
||||
const normalized = this.clampAnnotationRect(rect);
|
||||
return `x=${normalized.x}, y=${normalized.y}, width=${normalized.width}, height=${normalized.height}`;
|
||||
},
|
||||
|
||||
redactAnnotationText(value) {
|
||||
return String(value || "")
|
||||
.replace(/(<input\b(?=[^>]*\btype=(["'])?password\2?)[^>]*?)\svalue=(["'])[\s\S]*?\3/giu, "$1 value=\"[redacted]\"")
|
||||
.replace(/\b(password|passcode|token|secret|value)=((["'])[\s\S]{1,240}?\3)/giu, "$1=\"[redacted]\"");
|
||||
},
|
||||
|
||||
formatAnnotationMetadata(metadata = {}) {
|
||||
const lines = [];
|
||||
const target = metadata.target || {};
|
||||
const selector = target.selector || metadata.selector || "";
|
||||
const summary = target.summary || metadata.summary || "";
|
||||
const dom = this.redactAnnotationText(target.dom || metadata.dom || "").slice(0, ANNOTATION_DOM_LIMIT);
|
||||
|
||||
if (selector) {
|
||||
lines.push(`Selector: ${selector}`);
|
||||
}
|
||||
if (target.tagName || target.role || target.id || target.name || target.classes) {
|
||||
lines.push([
|
||||
"Element:",
|
||||
target.tagName ? `<${String(target.tagName).toLowerCase()}>` : "",
|
||||
target.role ? `role=${target.role}` : "",
|
||||
target.id ? `id=${target.id}` : "",
|
||||
target.name ? `name=${target.name}` : "",
|
||||
target.classes ? `class=${target.classes}` : "",
|
||||
].filter(Boolean).join(" "));
|
||||
}
|
||||
if (summary) {
|
||||
lines.push(`Summary: ${summary}`);
|
||||
}
|
||||
if (Array.isArray(metadata.elements) && metadata.elements.length) {
|
||||
lines.push("Intersecting elements:");
|
||||
metadata.elements.slice(0, 8).forEach((element, index) => {
|
||||
const elementLabel = [
|
||||
`${index + 1}.`,
|
||||
element.tagName ? `<${String(element.tagName).toLowerCase()}>` : "",
|
||||
element.selector || "",
|
||||
element.summary || "",
|
||||
].filter(Boolean).join(" ");
|
||||
lines.push(elementLabel);
|
||||
});
|
||||
}
|
||||
if (dom) {
|
||||
lines.push(`DOM: ${dom}`);
|
||||
}
|
||||
return lines.join("\n");
|
||||
},
|
||||
|
||||
buildAnnotationsPrompt() {
|
||||
const annotations = this.visibleAnnotations();
|
||||
if (!annotations.length) return "";
|
||||
const lines = [
|
||||
"Browser annotations",
|
||||
`Page title: ${this.activeTitle}`,
|
||||
`Page URL: ${this.activeAnnotationUrl()}`,
|
||||
`Browser id: ${this.activeBrowserId}`,
|
||||
"",
|
||||
];
|
||||
annotations.forEach((annotation, index) => {
|
||||
lines.push(
|
||||
`Annotation ${index + 1}`,
|
||||
`Comment: ${annotation.comment}`,
|
||||
`Selection kind: ${annotation.kind}`,
|
||||
`Coordinates: ${this.formatAnnotationRect(annotation.rect)}`,
|
||||
);
|
||||
const metadata = this.formatAnnotationMetadata(annotation.metadata);
|
||||
if (metadata) {
|
||||
lines.push(metadata);
|
||||
}
|
||||
lines.push("");
|
||||
});
|
||||
return lines.join("\n").trim();
|
||||
},
|
||||
|
||||
draftAnnotationsToChat() {
|
||||
const prompt = this.buildAnnotationsPrompt();
|
||||
if (!prompt) return;
|
||||
const existingMessage = String(chatInputStore.message || "").trim();
|
||||
chatInputStore.message = existingMessage ? `${existingMessage}\n\n${prompt}` : prompt;
|
||||
chatInputStore.adjustTextareaHeight?.();
|
||||
chatInputStore.focus?.();
|
||||
this.clearVisibleAnnotations();
|
||||
this.toggleAnnotationMode(false);
|
||||
},
|
||||
|
||||
async sendAnnotationsToChat() {
|
||||
const prompt = this.buildAnnotationsPrompt();
|
||||
if (!prompt) return;
|
||||
chatInputStore.message = prompt;
|
||||
chatInputStore.adjustTextareaHeight?.();
|
||||
try {
|
||||
if (typeof chatInputStore.sendMessage === "function") {
|
||||
await chatInputStore.sendMessage();
|
||||
} else if (typeof globalThis.sendMessage === "function") {
|
||||
await globalThis.sendMessage();
|
||||
} else {
|
||||
chatInputStore.focus?.();
|
||||
return;
|
||||
}
|
||||
this.clearVisibleAnnotations();
|
||||
this.toggleAnnotationMode(false);
|
||||
} catch (error) {
|
||||
this.error = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
},
|
||||
|
||||
currentViewportSize() {
|
||||
const stage = this._stageElement;
|
||||
if (!stage) return null;
|
||||
|
|
@ -910,6 +1346,7 @@ const model = {
|
|||
},
|
||||
|
||||
async sendMouse(eventType, event) {
|
||||
if (this.annotating) return;
|
||||
if (!this.activeBrowserId || !event?.currentTarget) return;
|
||||
const pointer = this.pointerCoordinatesFor(event);
|
||||
if (!pointer) return;
|
||||
|
|
@ -960,6 +1397,7 @@ const model = {
|
|||
},
|
||||
|
||||
async sendKey(event) {
|
||||
if (this.annotating) return;
|
||||
if (!this.activeBrowserId) return;
|
||||
if (event.ctrlKey || event.metaKey || event.altKey) return;
|
||||
const editable = ["INPUT", "TEXTAREA", "SELECT"].includes(event.target?.tagName);
|
||||
|
|
@ -982,6 +1420,11 @@ const model = {
|
|||
this._surfaceMounted = false;
|
||||
this._surfaceSwitching = false;
|
||||
this.commandInFlight = false;
|
||||
this.annotating = false;
|
||||
this.annotationBusy = false;
|
||||
this.annotationError = "";
|
||||
this.cancelAnnotationDraft();
|
||||
this.cancelAnnotationSelection();
|
||||
if (this.contextId) {
|
||||
try {
|
||||
await websocket.emit("browser_viewer_unsubscribe", { context_id: this.contextId });
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import asyncio
|
|||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from types import ModuleType, SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -11,6 +11,73 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
|
||||
class _TestAgentContext:
|
||||
@staticmethod
|
||||
def get(context_id):
|
||||
return None
|
||||
|
||||
|
||||
class _TestResponse(SimpleNamespace):
|
||||
def __init__(self, message="", break_loop=False, **kwargs):
|
||||
super().__init__(message=message, break_loop=break_loop, **kwargs)
|
||||
|
||||
|
||||
class _TestTool:
|
||||
def __init__(
|
||||
self,
|
||||
agent=None,
|
||||
name="",
|
||||
method=None,
|
||||
args=None,
|
||||
message="",
|
||||
loop_data=None,
|
||||
**kwargs,
|
||||
):
|
||||
self.agent = agent
|
||||
self.name = name
|
||||
self.method = method
|
||||
self.args = args or {}
|
||||
self.message = message
|
||||
self.loop_data = loop_data
|
||||
|
||||
|
||||
class _TestWsHandler:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.emitted = []
|
||||
|
||||
async def emit_to(self, sid, event, data, correlation_id=None):
|
||||
self.emitted.append((sid, event, data, correlation_id))
|
||||
|
||||
|
||||
class _TestWsResult(dict):
|
||||
@staticmethod
|
||||
def error(code="", message="", correlation_id=None):
|
||||
return _TestWsResult(
|
||||
{
|
||||
"ok": False,
|
||||
"code": code,
|
||||
"error": message,
|
||||
"correlation_id": correlation_id,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
sys.modules.setdefault("agent", SimpleNamespace(AgentContext=_TestAgentContext))
|
||||
sys.modules.setdefault("helpers.tool", SimpleNamespace(Response=_TestResponse, Tool=_TestTool))
|
||||
sys.modules.setdefault("helpers.ws", SimpleNamespace(WsHandler=_TestWsHandler))
|
||||
sys.modules.setdefault("helpers.ws_manager", SimpleNamespace(WsResult=_TestWsResult))
|
||||
_model_config_stub = ModuleType("plugins._model_config.helpers.model_config")
|
||||
_model_config_stub.get_presets = lambda: []
|
||||
_model_config_stub.get_preset_by_name = lambda name: None
|
||||
_model_config_stub.get_chat_model_config = lambda agent=None: {}
|
||||
sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_config_stub)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend():
|
||||
return "asyncio"
|
||||
|
||||
from plugins._browser.helpers.config import (
|
||||
build_browser_launch_config,
|
||||
get_browser_main_model_summary,
|
||||
|
|
@ -63,6 +130,8 @@ def test_browser_config_normalizes_extension_paths(tmp_path):
|
|||
|
||||
assert config == {
|
||||
"extension_paths": [str(extension_dir)],
|
||||
"default_homepage": "about:blank",
|
||||
"autofocus_active_page": True,
|
||||
"model_preset": "",
|
||||
}
|
||||
|
||||
|
|
@ -246,7 +315,7 @@ def test_browser_extension_manager_uses_modern_chrome_prodversion(monkeypatch):
|
|||
|
||||
|
||||
def test_browser_extension_menu_exposes_agent_and_url_paths():
|
||||
html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(
|
||||
html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
skill = PROJECT_ROOT / "skills" / "a0-browser-ext" / "SKILL.md"
|
||||
|
|
@ -279,7 +348,7 @@ def test_browser_viewer_allows_slow_extension_startup():
|
|||
|
||||
|
||||
def test_browser_ui_spinners_have_browser_local_animation():
|
||||
main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(
|
||||
main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
config_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "config.html").read_text(
|
||||
|
|
@ -305,7 +374,7 @@ def test_browser_extension_settings_stay_user_facing():
|
|||
|
||||
|
||||
def test_browser_viewer_uses_tabs_for_session_switching():
|
||||
main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(
|
||||
main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
browser_store = (
|
||||
|
|
@ -329,7 +398,7 @@ def test_browser_viewer_uses_cdp_screencast_transport():
|
|||
ws_browser = (PROJECT_ROOT / "plugins" / "_browser" / "api" / "ws_browser.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(
|
||||
main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
runtime = (
|
||||
|
|
@ -339,12 +408,12 @@ def test_browser_viewer_uses_cdp_screencast_transport():
|
|||
PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert 'runtime.call("screenshot"' not in ws_browser
|
||||
assert 'runtime.call("screenshot"' in ws_browser
|
||||
assert "SCREENCAST_QUALITY = 92" in ws_browser
|
||||
assert "initial_viewport = self._viewport_from_data(data)" in ws_browser
|
||||
assert '"set_viewport"' in ws_browser
|
||||
assert "start_screencast" in ws_browser
|
||||
assert "read_screencast_frame" in ws_browser
|
||||
assert "pop_screencast_frame" in ws_browser
|
||||
assert "stop_screencast" in ws_browser
|
||||
assert '"Page.startScreencast"' in runtime
|
||||
assert '"Page.screencastFrame"' in runtime
|
||||
|
|
@ -360,12 +429,54 @@ def test_browser_viewer_uses_cdp_screencast_transport():
|
|||
assert "viewport_height: initialViewport?.height" in browser_store
|
||||
assert "this.frameState = data.state || null" not in browser_store
|
||||
assert "overflow: hidden;" in main_html
|
||||
assert "object-fit: fill;" not in main_html
|
||||
assert "height: auto;" in main_html
|
||||
assert "object-fit: fill;" in main_html
|
||||
assert "image-rendering: auto;" in main_html
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
def test_browser_annotate_mode_ui_and_prompt_hooks():
|
||||
panel_html = (
|
||||
PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html"
|
||||
).read_text(encoding="utf-8")
|
||||
browser_store = (
|
||||
PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "Annotate" in panel_html
|
||||
assert "Annotating" in panel_html
|
||||
assert "browser-annotation-layer" in panel_html
|
||||
assert "browser-annotation-tray" in panel_html
|
||||
assert "Draft to chat" in panel_html
|
||||
assert "Send now" in panel_html
|
||||
assert "@pointerdown.stop.prevent=\"$store.browserPage.startAnnotationSelection($event)\"" in panel_html
|
||||
assert "@keydown.window=\"$store.browserPage.handleKeydown($event)\"" in panel_html
|
||||
assert "annotationComments: []" in browser_store
|
||||
assert '"browser_viewer_annotation"' in browser_store
|
||||
assert 'event?.key === "." && (event.metaKey || event.ctrlKey)' in browser_store
|
||||
assert "Browser annotations" in browser_store
|
||||
assert "Comment:" in browser_store
|
||||
assert "Coordinates:" in browser_store
|
||||
assert "Selector:" in browser_store
|
||||
assert "DOM:" in browser_store
|
||||
assert "value=\\\"[redacted]\\\"" in browser_store
|
||||
|
||||
|
||||
def test_browser_runtime_and_content_helper_expose_annotation_target():
|
||||
runtime = (
|
||||
PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"
|
||||
).read_text(encoding="utf-8")
|
||||
helper = (
|
||||
PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-page-content.js"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "async def annotation_target" in runtime
|
||||
assert "globalThis.__spaceBrowserPageContent__.annotate(payload || null)" in runtime
|
||||
assert "function annotate(payload = null)" in helper
|
||||
assert "annotate," in helper
|
||||
assert "sanitizeAnnotationDom" in helper
|
||||
assert "password" in helper
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_browser_screencast_acknowledges_and_drops_stale_frames():
|
||||
first_image = (
|
||||
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"
|
||||
|
|
@ -526,7 +637,7 @@ def test_browser_save_plugin_config_does_not_restart_runtimes_for_preset_only(mo
|
|||
assert restarted == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.anyio
|
||||
async def test_browser_tool_dispatches_direct_actions(monkeypatch):
|
||||
calls = []
|
||||
|
||||
|
|
@ -558,7 +669,7 @@ async def test_browser_tool_dispatches_direct_actions(monkeypatch):
|
|||
assert calls == [("content", (1, None))]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.anyio
|
||||
async def test_browser_viewer_subscribe_unregisters_stream(monkeypatch):
|
||||
class FakeRuntime:
|
||||
def __init__(self) -> None:
|
||||
|
|
@ -608,7 +719,7 @@ async def test_browser_viewer_subscribe_unregisters_stream(monkeypatch):
|
|||
assert ("sid-1", "ctx") not in ws_browser_module.WsBrowser._streams
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.anyio
|
||||
async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):
|
||||
calls = []
|
||||
|
||||
|
|
@ -642,11 +753,14 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):
|
|||
"sid-1",
|
||||
)
|
||||
|
||||
assert result == {"state": {"ok": True, "method": "set_viewport", "args": (7, 1280, 720)}}
|
||||
assert result == {
|
||||
"state": {"ok": True, "method": "set_viewport", "args": (7, 1280, 720)},
|
||||
"snapshot": None,
|
||||
}
|
||||
assert calls == [("set_viewport", (7, 1280, 720), {})]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.anyio
|
||||
async def test_browser_viewer_wheel_input_dispatches_scroll(monkeypatch):
|
||||
calls = []
|
||||
|
||||
|
|
@ -682,10 +796,68 @@ async def test_browser_viewer_wheel_input_dispatches_scroll(monkeypatch):
|
|||
"sid-1",
|
||||
)
|
||||
|
||||
assert result == {"state": {"ok": True, "method": "wheel", "args": (3, 320.0, 480.0, 0.0, 640.0)}}
|
||||
assert result == {
|
||||
"state": {"ok": True, "method": "wheel", "args": (3, 320.0, 480.0, 0.0, 640.0)},
|
||||
"snapshot": None,
|
||||
}
|
||||
assert calls == [("wheel", (3, 320.0, 480.0, 0.0, 640.0), {})]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_browser_viewer_annotation_dispatches_runtime(monkeypatch):
|
||||
calls = []
|
||||
|
||||
class FakeRuntime:
|
||||
async def call(self, method, *args, **kwargs):
|
||||
calls.append((method, args, kwargs))
|
||||
return {
|
||||
"kind": "element",
|
||||
"point": {"x": 320, "y": 180},
|
||||
"target": {"tagName": "BUTTON", "selector": "#save"},
|
||||
}
|
||||
|
||||
async def fake_get_runtime(context_id, create=True):
|
||||
assert context_id == "ctx"
|
||||
assert create is False
|
||||
return FakeRuntime()
|
||||
|
||||
monkeypatch.setattr(ws_browser_module, "get_runtime", fake_get_runtime)
|
||||
|
||||
handler = ws_browser_module.WsBrowser(
|
||||
SimpleNamespace(),
|
||||
threading.RLock(),
|
||||
manager=None,
|
||||
)
|
||||
|
||||
payload = {
|
||||
"kind": "element",
|
||||
"point": {"x": 320, "y": 180},
|
||||
"viewport": {"width": 1280, "height": 720},
|
||||
}
|
||||
result = await handler.process(
|
||||
"browser_viewer_annotation",
|
||||
{
|
||||
"context_id": "ctx",
|
||||
"browser_id": 4,
|
||||
"viewer_id": "viewer-1",
|
||||
"payload": payload,
|
||||
},
|
||||
"sid-1",
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"annotation": {
|
||||
"kind": "element",
|
||||
"point": {"x": 320, "y": 180},
|
||||
"target": {"tagName": "BUTTON", "selector": "#save"},
|
||||
},
|
||||
"context_id": "ctx",
|
||||
"browser_id": 4,
|
||||
"viewer_id": "viewer-1",
|
||||
}
|
||||
assert calls == [("annotation_target", (4, payload), {})]
|
||||
|
||||
|
||||
def test_browser_cleanup_extensions_follow_extensible_path_layout():
|
||||
extension = __import__("helpers.extension", fromlist=["_get_extension_classes"])
|
||||
remove_classes = extension._get_extension_classes( # type: ignore[attr-defined]
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import tempfile
|
|||
import threading
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Iterator
|
||||
|
||||
import pytest
|
||||
|
|
@ -14,6 +15,15 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
|
||||
class _TestAgentContext:
|
||||
@staticmethod
|
||||
def get(context_id):
|
||||
return None
|
||||
|
||||
|
||||
sys.modules.setdefault("agent", SimpleNamespace(AgentContext=_TestAgentContext))
|
||||
|
||||
from api.load_webui_extensions import LoadWebuiExtensions
|
||||
|
||||
|
||||
|
|
@ -69,6 +79,11 @@ def _new_handler() -> LoadWebuiExtensions:
|
|||
return LoadWebuiExtensions(app, threading.RLock())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend():
|
||||
return "asyncio"
|
||||
|
||||
|
||||
def _assert_surface_anchor_in_template(surface: str, template_rel_path: str) -> None:
|
||||
template_path = PROJECT_ROOT / template_rel_path
|
||||
template_html = template_path.read_text(encoding="utf-8")
|
||||
|
|
@ -118,7 +133,7 @@ def _temporary_probe_plugin(surface: str) -> Iterator[tuple[str, str]]:
|
|||
cache.clear("*(plugins)*")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize(
|
||||
("surface", "template_rel_path"),
|
||||
SURFACE_SCENARIOS,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue