mirror of
https://github.com/agent0ai/agent-zero.git
synced 2026-05-17 04:01:13 +00:00
6d29268cbd
Rename high-impact skills to task-oriented names and move plugin-owned skills into their owning plugin folders.\n\nAlign renamed skill frontmatter with the official SKILL.md standard by keeping trigger language in name/description metadata, replacing the old create-skill wizard with build-skill, and updating browser, A0 connector, computer-use, CLI setup, and scheduler skill references.\n\nTighten the recurring cross-provider guidance gaps surfaced by the evidence sweeps: memory requests now avoid promptinclude-file routing, scheduler prompts distinguish cron schedules from planned ISO dates, document questions prefer document_query, skills_tool search/read_file usage is clearer, normal notifications set info/priority 10, and local/host text editors preserve patch intent.\n\nUpdate regression tests for the renamed skills, plugin ownership, prompt budget reality, and standard frontmatter shape.
2783 lines
97 KiB
JavaScript
2783 lines
97 KiB
JavaScript
import { createStore } from "/js/AlpineStore.js";
|
|
import { callJsonApi } from "/js/api.js";
|
|
import { getNamespacedClient } from "/js/websocket.js";
|
|
import { getContext, setContext } from "/index.js";
|
|
import { copyToClipboard } from "/components/messages/action-buttons/simple-action-buttons.js";
|
|
import { store as chatInputStore } from "/components/chat/input/input-store.js";
|
|
import { store as pluginSettingsStore } from "/components/plugins/plugin-settings-store.js";
|
|
import { store as chatsStore } from "/components/sidebar/chats/chats-store.js";
|
|
import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";
|
|
import { openLatest as openLatestSurface, registerUrlHandler } from "/js/surfaces.js";
|
|
|
|
const websocket = getNamespacedClient("/ws");
|
|
websocket.addHandlers(["ws_webui"]);
|
|
|
|
const EXTENSIONS_ROOT = "/a0/usr/_browser/extensions";
|
|
const BROWSER_SUBSCRIBE_TIMEOUT_MS = 60000;
|
|
const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;
|
|
const BROWSER_CONFIG_REFRESH_MS = 15000;
|
|
const VIEWPORT_SYNC_DEBOUNCE_MS = 220;
|
|
const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;
|
|
const CANVAS_VIEWPORT_SETTLE_MS = 520;
|
|
const SURFACE_VIEWPORT_STABLE_FRAMES = 4;
|
|
const SURFACE_VIEWPORT_MAX_WAIT_MS = 1200;
|
|
const FRAME_REJECT_SYNC_COOLDOWN_MS = 600;
|
|
const ANNOTATION_DRAG_THRESHOLD = 6;
|
|
const ANNOTATION_MAX_COMMENTS = 24;
|
|
const ANNOTATION_DOM_LIMIT = 1200;
|
|
const ANNOTATION_TRAY_MARGIN = 10;
|
|
const BROWSER_VISUAL_SHORTCUT_KEYS = new Set(["a", "c", "insert", "v", "x", "y", "z"]);
|
|
const LOCAL_EDITABLE_SELECTOR = "input, textarea, select, [contenteditable]";
|
|
|
|
function makeViewerToken() {
|
|
return globalThis.crypto?.randomUUID?.()
|
|
|| `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
}
|
|
|
|
function firstOk(response) {
|
|
const result = response?.results?.find((item) => item?.ok);
|
|
if (result) {
|
|
const data = result.data || {};
|
|
if (data.browser_error) {
|
|
throw new Error(data.browser_error.error || data.browser_error.code || "Browser request failed");
|
|
}
|
|
return data;
|
|
}
|
|
const error = response?.results?.find((item) => !item?.ok)?.error;
|
|
if (error) throw new Error(error.error || error.code || "Browser request failed");
|
|
return {};
|
|
}
|
|
|
|
function normalizeBool(value, fallback = true) {
|
|
if (value === undefined || value === null || value === "") return fallback;
|
|
if (typeof value === "boolean") return value;
|
|
if (typeof value === "number") return Boolean(value);
|
|
const normalized = String(value).trim().toLowerCase();
|
|
if (["1", "true", "yes", "on", "enabled"].includes(normalized)) return true;
|
|
if (["0", "false", "no", "off", "disabled"].includes(normalized)) return false;
|
|
return fallback;
|
|
}
|
|
|
|
function elementFromTarget(target) {
|
|
if (!target) return null;
|
|
if (target.nodeType === 1) return target;
|
|
return target.parentElement || null;
|
|
}
|
|
|
|
function isLocalEditableTarget(target) {
|
|
const element = elementFromTarget(target);
|
|
const editable = element?.closest?.(LOCAL_EDITABLE_SELECTOR);
|
|
if (!editable) return false;
|
|
if (editable.matches?.("input, textarea, select")) return true;
|
|
const value = String(editable.getAttribute?.("contenteditable") || "").trim().toLowerCase();
|
|
return ["", "true", "plaintext-only"].includes(value);
|
|
}
|
|
|
|
function nextAnimationFrame() {
|
|
return new Promise((resolve) => {
|
|
const schedule = globalThis.requestAnimationFrame || ((callback) => globalThis.setTimeout(callback, 16));
|
|
schedule(() => resolve());
|
|
});
|
|
}
|
|
|
|
function loadFrameDimensions(src) {
|
|
return new Promise((resolve) => {
|
|
if (!src) {
|
|
resolve(null);
|
|
return;
|
|
}
|
|
|
|
const image = new Image();
|
|
let settled = false;
|
|
const finish = (dimensions) => {
|
|
if (settled) return;
|
|
settled = true;
|
|
resolve(dimensions);
|
|
};
|
|
|
|
image.onload = () => finish({
|
|
width: image.naturalWidth || 0,
|
|
height: image.naturalHeight || 0,
|
|
});
|
|
image.onerror = () => finish(null);
|
|
image.src = src;
|
|
|
|
if (image.complete) {
|
|
image.onload();
|
|
}
|
|
});
|
|
}
|
|
|
|
const model = {
|
|
loading: true,
|
|
error: "",
|
|
status: null,
|
|
contextId: "",
|
|
browsers: [],
|
|
activeBrowserId: null,
|
|
activeBrowserContextId: "",
|
|
address: "",
|
|
frameSrc: "",
|
|
frameState: null,
|
|
annotating: false,
|
|
annotationComments: [],
|
|
annotationDraft: null,
|
|
annotationDraftText: "",
|
|
annotationDragRect: null,
|
|
annotationBusy: false,
|
|
annotationError: "",
|
|
annotationTrayPosition: null,
|
|
annotationTrayDragging: false,
|
|
connected: false,
|
|
switchingBrowserId: null,
|
|
commandInFlight: false,
|
|
addressFocused: false,
|
|
_frameOff: null,
|
|
_stateOff: null,
|
|
_lastFrameAt: 0,
|
|
_lastFrameDimensions: null,
|
|
_pendingFrameSrc: "",
|
|
_pendingFrameOptions: null,
|
|
_frameRenderHandle: null,
|
|
_frameRenderCancel: null,
|
|
_frameRenderSequence: 0,
|
|
_floatingCleanup: null,
|
|
_stageElement: null,
|
|
_stageResizeObserver: null,
|
|
_viewportSyncTimer: null,
|
|
_lastFrameRejectSyncAt: 0,
|
|
_lastViewportKey: "",
|
|
_lastViewport: null,
|
|
_annotationPointer: null,
|
|
_annotationTrayDrag: null,
|
|
_annotationSequence: 0,
|
|
_mode: "",
|
|
_surfaceMounted: false,
|
|
_surfaceSwitching: false,
|
|
_surfaceHandoff: false,
|
|
_surfaceHandoffTimer: null,
|
|
_surfaceOpenedAt: 0,
|
|
_surfaceOpenSequence: 0,
|
|
_canvasSurfaceReadySequence: 0,
|
|
_canvasFirstFrameAcceptedSequence: 0,
|
|
_canvasFirstFrameNudgeSequence: 0,
|
|
_openPromise: null,
|
|
_openSignature: "",
|
|
_connectSequence: 0,
|
|
_viewerToken: "",
|
|
_contextCreatePromise: null,
|
|
_lastSelectedContextId: "",
|
|
_sessionRefreshPromise: null,
|
|
_sessionRefreshContextId: "",
|
|
extensionMenuOpen: false,
|
|
extensionInstallUrl: "",
|
|
extensionActionLoading: false,
|
|
extensionActionMessage: "",
|
|
extensionActionError: "",
|
|
extensionsRoot: "",
|
|
extensionsList: [],
|
|
extensionsListLoading: false,
|
|
extensionToggleLoadingPath: "",
|
|
modelPreset: "",
|
|
modelPresetOptions: [],
|
|
mainModelSummary: "",
|
|
modelPresetSaving: false,
|
|
browserInstallExpected: false,
|
|
defaultHomepage: "about:blank",
|
|
autofocusActivePage: true,
|
|
_commandInFlightCount: 0,
|
|
_closingBrowserIds: {},
|
|
_configLoadedAt: 0,
|
|
_configRefreshPromise: null,
|
|
_clipboardFallbackText: "",
|
|
|
|
async refreshStatus() {
|
|
this.status = await callJsonApi("/plugins/_browser/status", {});
|
|
this.browserInstallExpected = Boolean(this.status?.playwright?.install_required);
|
|
},
|
|
|
|
async refreshExtensionsList() {
|
|
this.extensionsListLoading = true;
|
|
try {
|
|
const response = await callJsonApi("/plugins/_browser/extensions", {
|
|
action: "list",
|
|
context_id: this.resolveContextId() || this.contextId,
|
|
});
|
|
if (!response?.ok) {
|
|
throw new Error(response?.error || "Could not load browser extensions.");
|
|
}
|
|
this.applyExtensionPayload(response);
|
|
} catch (error) {
|
|
this.extensionActionError = error instanceof Error ? error.message : String(error);
|
|
} finally {
|
|
this.extensionsListLoading = false;
|
|
}
|
|
},
|
|
|
|
applyExtensionPayload(response = {}) {
|
|
this.extensionsRoot = response.root || EXTENSIONS_ROOT;
|
|
this.extensionsList = Array.isArray(response.extensions) ? response.extensions : [];
|
|
this.defaultHomepage = String(response.default_homepage || "about:blank").trim() || "about:blank";
|
|
this.autofocusActivePage = normalizeBool(response.autofocus_active_page, true);
|
|
this.modelPreset = String(response.model_preset || "");
|
|
this.mainModelSummary = String(response.main_model_summary || "");
|
|
this.modelPresetOptions = Array.isArray(response.model_preset_options)
|
|
? response.model_preset_options
|
|
: [];
|
|
this._configLoadedAt = Date.now();
|
|
},
|
|
|
|
async ensureBrowserConfigLoaded(force = false) {
|
|
if (!force && this._configLoadedAt && Date.now() - this._configLoadedAt < BROWSER_CONFIG_REFRESH_MS) {
|
|
return;
|
|
}
|
|
if (this._configRefreshPromise) {
|
|
await this._configRefreshPromise;
|
|
return;
|
|
}
|
|
this._configRefreshPromise = (async () => {
|
|
const response = await callJsonApi("/plugins/_browser/extensions", {
|
|
action: "list",
|
|
context_id: this.resolveContextId() || this.contextId,
|
|
});
|
|
if (!response?.ok) {
|
|
throw new Error(response?.error || "Could not load browser settings.");
|
|
}
|
|
this.applyExtensionPayload(response);
|
|
})();
|
|
try {
|
|
await this._configRefreshPromise;
|
|
} finally {
|
|
this._configRefreshPromise = null;
|
|
}
|
|
},
|
|
|
|
async allowsToolAutofocus() {
|
|
try {
|
|
await this.ensureBrowserConfigLoaded();
|
|
} catch (error) {
|
|
console.warn("Browser autofocus setting could not be loaded", error);
|
|
}
|
|
return this.autofocusActivePage !== false;
|
|
},
|
|
|
|
handleSelectedContextChange(contextId = "") {
|
|
const selectedContextId = this.normalizeContextId(contextId || this.resolveContextId());
|
|
if (selectedContextId === this._lastSelectedContextId) return;
|
|
this._lastSelectedContextId = selectedContextId;
|
|
if (!this._surfaceMounted) return;
|
|
void this.syncViewerToSelectedContext(selectedContextId);
|
|
},
|
|
|
|
async refreshBrowserSessions(contextId = "") {
|
|
const requestedContextId = this.normalizeContextId(contextId || this.resolveContextId());
|
|
if (this._sessionRefreshPromise) {
|
|
const inFlightContextId = this._sessionRefreshContextId;
|
|
await this._sessionRefreshPromise;
|
|
if (requestedContextId && requestedContextId !== inFlightContextId) {
|
|
return await this.refreshBrowserSessions(requestedContextId);
|
|
}
|
|
return;
|
|
}
|
|
this._sessionRefreshContextId = requestedContextId;
|
|
this._sessionRefreshPromise = (async () => {
|
|
const response = await websocket.request(
|
|
"browser_viewer_sessions",
|
|
{ context_id: requestedContextId },
|
|
{ timeoutMs: 10000 },
|
|
);
|
|
const data = firstOk(response);
|
|
this.applyBrowserListing(data.browsers || [], data.context_id || "", {
|
|
replaceAll: Boolean(data.all_browsers),
|
|
});
|
|
})();
|
|
try {
|
|
await this._sessionRefreshPromise;
|
|
} catch (error) {
|
|
console.warn("Browser session refresh failed", error);
|
|
} finally {
|
|
this._sessionRefreshPromise = null;
|
|
this._sessionRefreshContextId = "";
|
|
}
|
|
},
|
|
|
|
async syncViewerToSelectedContext(contextId = "") {
|
|
const selectedContextId = this.normalizeContextId(contextId || this.resolveContextId());
|
|
if (!selectedContextId) return;
|
|
await this.refreshBrowserSessions(selectedContextId);
|
|
if (!this._surfaceMounted || !this.isVisibleBrowserSurface()) return;
|
|
|
|
const targetBrowserId = this.firstBrowserInContext(selectedContextId)?.id || null;
|
|
if (
|
|
this.normalizeContextId(this.contextId) === selectedContextId
|
|
&& (
|
|
!targetBrowserId
|
|
|| this.sameBrowserTab(targetBrowserId, selectedContextId, this.activeBrowserId, this.activeBrowserContextId)
|
|
)
|
|
) {
|
|
return;
|
|
}
|
|
|
|
this.loading = true;
|
|
this.error = "";
|
|
this.resetRenderedFrame();
|
|
this.resetViewportTracking();
|
|
this._surfaceSwitching = Boolean(targetBrowserId);
|
|
this.switchingBrowserId = targetBrowserId;
|
|
try {
|
|
await this.connectViewer({
|
|
browserId: targetBrowserId,
|
|
contextId: selectedContextId,
|
|
initialViewport: this.currentViewportSize(),
|
|
});
|
|
await this.syncViewportAfterSurfaceOpen(this._surfaceOpenSequence);
|
|
} catch (error) {
|
|
this.error = error instanceof Error ? error.message : String(error);
|
|
} finally {
|
|
this.loading = false;
|
|
this._surfaceSwitching = false;
|
|
}
|
|
},
|
|
|
|
toggleExtensionsMenu() {
|
|
this.extensionMenuOpen = !this.extensionMenuOpen;
|
|
if (this.extensionMenuOpen) {
|
|
this.extensionActionMessage = "";
|
|
this.extensionActionError = "";
|
|
void this.refreshExtensionsList();
|
|
}
|
|
},
|
|
|
|
closeExtensionsMenu() {
|
|
this.extensionMenuOpen = false;
|
|
},
|
|
|
|
resolveContextId() {
|
|
const urlContext = new URLSearchParams(globalThis.location?.search || "").get("ctxid");
|
|
return getContext() || urlContext || chatsStore.selected || "";
|
|
},
|
|
|
|
normalizeContextId(contextId = "") {
|
|
return String(contextId || "").trim();
|
|
},
|
|
|
|
async ensureContextId() {
|
|
const existingContextId = String(this.resolveContextId() || "").trim();
|
|
if (existingContextId) {
|
|
this.contextId = existingContextId;
|
|
return existingContextId;
|
|
}
|
|
|
|
if (!this._contextCreatePromise) {
|
|
this._contextCreatePromise = this.createChatContextForBrowser();
|
|
}
|
|
|
|
try {
|
|
const contextId = await this._contextCreatePromise;
|
|
this.contextId = contextId;
|
|
return contextId;
|
|
} finally {
|
|
this._contextCreatePromise = null;
|
|
}
|
|
},
|
|
|
|
async contextIdForNewBrowser() {
|
|
return await this.ensureContextId();
|
|
},
|
|
|
|
async contextIdForActiveBrowser() {
|
|
const activeContextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId);
|
|
if (activeContextId) return activeContextId;
|
|
return await this.ensureContextId();
|
|
},
|
|
|
|
async createChatContextForBrowser() {
|
|
const response = await callJsonApi("/chat_create", {
|
|
current_context: this.resolveContextId() || "",
|
|
});
|
|
const selectedContextId = String(this.resolveContextId() || "").trim();
|
|
if (selectedContextId) return selectedContextId;
|
|
|
|
const contextId = String(response?.ctxid || "").trim();
|
|
if (!response?.ok || !contextId) {
|
|
throw new Error(response?.error || "Could not create a chat for Browser.");
|
|
}
|
|
|
|
setContext(contextId);
|
|
chatsStore.setSelected?.(contextId);
|
|
|
|
return contextId;
|
|
},
|
|
|
|
async openExtensionsSettings() {
|
|
if (!pluginSettingsStore?.openConfig) {
|
|
this.error = "Browser settings are unavailable.";
|
|
return;
|
|
}
|
|
try {
|
|
this.closeExtensionsMenu();
|
|
await pluginSettingsStore.openConfig("_browser");
|
|
await this.refreshAfterSettingsClose();
|
|
} catch (error) {
|
|
this.error = error instanceof Error ? error.message : String(error);
|
|
}
|
|
},
|
|
|
|
async refreshAfterSettingsClose() {
|
|
this.loading = true;
|
|
this.error = "";
|
|
try {
|
|
await this.refreshStatus();
|
|
await this.refreshExtensionsList();
|
|
this.connected = false;
|
|
this.browsers = [];
|
|
this.setActiveBrowserId(null);
|
|
this.address = "";
|
|
this.frameState = null;
|
|
this.frameSrc = "";
|
|
if (this.contextId) {
|
|
await this.connectViewer();
|
|
}
|
|
} finally {
|
|
this.loading = false;
|
|
}
|
|
},
|
|
|
|
createExtensionWithAgent() {
|
|
this._prefillAgentPrompt(
|
|
[
|
|
"Use the browser-extension-control skill to create a new Chrome extension for Agent Zero's Browser.",
|
|
"Start by asking me for the extension name, purpose, target websites, and required permissions.",
|
|
`Create it under ${this.extensionsRoot || EXTENSIONS_ROOT}/<extension-slug> and keep permissions minimal.`,
|
|
].join("\n")
|
|
);
|
|
},
|
|
|
|
askAgentInstallExtension() {
|
|
const url = String(this.extensionInstallUrl || "").trim();
|
|
const prompt = url
|
|
? [
|
|
"Use the browser-extension-control skill to review and optionally install this Chrome Web Store extension for Agent Zero's Browser.",
|
|
`Chrome Web Store URL or id: ${url}`,
|
|
"Explain the permissions and any sandbox risk before enabling it.",
|
|
].join("\n")
|
|
: [
|
|
"Use the browser-extension-control skill to help me install and review a Chrome Web Store extension for Agent Zero's Browser.",
|
|
"Ask me for the Chrome Web Store URL or extension id first.",
|
|
"Explain the permissions and any sandbox risk before enabling it.",
|
|
].join("\n");
|
|
this._prefillAgentPrompt(prompt);
|
|
},
|
|
|
|
async installExtensionFromUrl() {
|
|
const url = String(this.extensionInstallUrl || "").trim();
|
|
this.extensionActionMessage = "";
|
|
this.extensionActionError = "";
|
|
if (!url) {
|
|
this.extensionActionError = "Paste a Chrome Web Store URL or extension id first.";
|
|
return;
|
|
}
|
|
|
|
this.extensionActionLoading = true;
|
|
try {
|
|
const response = await callJsonApi("/plugins/_browser/extensions", {
|
|
action: "install_web_store",
|
|
context_id: this.resolveContextId() || this.contextId,
|
|
url,
|
|
});
|
|
if (!response?.ok) {
|
|
throw new Error(response?.error || "Install failed.");
|
|
}
|
|
this.applyExtensionPayload(response);
|
|
this.extensionInstallUrl = "";
|
|
this.extensionActionMessage = `Installed ${response.name || response.id}.`;
|
|
await this.refreshAfterSettingsClose();
|
|
} catch (error) {
|
|
this.extensionActionError = error instanceof Error ? error.message : String(error);
|
|
} finally {
|
|
this.extensionActionLoading = false;
|
|
}
|
|
},
|
|
|
|
async setExtensionEnabled(extension, enabled, input = null) {
|
|
const path = String(extension?.path || "");
|
|
if (!path) return;
|
|
const previous = Boolean(extension?.enabled);
|
|
this.extensionActionMessage = "";
|
|
this.extensionActionError = "";
|
|
this.extensionToggleLoadingPath = path;
|
|
try {
|
|
const response = await callJsonApi("/plugins/_browser/extensions", {
|
|
action: "set_extension_enabled",
|
|
context_id: this.resolveContextId() || this.contextId,
|
|
path,
|
|
enabled: Boolean(enabled),
|
|
});
|
|
if (!response?.ok) {
|
|
throw new Error(response?.error || "Could not update extension.");
|
|
}
|
|
this.applyExtensionPayload(response);
|
|
this.extensionActionMessage = `${enabled ? "Enabled" : "Disabled"} ${extension.name || "extension"}.`;
|
|
await this.refreshAfterSettingsClose();
|
|
} catch (error) {
|
|
if (input) input.checked = previous;
|
|
this.extensionActionError = error instanceof Error ? error.message : String(error);
|
|
} finally {
|
|
this.extensionToggleLoadingPath = "";
|
|
}
|
|
},
|
|
|
|
async setBrowserModelPreset(value) {
|
|
const presetName = String(value || "");
|
|
this.modelPreset = presetName;
|
|
this.extensionActionMessage = "";
|
|
this.extensionActionError = "";
|
|
this.modelPresetSaving = true;
|
|
try {
|
|
const response = await callJsonApi("/plugins/_browser/extensions", {
|
|
action: "set_model_preset",
|
|
context_id: this.resolveContextId() || this.contextId,
|
|
model_preset: presetName,
|
|
});
|
|
if (!response?.ok) {
|
|
throw new Error(response?.error || "Could not update browser model preset.");
|
|
}
|
|
this.applyExtensionPayload(response);
|
|
this.extensionActionMessage = "Browser model preset updated.";
|
|
} catch (error) {
|
|
this.extensionActionError = error instanceof Error ? error.message : String(error);
|
|
await this.refreshExtensionsList();
|
|
} finally {
|
|
this.modelPresetSaving = false;
|
|
}
|
|
},
|
|
|
|
modelPresetSummary() {
|
|
if (!this.modelPreset) {
|
|
return this.mainModelSummary ? `Using ${this.mainModelSummary}` : "Using Main Model";
|
|
}
|
|
const option = this.modelPresetOptions.find((preset) => preset?.name === this.modelPreset);
|
|
return option?.summary || option?.label || this.modelPreset;
|
|
},
|
|
|
|
hasExtensionInstallUrl() {
|
|
return Boolean(String(this.extensionInstallUrl || "").trim());
|
|
},
|
|
|
|
extensionAssistantActionLabel() {
|
|
return "Scan with A0";
|
|
},
|
|
|
|
extensionVersionLabel(extension) {
|
|
const version = String(extension?.version || "").trim();
|
|
return version ? `v${version}` : "Unpacked extension";
|
|
},
|
|
|
|
extensionOpenUrl(extension) {
|
|
return String(extension?.open_url || extension?.ui?.open_url || "").trim();
|
|
},
|
|
|
|
extensionHasOpenUi(extension) {
|
|
return Boolean(this.extensionOpenUrl(extension));
|
|
},
|
|
|
|
extensionOpenTitle(extension) {
|
|
const label = String(extension?.open_label || extension?.ui?.open_label || "Extension UI").trim();
|
|
const name = String(extension?.name || "extension").trim();
|
|
if (!extension?.enabled) {
|
|
return `Enable ${name} before opening ${label}.`;
|
|
}
|
|
return `Open ${label} for ${name}`;
|
|
},
|
|
|
|
async openExtensionUi(extension) {
|
|
const url = this.extensionOpenUrl(extension);
|
|
if (!url) return;
|
|
this.extensionActionMessage = "";
|
|
this.extensionActionError = "";
|
|
if (!extension?.enabled) {
|
|
this.extensionActionError = `Enable ${extension?.name || "this extension"} before opening it.`;
|
|
return;
|
|
}
|
|
this.closeExtensionsMenu();
|
|
await this.command("open", { url });
|
|
},
|
|
|
|
_prefillAgentPrompt(prompt) {
|
|
chatInputStore.message = prompt;
|
|
chatInputStore.adjustTextareaHeight?.();
|
|
chatInputStore.focus?.();
|
|
this.closeExtensionsMenu();
|
|
},
|
|
|
|
async onOpen(element = null, options = {}) {
|
|
const requestedBrowserId = this.normalizeBrowserId(
|
|
options.requestedBrowserId ?? options.browserId ?? options.browser_id,
|
|
);
|
|
const requestedContextId = this.normalizeContextId(
|
|
options.requestedContextId ?? options.contextId ?? options.context_id,
|
|
);
|
|
const nextMode = options?.mode === "modal" ? "modal" : "canvas";
|
|
if (nextMode === "canvas" && !this.isCanvasSurfaceVisible(element)) {
|
|
return;
|
|
}
|
|
const openSignature = this.surfaceOpenSignature(element, nextMode, requestedBrowserId, requestedContextId);
|
|
if (this._openPromise && this._openSignature === openSignature) {
|
|
return await this._openPromise;
|
|
}
|
|
const promise = this.openSurface(element, {
|
|
...options,
|
|
requestedBrowserId,
|
|
requestedContextId,
|
|
nextMode,
|
|
});
|
|
this._openPromise = promise;
|
|
this._openSignature = openSignature;
|
|
try {
|
|
return await promise;
|
|
} finally {
|
|
if (this._openPromise === promise) {
|
|
this._openPromise = null;
|
|
this._openSignature = "";
|
|
}
|
|
}
|
|
},
|
|
|
|
async openSurface(element = null, options = {}) {
|
|
this.loading = true;
|
|
this.error = "";
|
|
const requestedBrowserId = this.normalizeBrowserId(
|
|
options.requestedBrowserId ?? options.browserId ?? options.browser_id,
|
|
);
|
|
const requestedContextId = this.normalizeContextId(
|
|
options.requestedContextId ?? options.contextId ?? options.context_id,
|
|
);
|
|
let targetContextId = requestedContextId
|
|
|| this.contextIdForBrowserId(requestedBrowserId)
|
|
|| this.resolveContextId();
|
|
const nextMode = options?.nextMode || (options?.mode === "modal" ? "modal" : "canvas");
|
|
if (nextMode === "canvas" && !this.isCanvasSurfaceVisible(element)) {
|
|
this.loading = false;
|
|
return;
|
|
}
|
|
const surfaceSequence = this._surfaceOpenSequence + 1;
|
|
this._surfaceOpenSequence = surfaceSequence;
|
|
this.prepareSurfaceOpen(nextMode, requestedBrowserId, requestedContextId);
|
|
if (nextMode === "modal") {
|
|
this.setupFloatingModal(element);
|
|
} else {
|
|
this.setupCanvasSurface(element);
|
|
}
|
|
try {
|
|
if (!targetContextId && !this.activeBrowserContextId && !this.contextId) {
|
|
targetContextId = await this.ensureContextId();
|
|
}
|
|
if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;
|
|
await this.refreshStatus();
|
|
if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;
|
|
const viewport = await this.waitForSurfaceViewport({ sequence: surfaceSequence });
|
|
if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;
|
|
if (nextMode === "canvas" && !viewport) return;
|
|
this.resetRenderedFrameIfViewportChanged(viewport, requestedBrowserId, targetContextId);
|
|
await this.connectViewer({
|
|
browserId: requestedBrowserId,
|
|
contextId: targetContextId,
|
|
initialViewport: viewport,
|
|
});
|
|
if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;
|
|
await this.syncViewportAfterSurfaceOpen(surfaceSequence);
|
|
} catch (error) {
|
|
if (this.isCurrentSurfaceOpen(surfaceSequence)) {
|
|
this.error = error instanceof Error ? error.message : String(error);
|
|
}
|
|
} finally {
|
|
if (this.isCurrentSurfaceOpen(surfaceSequence)) {
|
|
this.loading = false;
|
|
if (this._mode === "canvas") {
|
|
this._canvasSurfaceReadySequence = surfaceSequence;
|
|
this.scheduleCanvasWidthNudgeAfterFirstFrame();
|
|
}
|
|
}
|
|
}
|
|
},
|
|
|
|
surfaceOpenSignature(element = null, mode = "", browserId = null, contextId = "") {
|
|
const root = element || globalThis.document?.querySelector(".browser-panel");
|
|
if (root && !root.__browserSurfaceOpenId) {
|
|
root.__browserSurfaceOpenId = makeViewerToken();
|
|
}
|
|
return [
|
|
mode || "",
|
|
this.normalizeContextId(contextId) || "",
|
|
this.normalizeBrowserId(browserId) || "",
|
|
root?.__browserSurfaceOpenId || "",
|
|
].join(":");
|
|
},
|
|
|
|
isCurrentSurfaceOpen(sequence) {
|
|
return this._surfaceMounted && sequence === this._surfaceOpenSequence;
|
|
},
|
|
|
|
beginSurfaceHandoff() {
|
|
if (this._surfaceHandoffTimer) {
|
|
globalThis.clearTimeout(this._surfaceHandoffTimer);
|
|
}
|
|
this._surfaceHandoff = true;
|
|
this._surfaceHandoffTimer = globalThis.setTimeout(() => {
|
|
this._surfaceHandoff = false;
|
|
this._surfaceHandoffTimer = null;
|
|
}, 3000);
|
|
},
|
|
|
|
finishSurfaceHandoff() {
|
|
if (this._surfaceHandoffTimer) {
|
|
globalThis.clearTimeout(this._surfaceHandoffTimer);
|
|
this._surfaceHandoffTimer = null;
|
|
}
|
|
this._surfaceHandoff = false;
|
|
},
|
|
|
|
cancelSurfaceHandoff() {
|
|
if (this._surfaceHandoffTimer) {
|
|
globalThis.clearTimeout(this._surfaceHandoffTimer);
|
|
this._surfaceHandoffTimer = null;
|
|
}
|
|
this._surfaceHandoff = false;
|
|
},
|
|
|
|
releaseSurfaceBindings() {
|
|
this._floatingCleanup?.();
|
|
this._floatingCleanup = null;
|
|
this._stageResizeObserver?.disconnect?.();
|
|
this._stageResizeObserver = null;
|
|
this._stageElement = null;
|
|
},
|
|
|
|
isCanvasSurfaceVisible(element = null) {
|
|
const root = element
|
|
|| globalThis.document?.querySelector?.(".browser-canvas-surface .browser-panel")
|
|
|| globalThis.document?.querySelector?.(".browser-panel");
|
|
if (!root?.isConnected) return false;
|
|
const surface = root.closest?.(".browser-canvas-surface");
|
|
const stage = root.querySelector?.(".browser-stage") || root;
|
|
const surfaceStyle = surface ? globalThis.getComputedStyle?.(surface) : null;
|
|
const rootStyle = globalThis.getComputedStyle?.(root);
|
|
if (surfaceStyle?.display === "none" || surfaceStyle?.visibility === "hidden") return false;
|
|
if (rootStyle?.display === "none" || rootStyle?.visibility === "hidden") return false;
|
|
const rect = stage.getBoundingClientRect?.();
|
|
return Boolean(rect && Math.round(rect.width || 0) >= 80 && Math.round(rect.height || 0) >= 80);
|
|
},
|
|
|
|
isVisibleBrowserSurface() {
|
|
if (!this._surfaceMounted) return false;
|
|
if (this._mode === "canvas") {
|
|
return Boolean(rightCanvasStore?.isSurfaceVisible?.("browser"))
|
|
&& this.isCanvasSurfaceVisible(globalThis.document?.querySelector?.(".browser-canvas-surface .browser-panel"));
|
|
}
|
|
|
|
const panel = globalThis.document?.querySelector?.(".modal .browser-panel");
|
|
const modal = panel?.closest?.(".modal");
|
|
if (!panel || !modal) return false;
|
|
if (modal.classList.contains("modal-surface-parked") || modal.classList.contains("surface-modal-parked")) {
|
|
return false;
|
|
}
|
|
const panelStyle = globalThis.getComputedStyle?.(panel);
|
|
if (panelStyle?.display === "none" || panelStyle?.visibility === "hidden") return false;
|
|
const rect = panel.getBoundingClientRect?.();
|
|
return Boolean(rect && Math.round(rect.width || 0) >= 80 && Math.round(rect.height || 0) >= 80);
|
|
},
|
|
|
|
prepareSurfaceOpen(nextMode, requestedBrowserId = null, requestedContextId = "") {
|
|
const targetBrowserId = requestedBrowserId || this.activeBrowserId || this.firstBrowserId(requestedContextId);
|
|
const targetContextId = this.normalizeContextId(
|
|
requestedContextId
|
|
|| this.contextIdForBrowserId(targetBrowserId)
|
|
|| this.resolveContextId()
|
|
|| this.activeBrowserContextId
|
|
|| this.contextId,
|
|
);
|
|
const targetChanged = Boolean(
|
|
targetBrowserId
|
|
&& this.activeBrowserId
|
|
&& !this.sameBrowserTab(targetBrowserId, targetContextId, this.activeBrowserId, this.activeBrowserContextId),
|
|
);
|
|
this._mode = nextMode;
|
|
this._surfaceMounted = true;
|
|
this._surfaceOpenedAt = Date.now();
|
|
this._lastViewportKey = "";
|
|
if (this.frameSrc && !targetChanged) {
|
|
this._surfaceSwitching = false;
|
|
this.switchingBrowserId = null;
|
|
return;
|
|
}
|
|
if (!targetBrowserId) return;
|
|
|
|
this.resetRenderedFrame();
|
|
this.resetViewportTracking();
|
|
this._surfaceSwitching = Boolean(targetBrowserId);
|
|
this.switchingBrowserId = targetBrowserId;
|
|
},
|
|
|
|
resetViewportTracking() {
|
|
this._lastViewportKey = "";
|
|
this._lastViewport = null;
|
|
},
|
|
|
|
resetRenderedFrame() {
|
|
this.cancelFrameRender();
|
|
this.frameSrc = "";
|
|
this._lastFrameDimensions = null;
|
|
this._lastFrameAt = 0;
|
|
},
|
|
|
|
resetRenderedFrameIfViewportChanged(viewport = null, requestedBrowserId = null, requestedContextId = "") {
|
|
if (!viewport || !this.frameSrc || !this._lastViewport) return;
|
|
const targetBrowserId = requestedBrowserId || this.activeBrowserId || this.firstBrowserId();
|
|
const targetContextId = this.normalizeContextId(requestedContextId || this.contextIdForBrowserId(targetBrowserId) || this.activeBrowserContextId);
|
|
if (!this.sameBrowserTab(this._lastViewport.browserId, this._lastViewport.contextId, targetBrowserId, targetContextId)) return;
|
|
const changed = Math.abs(this._lastViewport.width - viewport.width) > VIEWPORT_SYNC_SIZE_TOLERANCE
|
|
|| Math.abs(this._lastViewport.height - viewport.height) > VIEWPORT_SYNC_SIZE_TOLERANCE;
|
|
if (!changed) return;
|
|
|
|
this.cancelFrameRender();
|
|
this.resetViewportTracking();
|
|
this._surfaceSwitching = true;
|
|
this.switchingBrowserId = targetBrowserId;
|
|
},
|
|
|
|
async waitForSurfaceViewport(options = {}) {
|
|
const sequence = Number(options.sequence || 0);
|
|
const startedAt = Date.now();
|
|
let lastKey = "";
|
|
let stableCount = 0;
|
|
while (Date.now() - startedAt <= SURFACE_VIEWPORT_MAX_WAIT_MS) {
|
|
await nextAnimationFrame();
|
|
if (sequence && !this.isCurrentSurfaceOpen(sequence)) {
|
|
return null;
|
|
}
|
|
const viewport = this.surfaceViewportMeasurement();
|
|
if (!viewport) continue;
|
|
const key = `${viewport.rawWidth}x${viewport.rawHeight}`;
|
|
if (key === lastKey) {
|
|
stableCount += 1;
|
|
const canvasSettled = this._mode !== "canvas"
|
|
|| !this._surfaceOpenedAt
|
|
|| Date.now() - this._surfaceOpenedAt >= CANVAS_VIEWPORT_SETTLE_MS;
|
|
if (canvasSettled && stableCount >= SURFACE_VIEWPORT_STABLE_FRAMES) {
|
|
return { width: viewport.width, height: viewport.height };
|
|
}
|
|
} else {
|
|
stableCount = 0;
|
|
lastKey = key;
|
|
}
|
|
}
|
|
const fallbackViewport = this.currentViewportSize();
|
|
return fallbackViewport;
|
|
},
|
|
|
|
async syncViewportAfterSurfaceOpen(sequence = this._surfaceOpenSequence) {
|
|
if (!this.connected || !this.activeBrowserId) return;
|
|
await this.waitForSurfaceViewport({ sequence });
|
|
if (!this.isCurrentSurfaceOpen(sequence)) {
|
|
return;
|
|
}
|
|
await this.syncViewport(true, { restartStream: this._mode === "canvas" });
|
|
if (this._mode !== "canvas") return;
|
|
this.scheduleViewportSyncForSurface(sequence, 240);
|
|
this.scheduleViewportSyncForSurface(sequence, 520);
|
|
},
|
|
|
|
scheduleViewportSyncForSurface(sequence, delayMs = 0) {
|
|
globalThis.setTimeout?.(() => {
|
|
if (!this.isCurrentSurfaceOpen(sequence) || this._mode !== "canvas") {
|
|
return;
|
|
}
|
|
this.queueViewportSync(true);
|
|
}, delayMs);
|
|
},
|
|
|
|
async connectViewer(options = {}) {
|
|
let contextId = "";
|
|
const requestedBrowserId = this.normalizeBrowserId(options.browserId ?? this.activeBrowserId);
|
|
const requestedContextId = this.normalizeContextId(
|
|
options.contextId
|
|
?? options.context_id
|
|
?? this.contextIdForBrowserId(requestedBrowserId)
|
|
?? this.activeBrowserContextId
|
|
?? this.contextId
|
|
);
|
|
try {
|
|
contextId = requestedContextId || await this.ensureContextId();
|
|
} catch (error) {
|
|
this.connected = false;
|
|
this.switchingBrowserId = null;
|
|
this._surfaceSwitching = false;
|
|
throw error;
|
|
}
|
|
if (!contextId) {
|
|
this.connected = false;
|
|
this.error = "Could not create a chat for Browser.";
|
|
this.switchingBrowserId = null;
|
|
this._surfaceSwitching = false;
|
|
return;
|
|
}
|
|
const previousContextId = this.normalizeContextId(this.contextId);
|
|
if (previousContextId && previousContextId !== contextId) {
|
|
try {
|
|
await websocket.emit("browser_viewer_unsubscribe", { context_id: previousContextId });
|
|
} catch {}
|
|
}
|
|
this.contextId = contextId;
|
|
const sequence = this._connectSequence + 1;
|
|
const viewerToken = makeViewerToken();
|
|
this._connectSequence = sequence;
|
|
this._viewerToken = viewerToken;
|
|
this.error = "";
|
|
await this._bindSocketEvents();
|
|
if (sequence !== this._connectSequence || viewerToken !== this._viewerToken) {
|
|
return;
|
|
}
|
|
const initialViewport = options.initialViewport || this.currentViewportSize();
|
|
let response;
|
|
try {
|
|
response = await websocket.request(
|
|
"browser_viewer_subscribe",
|
|
{
|
|
context_id: contextId,
|
|
browser_id: requestedBrowserId,
|
|
viewer_id: viewerToken,
|
|
create_browser: Boolean(options.createBrowser || options.create_browser),
|
|
viewport_width: initialViewport?.width,
|
|
viewport_height: initialViewport?.height,
|
|
},
|
|
{
|
|
timeoutMs: this.browserInstallExpected
|
|
? BROWSER_FIRST_INSTALL_TIMEOUT_MS
|
|
: BROWSER_SUBSCRIBE_TIMEOUT_MS,
|
|
},
|
|
);
|
|
} catch (error) {
|
|
if (sequence === this._connectSequence && viewerToken === this._viewerToken) {
|
|
this.switchingBrowserId = null;
|
|
this._surfaceSwitching = false;
|
|
throw error;
|
|
}
|
|
return;
|
|
}
|
|
if (sequence !== this._connectSequence || viewerToken !== this._viewerToken) {
|
|
return;
|
|
}
|
|
const data = firstOk(response);
|
|
this.applyBrowserListing(data.browsers || [], contextId, { replaceAll: Boolean(data.all_browsers) });
|
|
this.setActiveBrowserId(
|
|
data.active_browser_id || requestedBrowserId || this.activeBrowserId || null,
|
|
data.active_browser_context_id || contextId,
|
|
);
|
|
this.applySnapshot(data.snapshot);
|
|
this.connected = true;
|
|
this.browserInstallExpected = false;
|
|
},
|
|
|
|
async _bindSocketEvents() {
|
|
if (!this._frameOff) {
|
|
const frameHandler = ({ data }) => {
|
|
if (data?.context_id !== this.contextId) return;
|
|
if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
|
|
const incomingContextId = this.normalizeContextId(data.context_id || this.contextId);
|
|
const incomingBrowserId = this.normalizeBrowserId(data.browser_id || data.state?.id);
|
|
this.applyBrowserListing(data.browsers || [], incomingContextId, { replaceContext: true });
|
|
if (incomingBrowserId && !this.activeBrowserId) {
|
|
this.setActiveBrowserId(incomingBrowserId, incomingContextId);
|
|
}
|
|
if (
|
|
incomingBrowserId
|
|
&& this.activeBrowserId
|
|
&& !this.sameBrowserTab(incomingBrowserId, incomingContextId, this.activeBrowserId, this.activeBrowserContextId)
|
|
) {
|
|
return;
|
|
}
|
|
if (data.state) {
|
|
this.frameState = data.state;
|
|
}
|
|
if (!this.addressFocused && data.state?.currentUrl) {
|
|
this.address = data.state.currentUrl;
|
|
}
|
|
if (data.image) {
|
|
const frameBrowserId = incomingBrowserId || this.activeBrowserId;
|
|
this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, {
|
|
browserId: frameBrowserId,
|
|
contextId: incomingContextId,
|
|
onAccepted: () => {
|
|
if (
|
|
this.sameBrowserId(this.switchingBrowserId, frameBrowserId)
|
|
&& this.normalizeContextId(this.activeBrowserContextId) === incomingContextId
|
|
) {
|
|
this.switchingBrowserId = null;
|
|
}
|
|
this._surfaceSwitching = false;
|
|
},
|
|
});
|
|
} else if (!data.state) {
|
|
this.cancelFrameRender();
|
|
this.frameSrc = "";
|
|
}
|
|
if (!data.image && !data.state) {
|
|
if (!this.activeBrowserId) {
|
|
this.setActiveBrowserId(null, "");
|
|
this.frameState = null;
|
|
this.frameSrc = "";
|
|
}
|
|
}
|
|
this._lastFrameAt = Date.now();
|
|
};
|
|
await websocket.on("browser_viewer_frame", frameHandler);
|
|
this._frameOff = () => websocket.off("browser_viewer_frame", frameHandler);
|
|
}
|
|
if (!this._stateOff) {
|
|
const stateHandler = ({ data }) => {
|
|
if (data?.context_id !== this.contextId) return;
|
|
if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
|
|
const commandContextId = this.normalizeContextId(data.active_browser_context_id || data.context_id || this.contextId);
|
|
this.applyBrowserListing(data.browsers || [], commandContextId, { replaceAll: Boolean(data.all_browsers) });
|
|
const command = String(data.command || "").toLowerCase();
|
|
const commandBrowserId = this.normalizeBrowserId(data.browser_id);
|
|
const result = data.result || {};
|
|
const resultState = this.stateFromCommandResult(result);
|
|
const resultContextId = this.normalizeContextId(
|
|
result.context_id
|
|
|| result.state?.context_id
|
|
|| commandContextId
|
|
);
|
|
const preferredBrowserId = this.normalizeBrowserId(
|
|
result.id
|
|
|| result.state?.id
|
|
|| data.last_interacted_browser_id
|
|
|| this.activeBrowserId
|
|
|| this.firstBrowserId(resultContextId)
|
|
);
|
|
if (
|
|
!this.activeBrowserId
|
|
|| command === "open"
|
|
|| command === "close"
|
|
|| this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)
|
|
) {
|
|
this.setActiveBrowserId(preferredBrowserId, resultContextId);
|
|
}
|
|
this.applyActiveFrameState(resultState || this.browserById(this.activeBrowserId, this.activeBrowserContextId));
|
|
this.applySnapshot(data.snapshot);
|
|
};
|
|
await websocket.on("browser_viewer_state", stateHandler);
|
|
this._stateOff = () => websocket.off("browser_viewer_state", stateHandler);
|
|
}
|
|
},
|
|
|
|
queueFrameRender(frameSrc, options = {}) {
|
|
this._pendingFrameSrc = frameSrc;
|
|
this._pendingFrameOptions = options || null;
|
|
if (this._frameRenderHandle) return;
|
|
const schedule = globalThis.requestAnimationFrame?.bind(globalThis);
|
|
if (schedule) {
|
|
this._frameRenderCancel = globalThis.cancelAnimationFrame?.bind(globalThis) || null;
|
|
this._frameRenderHandle = schedule(() => this.flushFrameRender());
|
|
return;
|
|
}
|
|
this._frameRenderCancel = globalThis.clearTimeout?.bind(globalThis) || null;
|
|
this._frameRenderHandle = globalThis.setTimeout(() => this.flushFrameRender(), 16);
|
|
},
|
|
|
|
flushFrameRender() {
|
|
this._frameRenderHandle = null;
|
|
this._frameRenderCancel = null;
|
|
const frameSrc = this._pendingFrameSrc || "";
|
|
const options = this._pendingFrameOptions || {};
|
|
this._pendingFrameSrc = "";
|
|
this._pendingFrameOptions = null;
|
|
const sequence = this._frameRenderSequence + 1;
|
|
const surfaceSequence = this._surfaceOpenSequence;
|
|
this._frameRenderSequence = sequence;
|
|
void this.renderDecodedFrame(frameSrc, options, sequence, surfaceSequence);
|
|
},
|
|
|
|
async renderDecodedFrame(frameSrc, options = {}, sequence = 0, surfaceSequence = this._surfaceOpenSequence) {
|
|
if (!frameSrc) {
|
|
if (sequence === this._frameRenderSequence) {
|
|
this.frameSrc = "";
|
|
}
|
|
return;
|
|
}
|
|
const dimensions = await loadFrameDimensions(frameSrc);
|
|
if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) {
|
|
return;
|
|
}
|
|
const viewport = this.currentViewportSize() || this._lastViewport;
|
|
if (!this.frameMatchesViewport(dimensions, viewport)) {
|
|
this.requestViewportSyncAfterRejectedFrame();
|
|
if (!this.shouldAcceptMismatchedFrame(dimensions)) {
|
|
return;
|
|
}
|
|
}
|
|
this.frameSrc = frameSrc;
|
|
this._lastFrameDimensions = dimensions;
|
|
this._lastFrameAt = Date.now();
|
|
options?.onAccepted?.();
|
|
this._canvasFirstFrameAcceptedSequence = surfaceSequence;
|
|
this.scheduleCanvasWidthNudgeAfterFirstFrame();
|
|
},
|
|
|
|
shouldAcceptMismatchedFrame(dimensions = null) {
|
|
return Boolean(
|
|
dimensions?.width
|
|
&& dimensions?.height
|
|
&& (!this.frameSrc || this._surfaceSwitching || this.isSwitchingBrowser())
|
|
);
|
|
},
|
|
|
|
scheduleCanvasWidthNudgeAfterFirstFrame() {
|
|
const surfaceSequence = this._surfaceOpenSequence;
|
|
if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {
|
|
return;
|
|
}
|
|
if (this._canvasFirstFrameNudgeSequence === surfaceSequence) {
|
|
return;
|
|
}
|
|
if (
|
|
this._canvasSurfaceReadySequence !== surfaceSequence
|
|
|| this._canvasFirstFrameAcceptedSequence !== surfaceSequence
|
|
) {
|
|
return;
|
|
}
|
|
this._canvasFirstFrameNudgeSequence = surfaceSequence;
|
|
|
|
void (async () => {
|
|
await nextAnimationFrame();
|
|
await nextAnimationFrame();
|
|
if (!this.isCurrentSurfaceOpen(surfaceSequence) || this._mode !== "canvas") {
|
|
return;
|
|
}
|
|
this.forceRightCanvasWidthNudge();
|
|
})();
|
|
},
|
|
|
|
forceRightCanvasWidthNudge() {
|
|
const canvas = rightCanvasStore;
|
|
if (!canvas || canvas.isMobileMode || !canvas.isOpen || canvas.activeSurfaceId !== "browser") {
|
|
return;
|
|
}
|
|
|
|
const currentWidth = Number(canvas.width || 0);
|
|
if (!Number.isFinite(currentWidth) || currentWidth <= 0) {
|
|
return;
|
|
}
|
|
const maxWidth = Number(canvas.maxWidth?.() || currentWidth);
|
|
const minWidth = Number(canvas.minWidth || 420);
|
|
const direction = currentWidth < maxWidth ? 1 : -1;
|
|
const nudgedWidth = currentWidth + direction;
|
|
if (nudgedWidth < minWidth || nudgedWidth > maxWidth || nudgedWidth === currentWidth) {
|
|
return;
|
|
}
|
|
|
|
canvas.setWidth?.(nudgedWidth, { persist: false });
|
|
this.queueViewportSync(true);
|
|
},
|
|
|
|
frameMatchesViewport(dimensions = null, viewport = null) {
|
|
if (!dimensions?.width || !dimensions?.height || !viewport?.width || !viewport?.height) {
|
|
return false;
|
|
}
|
|
return Math.abs(Number(dimensions.width) - Number(viewport.width)) <= VIEWPORT_SYNC_SIZE_TOLERANCE
|
|
&& Math.abs(Number(dimensions.height) - Number(viewport.height)) <= VIEWPORT_SYNC_SIZE_TOLERANCE;
|
|
},
|
|
|
|
requestViewportSyncAfterRejectedFrame() {
|
|
const now = Date.now();
|
|
if (now - this._lastFrameRejectSyncAt < FRAME_REJECT_SYNC_COOLDOWN_MS) {
|
|
return;
|
|
}
|
|
this._lastFrameRejectSyncAt = now;
|
|
this.queueViewportSync(true);
|
|
},
|
|
|
|
clearRenderedFrameIfViewportChanged() {
|
|
const viewport = this.currentViewportSize();
|
|
if (!this.frameSrc || !this._lastFrameDimensions || !viewport) return;
|
|
if (this.frameMatchesViewport(this._lastFrameDimensions, viewport)) return;
|
|
this.cancelFrameRender();
|
|
this.resetViewportTracking();
|
|
if (this.activeBrowserId) {
|
|
this._surfaceSwitching = true;
|
|
this.switchingBrowserId = this.activeBrowserId;
|
|
}
|
|
},
|
|
|
|
cancelFrameRender() {
|
|
if (this._frameRenderHandle && this._frameRenderCancel) {
|
|
this._frameRenderCancel(this._frameRenderHandle);
|
|
}
|
|
this._frameRenderHandle = null;
|
|
this._frameRenderCancel = null;
|
|
this._pendingFrameSrc = "";
|
|
this._pendingFrameOptions = null;
|
|
this._frameRenderSequence += 1;
|
|
},
|
|
|
|
beginCommand() {
|
|
this._commandInFlightCount += 1;
|
|
this.commandInFlight = true;
|
|
},
|
|
|
|
finishCommand() {
|
|
this._commandInFlightCount = Math.max(0, this._commandInFlightCount - 1);
|
|
this.commandInFlight = this._commandInFlightCount > 0;
|
|
},
|
|
|
|
async command(command, extra = {}) {
|
|
this.error = "";
|
|
this.annotationError = "";
|
|
this.beginCommand();
|
|
const previousActiveBrowserId = this.activeBrowserId;
|
|
const previousActiveContextId = this.activeBrowserContextId;
|
|
const commandName = String(command || "").toLowerCase();
|
|
try {
|
|
const targetContextId = commandName === "open"
|
|
? await this.contextIdForNewBrowser()
|
|
: this.normalizeContextId(extra.context_id || extra.contextId) || await this.contextIdForActiveBrowser();
|
|
const targetBrowserId = this.normalizeBrowserId(extra.browser_id ?? this.activeBrowserId);
|
|
this.contextId = targetContextId;
|
|
const response = await websocket.request(
|
|
"browser_viewer_command",
|
|
{
|
|
...extra,
|
|
context_id: targetContextId,
|
|
browser_id: targetBrowserId,
|
|
viewer_id: this._viewerToken,
|
|
command,
|
|
},
|
|
{ timeoutMs: 20000 },
|
|
);
|
|
const data = firstOk(response);
|
|
this.applyBrowserListing(data.browsers || [], targetContextId, { replaceAll: Boolean(data.all_browsers) });
|
|
const result = data.result || {};
|
|
const resultContextId = this.normalizeContextId(
|
|
result.context_id
|
|
|| result.state?.context_id
|
|
|| data.active_browser_context_id
|
|
|| targetContextId
|
|
);
|
|
const preferredBrowser = this.browserById(
|
|
result.id
|
|
|| result.state?.id
|
|
|| result.last_interacted_browser_id
|
|
|| data.last_interacted_browser_id,
|
|
resultContextId,
|
|
)
|
|
|| this.browserById(this.activeBrowserId, this.activeBrowserContextId)
|
|
|| this.firstBrowser(resultContextId)
|
|
|| this.firstBrowser();
|
|
this.setActiveBrowserId(preferredBrowser?.id || null, preferredBrowser?.context_id || resultContextId);
|
|
this.applyActiveFrameState(
|
|
this.stateFromCommandResult(result)
|
|
|| this.browserById(this.activeBrowserId, this.activeBrowserContextId)
|
|
);
|
|
if (!this.activeBrowserId) {
|
|
this.frameState = null;
|
|
this.frameSrc = "";
|
|
}
|
|
if (result.state?.currentUrl || result.currentUrl) {
|
|
this.address = result.state?.currentUrl || result.currentUrl;
|
|
}
|
|
this.applySnapshot(data.snapshot);
|
|
if (["navigate", "back", "forward", "reload", "close"].includes(commandName)) {
|
|
this.clearAnnotationsForBrowser(previousActiveBrowserId, null, previousActiveContextId);
|
|
this.cancelAnnotationDraft();
|
|
}
|
|
const activeChanged = this.activeBrowserId
|
|
&& !this.sameBrowserTab(
|
|
this.activeBrowserId,
|
|
this.activeBrowserContextId,
|
|
previousActiveBrowserId,
|
|
previousActiveContextId,
|
|
);
|
|
if ((commandName === "open" || commandName === "close" || activeChanged) && this.contextId && this.activeBrowserId) {
|
|
await this.connectViewer({
|
|
browserId: this.activeBrowserId,
|
|
contextId: this.activeBrowserContextId,
|
|
});
|
|
} else if (["navigate", "back", "forward", "reload"].includes(commandName)) {
|
|
await this.restartCanvasStreamAfterPageChange();
|
|
}
|
|
} catch (error) {
|
|
this.error = error instanceof Error ? error.message : String(error);
|
|
} finally {
|
|
this.finishCommand();
|
|
}
|
|
},
|
|
|
|
async restartCanvasStreamAfterPageChange() {
|
|
const surfaceSequence = this._surfaceOpenSequence;
|
|
if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {
|
|
return;
|
|
}
|
|
await this.waitForSurfaceViewport({ sequence: surfaceSequence });
|
|
if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {
|
|
return;
|
|
}
|
|
await this.syncViewport(true, { restartStream: true });
|
|
},
|
|
|
|
async go() {
|
|
const url = String(this.address || "").trim();
|
|
if (!url) return;
|
|
this.addressFocused = false;
|
|
globalThis.document?.activeElement?.blur?.();
|
|
if (this.activeBrowserId) {
|
|
await this.command("navigate", { url });
|
|
} else {
|
|
await this.command("open", { url });
|
|
}
|
|
},
|
|
|
|
async openUrlIntent(url = "", options = {}) {
|
|
if (this._openPromise) {
|
|
try {
|
|
await this._openPromise;
|
|
} catch {}
|
|
}
|
|
if (!this._surfaceMounted) return false;
|
|
const targetUrl = String(url || "").trim();
|
|
if (targetUrl) {
|
|
await this.command("open", {
|
|
url: targetUrl,
|
|
source: options?.source || "desktop-url",
|
|
});
|
|
return true;
|
|
}
|
|
if (!this.activeBrowserId) {
|
|
await this.command("open");
|
|
}
|
|
return true;
|
|
},
|
|
|
|
onAddressFocus() {
|
|
this.addressFocused = true;
|
|
},
|
|
|
|
onAddressBlur() {
|
|
this.addressFocused = false;
|
|
if (this.frameState?.currentUrl && !String(this.address || "").trim()) {
|
|
this.address = this.frameState.currentUrl;
|
|
}
|
|
},
|
|
|
|
async selectBrowser(id, contextId = "") {
|
|
const targetId = this.normalizeBrowserId(id);
|
|
const targetContextId = this.normalizeContextId(contextId || this.contextIdForBrowserId(targetId));
|
|
if (!targetId) {
|
|
await this.openNewBrowser();
|
|
return;
|
|
}
|
|
if (
|
|
this.sameBrowserTab(targetId, targetContextId, this.activeBrowserId, this.activeBrowserContextId)
|
|
&& this.connected
|
|
&& !this.isSwitchingBrowser()
|
|
) {
|
|
return;
|
|
}
|
|
const browser = this.browserById(targetId, targetContextId);
|
|
this.error = "";
|
|
this.switchingBrowserId = targetId;
|
|
this.cancelFrameRender();
|
|
this.frameSrc = "";
|
|
this.frameState = browser || null;
|
|
if (!this.addressFocused && browser?.currentUrl) {
|
|
this.address = browser.currentUrl;
|
|
}
|
|
this.setActiveBrowserId(targetId, targetContextId);
|
|
if (this.activeBrowserContextId) {
|
|
try {
|
|
await this.connectViewer({ browserId: targetId, contextId: targetContextId });
|
|
} catch (error) {
|
|
if (
|
|
this.sameBrowserId(this.switchingBrowserId, targetId)
|
|
&& this.normalizeContextId(this.activeBrowserContextId) === targetContextId
|
|
) {
|
|
this.switchingBrowserId = null;
|
|
}
|
|
this.error = error instanceof Error ? error.message : String(error);
|
|
}
|
|
}
|
|
},
|
|
|
|
async openNewBrowser() {
|
|
await this.command("open");
|
|
},
|
|
|
|
isClosingBrowser(id, contextId = "") {
|
|
const browserId = this.normalizeBrowserId(id);
|
|
const key = this.browserTabKey({
|
|
id: browserId,
|
|
context_id: contextId || this.contextIdForBrowserId(browserId),
|
|
});
|
|
return Boolean(key && this._closingBrowserIds[key]);
|
|
},
|
|
|
|
markBrowserClosing(id, contextId = "", closing = true) {
|
|
const browserId = this.normalizeBrowserId(id);
|
|
if (!browserId) return;
|
|
const key = this.browserTabKey({
|
|
id: browserId,
|
|
context_id: contextId || this.contextIdForBrowserId(browserId),
|
|
});
|
|
if (!key) return;
|
|
const nextClosing = { ...this._closingBrowserIds };
|
|
if (closing) {
|
|
nextClosing[key] = true;
|
|
} else {
|
|
delete nextClosing[key];
|
|
}
|
|
this._closingBrowserIds = nextClosing;
|
|
},
|
|
|
|
async closeBrowser(id, contextId = "") {
|
|
const browserId = this.normalizeBrowserId(id);
|
|
const browserContextId = this.normalizeContextId(contextId || this.contextIdForBrowserId(browserId));
|
|
if (!browserId || !browserContextId || this.isClosingBrowser(browserId, browserContextId)) return;
|
|
this.markBrowserClosing(browserId, browserContextId, true);
|
|
try {
|
|
await this.command("close", { browser_id: browserId, context_id: browserContextId });
|
|
} finally {
|
|
this.markBrowserClosing(browserId, browserContextId, false);
|
|
}
|
|
},
|
|
|
|
isActiveBrowser(browser) {
|
|
return this.sameBrowserTab(browser?.id, browser?.context_id, this.activeBrowserId, this.activeBrowserContextId);
|
|
},
|
|
|
|
browserTabTitle(browser) {
|
|
const title = String(browser?.title || "").trim();
|
|
const url = String(browser?.currentUrl || "").trim();
|
|
return title || url || "about:blank";
|
|
},
|
|
|
|
browserTabLabel(browser) {
|
|
const id = browser?.id ? `#${browser.id}` : "Browser";
|
|
return [id, this.browserTabTitle(browser)].filter(Boolean).join(" ");
|
|
},
|
|
|
|
browserTabTooltip(browser) {
|
|
const chatTitle = this.browserChatTitle(browser);
|
|
return [this.browserTabLabel(browser), chatTitle ? `Chat: ${chatTitle}` : ""]
|
|
.filter(Boolean)
|
|
.join("\n");
|
|
},
|
|
|
|
browserTabKey(browser = {}) {
|
|
const id = this.normalizeBrowserId(browser?.id ?? browser);
|
|
const contextId = this.normalizeContextId(browser?.context_id || browser?.contextId || this.activeBrowserContextId || this.contextId);
|
|
return id && contextId ? `${contextId}:${id}` : "";
|
|
},
|
|
|
|
browserChatTitle(browser = {}) {
|
|
const contextId = this.normalizeContextId(browser?.context_id || browser?.contextId);
|
|
if (!contextId) return "";
|
|
const context = chatsStore.contexts?.find?.((item) => item?.id === contextId);
|
|
return String(context?.name || context?.title || "").trim();
|
|
},
|
|
|
|
firstBrowser(contextId = "") {
|
|
const normalizedContextId = this.normalizeContextId(contextId);
|
|
const browsers = Array.isArray(this.browsers) ? this.browsers : [];
|
|
if (normalizedContextId) {
|
|
const scoped = browsers.find((browser) => this.normalizeContextId(browser?.context_id) === normalizedContextId);
|
|
if (scoped) return scoped;
|
|
}
|
|
return browsers[0] || null;
|
|
},
|
|
|
|
firstBrowserInContext(contextId = "") {
|
|
const normalizedContextId = this.normalizeContextId(contextId);
|
|
if (!normalizedContextId || !Array.isArray(this.browsers)) return null;
|
|
return this.browsers.find((browser) => this.normalizeContextId(browser?.context_id) === normalizedContextId) || null;
|
|
},
|
|
|
|
firstBrowserId(contextId = "") {
|
|
return this.firstBrowser(contextId)?.id || null;
|
|
},
|
|
|
|
normalizeBrowserId(id) {
|
|
return Number(id) || null;
|
|
},
|
|
|
|
sameBrowserId(left, right) {
|
|
const leftId = this.normalizeBrowserId(left);
|
|
const rightId = this.normalizeBrowserId(right);
|
|
return Boolean(leftId && rightId && leftId === rightId);
|
|
},
|
|
|
|
sameBrowserTab(leftId, leftContextId, rightId, rightContextId) {
|
|
return this.sameBrowserId(leftId, rightId)
|
|
&& this.normalizeContextId(leftContextId) === this.normalizeContextId(rightContextId);
|
|
},
|
|
|
|
browserById(id, contextId = "") {
|
|
const numeric = this.normalizeBrowserId(id);
|
|
if (!numeric || !Array.isArray(this.browsers)) return null;
|
|
const normalizedContextId = this.normalizeContextId(contextId);
|
|
return this.browsers.find((browser) => (
|
|
Number(browser?.id) === numeric
|
|
&& (!normalizedContextId || this.normalizeContextId(browser?.context_id) === normalizedContextId)
|
|
)) || null;
|
|
},
|
|
|
|
contextIdForBrowserId(id) {
|
|
const numeric = this.normalizeBrowserId(id);
|
|
if (!numeric) return "";
|
|
if (this.sameBrowserId(numeric, this.activeBrowserId) && this.activeBrowserContextId) {
|
|
return this.activeBrowserContextId;
|
|
}
|
|
return this.normalizeContextId(this.browserById(numeric)?.context_id);
|
|
},
|
|
|
|
applyBrowserListing(browsers = [], fallbackContextId = "", options = {}) {
|
|
const incoming = Array.isArray(browsers)
|
|
? browsers.map((browser) => ({
|
|
...browser,
|
|
context_id: this.normalizeContextId(browser?.context_id || fallbackContextId),
|
|
})).filter((browser) => browser.id && browser.context_id)
|
|
: [];
|
|
const incomingKeys = new Set(incoming.map((browser) => this.browserTabKey(browser)));
|
|
const fallback = this.normalizeContextId(fallbackContextId);
|
|
const existing = Array.isArray(this.browsers) ? this.browsers : [];
|
|
const retained = options.replaceAll
|
|
? []
|
|
: existing.filter((browser) => {
|
|
const key = this.browserTabKey(browser);
|
|
if (incomingKeys.has(key)) return false;
|
|
if (options.replaceContext && fallback && this.normalizeContextId(browser?.context_id) === fallback) return false;
|
|
return true;
|
|
});
|
|
this.browsers = [...retained, ...incoming];
|
|
},
|
|
|
|
stateFromCommandResult(result = {}) {
|
|
if (result?.state?.id || result?.state?.currentUrl || result?.state?.title) {
|
|
return result.state;
|
|
}
|
|
if (result?.id || result?.currentUrl || result?.title) {
|
|
return result;
|
|
}
|
|
return null;
|
|
},
|
|
|
|
applyActiveFrameState(nextState = null) {
|
|
if (!nextState) return;
|
|
const stateId = this.normalizeBrowserId(nextState.id);
|
|
const stateContextId = this.normalizeContextId(nextState.context_id || this.activeBrowserContextId);
|
|
if (
|
|
stateId
|
|
&& this.activeBrowserId
|
|
&& !this.sameBrowserTab(stateId, stateContextId, this.activeBrowserId, this.activeBrowserContextId)
|
|
) {
|
|
return;
|
|
}
|
|
const previousUrl = String(this.frameState?.currentUrl || "");
|
|
const nextUrl = String(nextState.currentUrl || "");
|
|
this.frameState = nextState;
|
|
if (previousUrl && nextUrl && previousUrl !== nextUrl) {
|
|
this.cancelAnnotationDraft();
|
|
}
|
|
if (!this.addressFocused && nextState.currentUrl) {
|
|
this.address = nextState.currentUrl;
|
|
}
|
|
},
|
|
|
|
applySnapshot(snapshot = null) {
|
|
if (!snapshot?.image) return;
|
|
const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id);
|
|
const snapshotContextId = this.normalizeContextId(snapshot.context_id || snapshot.state?.context_id || this.activeBrowserContextId);
|
|
if (
|
|
snapshotId
|
|
&& this.activeBrowserId
|
|
&& !this.sameBrowserTab(snapshotId, snapshotContextId, this.activeBrowserId, this.activeBrowserContextId)
|
|
) {
|
|
return;
|
|
}
|
|
if (snapshot.state) {
|
|
this.applyActiveFrameState(snapshot.state);
|
|
}
|
|
const frameBrowserId = snapshotId || this.activeBrowserId;
|
|
this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
|
|
browserId: frameBrowserId,
|
|
contextId: snapshotContextId,
|
|
onAccepted: () => {
|
|
if (
|
|
this.sameBrowserId(this.switchingBrowserId, frameBrowserId)
|
|
&& this.normalizeContextId(this.activeBrowserContextId) === snapshotContextId
|
|
) {
|
|
this.switchingBrowserId = null;
|
|
}
|
|
this._surfaceSwitching = false;
|
|
},
|
|
});
|
|
},
|
|
|
|
isSwitchingBrowser() {
|
|
return Boolean(
|
|
this.switchingBrowserId
|
|
&& this.sameBrowserId(this.switchingBrowserId, this.activeBrowserId)
|
|
&& this.normalizeContextId(this.contextId) === this.normalizeContextId(this.activeBrowserContextId)
|
|
);
|
|
},
|
|
|
|
isBusy() {
|
|
return Boolean(this.loading || this.commandInFlight || this._surfaceSwitching || this.isSwitchingBrowser());
|
|
},
|
|
|
|
setActiveBrowserId(id, contextId = "") {
|
|
const previous = this.activeBrowserId;
|
|
const previousContextId = this.activeBrowserContextId;
|
|
const numeric = this.normalizeBrowserId(id);
|
|
const normalizedContextId = this.normalizeContextId(contextId || this.contextIdForBrowserId(numeric));
|
|
const exists = !numeric
|
|
|| !Array.isArray(this.browsers)
|
|
|| this.browsers.some((browser) => (
|
|
Number(browser.id) === numeric
|
|
&& (!normalizedContextId || this.normalizeContextId(browser.context_id) === normalizedContextId)
|
|
));
|
|
this.activeBrowserId = exists ? numeric : null;
|
|
this.activeBrowserContextId = this.activeBrowserId ? normalizedContextId : "";
|
|
this.contextId = this.activeBrowserContextId || this.contextId;
|
|
if (this.activeBrowserId !== previous || this.activeBrowserContextId !== previousContextId) {
|
|
this._lastViewportKey = "";
|
|
this._lastViewport = null;
|
|
this.cancelAnnotationDraft();
|
|
}
|
|
},
|
|
|
|
pointerCoordinatesFor(event, element = null) {
|
|
const target = element || event?.currentTarget;
|
|
if (!target) return null;
|
|
const rect = target.getBoundingClientRect();
|
|
const naturalWidth = target.naturalWidth || rect.width;
|
|
const naturalHeight = target.naturalHeight || rect.height;
|
|
let contentLeft = rect.left;
|
|
let contentTop = rect.top;
|
|
let contentWidth = rect.width;
|
|
let contentHeight = rect.height;
|
|
|
|
const objectFit = globalThis.getComputedStyle?.(target)?.objectFit || "";
|
|
if (
|
|
target.matches?.(".browser-frame")
|
|
&& ["contain", "scale-down"].includes(objectFit)
|
|
&& naturalWidth > 0
|
|
&& naturalHeight > 0
|
|
&& rect.width > 0
|
|
&& rect.height > 0
|
|
) {
|
|
const naturalRatio = naturalWidth / naturalHeight;
|
|
const rectRatio = rect.width / rect.height;
|
|
if (naturalRatio > rectRatio) {
|
|
contentWidth = rect.width;
|
|
contentHeight = rect.width / naturalRatio;
|
|
contentTop = rect.top + (rect.height - contentHeight) / 2;
|
|
} else {
|
|
contentHeight = rect.height;
|
|
contentWidth = rect.height * naturalRatio;
|
|
contentLeft = rect.left + (rect.width - contentWidth) / 2;
|
|
}
|
|
}
|
|
|
|
const relativeX = (event.clientX - contentLeft) / Math.max(1, contentWidth);
|
|
const relativeY = (event.clientY - contentTop) / Math.max(1, contentHeight);
|
|
return {
|
|
x: Math.max(0, Math.min(naturalWidth, relativeX * naturalWidth)),
|
|
y: Math.max(0, Math.min(naturalHeight, relativeY * naturalHeight)),
|
|
};
|
|
},
|
|
|
|
handleKeydown(event) {
|
|
const annotateShortcut = event?.key === "." && (event.metaKey || event.ctrlKey) && !event.altKey;
|
|
if (annotateShortcut && this._surfaceMounted) {
|
|
event.preventDefault();
|
|
event.stopPropagation?.();
|
|
this.toggleAnnotationMode();
|
|
return;
|
|
}
|
|
|
|
if (this.annotating) {
|
|
if (event?.key === "Escape") {
|
|
event.preventDefault();
|
|
if (this.annotationDraft || this.annotationDragRect) {
|
|
this.cancelAnnotationDraft();
|
|
} else {
|
|
this.toggleAnnotationMode(false);
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (this.handleVisualBrowserShortcut(event)) {
|
|
return;
|
|
}
|
|
|
|
void this.sendKey(event);
|
|
},
|
|
|
|
handleVisualBrowserShortcut(event) {
|
|
const shortcut = this.visualBrowserShortcut(event);
|
|
if (!shortcut) return false;
|
|
event.preventDefault();
|
|
event.stopPropagation?.();
|
|
|
|
if (shortcut.action === "paste") {
|
|
void this.pasteHostClipboardToBrowser();
|
|
return true;
|
|
}
|
|
if (shortcut.action === "copy" || shortcut.action === "cut") {
|
|
void this.copyBrowserClipboardToHost(shortcut.action);
|
|
return true;
|
|
}
|
|
if (shortcut.key) {
|
|
void this.sendShortcut(shortcut.key);
|
|
return true;
|
|
}
|
|
return false;
|
|
},
|
|
|
|
visualBrowserShortcut(event) {
|
|
if (!this.shouldHandleVisualBrowserShortcut(event)) return null;
|
|
const key = String(event?.key || "").toLowerCase();
|
|
const primary = Boolean(event?.ctrlKey || event?.metaKey);
|
|
const shift = Boolean(event?.shiftKey);
|
|
|
|
if (!primary && shift && key === "insert") {
|
|
return { action: "paste" };
|
|
}
|
|
if (!primary || event?.altKey) return null;
|
|
|
|
if (key === "v") return { action: "paste" };
|
|
if (!shift && (key === "c" || key === "insert")) return { action: "copy" };
|
|
if (!shift && key === "x") return { action: "cut" };
|
|
if (!shift && key === "a") return { key: "Control+A" };
|
|
if (key === "z") return { key: shift ? "Control+Shift+Z" : "Control+Z" };
|
|
if (!shift && key === "y") return { key: "Control+Y" };
|
|
return null;
|
|
},
|
|
|
|
shouldHandleVisualBrowserShortcut(event) {
|
|
if (!this._surfaceMounted || !this.activeBrowserId || this.annotating) return false;
|
|
if (isLocalEditableTarget(event?.target)) return false;
|
|
const key = String(event?.key || "").toLowerCase();
|
|
if (!BROWSER_VISUAL_SHORTCUT_KEYS.has(key)) return false;
|
|
return Boolean(this.visualBrowserStageForEvent(event));
|
|
},
|
|
|
|
visualBrowserStageForEvent(event) {
|
|
const element = elementFromTarget(event?.target);
|
|
const blockingUi = element?.closest?.(
|
|
".browser-toolbar, .browser-meta, .browser-extension-dropdown, .browser-annotation-popover, .browser-annotation-tray, button, a",
|
|
);
|
|
if (blockingUi) return null;
|
|
|
|
const stage = element?.closest?.(".browser-stage");
|
|
if (stage?.closest?.(".browser-panel")) return stage;
|
|
|
|
const activeElement = globalThis.document?.activeElement;
|
|
const activeStage = activeElement?.closest?.(".browser-stage");
|
|
if (activeStage?.closest?.(".browser-panel")) return activeStage;
|
|
return null;
|
|
},
|
|
|
|
handleStageWheel(event) {
|
|
if (this.annotating) return;
|
|
void this.sendWheel(event);
|
|
},
|
|
|
|
toggleAnnotationMode(force = null) {
|
|
const nextValue = force === null ? !this.annotating : Boolean(force);
|
|
if (nextValue && !this.canAnnotate()) return;
|
|
|
|
this.annotating = nextValue;
|
|
this.annotationError = "";
|
|
this.closeExtensionsMenu();
|
|
if (!nextValue) {
|
|
this.cancelAnnotationDraft();
|
|
this.annotationDragRect = null;
|
|
this._annotationPointer = null;
|
|
} else {
|
|
this._stageElement?.focus?.({ preventScroll: true });
|
|
}
|
|
},
|
|
|
|
canAnnotate() {
|
|
return Boolean(this.activeBrowserId && this.frameSrc && !this.isBusy());
|
|
},
|
|
|
|
activeAnnotationUrl() {
|
|
return String(this.frameState?.currentUrl || this.address || "about:blank");
|
|
},
|
|
|
|
visibleAnnotations() {
|
|
const browserId = this.normalizeBrowserId(this.activeBrowserId);
|
|
const contextId = this.normalizeContextId(this.activeBrowserContextId);
|
|
const url = this.activeAnnotationUrl();
|
|
return this.annotationComments.filter((annotation) => (
|
|
this.sameBrowserTab(annotation.browserId, annotation.contextId, browserId, contextId)
|
|
&& String(annotation.url || "") === url
|
|
));
|
|
},
|
|
|
|
nextAnnotationIndex() {
|
|
return this.visibleAnnotations().length + 1;
|
|
},
|
|
|
|
annotationTrayStyle() {
|
|
if (!this.annotationTrayPosition) return {};
|
|
const position = this.clampAnnotationTrayPosition(this.annotationTrayPosition);
|
|
return {
|
|
left: `${position.x}px`,
|
|
top: `${position.y}px`,
|
|
right: "auto",
|
|
bottom: "auto",
|
|
};
|
|
},
|
|
|
|
clampAnnotationTrayPosition(position = {}) {
|
|
const stageRect = this._stageElement?.getBoundingClientRect?.();
|
|
const stageWidth = Math.max(1, Number(stageRect?.width || 0));
|
|
const stageHeight = Math.max(1, Number(stageRect?.height || 0));
|
|
const width = Math.max(180, Number(position.width || 0));
|
|
const height = Math.max(90, Number(position.height || 0));
|
|
const maxX = Math.max(ANNOTATION_TRAY_MARGIN, stageWidth - width - ANNOTATION_TRAY_MARGIN);
|
|
const maxY = Math.max(ANNOTATION_TRAY_MARGIN, stageHeight - height - ANNOTATION_TRAY_MARGIN);
|
|
return {
|
|
x: Math.min(Math.max(ANNOTATION_TRAY_MARGIN, Number(position.x || 0)), maxX),
|
|
y: Math.min(Math.max(ANNOTATION_TRAY_MARGIN, Number(position.y || 0)), maxY),
|
|
width,
|
|
height,
|
|
};
|
|
},
|
|
|
|
startAnnotationTrayDrag(event) {
|
|
if (event.button !== 0) return;
|
|
if (event.target?.closest?.("button, input, select, textarea, a")) return;
|
|
const tray = event.currentTarget?.closest?.(".browser-annotation-tray");
|
|
const stageRect = this._stageElement?.getBoundingClientRect?.();
|
|
const trayRect = tray?.getBoundingClientRect?.();
|
|
if (!tray || !stageRect || !trayRect) return;
|
|
|
|
const position = this.clampAnnotationTrayPosition({
|
|
x: trayRect.left - stageRect.left,
|
|
y: trayRect.top - stageRect.top,
|
|
width: trayRect.width,
|
|
height: trayRect.height,
|
|
});
|
|
this.annotationTrayPosition = position;
|
|
this.annotationTrayDragging = true;
|
|
this._annotationTrayDrag = {
|
|
id: event.pointerId,
|
|
target: event.currentTarget,
|
|
x: event.clientX,
|
|
y: event.clientY,
|
|
startX: position.x,
|
|
startY: position.y,
|
|
width: position.width,
|
|
height: position.height,
|
|
};
|
|
event.currentTarget?.setPointerCapture?.(event.pointerId);
|
|
event.preventDefault();
|
|
},
|
|
|
|
moveAnnotationTrayDrag(event) {
|
|
const drag = this._annotationTrayDrag;
|
|
if (!drag || event.pointerId !== drag.id) return;
|
|
this.annotationTrayPosition = this.clampAnnotationTrayPosition({
|
|
x: drag.startX + event.clientX - drag.x,
|
|
y: drag.startY + event.clientY - drag.y,
|
|
width: drag.width,
|
|
height: drag.height,
|
|
});
|
|
event.preventDefault();
|
|
},
|
|
|
|
finishAnnotationTrayDrag(event = null) {
|
|
const drag = this._annotationTrayDrag;
|
|
if (!drag || (event?.pointerId && event.pointerId !== drag.id)) return;
|
|
try {
|
|
drag.target?.releasePointerCapture?.(drag.id);
|
|
} catch {}
|
|
this._annotationTrayDrag = null;
|
|
this.annotationTrayDragging = false;
|
|
},
|
|
|
|
resetAnnotationTrayPosition() {
|
|
this.finishAnnotationTrayDrag();
|
|
this.annotationTrayPosition = null;
|
|
},
|
|
|
|
clearVisibleAnnotations() {
|
|
this.clearAnnotationsForBrowser(this.activeBrowserId, this.activeAnnotationUrl(), this.activeBrowserContextId);
|
|
this.resetAnnotationTrayPosition();
|
|
},
|
|
|
|
clearAnnotationsForBrowser(browserId, url = null, contextId = "") {
|
|
const numericBrowserId = this.normalizeBrowserId(browserId);
|
|
const normalizedContextId = this.normalizeContextId(contextId || this.activeBrowserContextId);
|
|
if (!numericBrowserId) return;
|
|
this.annotationComments = this.annotationComments.filter((annotation) => {
|
|
if (!this.sameBrowserTab(annotation.browserId, annotation.contextId, numericBrowserId, normalizedContextId)) return true;
|
|
return url ? String(annotation.url || "") !== String(url) : false;
|
|
});
|
|
},
|
|
|
|
annotationBoxStyle(rect = {}) {
|
|
const viewport = this.currentViewportSize() || this._lastViewport || {};
|
|
const width = Math.max(1, Number(viewport.width || rect.width || 1));
|
|
const height = Math.max(1, Number(viewport.height || rect.height || 1));
|
|
const normalized = this.clampAnnotationRect(rect);
|
|
return [
|
|
`left: ${(normalized.x / width) * 100}%`,
|
|
`top: ${(normalized.y / height) * 100}%`,
|
|
`width: ${(Math.max(1, normalized.width) / width) * 100}%`,
|
|
`height: ${(Math.max(1, normalized.height) / height) * 100}%`,
|
|
].join("; ");
|
|
},
|
|
|
|
annotationPopoverStyle() {
|
|
const rect = this.annotationDraft?.rect || this.annotationDragRect || {};
|
|
const viewport = this.currentViewportSize() || this._lastViewport || {};
|
|
const width = Math.max(1, Number(viewport.width || 1));
|
|
const height = Math.max(1, Number(viewport.height || 1));
|
|
const popoverWidth = Math.min(320, Math.max(240, width - 20));
|
|
const popoverHeight = 190;
|
|
const nextLeft = Math.min(
|
|
Math.max(10, Number(rect.x || 0) + Number(rect.width || 0) + 10),
|
|
Math.max(10, width - popoverWidth - 10),
|
|
);
|
|
const nextTop = Math.min(
|
|
Math.max(10, Number(rect.y || 0) + Number(rect.height || 0) + 10),
|
|
Math.max(10, height - popoverHeight - 10),
|
|
);
|
|
return [
|
|
`left: ${(nextLeft / width) * 100}%`,
|
|
`top: ${(nextTop / height) * 100}%`,
|
|
`width: min(${popoverWidth}px, calc(100% - 20px))`,
|
|
].join("; ");
|
|
},
|
|
|
|
annotationDraftTitle() {
|
|
if (!this.annotationDraft) return "Annotation";
|
|
return this.annotationDraft.kind === "area" ? "Area annotation" : "Element annotation";
|
|
},
|
|
|
|
stagePointForEvent(event) {
|
|
const image = this._stageElement?.querySelector?.(".browser-frame") || null;
|
|
return this.pointerCoordinatesFor(event, image);
|
|
},
|
|
|
|
normalizeAnnotationRect(start = {}, end = {}) {
|
|
const x1 = Number(start.x || 0);
|
|
const y1 = Number(start.y || 0);
|
|
const x2 = Number(end.x || x1);
|
|
const y2 = Number(end.y || y1);
|
|
return this.clampAnnotationRect({
|
|
x: Math.min(x1, x2),
|
|
y: Math.min(y1, y2),
|
|
width: Math.abs(x2 - x1),
|
|
height: Math.abs(y2 - y1),
|
|
});
|
|
},
|
|
|
|
clampAnnotationRect(rect = {}) {
|
|
const viewport = this.currentViewportSize() || this._lastViewport || {};
|
|
const viewportWidth = Math.max(1, Number(viewport.width || rect.x + rect.width || 1));
|
|
const viewportHeight = Math.max(1, Number(viewport.height || rect.y + rect.height || 1));
|
|
const x = Math.max(0, Math.min(viewportWidth, Number(rect.x || 0)));
|
|
const y = Math.max(0, Math.min(viewportHeight, Number(rect.y || 0)));
|
|
const width = Math.max(1, Math.min(viewportWidth - x, Number(rect.width || 1)));
|
|
const height = Math.max(1, Math.min(viewportHeight - y, Number(rect.height || 1)));
|
|
return {
|
|
x: Math.round(x),
|
|
y: Math.round(y),
|
|
width: Math.round(width),
|
|
height: Math.round(height),
|
|
};
|
|
},
|
|
|
|
startAnnotationSelection(event) {
|
|
if (!this.annotating || this.annotationBusy || !this.canAnnotate()) return;
|
|
const point = this.stagePointForEvent(event);
|
|
if (!point) return;
|
|
this.cancelAnnotationDraft();
|
|
this.annotationError = "";
|
|
this._annotationPointer = {
|
|
id: event.pointerId,
|
|
start: point,
|
|
last: point,
|
|
};
|
|
this.annotationDragRect = this.clampAnnotationRect({
|
|
x: point.x,
|
|
y: point.y,
|
|
width: 1,
|
|
height: 1,
|
|
});
|
|
event.currentTarget?.setPointerCapture?.(event.pointerId);
|
|
},
|
|
|
|
moveAnnotationSelection(event) {
|
|
if (!this.annotating || !this._annotationPointer) return;
|
|
if (event.pointerId !== this._annotationPointer.id) return;
|
|
const point = this.stagePointForEvent(event);
|
|
if (!point) return;
|
|
this._annotationPointer.last = point;
|
|
this.annotationDragRect = this.normalizeAnnotationRect(this._annotationPointer.start, point);
|
|
},
|
|
|
|
async finishAnnotationSelection(event) {
|
|
if (!this.annotating || !this._annotationPointer) return;
|
|
if (event.pointerId !== this._annotationPointer.id) return;
|
|
const pointer = this._annotationPointer;
|
|
this._annotationPointer = null;
|
|
event.currentTarget?.releasePointerCapture?.(event.pointerId);
|
|
const endPoint = this.stagePointForEvent(event) || pointer.last || pointer.start;
|
|
const rect = this.normalizeAnnotationRect(pointer.start, endPoint);
|
|
this.annotationDragRect = null;
|
|
const isDrag = rect.width >= ANNOTATION_DRAG_THRESHOLD || rect.height >= ANNOTATION_DRAG_THRESHOLD;
|
|
const point = {
|
|
x: Math.round(endPoint.x),
|
|
y: Math.round(endPoint.y),
|
|
};
|
|
const payload = {
|
|
kind: isDrag ? "area" : "element",
|
|
point,
|
|
rect: isDrag ? rect : null,
|
|
viewport: this.currentViewportSize(),
|
|
url: this.activeAnnotationUrl(),
|
|
title: this.activeTitle,
|
|
};
|
|
await this.createAnnotationDraft(payload, isDrag ? rect : {
|
|
x: point.x - 10,
|
|
y: point.y - 10,
|
|
width: 20,
|
|
height: 20,
|
|
});
|
|
},
|
|
|
|
cancelAnnotationSelection(event = null) {
|
|
if (event && this._annotationPointer?.id === event.pointerId) {
|
|
event.currentTarget?.releasePointerCapture?.(event.pointerId);
|
|
}
|
|
this._annotationPointer = null;
|
|
this.annotationDragRect = null;
|
|
},
|
|
|
|
cancelAnnotationDraft() {
|
|
this.annotationDraft = null;
|
|
this.annotationDraftText = "";
|
|
this.annotationDragRect = null;
|
|
},
|
|
|
|
async createAnnotationDraft(payload, fallbackRect) {
|
|
const contextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId);
|
|
if (!this.activeBrowserId || !contextId) return;
|
|
const sequence = this._annotationSequence + 1;
|
|
const browserId = this.activeBrowserId;
|
|
const url = this.activeAnnotationUrl();
|
|
const title = this.activeTitle;
|
|
this._annotationSequence = sequence;
|
|
this.annotationBusy = true;
|
|
this.annotationError = "";
|
|
try {
|
|
const response = await websocket.request(
|
|
"browser_viewer_annotation",
|
|
{
|
|
context_id: contextId,
|
|
browser_id: browserId,
|
|
viewer_id: this._viewerToken,
|
|
payload,
|
|
},
|
|
{ timeoutMs: 10000 },
|
|
);
|
|
if (sequence !== this._annotationSequence) return;
|
|
const data = firstOk(response);
|
|
const metadata = data.annotation || {};
|
|
this.annotationDraft = {
|
|
id: makeViewerToken(),
|
|
browserId,
|
|
contextId,
|
|
url,
|
|
title,
|
|
kind: metadata.kind || payload.kind,
|
|
rect: this.annotationRectFromMetadata(metadata, fallbackRect),
|
|
metadata,
|
|
createdAt: Date.now(),
|
|
};
|
|
this.annotationDraftText = "";
|
|
} catch (error) {
|
|
this.annotationError = error instanceof Error ? error.message : String(error);
|
|
this.error = this.annotationError;
|
|
} finally {
|
|
if (sequence === this._annotationSequence) {
|
|
this.annotationBusy = false;
|
|
}
|
|
}
|
|
},
|
|
|
|
annotationRectFromMetadata(metadata = {}, fallbackRect = {}) {
|
|
const targetRect = metadata?.target?.rect || metadata?.rect || null;
|
|
return this.clampAnnotationRect(targetRect || fallbackRect);
|
|
},
|
|
|
|
addAnnotationComment() {
|
|
const comment = String(this.annotationDraftText || "").trim();
|
|
if (!this.annotationDraft || !comment) return;
|
|
if (this.visibleAnnotations().length >= ANNOTATION_MAX_COMMENTS) {
|
|
this.annotationError = `Keep each batch to ${ANNOTATION_MAX_COMMENTS} annotations or fewer.`;
|
|
this.error = this.annotationError;
|
|
return;
|
|
}
|
|
this.annotationComments = [
|
|
...this.annotationComments,
|
|
{
|
|
...this.annotationDraft,
|
|
comment,
|
|
index: this.nextAnnotationIndex(),
|
|
},
|
|
];
|
|
this.cancelAnnotationDraft();
|
|
},
|
|
|
|
removeAnnotationComment(annotationId) {
|
|
this.annotationComments = this.annotationComments.filter((annotation) => annotation.id !== annotationId);
|
|
if (!this.visibleAnnotations().length) {
|
|
this.resetAnnotationTrayPosition();
|
|
}
|
|
},
|
|
|
|
annotationChipLabel(annotation) {
|
|
const prefix = annotation?.kind === "area" ? "Area" : "Element";
|
|
return `${prefix} ${annotation?.index || ""}`.trim();
|
|
},
|
|
|
|
formatAnnotationRect(rect = {}) {
|
|
const normalized = this.clampAnnotationRect(rect);
|
|
return `x=${normalized.x}, y=${normalized.y}, width=${normalized.width}, height=${normalized.height}`;
|
|
},
|
|
|
|
redactAnnotationText(value) {
|
|
return String(value || "")
|
|
.replace(/(<input\b(?=[^>]*\btype=(["'])?password\2?)[^>]*?)\svalue=(["'])[\s\S]*?\3/giu, "$1 value=\"[redacted]\"")
|
|
.replace(/\b(password|passcode|token|secret|value)=((["'])[\s\S]{1,240}?\3)/giu, "$1=\"[redacted]\"");
|
|
},
|
|
|
|
formatAnnotationMetadata(metadata = {}) {
|
|
const lines = [];
|
|
const target = metadata.target || {};
|
|
const selector = target.selector || metadata.selector || "";
|
|
const summary = target.summary || metadata.summary || "";
|
|
const dom = this.redactAnnotationText(target.dom || metadata.dom || "").slice(0, ANNOTATION_DOM_LIMIT);
|
|
|
|
if (selector) {
|
|
lines.push(`Selector: ${selector}`);
|
|
}
|
|
if (target.tagName || target.role || target.id || target.name || target.classes) {
|
|
lines.push([
|
|
"Element:",
|
|
target.tagName ? `<${String(target.tagName).toLowerCase()}>` : "",
|
|
target.role ? `role=${target.role}` : "",
|
|
target.id ? `id=${target.id}` : "",
|
|
target.name ? `name=${target.name}` : "",
|
|
target.classes ? `class=${target.classes}` : "",
|
|
].filter(Boolean).join(" "));
|
|
}
|
|
if (summary) {
|
|
lines.push(`Summary: ${summary}`);
|
|
}
|
|
if (Array.isArray(metadata.elements) && metadata.elements.length) {
|
|
lines.push("Intersecting elements:");
|
|
metadata.elements.slice(0, 8).forEach((element, index) => {
|
|
const elementLabel = [
|
|
`${index + 1}.`,
|
|
element.tagName ? `<${String(element.tagName).toLowerCase()}>` : "",
|
|
element.selector || "",
|
|
element.summary || "",
|
|
].filter(Boolean).join(" ");
|
|
lines.push(elementLabel);
|
|
});
|
|
}
|
|
if (dom) {
|
|
lines.push(`DOM: ${dom}`);
|
|
}
|
|
return lines.join("\n");
|
|
},
|
|
|
|
buildAnnotationsPrompt() {
|
|
const annotations = this.visibleAnnotations();
|
|
if (!annotations.length) return "";
|
|
const lines = [
|
|
"Browser annotations",
|
|
`Page title: ${this.activeTitle}`,
|
|
`Page URL: ${this.activeAnnotationUrl()}`,
|
|
`Browser id: ${this.activeBrowserId}`,
|
|
"",
|
|
];
|
|
annotations.forEach((annotation, index) => {
|
|
lines.push(
|
|
`Annotation ${index + 1}`,
|
|
`Comment: ${annotation.comment}`,
|
|
`Selection kind: ${annotation.kind}`,
|
|
`Coordinates: ${this.formatAnnotationRect(annotation.rect)}`,
|
|
);
|
|
const metadata = this.formatAnnotationMetadata(annotation.metadata);
|
|
if (metadata) {
|
|
lines.push(metadata);
|
|
}
|
|
lines.push("");
|
|
});
|
|
return lines.join("\n").trim();
|
|
},
|
|
|
|
draftAnnotationsToChat() {
|
|
const prompt = this.buildAnnotationsPrompt();
|
|
if (!prompt) return;
|
|
const existingMessage = String(chatInputStore.message || "").trim();
|
|
chatInputStore.message = existingMessage ? `${existingMessage}\n\n${prompt}` : prompt;
|
|
chatInputStore.adjustTextareaHeight?.();
|
|
chatInputStore.focus?.();
|
|
this.clearVisibleAnnotations();
|
|
this.toggleAnnotationMode(false);
|
|
},
|
|
|
|
async sendAnnotationsToChat() {
|
|
const prompt = this.buildAnnotationsPrompt();
|
|
if (!prompt) return;
|
|
chatInputStore.message = prompt;
|
|
chatInputStore.adjustTextareaHeight?.();
|
|
try {
|
|
if (typeof chatInputStore.sendMessage === "function") {
|
|
await chatInputStore.sendMessage();
|
|
} else if (typeof globalThis.sendMessage === "function") {
|
|
await globalThis.sendMessage();
|
|
} else {
|
|
chatInputStore.focus?.();
|
|
return;
|
|
}
|
|
this.clearVisibleAnnotations();
|
|
this.toggleAnnotationMode(false);
|
|
} catch (error) {
|
|
this.error = error instanceof Error ? error.message : String(error);
|
|
}
|
|
},
|
|
|
|
currentViewportSize() {
|
|
const measurement = this.surfaceViewportMeasurement();
|
|
if (!measurement) return null;
|
|
return {
|
|
width: measurement.width,
|
|
height: measurement.height,
|
|
};
|
|
},
|
|
|
|
surfaceViewportMeasurement() {
|
|
const stage = this._stageElement;
|
|
if (!stage) return null;
|
|
const rect = stage.getBoundingClientRect?.();
|
|
const rawWidth = Math.round(rect?.width || stage.clientWidth || 0);
|
|
const rawHeight = Math.round(rect?.height || stage.clientHeight || 0);
|
|
if (rawWidth < 80 || rawHeight < 80) return null;
|
|
return {
|
|
rawWidth,
|
|
rawHeight,
|
|
width: Math.max(320, rawWidth),
|
|
height: Math.max(200, rawHeight),
|
|
};
|
|
},
|
|
|
|
queueViewportSync(force = false) {
|
|
this.clearRenderedFrameIfViewportChanged();
|
|
if (this._viewportSyncTimer) {
|
|
globalThis.clearTimeout(this._viewportSyncTimer);
|
|
}
|
|
this._viewportSyncTimer = globalThis.setTimeout(() => {
|
|
this._viewportSyncTimer = null;
|
|
void this.syncViewport(force);
|
|
}, force ? 0 : VIEWPORT_SYNC_DEBOUNCE_MS);
|
|
},
|
|
|
|
async syncViewport(force = false, options = {}) {
|
|
const restartStream = Boolean(options.restartStream);
|
|
const contextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId);
|
|
if (!contextId || !this.activeBrowserId) {
|
|
return;
|
|
}
|
|
const viewport = this.currentViewportSize();
|
|
if (!viewport) {
|
|
return;
|
|
}
|
|
const key = `${contextId}:${this.activeBrowserId}:${viewport.width}x${viewport.height}`;
|
|
if (
|
|
(!restartStream && this._lastViewportKey === key)
|
|
|| (
|
|
!force
|
|
&& !restartStream
|
|
&& this._lastViewport
|
|
&& this.sameBrowserTab(this._lastViewport.browserId, this._lastViewport.contextId, this.activeBrowserId, contextId)
|
|
&& Math.abs(this._lastViewport.width - viewport.width) <= VIEWPORT_SYNC_SIZE_TOLERANCE
|
|
&& Math.abs(this._lastViewport.height - viewport.height) <= VIEWPORT_SYNC_SIZE_TOLERANCE
|
|
)
|
|
) {
|
|
return;
|
|
}
|
|
try {
|
|
await websocket.emit("browser_viewer_input", {
|
|
context_id: contextId,
|
|
browser_id: this.activeBrowserId,
|
|
viewer_id: this._viewerToken,
|
|
input_type: "viewport",
|
|
width: viewport.width,
|
|
height: viewport.height,
|
|
restart_stream: restartStream,
|
|
});
|
|
this._lastViewportKey = key;
|
|
this._lastViewport = {
|
|
browserId: this.activeBrowserId,
|
|
contextId,
|
|
width: viewport.width,
|
|
height: viewport.height,
|
|
};
|
|
} catch (error) {
|
|
this._lastViewportKey = "";
|
|
this._lastViewport = null;
|
|
console.warn("Browser viewport sync failed", error);
|
|
}
|
|
},
|
|
|
|
async sendMouse(eventType, event) {
|
|
if (this.annotating) return;
|
|
const contextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId);
|
|
if (!contextId || !this.activeBrowserId || !event?.currentTarget) return;
|
|
const pointer = this.pointerCoordinatesFor(event);
|
|
if (!pointer) return;
|
|
const payload = {
|
|
context_id: contextId,
|
|
browser_id: this.activeBrowserId,
|
|
viewer_id: this._viewerToken,
|
|
input_type: "mouse",
|
|
event_type: eventType,
|
|
x: pointer.x,
|
|
y: pointer.y,
|
|
button: "left",
|
|
};
|
|
if (eventType === "click") {
|
|
try {
|
|
const response = await websocket.request("browser_viewer_input", payload, { timeoutMs: 10000 });
|
|
const data = firstOk(response);
|
|
this.applyActiveFrameState(data.state);
|
|
this.applySnapshot(data.snapshot);
|
|
} catch (error) {
|
|
this.error = error instanceof Error ? error.message : String(error);
|
|
}
|
|
return;
|
|
}
|
|
await websocket.emit("browser_viewer_input", payload);
|
|
},
|
|
|
|
async sendWheel(event) {
|
|
const contextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId);
|
|
if (!contextId || !this.activeBrowserId || !event) return;
|
|
const image = event.currentTarget?.querySelector?.(".browser-frame") || event.target?.closest?.(".browser-frame");
|
|
const pointer = this.pointerCoordinatesFor(event, image);
|
|
if (!pointer) return;
|
|
const payload = {
|
|
context_id: contextId,
|
|
browser_id: this.activeBrowserId,
|
|
viewer_id: this._viewerToken,
|
|
input_type: "wheel",
|
|
x: pointer.x,
|
|
y: pointer.y,
|
|
delta_x: Number(event.deltaX || 0),
|
|
delta_y: Number(event.deltaY || 0),
|
|
};
|
|
try {
|
|
await websocket.emit("browser_viewer_input", payload);
|
|
} catch (error) {
|
|
this.error = error instanceof Error ? error.message : String(error);
|
|
}
|
|
},
|
|
|
|
async sendKey(event) {
|
|
if (this.annotating) return;
|
|
const contextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId);
|
|
if (!contextId || !this.activeBrowserId) return;
|
|
if (event.ctrlKey || event.metaKey || event.altKey) return;
|
|
const editable = ["INPUT", "TEXTAREA", "SELECT"].includes(event.target?.tagName);
|
|
if (editable) return;
|
|
event.preventDefault();
|
|
const printable = event.key && event.key.length === 1;
|
|
await websocket.emit("browser_viewer_input", {
|
|
context_id: contextId,
|
|
browser_id: this.activeBrowserId,
|
|
input_type: "keyboard",
|
|
key: printable ? "" : event.key,
|
|
text: printable ? event.key : "",
|
|
});
|
|
},
|
|
|
|
async sendShortcut(key) {
|
|
const contextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId);
|
|
if (!contextId || !this.activeBrowserId || !key) return;
|
|
await websocket.emit("browser_viewer_input", {
|
|
context_id: contextId,
|
|
browser_id: this.activeBrowserId,
|
|
viewer_id: this._viewerToken,
|
|
input_type: "keyboard",
|
|
key,
|
|
text: "",
|
|
});
|
|
},
|
|
|
|
async pasteHostClipboardToBrowser() {
|
|
try {
|
|
const text = await this.readHostClipboardText();
|
|
if (!text) return;
|
|
await this.sendClipboard("paste", text);
|
|
} catch (error) {
|
|
this.error = "Browser paste needs clipboard permission in this tab.";
|
|
globalThis.justToast?.(this.error, "warning", 2200, "browser-clipboard");
|
|
console.warn("Browser clipboard paste failed", error);
|
|
}
|
|
},
|
|
|
|
async copyBrowserClipboardToHost(action = "copy") {
|
|
try {
|
|
const clipboard = await this.sendClipboard(action);
|
|
const text = String(clipboard?.text || clipboard?.clipboard_text || "");
|
|
if (!text) return;
|
|
await copyToClipboard(text);
|
|
this._clipboardFallbackText = text;
|
|
const message = action === "cut" ? "Cut from Browser" : "Copied from Browser";
|
|
globalThis.justToast?.(message, "success", 1200, "browser-clipboard");
|
|
} catch (error) {
|
|
this.error = action === "cut"
|
|
? "Browser cut failed."
|
|
: "Browser copy failed.";
|
|
globalThis.justToast?.(this.error, "warning", 1800, "browser-clipboard");
|
|
console.warn("Browser clipboard copy failed", error);
|
|
}
|
|
},
|
|
|
|
async readHostClipboardText() {
|
|
const clipboard = globalThis.navigator?.clipboard;
|
|
if (clipboard?.readText && globalThis.isSecureContext) {
|
|
try {
|
|
return await clipboard.readText();
|
|
} catch (error) {
|
|
if (this._clipboardFallbackText) return this._clipboardFallbackText;
|
|
throw error;
|
|
}
|
|
}
|
|
return this._clipboardFallbackText || "";
|
|
},
|
|
|
|
async sendClipboard(action = "copy", text = "") {
|
|
const contextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId);
|
|
if (!contextId || !this.activeBrowserId) return {};
|
|
const response = await websocket.request(
|
|
"browser_viewer_input",
|
|
{
|
|
context_id: contextId,
|
|
browser_id: this.activeBrowserId,
|
|
viewer_id: this._viewerToken,
|
|
input_type: "clipboard",
|
|
action,
|
|
text,
|
|
},
|
|
{ timeoutMs: 10000 },
|
|
);
|
|
const data = firstOk(response);
|
|
this.applyActiveFrameState(data.state);
|
|
this.applySnapshot(data.snapshot);
|
|
return data.clipboard || {};
|
|
},
|
|
|
|
async cleanup() {
|
|
if (this._surfaceHandoff) {
|
|
this.releaseSurfaceBindings();
|
|
this.extensionMenuOpen = false;
|
|
return;
|
|
}
|
|
this._surfaceOpenSequence += 1;
|
|
this._openPromise = null;
|
|
this._openSignature = "";
|
|
this._connectSequence += 1;
|
|
this._viewerToken = "";
|
|
this.switchingBrowserId = null;
|
|
this._surfaceMounted = false;
|
|
this._surfaceSwitching = false;
|
|
this.commandInFlight = false;
|
|
this._commandInFlightCount = 0;
|
|
this._closingBrowserIds = {};
|
|
this.annotating = false;
|
|
this.annotationBusy = false;
|
|
this.annotationError = "";
|
|
this.cancelAnnotationDraft();
|
|
this.cancelAnnotationSelection();
|
|
this.resetAnnotationTrayPosition();
|
|
if (this.contextId) {
|
|
try {
|
|
await websocket.emit("browser_viewer_unsubscribe", { context_id: this.contextId });
|
|
} catch {}
|
|
}
|
|
this._frameOff?.();
|
|
this._stateOff?.();
|
|
this._frameOff = null;
|
|
this._stateOff = null;
|
|
this.resetRenderedFrame();
|
|
this.releaseSurfaceBindings();
|
|
if (this._viewportSyncTimer) {
|
|
globalThis.clearTimeout(this._viewportSyncTimer);
|
|
this._viewportSyncTimer = null;
|
|
}
|
|
this.resetViewportTracking();
|
|
this.extensionMenuOpen = false;
|
|
this.extensionActionLoading = false;
|
|
this.extensionsListLoading = false;
|
|
this.extensionToggleLoadingPath = "";
|
|
this.modelPresetSaving = false;
|
|
this.connected = false;
|
|
},
|
|
|
|
setupFloatingModal(element = null) {
|
|
this._floatingCleanup?.();
|
|
const root = element || globalThis.document?.querySelector(".browser-panel");
|
|
const modal = root?.closest?.(".modal");
|
|
const inner = modal?.querySelector?.(".modal-inner");
|
|
const body = modal?.querySelector?.(".modal-bd");
|
|
const header = modal?.querySelector?.(".modal-header");
|
|
const stage = root?.querySelector?.(".browser-stage");
|
|
if (!modal || !inner || !header) return;
|
|
modal.classList.add("surface-floating", "modal-floating");
|
|
inner.classList.add("surface-modal", "browser-modal");
|
|
body?.classList?.add("browser-modal-body");
|
|
this._stageElement = stage || null;
|
|
|
|
const rect = inner.getBoundingClientRect();
|
|
inner.style.left = `${Math.max(8, rect.left)}px`;
|
|
inner.style.top = `${Math.max(8, rect.top)}px`;
|
|
inner.style.transform = "none";
|
|
|
|
let drag = null;
|
|
let resizeObserver = null;
|
|
let beforeFocusBounds = null;
|
|
const viewportGap = 8;
|
|
const currentBounds = () => {
|
|
const bounds = inner.getBoundingClientRect();
|
|
return {
|
|
left: bounds.left,
|
|
top: bounds.top,
|
|
width: bounds.width,
|
|
height: bounds.height,
|
|
};
|
|
};
|
|
const normalizedBounds = (bounds = {}) => {
|
|
const maxWidth = Math.max(320, globalThis.innerWidth - viewportGap * 2);
|
|
const maxHeight = Math.max(300, globalThis.innerHeight - viewportGap * 2);
|
|
const width = Math.min(Math.max(320, Number(bounds.width || 320)), maxWidth);
|
|
const height = Math.min(Math.max(300, Number(bounds.height || 300)), maxHeight);
|
|
return {
|
|
left: Math.min(
|
|
Math.max(viewportGap, Number(bounds.left || viewportGap)),
|
|
Math.max(viewportGap, globalThis.innerWidth - width - viewportGap),
|
|
),
|
|
top: Math.min(
|
|
Math.max(viewportGap, Number(bounds.top || viewportGap)),
|
|
Math.max(viewportGap, globalThis.innerHeight - height - viewportGap),
|
|
),
|
|
width,
|
|
height,
|
|
};
|
|
};
|
|
const setBounds = (bounds = {}) => {
|
|
const next = normalizedBounds(bounds);
|
|
inner.style.position = "fixed";
|
|
inner.style.transform = "none";
|
|
inner.style.left = `${Math.round(next.left)}px`;
|
|
inner.style.top = `${Math.round(next.top)}px`;
|
|
inner.style.width = `${Math.round(next.width)}px`;
|
|
inner.style.height = `${Math.round(next.height)}px`;
|
|
inner.style.maxWidth = `${Math.max(320, globalThis.innerWidth - viewportGap * 2)}px`;
|
|
inner.style.maxHeight = `${Math.max(300, globalThis.innerHeight - viewportGap * 2)}px`;
|
|
this.queueViewportSync();
|
|
return next;
|
|
};
|
|
const focusBounds = () => ({
|
|
left: viewportGap,
|
|
top: viewportGap,
|
|
width: globalThis.innerWidth - viewportGap * 2,
|
|
height: globalThis.innerHeight - viewportGap * 2,
|
|
});
|
|
const clampPosition = (left, top) => {
|
|
const bounds = inner.getBoundingClientRect();
|
|
const maxLeft = Math.max(viewportGap, globalThis.innerWidth - bounds.width - viewportGap);
|
|
const maxTop = Math.max(viewportGap, globalThis.innerHeight - bounds.height - viewportGap);
|
|
return {
|
|
left: Math.min(Math.max(viewportGap, left), maxLeft),
|
|
top: Math.min(Math.max(viewportGap, top), maxTop),
|
|
};
|
|
};
|
|
const clampGeometry = () => {
|
|
if (inner.classList.contains("is-focus-mode")) {
|
|
setBounds(focusBounds());
|
|
return;
|
|
}
|
|
setBounds(currentBounds());
|
|
};
|
|
clampGeometry();
|
|
|
|
const focusButton = globalThis.document.createElement("button");
|
|
focusButton.type = "button";
|
|
focusButton.className = "surface-button browser-modal-focus-button";
|
|
focusButton.innerHTML = '<span class="material-symbols-outlined" aria-hidden="true">fullscreen</span>';
|
|
const updateFocusButton = (active) => {
|
|
const label = active ? "Restore size" : "Focus mode";
|
|
focusButton.setAttribute("aria-label", label);
|
|
focusButton.setAttribute("title", label);
|
|
focusButton.querySelector(".material-symbols-outlined").textContent = active ? "fullscreen_exit" : "fullscreen";
|
|
};
|
|
const setFocusMode = (enabled) => {
|
|
if (enabled) {
|
|
beforeFocusBounds = currentBounds();
|
|
inner.classList.add("is-focus-mode");
|
|
setBounds(focusBounds());
|
|
updateFocusButton(true);
|
|
return;
|
|
}
|
|
inner.classList.remove("is-focus-mode");
|
|
setBounds(beforeFocusBounds || currentBounds());
|
|
beforeFocusBounds = null;
|
|
updateFocusButton(false);
|
|
};
|
|
updateFocusButton(false);
|
|
const closeButton = inner.querySelector(".modal-close");
|
|
if (closeButton) {
|
|
closeButton.insertAdjacentElement("beforebegin", focusButton);
|
|
} else {
|
|
header.appendChild(focusButton);
|
|
}
|
|
const onFocusClick = () => setFocusMode(!inner.classList.contains("is-focus-mode"));
|
|
focusButton.addEventListener("click", onFocusClick);
|
|
|
|
globalThis.addEventListener("resize", clampGeometry);
|
|
if (globalThis.ResizeObserver) {
|
|
resizeObserver = new ResizeObserver(clampGeometry);
|
|
resizeObserver.observe(inner);
|
|
if (stage) {
|
|
this._stageResizeObserver?.disconnect?.();
|
|
this._stageResizeObserver = new ResizeObserver(() => {
|
|
this.queueViewportSync();
|
|
});
|
|
this._stageResizeObserver.observe(stage);
|
|
}
|
|
}
|
|
const surfaceSequence = this._surfaceOpenSequence;
|
|
globalThis.requestAnimationFrame(() => {
|
|
if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;
|
|
this.queueViewportSync(true);
|
|
});
|
|
|
|
const onPointerMove = (event) => {
|
|
if (!drag) return;
|
|
const next = clampPosition(
|
|
drag.left + event.clientX - drag.x,
|
|
drag.top + event.clientY - drag.y,
|
|
);
|
|
inner.style.left = `${next.left}px`;
|
|
inner.style.top = `${next.top}px`;
|
|
clampGeometry();
|
|
};
|
|
const onPointerUp = () => {
|
|
drag = null;
|
|
globalThis.removeEventListener("pointermove", onPointerMove);
|
|
globalThis.removeEventListener("pointerup", onPointerUp);
|
|
try {
|
|
header.releasePointerCapture?.(header.__browserPanelPointerId || 0);
|
|
} catch {}
|
|
};
|
|
const onPointerDown = (event) => {
|
|
if (event.button !== 0) return;
|
|
if (event.target?.closest?.("button, input, select, textarea, a")) return;
|
|
if (inner.classList.contains("is-focus-mode")) return;
|
|
const current = inner.getBoundingClientRect();
|
|
drag = {
|
|
x: event.clientX,
|
|
y: event.clientY,
|
|
left: current.left,
|
|
top: current.top,
|
|
};
|
|
header.__browserPanelPointerId = event.pointerId;
|
|
header.setPointerCapture?.(event.pointerId);
|
|
globalThis.addEventListener("pointermove", onPointerMove);
|
|
globalThis.addEventListener("pointerup", onPointerUp);
|
|
event.preventDefault();
|
|
};
|
|
header.addEventListener("pointerdown", onPointerDown);
|
|
|
|
this._floatingCleanup = () => {
|
|
focusButton.removeEventListener("click", onFocusClick);
|
|
focusButton.remove();
|
|
header.removeEventListener("pointerdown", onPointerDown);
|
|
globalThis.removeEventListener("pointermove", onPointerMove);
|
|
globalThis.removeEventListener("pointerup", onPointerUp);
|
|
globalThis.removeEventListener("resize", clampGeometry);
|
|
resizeObserver?.disconnect?.();
|
|
this._stageResizeObserver?.disconnect?.();
|
|
this._stageResizeObserver = null;
|
|
inner.classList.remove("is-focus-mode");
|
|
};
|
|
},
|
|
|
|
setupCanvasSurface(element = null) {
|
|
const surfaceSequence = this._surfaceOpenSequence;
|
|
this._floatingCleanup?.();
|
|
this._floatingCleanup = null;
|
|
this._stageResizeObserver?.disconnect?.();
|
|
const root = element || globalThis.document?.querySelector(".browser-panel");
|
|
const stage = root?.querySelector?.(".browser-stage");
|
|
this._stageElement = stage || null;
|
|
if (stage && globalThis.ResizeObserver) {
|
|
this._stageResizeObserver = new ResizeObserver(() => {
|
|
this.queueViewportSync();
|
|
});
|
|
this._stageResizeObserver.observe(stage);
|
|
}
|
|
globalThis.requestAnimationFrame?.(() => {
|
|
if (!this.isCurrentSurfaceOpen(surfaceSequence) || this._mode !== "canvas") return;
|
|
this.queueViewportSync(true);
|
|
});
|
|
},
|
|
|
|
get activeTitle() {
|
|
return this.frameState?.title || "Browser";
|
|
},
|
|
|
|
get activeUrl() {
|
|
return this.frameState?.currentUrl || this.address || "about:blank";
|
|
},
|
|
|
|
loadingMessage() {
|
|
if (this.browserInstallExpected) {
|
|
const cacheDir = this.status?.playwright?.cache_dir || "/a0/tmp/playwright";
|
|
return `Installing Chromium for the first Browser run. This can take a few minutes; future starts reuse ${cacheDir}.`;
|
|
}
|
|
return "Loading";
|
|
},
|
|
};
|
|
|
|
export const store = createStore("browserPage", model);
|
|
|
|
registerUrlHandler(async (intent = {}) => {
|
|
const url = String(intent.url || "").trim();
|
|
const payload = { url, source: intent.source || "surface-url-intent" };
|
|
await openLatestSurface("browser", payload);
|
|
return await store.openUrlIntent(url, { source: payload.source });
|
|
});
|