From 8e84735ecfdb2df0d2d6a5f72331d65f44b8dea8 Mon Sep 17 00:00:00 2001 From: go0p Date: Thu, 3 Sep 2026 17:04:00 +0800 Subject: [PATCH] feat(browser): add local handoff presentation --- src/app/background/index.ts | 17 +- src/features/engine-bridge/service.test.ts | 38 +++ src/features/engine-bridge/service.ts | 18 +- .../grants/capability-handlers/handoff.ts | 16 +- .../grants/capability-handlers/page.ts | 8 +- src/features/handoff/service.test.ts | 110 +++++++ src/features/handoff/service.ts | 299 ++++++++++++++++++ src/protocol/bridge.ts | 10 + src/protocol/capabilities.test.ts | 18 ++ src/protocol/capabilities.ts | 26 +- src/types/messages.ts | 2 + 11 files changed, 536 insertions(+), 26 deletions(-) create mode 100644 src/features/handoff/service.test.ts create mode 100644 src/features/handoff/service.ts diff --git a/src/app/background/index.ts b/src/app/background/index.ts index 9ec8876..0b4fe5d 100644 --- a/src/app/background/index.ts +++ b/src/app/background/index.ts @@ -64,6 +64,7 @@ import { handleCookieRequest } from './handlers/cookies'; import { handleUserAgentRequest } from './handlers/user-agent'; import { handleRecordingRequest } from './handlers/recording'; import { handleTransformRequest } from './handlers/transform'; +import { resolveHandoff } from '@/features/handoff/service'; function originOf(url: string): string { const parsed = new URL(url); @@ -299,21 +300,7 @@ async function handleRequest(request: ExtensionRequest, sender: Browser.runtime. } case 'handoff.resolve': { const input = request.payload; - const state = await updateState((current) => { - if (!current.handoff || current.handoff.id !== input.id || current.handoff.state !== 'waiting_for_user') { - throw new ExtensionError('handoff_not_waiting', '人工接管请求不存在或已经结束'); - } - return { - ...current, - handoff: { ...current.handoff, state: input.outcome, resolvedAt: Date.now() }, - }; - }); - const handoff = state.handoff!; - await setAgentRuntimeState( - input.outcome === 'completed' ? 'running' : 'paused', - await browserInstanceAccess('browser.tabs.read'), - ); - await browser.action.setBadgeText({ text: '', tabId: handoff.target.tabId }); + const { state, handoff } = await resolveHandoff(input.id, input.outcome); engineBridge.emitEvent('browser.handoff.changed', handoff); void appendAuditEvent({ category: 'handoff', action: `handoff.${input.outcome}`, outcome: input.outcome === 'completed' ? 'success' : 'cancelled', diff --git a/src/features/engine-bridge/service.test.ts b/src/features/engine-bridge/service.test.ts index 1f2de71..7002c91 100644 --- a/src/features/engine-bridge/service.test.ts +++ b/src/features/engine-bridge/service.test.ts @@ -87,9 +87,26 @@ vi.mock('@/platform/storage/state', () => ({ vi.mock('@/protocol/capabilities', () => ({ BRIDGE_CAPABILITIES: [], + capabilityVisibleToAgent: vi.fn((method: string) => ![ + 'browser.thumbnail', + 'browser.handoff.presentation.get', + 'browser.handoff.focus', + 'browser.handoff.resolve', + ].includes(method)), getBridgeCapabilityCatalog: vi.fn(async () => ({ version: 1, capabilities: [] })), })); +vi.mock('@/features/grants/capability-context', () => ({ + browserInstanceAccess: vi.fn(async () => ({ + id: 'paired-browser-instance', + taskId: 'paired-browser-instance', + targets: [], + scopes: ['browser.tabs.read'], + createdAt: 0, + expiresAt: Number.MAX_SAFE_INTEGER, + })), +})); + vi.mock('@/features/grants/service', () => ({ routeCapability: vi.fn(async () => ({ ok: true })), })); @@ -129,6 +146,7 @@ import { BRIDGE_HEARTBEAT_TIMEOUT_MS, EngineBridge, } from './service'; +import { beginAgentAction } from '@/features/agent-runtime/service'; import { BRIDGE_CHUNK_TIMEOUT_MS, BRIDGE_PROTOCOL_VERSION, @@ -214,6 +232,26 @@ describe('Engine Bridge transport lifecycle', () => { expect(socket.readyState).toBe(FakeWebSocket.CLOSED); }); + it('routes local UI capabilities without entering the paused Agent action gate', async () => { + const bridge = new EngineBridge(); + const socket = await connect(bridge); + + socket.receive({ + type: 'request', + id: 'local-ui-1', + method: 'browser.handoff.presentation.get', + params: { handoffId: 'handoff-1' }, + }); + await vi.advanceTimersByTimeAsync(0); + + expect(socket.sent.map((item) => JSON.parse(item)).find((item) => item.id === 'local-ui-1')).toMatchObject({ + type: 'response', + id: 'local-ui-1', + result: { ok: true }, + }); + expect(beginAgentAction).not.toHaveBeenCalled(); + }); + it('closes a half-open connection and rejects pending calls after missed heartbeats', async () => { const bridge = new EngineBridge(); const socket = await connect(bridge); diff --git a/src/features/engine-bridge/service.ts b/src/features/engine-bridge/service.ts index bec55c9..de4457e 100644 --- a/src/features/engine-bridge/service.ts +++ b/src/features/engine-bridge/service.ts @@ -1,7 +1,11 @@ import { browser } from 'wxt/browser'; import type { BridgeEnvelope } from '@/types/messages'; import type { BridgeConfig, BridgePairingStatus, BridgePublicKey, BridgeStatus } from '@/types/models'; -import { BRIDGE_CAPABILITIES, getBridgeCapabilityCatalog } from '@/protocol/capabilities'; +import { + BRIDGE_CAPABILITIES, + capabilityVisibleToAgent, + getBridgeCapabilityCatalog, +} from '@/protocol/capabilities'; import { BRIDGE_CHUNK_BYTES, BRIDGE_CHUNK_THRESHOLD_BYTES, BRIDGE_CHUNK_TIMEOUT_MS, BRIDGE_MAX_CHUNK_TRANSFERS, BRIDGE_MAX_MESSAGE_BYTES, BRIDGE_PROTOCOL_VERSION, parseBridgeEnvelope, @@ -525,11 +529,13 @@ export class EngineBridge { try { const grant = await browserInstanceAccess('browser.tabs.read'); taskId = grant.taskId; - actionId = (await beginAgentAction(grant, { - requestId: message.id, - method: message.method, - targetTabId, - })).id; + if (capabilityVisibleToAgent(message.method)) { + actionId = (await beginAgentAction(grant, { + requestId: message.id, + method: message.method, + targetTabId, + })).id; + } const operation = routeCapability(message.method, message.params, (method, params) => this.requestEngine(method, params)); const result = await Promise.race([operation, cancelled]); const durationMs = performance.now() - startedAt; diff --git a/src/features/grants/capability-handlers/handoff.ts b/src/features/grants/capability-handlers/handoff.ts index b7d0be3..6aba670 100644 --- a/src/features/grants/capability-handlers/handoff.ts +++ b/src/features/grants/capability-handlers/handoff.ts @@ -2,16 +2,29 @@ import { browser } from 'wxt/browser'; import type { HandoffReason } from '@/types/models'; import type { CapabilityDomainHandler } from '../capability-context'; import { allowedTarget } from '../capability-context'; -import { activateTab } from '@/platform/browser/targets'; import { getTab } from '@/platform/browser/targets'; import { getState, updateState } from '@/platform/storage/state'; import { setAgentRuntimeState } from '@/features/agent-runtime/service'; import { ExtensionError } from '@/shared/errors'; import { HANDOFF_CAPABILITY_DOMAIN } from '../capability-domains'; +import { focusHandoff, getHandoffPresentation, resolveHandoff } from '@/features/handoff/service'; export const handoffCapabilityHandler: CapabilityDomainHandler = { ...HANDOFF_CAPABILITY_DOMAIN, async handle({ method, input, grant }) { + if (method === 'browser.handoff.presentation.get') { + return getHandoffPresentation(String(input.handoffId || ''), grant); + } + if (method === 'browser.handoff.focus') { + return focusHandoff(String(input.handoffId || ''), grant); + } + if (method === 'browser.handoff.resolve') { + return resolveHandoff( + String(input.handoffId || ''), + input.outcome === 'cancelled' ? 'cancelled' : 'completed', + grant, + ); + } if (method === 'browser.handoff.status') { const handoff = (await getState()).handoff; return handoff?.taskId === grant.taskId ? handoff : { state: 'idle' }; @@ -50,7 +63,6 @@ export const handoffCapabilityHandler: CapabilityDomainHandler = { }, }; }); - await activateTab(resolvedTarget.tabId); await browser.action.setBadgeBackgroundColor({ color: '#ee7815' }); await browser.action.setBadgeText({ text: '待确认', tabId: resolvedTarget.tabId }); await setAgentRuntimeState('waiting_for_human', grant); diff --git a/src/features/grants/capability-handlers/page.ts b/src/features/grants/capability-handlers/page.ts index 348ca68..2ba760c 100644 --- a/src/features/grants/capability-handlers/page.ts +++ b/src/features/grants/capability-handlers/page.ts @@ -14,6 +14,7 @@ import { listCookies } from '@/features/cookies/service'; import { resolveTabCookieStoreId } from '@/platform/browser/isolation'; import { ExtensionError } from '@/shared/errors'; import { PAGE_CAPABILITY_DOMAIN } from '../capability-domains'; +import { getState } from '@/platform/storage/state'; export const pageCapabilityHandler: CapabilityDomainHandler = { ...PAGE_CAPABILITY_DOMAIN, @@ -63,7 +64,12 @@ export const pageCapabilityHandler: CapabilityDomainHandler = { await browser.action.setBadgeBackgroundColor({ color: '#f28c28' }); await browser.action.setBadgeText({ text: '接管', tabId: target.tabId }); globalThis.setTimeout( - () => void browser.action.setBadgeText({ text: '', tabId: target.tabId }), + () => void getState() + .then((state) => browser.action.setBadgeText({ + text: state.bridge.managedInstance?.badge || '', + tabId: target.tabId, + })) + .catch(() => undefined), 10_000, ); return { activated: true, target }; diff --git a/src/features/handoff/service.test.ts b/src/features/handoff/service.test.ts new file mode 100644 index 0000000..d02a776 --- /dev/null +++ b/src/features/handoff/service.test.ts @@ -0,0 +1,110 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const fixture = vi.hoisted(() => ({ + state: {} as Record, + activateTab: vi.fn(async () => undefined), + getFrame: vi.fn(), + getTab: vi.fn(), + resolveDocumentTarget: vi.fn(), + executeScript: vi.fn(), + scriptingTarget: vi.fn((target) => target), +})); + +vi.mock('wxt/browser', () => ({ + browser: { + storage: {}, + webNavigation: { getFrame: fixture.getFrame }, + scripting: { executeScript: fixture.executeScript }, + }, +})); +vi.mock('@/platform/storage/state', () => ({ + getState: vi.fn(async () => structuredClone(fixture.state)), + updateState: vi.fn(), +})); +vi.mock('@/platform/browser/targets', () => ({ + activateTab: fixture.activateTab, + getTab: fixture.getTab, + resolveDocumentTarget: fixture.resolveDocumentTarget, + scriptingTarget: fixture.scriptingTarget, +})); + +import { ExtensionError } from '@/shared/errors'; +import { focusHandoff, getHandoffPresentation, isSafeHandoffPresentationDataUrl } from './service'; + +const grant = { + id: 'paired-browser-instance', + taskId: 'paired-browser-instance', + targets: [], + scopes: [], + createdAt: 0, + expiresAt: Number.MAX_SAFE_INTEGER, +}; + +function waitingHandoff(origin = 'https://passport.example.test') { + return { + handoff: { + id: 'handoff-1', + taskId: 'paired-browser-instance', + state: 'waiting_for_user', + reason: 'qr_code', + target: { + tabId: 7, + frameId: 0, + documentId: 'document-old', + origin, + grantedUrl: `${origin}/login`, + title: 'Sign in', + }, + }, + }; +} + +describe('handoff presentation data URL validation', () => { + beforeEach(() => { + vi.clearAllMocks(); + fixture.state = {}; + }); + + it('accepts bounded raster data and rejects executable or oversized content', () => { + expect(isSafeHandoffPresentationDataUrl('data:image/png;base64,AAAA')).toBe(true); + expect(isSafeHandoffPresentationDataUrl('data:image/svg+xml,')).toBe(false); + expect(isSafeHandoffPresentationDataUrl(`data:image/png;base64,${'AAAA'.repeat(350_000)}`)).toBe(false); + }); + + it('focuses only the waiting handoff owned by the local paired task', async () => { + fixture.state = { + handoff: { + id: 'handoff-1', + taskId: 'paired-browser-instance', + state: 'waiting_for_user', + target: { tabId: 7 }, + }, + }; + + await expect(focusHandoff('handoff-1', grant)).resolves.toEqual({ focused: true, tabId: 7 }); + expect(fixture.activateTab).toHaveBeenCalledWith(7); + await expect(focusHandoff('other-handoff', grant)).rejects.toMatchObject({ code: 'handoff_not_waiting' }); + }); + + it('rebinds presentation reads after a same-origin document refresh', async () => { + fixture.state = waitingHandoff(); + fixture.resolveDocumentTarget + .mockRejectedValueOnce(new ExtensionError('stale_document', 'stale')) + .mockResolvedValueOnce({ tabId: 7, frameId: 0, documentId: 'document-new' }); + fixture.getFrame.mockResolvedValue({ url: 'https://passport.example.test/login?refreshed=1' }); + fixture.getTab.mockResolvedValue({ id: 7 }); + fixture.executeScript.mockResolvedValue([]); + + await expect(getHandoffPresentation('handoff-1', grant)).resolves.toMatchObject({ state: 'not_found' }); + expect(fixture.resolveDocumentTarget).toHaveBeenLastCalledWith({ tabId: 7, frameId: 0 }); + }); + + it('reports a changed page instead of leaking a stale-document error', async () => { + fixture.state = waitingHandoff(); + fixture.resolveDocumentTarget.mockRejectedValueOnce(new ExtensionError('stale_document', 'stale')); + fixture.getFrame.mockResolvedValue({ url: 'https://www.example.test/' }); + + await expect(getHandoffPresentation('handoff-1', grant)).resolves.toMatchObject({ state: 'page_changed' }); + expect(fixture.resolveDocumentTarget).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/features/handoff/service.ts b/src/features/handoff/service.ts new file mode 100644 index 0000000..1bf38fb --- /dev/null +++ b/src/features/handoff/service.ts @@ -0,0 +1,299 @@ +import { browser, type Browser } from 'wxt/browser'; +import { setAgentRuntimeState } from '@/features/agent-runtime/service'; +import { browserInstanceAccess } from '@/features/grants/capability-context'; +import { activateTab, getTab, resolveDocumentTarget, scriptingTarget } from '@/platform/browser/targets'; +import { getState, updateState } from '@/platform/storage/state'; +import { ExtensionError } from '@/shared/errors'; +import type { BridgeGrant, HandoffState, HumanHandoff } from '@/types/models'; + +const MAX_PRESENTATION_BYTES = 1024 * 1024; +const SAFE_RASTER_DATA_URL = /^data:image\/(?:png|jpeg|webp);base64,/i; + +interface PageQrCandidate { + dataUrl?: string; + source: 'image' | 'canvas' | 'svg' | 'background' | 'screenshot'; + rect: { x: number; y: number; width: number; height: number }; + viewport: { width: number; height: number; devicePixelRatio: number }; + title: string; + url: string; +} + +export interface HandoffPresentation { + handoffId: string; + state: HandoffState | 'not_found' | 'page_changed'; + title: string; + url: string; + capturedAt: number; + source?: PageQrCandidate['source']; + dataUrl?: string; +} + +async function resolvePresentationTarget(target: HumanHandoff['target']) { + try { + return await resolveDocumentTarget(target); + } catch (error) { + if (!(error instanceof ExtensionError) || !['stale_document', 'target_unavailable'].includes(error.code)) throw error; + } + + const frame = await browser.webNavigation.getFrame({ + tabId: target.tabId, + frameId: target.frameId, + }).catch(() => null); + if (!frame?.url || !/^https?:/i.test(frame.url) || new URL(frame.url).origin !== target.origin) return undefined; + return resolveDocumentTarget({ tabId: target.tabId, frameId: target.frameId }).catch(() => undefined); +} + +function dataUrlBytes(value: string): number { + const comma = value.indexOf(','); + return comma < 0 ? Number.MAX_SAFE_INTEGER : Math.ceil((value.length - comma - 1) * 0.75); +} + +export function isSafeHandoffPresentationDataUrl(value: unknown): value is string { + return typeof value === 'string' + && SAFE_RASTER_DATA_URL.test(value) + && dataUrlBytes(value) <= MAX_PRESENTATION_BYTES; +} + +async function findQrCandidateInPage(): Promise { + const resolvePresentationDataUrl = async (source: string, width: number, height: number): Promise => { + const rasterize = async (url: string): Promise => { + const image = new Image(); + image.decoding = 'async'; + await new Promise((resolve, reject) => { + image.onload = () => resolve(); + image.onerror = () => reject(new Error('image load failed')); + image.src = url; + }); + const canvas = document.createElement('canvas'); + canvas.width = Math.min(1024, Math.max(1, image.naturalWidth || Math.round(width))); + canvas.height = Math.min(1024, Math.max(1, image.naturalHeight || Math.round(height))); + canvas.getContext('2d')?.drawImage(image, 0, 0, canvas.width, canvas.height); + return canvas.toDataURL('image/png'); + }; + + try { + if (/^data:image\/(?:png|jpeg|webp);base64,/i.test(source)) return source; + if (/^data:image\/svg\+xml/i.test(source)) return rasterize(source); + if (!/^(?:blob:|https?:)/i.test(source)) return undefined; + const response = await fetch(source); + if (!response.ok) return undefined; + const blob = await response.blob(); + if (blob.size > 1024 * 1024) return undefined; + const localUrl = URL.createObjectURL(blob); + try { + if (/^image\/(?:png|jpeg|webp)$/i.test(blob.type)) { + return await new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(String(reader.result || '')); + reader.onerror = () => reject(reader.error); + reader.readAsDataURL(blob); + }); + } + if (blob.type === 'image/svg+xml') return await rasterize(localUrl); + } finally { + URL.revokeObjectURL(localUrl); + } + } catch { + return undefined; + } + return undefined; + }; + + const keywords = /(?:^|[^a-z])(qr|qrcode|scan)(?:[^a-z]|$)|二维码|扫码|扫码登录/i; + const selector = 'img,canvas,svg,[role="img"],[class*="qr" i],[id*="qr" i]'; + const seen = new Set(); + const candidates: Array<{ element: Element; score: number; rect: DOMRect }> = []; + + for (const element of document.querySelectorAll(selector)) { + if (seen.has(element)) continue; + seen.add(element); + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + if ( + rect.width < 96 || rect.height < 96 + || rect.bottom <= 0 || rect.right <= 0 + || rect.top >= innerHeight || rect.left >= innerWidth + || style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0 + ) continue; + const ratio = rect.width / rect.height; + if (ratio < 0.72 || ratio > 1.38) continue; + + const ownText = [ + element.id, + element.getAttribute('class'), + element.getAttribute('alt'), + element.getAttribute('aria-label'), + element.getAttribute('title'), + ].filter(Boolean).join(' '); + let contextText = ''; + let parent: Element | null = element; + for (let depth = 0; parent && depth < 4; depth += 1, parent = parent.parentElement) { + contextText += ` ${parent.textContent || ''}`; + if (contextText.length >= 500) break; + } + const inDialog = Boolean(element.closest('dialog,[role="dialog"],[aria-modal="true"]')); + const score = (keywords.test(ownText) ? 8 : 0) + + (keywords.test(contextText.slice(0, 500)) ? 5 : 0) + + (Math.abs(1 - ratio) < 0.12 ? 4 : 2) + + (inDialog ? 2 : 0) + + (element instanceof HTMLCanvasElement || element instanceof SVGElement ? 1 : 0); + if (score >= 6) candidates.push({ element, score, rect }); + } + + candidates.sort((left, right) => right.score - left.score || right.rect.width - left.rect.width); + for (const { element, rect } of candidates.slice(0, 8)) { + let source: PageQrCandidate['source'] = 'screenshot'; + let dataUrl: string | undefined; + try { + if (element instanceof HTMLCanvasElement) { + source = 'canvas'; + dataUrl = element.toDataURL('image/png'); + } else if (element instanceof SVGElement) { + source = 'svg'; + const svg = new XMLSerializer().serializeToString(element); + dataUrl = await resolvePresentationDataUrl( + `data:image/svg+xml;charset=utf-8,${encodeURIComponent(svg)}`, + rect.width, + rect.height, + ); + } else { + const imageSource = element instanceof HTMLImageElement + ? element.currentSrc || element.src + : getComputedStyle(element).backgroundImage.match(/^url\(["']?(.*?)["']?\)$/)?.[1] || ''; + source = element instanceof HTMLImageElement ? 'image' : 'background'; + dataUrl = await resolvePresentationDataUrl(imageSource, rect.width, rect.height); + } + } catch { + dataUrl = undefined; + } + return { + dataUrl, + source: dataUrl ? source : 'screenshot', + rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }, + viewport: { width: innerWidth, height: innerHeight, devicePixelRatio: devicePixelRatio || 1 }, + title: document.title, + url: location.href, + }; + } + return undefined; +} + +async function cropVisibleTab( + tab: Awaited>, + candidate: PageQrCandidate, +): Promise { + if (!tab.active || typeof OffscreenCanvas === 'undefined') return undefined; + const screenshot = await browser.tabs.captureVisibleTab(tab.windowId, { format: 'png' }); + const bitmap = await createImageBitmap(await (await fetch(screenshot)).blob()); + const scaleX = bitmap.width / candidate.viewport.width; + const scaleY = bitmap.height / candidate.viewport.height; + const padding = 12; + const x = Math.max(0, Math.floor((candidate.rect.x - padding) * scaleX)); + const y = Math.max(0, Math.floor((candidate.rect.y - padding) * scaleY)); + const width = Math.min(bitmap.width - x, Math.ceil((candidate.rect.width + padding * 2) * scaleX)); + const height = Math.min(bitmap.height - y, Math.ceil((candidate.rect.height + padding * 2) * scaleY)); + if (width < 1 || height < 1) return undefined; + const canvas = new OffscreenCanvas(width, height); + const context = canvas.getContext('2d'); + if (!context) return undefined; + context.fillStyle = '#fff'; + context.fillRect(0, 0, width, height); + context.drawImage(bitmap, x, y, width, height, 0, 0, width, height); + bitmap.close(); + const bytes = new Uint8Array(await (await canvas.convertToBlob({ type: 'image/png' })).arrayBuffer()); + if (bytes.byteLength > MAX_PRESENTATION_BYTES) return undefined; + let binary = ''; + for (let offset = 0; offset < bytes.length; offset += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); + } + return `data:image/png;base64,${btoa(binary)}`; +} + +export async function getHandoffPresentation(handoffId: string, grant: BridgeGrant): Promise { + const handoff = (await getState()).handoff; + if (!handoff || handoff.id !== handoffId || handoff.taskId !== grant.taskId) { + throw new ExtensionError('handoff_not_found', '人工接管请求不存在'); + } + const base = { + handoffId, + state: handoff.state, + title: handoff.target.title || '', + url: handoff.target.grantedUrl || '', + capturedAt: Date.now(), + }; + if (handoff.state !== 'waiting_for_user' || handoff.reason !== 'qr_code') return base; + + const target = await resolvePresentationTarget(handoff.target); + if (!target) return { ...base, state: 'page_changed' }; + const tab = await getTab(target.tabId); + const results = await browser.scripting.executeScript({ + target: scriptingTarget(target), + world: 'MAIN', + func: findQrCandidateInPage, + }) as Array>; + if (results.length !== 1 || !results[0]?.result) return { ...base, state: 'not_found' }; + + const candidate = results[0].result; + const directDataUrl = isSafeHandoffPresentationDataUrl(candidate.dataUrl) ? candidate.dataUrl : undefined; + const dataUrl = directDataUrl + ? directDataUrl + : target.frameId === 0 + ? await cropVisibleTab(tab, candidate).catch(() => undefined) + : undefined; + if (!isSafeHandoffPresentationDataUrl(dataUrl)) { + return { ...base, state: 'not_found', title: candidate.title || base.title, url: candidate.url || base.url }; + } + return { + ...base, + state: 'waiting_for_user', + title: candidate.title || base.title, + url: candidate.url || base.url, + source: directDataUrl ? candidate.source : 'screenshot', + dataUrl, + }; +} + +export async function focusHandoff(handoffId: string, grant: BridgeGrant): Promise<{ focused: true; tabId: number }> { + const handoff = (await getState()).handoff; + if ( + !handoff + || handoff.id !== handoffId + || handoff.taskId !== grant.taskId + || handoff.state !== 'waiting_for_user' + ) { + throw new ExtensionError('handoff_not_waiting', '人工接管请求不存在或已经结束'); + } + await activateTab(handoff.target.tabId); + return { focused: true, tabId: handoff.target.tabId }; +} + +export async function resolveHandoff( + handoffId: string, + outcome: Extract, + grant?: BridgeGrant, +): Promise<{ state: Awaited>; handoff: HumanHandoff }> { + const state = await updateState((current) => { + if ( + !current.handoff + || current.handoff.id !== handoffId + || current.handoff.state !== 'waiting_for_user' + || (grant && current.handoff.taskId !== grant.taskId) + ) { + throw new ExtensionError('handoff_not_waiting', '人工接管请求不存在或已经结束'); + } + return { + ...current, + handoff: { ...current.handoff, state: outcome, resolvedAt: Date.now() }, + }; + }); + const handoff = state.handoff!; + await setAgentRuntimeState( + outcome === 'completed' ? 'running' : 'paused', + grant || await browserInstanceAccess('browser.tabs.read'), + ); + await browser.action.setBadgeText({ + text: state.bridge.managedInstance?.badge || '', + tabId: handoff.target.tabId, + }).catch(() => undefined); + return { state, handoff }; +} diff --git a/src/protocol/bridge.ts b/src/protocol/bridge.ts index 466bd23..6899780 100644 --- a/src/protocol/bridge.ts +++ b/src/protocol/bridge.ts @@ -193,6 +193,16 @@ export const capabilityParams = { message: v.optional(v.pipe(v.string(), v.trim(), v.maxLength(500)), ''), }), 'browser.handoff.status': v.optional(v.strictObject({})), + 'browser.handoff.presentation.get': v.strictObject({ + handoffId: id, + }), + 'browser.handoff.focus': v.strictObject({ + handoffId: id, + }), + 'browser.handoff.resolve': v.strictObject({ + handoffId: id, + outcome: v.picklist(['completed', 'cancelled']), + }), 'browser.network.start': v.optional(v.strictObject({ ...targetFields, captureHeaders: v.optional(v.boolean()), diff --git a/src/protocol/capabilities.test.ts b/src/protocol/capabilities.test.ts index 0d288f8..ce49a23 100644 --- a/src/protocol/capabilities.test.ts +++ b/src/protocol/capabilities.test.ts @@ -3,6 +3,7 @@ import { BRIDGE_CAPABILITIES, canonicalCapabilityCatalogPayload, capabilityBaseScope, + capabilityVisibleToAgent, getBridgeCapabilityCatalog, } from './capabilities'; @@ -34,6 +35,23 @@ describe('versioned Bridge capability catalog', () => { expect(JSON.stringify(evalCapability?.paramsSchema)).toContain('"mode"'); expect(JSON.stringify(evalCapability?.paramsSchema)).toContain('"program"'); expect(capabilityBaseScope('browser.profile.validate')).toBe('browser.transform.execute'); + expect(catalog.capabilities.find((capability) => capability.method === 'browser.thumbnail')).toMatchObject({ + agentVisible: false, + }); + expect(catalog.capabilities.find((capability) => capability.method === 'browser.handoff.presentation.get')).toMatchObject({ + agentVisible: false, + }); + expect(catalog.capabilities.find((capability) => capability.method === 'browser.handoff.focus')).toMatchObject({ + agentVisible: false, + }); + expect(catalog.capabilities.find((capability) => capability.method === 'browser.handoff.resolve')).toMatchObject({ + agentVisible: false, + }); + expect(capabilityVisibleToAgent('browser.handoff.presentation.get')).toBe(false); + expect(capabilityVisibleToAgent('browser.handoff.focus')).toBe(false); + expect(capabilityVisibleToAgent('browser.handoff.resolve')).toBe(false); + expect(capabilityVisibleToAgent('browser.thumbnail')).toBe(false); + expect(capabilityVisibleToAgent('browser.context')).toBe(true); expect(catalog.capabilities.find((capability) => capability.method === 'browser.transform.recovery.capture')).toMatchObject({ access: 'dangerous', scopes: ['browser.transform.manage', 'browser.debugger.control', 'browser.callable.execute'], diff --git a/src/protocol/capabilities.ts b/src/protocol/capabilities.ts index 3c2b764..93b271e 100644 --- a/src/protocol/capabilities.ts +++ b/src/protocol/capabilities.ts @@ -20,6 +20,7 @@ export type BridgeCapabilityMethod = keyof typeof capabilityParams; interface CapabilityMetadata { domain: BridgeCapabilityDomain; access: BridgeCapabilityAccess; + agentVisible?: boolean; summary: string; scopes: CapabilityScope[]; conditionalScopes?: BridgeCapabilityScopeCondition[]; @@ -42,7 +43,7 @@ const CAPABILITY_METADATA = { }, 'browser.thumbnail': { domain: 'page', access: 'read', summary: '读取当前可见标签页的低清预览图，供 Yakit 实例列表展示', - scopes: ['browser.tabs.read'], targetMode: 'tab', defaultTimeoutMs: READ_TIMEOUT_MS, + scopes: ['browser.tabs.read'], targetMode: 'tab', defaultTimeoutMs: READ_TIMEOUT_MS, agentVisible: false, }, 'browser.isolation.inspect': { domain: 'isolation', access: 'read', summary: '读取浏览器实例内标签页的 Cookie Store 与身份隔离上下文', @@ -168,13 +169,29 @@ const CAPABILITY_METADATA = { scopes: ['browser.instance.close'], targetMode: 'none', defaultTimeoutMs: READ_TIMEOUT_MS, }, 'browser.handoff.request': { - domain: 'handoff', access: 'write', summary: '请求用户完成扫码、MFA、验证码或设备确认', + domain: 'handoff', access: 'write', + summary: '页面需要用户扫码、MFA、验证码或设备确认时调用；Yakit 会在本地呈现交互内容，Agent 只等待结果', scopes: ['browser.human.takeover'], targetMode: 'document', defaultTimeoutMs: READ_TIMEOUT_MS, }, 'browser.handoff.status': { domain: 'handoff', access: 'read', summary: '读取当前人工接管状态', scopes: ['browser.human.takeover'], targetMode: 'none', defaultTimeoutMs: READ_TIMEOUT_MS, }, + 'browser.handoff.presentation.get': { + domain: 'handoff', access: 'sensitive-read', agentVisible: false, + summary: '仅在本机提取当前扫码接管的二维码展示数据', + scopes: ['browser.human.takeover'], targetMode: 'none', defaultTimeoutMs: READ_TIMEOUT_MS, + }, + 'browser.handoff.focus': { + domain: 'handoff', access: 'write', agentVisible: false, + summary: '二维码无法在本地呈现时，由 Yakit 将对应浏览器实例切换到前台', + scopes: ['browser.human.takeover'], targetMode: 'none', defaultTimeoutMs: READ_TIMEOUT_MS, + }, + 'browser.handoff.resolve': { + domain: 'handoff', access: 'write', agentVisible: false, + summary: '由 Yakit 本地界面完成或取消人工接管', + scopes: ['browser.human.takeover'], targetMode: 'none', defaultTimeoutMs: READ_TIMEOUT_MS, + }, 'browser.network.start': { domain: 'network', access: 'control', summary: '启动有界网络捕获，可选采集请求头和 Body', scopes: ['browser.network.capture'], @@ -458,6 +475,11 @@ export function capabilityBaseScope(method: string): CapabilityScope | undefined return CAPABILITY_METADATA[method as BridgeCapabilityMethod]?.scopes[0]; } +export function capabilityVisibleToAgent(method: string): boolean { + const metadata = CAPABILITY_METADATA[method as BridgeCapabilityMethod] as CapabilityMetadata | undefined; + return metadata?.agentVisible !== false; +} + export function isControlScopeSet(scopes: readonly CapabilityScope[]): boolean { return scopes.some((scope) => !READ_CAPABILITY_SCOPES.includes(scope)); } diff --git a/src/types/messages.ts b/src/types/messages.ts index 09c6320..62083e4 100644 --- a/src/types/messages.ts +++ b/src/types/messages.ts @@ -306,6 +306,8 @@ export interface BridgeCapabilityDescriptor { method: string; domain: BridgeCapabilityDomain; access: BridgeCapabilityAccess; + /** False keeps local presentation/control methods out of the Agent tool catalog. */ + agentVisible?: boolean; summary: string; scopes: CapabilityScope[]; conditionalScopes?: BridgeCapabilityScopeCondition[];