feat: advance browser agent integration workflows

This commit is contained in:
go0p
2026-08-06 15:08:07 +08:00
parent c7891a6a9d
commit b11efcc79c
215 changed files with 35137 additions and 5442 deletions
+47 -75
View File
@@ -1,6 +1,6 @@
import { useEffect, useMemo, useRef, useState } from 'react';
import { useEffect, useRef, useState } from 'react';
import {
AlertTriangle, Braces, Check, ChevronLeft, ChevronRight, Copy, ExternalLink, GripVertical,
AlertTriangle, Braces, Check, Copy, ExternalLink,
EyeOff, Network, Pause, Play, Radio, RefreshCw, Settings, ShieldCheck, X,
} from 'lucide-react';
import { browser } from 'wxt/browser';
@@ -12,32 +12,25 @@ import { READ_CAPABILITY_SCOPES, isControlScopeSet } from '@/protocol/capabiliti
import { AGENT_RUNTIME_STORAGE_KEY, isStateStorageChange } from '@/protocol/storage';
import type { ActiveTabInfo, AgentRuntime, BridgeStatus, ExtensionState, PageContext } from '@/types/models';
import { errorMessage, request } from '@/platform/messaging/runtime';
import { isFloatingPanelShortcut, mergeFloatingTabUpdate } from './host-controller';
interface FloatingPanelProps {
initialState: ExtensionState;
initialTab?: ActiveTabInfo;
initialBridge: BridgeStatus;
yakIconUrl: string;
embedded?: boolean;
hostChannel: string;
}
export function FloatingPanel({ initialState, initialTab, initialBridge, yakIconUrl, embedded = false }: FloatingPanelProps) {
export function FloatingPanel({ initialState, initialTab, initialBridge, hostChannel }: FloatingPanelProps) {
const [state, setState] = useState(initialState);
const [bridge, setBridge] = useState(initialBridge);
const [tab] = useState(initialTab);
const [expanded, setExpanded] = useState(embedded);
const [side, setSide] = useState(initialState.floatingPanel.side);
const [y, setY] = useState(initialState.floatingPanel.y);
const [tab, setTab] = useState(initialTab);
const [busy, setBusy] = useState(false);
const [notice, setNotice] = useState('');
const [context, setContext] = useState<PageContext>();
const [runtime, setRuntime] = useState<AgentRuntime>({ state: 'idle', updatedAt: Date.now(), actions: [] });
const drag = useRef<{ pointerId: number; startX: number; startY: number; moved: boolean } | undefined>(undefined);
const bodyRef = useRef<HTMLDivElement>(null);
const activeProfile = useMemo(
() => state.proxyProfiles.find((profile) => profile.id === state.activeProxyId),
[state],
);
const grantActive = Boolean(
state.activeGrant && state.activeGrant.expiresAt > Date.now() && tab && state.activeGrant.targets.some((target) => target.tabId === tab.id),
);
@@ -50,8 +43,6 @@ export function FloatingPanel({ initialState, initialTab, initialBridge, yakIcon
if (isStateStorageChange(changes)) {
void request('state.get').then((next) => {
setState(next);
setSide(next.floatingPanel.side);
setY(next.floatingPanel.y);
}).catch(() => undefined);
}
if (AGENT_RUNTIME_STORAGE_KEY in changes) void request('agent.runtime.get').then(setRuntime).catch(() => undefined);
@@ -69,20 +60,50 @@ export function FloatingPanel({ initialState, initialTab, initialBridge, yakIcon
return () => browser.runtime.onMessage.removeListener(listener);
}, []);
// Embedded mode: report natural content height so the host shell can size the iframe (no dead space, internal scroll when clamped).
useEffect(() => {
if (!embedded) return undefined;
const listener = (event: MessageEvent) => {
const input = event.data as {
channel?: string;
token?: string;
type?: string;
tab?: { tabId: number; title?: string; url?: string };
};
if (event.source !== window.parent || input?.channel !== 'yakit-floating-host'
|| input.token !== hostChannel || input.type !== 'tab.changed' || !input.tab) return;
setTab((current) => mergeFloatingTabUpdate(current, input.tab!));
setContext(undefined);
};
globalThis.addEventListener('message', listener);
return () => globalThis.removeEventListener('message', listener);
}, [hostChannel]);
useEffect(() => {
const listener = (event: KeyboardEvent) => {
const target = event.target as HTMLElement | null;
const editable = Boolean(target?.isContentEditable || target?.closest('input, textarea, select, [contenteditable="true"]'));
const shortcut = isFloatingPanelShortcut(state.floatingPanel, event, editable);
if (!shortcut && event.key !== 'Escape') return;
event.preventDefault();
window.parent.postMessage({ channel: 'yakit-floating-host', token: hostChannel, type: 'collapse' }, '*');
};
globalThis.addEventListener('keydown', listener);
return () => globalThis.removeEventListener('keydown', listener);
}, [hostChannel, state.floatingPanel]);
// The content-script host owns header, drag, placement and collapse. This
// document reports body height only, so those responsibilities never exist twice.
useEffect(() => {
const post = () => {
const header = document.querySelector('.floating-panel__header');
const body = document.querySelector('.floating-panel__body');
const height = (header?.getBoundingClientRect().height || 46) + (body?.scrollHeight || 0);
window.parent.postMessage({ channel: 'yakit-floating-host', type: 'resize', height: Math.ceil(height) }, '*');
const height = bodyRef.current?.scrollHeight || 0;
window.parent.postMessage({
channel: 'yakit-floating-host', token: hostChannel, type: 'resize', height: Math.ceil(height),
}, '*');
};
post();
const observer = new ResizeObserver(post);
observer.observe(document.body);
if (bodyRef.current) observer.observe(bodyRef.current);
return () => observer.disconnect();
}, [embedded]);
}, [hostChannel]);
const run = async (task: () => Promise<void>) => {
setBusy(true);
@@ -113,59 +134,11 @@ export function FloatingPanel({ initialState, initialTab, initialBridge, yakIcon
}));
});
const onPointerDown = (event: React.PointerEvent<HTMLElement>) => {
if (event.button !== 0) return;
drag.current = { pointerId: event.pointerId, startX: event.clientX, startY: event.clientY, moved: false };
event.currentTarget.setPointerCapture(event.pointerId);
};
const onPointerMove = (event: React.PointerEvent<HTMLElement>) => {
const current = drag.current;
if (!current || current.pointerId !== event.pointerId) return;
if (Math.hypot(event.clientX - current.startX, event.clientY - current.startY) > 4) current.moved = true;
if (!current.moved) return;
setY(Math.min(Math.max(event.clientY / window.innerHeight, 0.08), 0.92));
setSide(event.clientX < window.innerWidth / 2 ? 'left' : 'right');
};
const onPointerUp = (event: React.PointerEvent<HTMLElement>) => {
const current = drag.current;
if (!current || current.pointerId !== event.pointerId) return;
drag.current = undefined;
if (current.moved) {
const nextSide = event.clientX < window.innerWidth / 2 ? 'left' : 'right';
const nextY = Math.min(Math.max(event.clientY / window.innerHeight, 0.08), 0.92);
setSide(nextSide);
setY(nextY);
void request('panel.update', { side: nextSide, y: nextY }).then(setState).catch(() => undefined);
} else {
setExpanded((value) => !value);
}
};
if (!state.floatingPanel.enabled) return null;
const collapseEmbedded = () => window.parent.postMessage({ channel: 'yakit-floating-host', type: 'collapse' }, '*');
return (
<div className={`floating-panel floating-panel--${side} ${embedded ? 'floating-panel--embedded' : ''} ${expanded ? 'is-expanded' : ''}`} style={embedded ? undefined : { top: `${y * 100}%` }}>
<div className="floating-panel__header" onClick={embedded ? collapseEmbedded : undefined} onPointerDown={embedded ? undefined : onPointerDown} onPointerMove={embedded ? undefined : onPointerMove} onPointerUp={embedded ? undefined : onPointerUp}>
<button className="floating-panel__brand" aria-label={expanded ? '收起 Yakit Browser Agent' : '展开 Yakit Browser Agent'}>
<img src={yakIconUrl} alt="Yak" draggable={false} />
<span className={`floating-panel__signal ${bridge.state}`} />
</button>
{expanded && <>
<div className="floating-panel__title">
<strong>Yakit Browser Agent</strong>
<span>{activeProfile?.name || (state.activeProxyId === 'auto' ? '自动切换' : '浏览器工具')}</span>
</div>
<GripVertical className="floating-panel__grip" size={15} aria-hidden="true" />
{side === 'right' ? <ChevronRight size={15} /> : <ChevronLeft size={15} />}
</>}
</div>
{expanded && (
<div className="floating-panel__body">
<div className="floating-panel floating-panel--embedded is-expanded">
<div ref={bodyRef} className="floating-panel__body">
<Tabs key={handoff?.id || 'default'} defaultValue={handoff ? 'agent' : 'proxy'}>
<TabsList className="floating-tabs">
<TabsTrigger value="proxy"><Network size={13} />代理</TabsTrigger>
@@ -213,7 +186,6 @@ export function FloatingPanel({ initialState, initialTab, initialBridge, yakIcon
</Tabs>
{notice && <div className="floating-notice">{notice}</div>}
</div>
)}
</div>
);
}
@@ -0,0 +1,119 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import type { ActiveTabInfo, ExtensionState, FloatingPanelPreferences } from '@/types/models';
import {
createLazyUnloadController,
floatingPanelVisible,
isFloatingPanelShortcut,
mergeFloatingTabUpdate,
resolvePanelPlacement,
shouldCollapseForFullscreen,
} from './host-controller';
const preferences: FloatingPanelPreferences = {
enabled: true,
side: 'right',
y: 0.46,
displayMode: 'always',
siteMode: 'all',
siteOrigins: [],
shortcutEnabled: true,
autoCollapseFullscreen: true,
};
const tab: ActiveTabInfo = {
id: 7,
windowId: 1,
title: 'Before',
url: 'https://example.test/before',
incognito: false,
isolationContextId: 'browser-profile:default',
};
function state(overrides: Partial<ExtensionState> = {}): ExtensionState {
return {
version: 7,
proxyProfiles: [],
proxyRules: [],
proxyRuleSources: [],
proxyRouting: { defaultProfileId: 'direct', failMode: 'closed' },
proxyRuntime: { dirty: false, compiledBytes: 0, manualRuleCount: 0, sourceRuleCount: 0, warnings: [] },
activeProxyId: 'direct',
customUserAgentProfiles: [],
userAgentAssignments: [],
bridge: {
transport: 'websocket', nativeHost: 'com.yaklang.browser_agent',
endpoint: 'ws://127.0.0.1:64333/extension', autoConnect: false, installationId: 'fixture-installation',
},
floatingPanel: preferences,
...overrides,
};
}
describe('floating panel host controller', () => {
beforeEach(() => vi.useRealTimers());
it('updates URL and title for the same SPA tab without changing its isolation identity', () => {
expect(mergeFloatingTabUpdate(tab, {
tabId: tab.id,
title: 'Account · 42',
url: 'https://example.test/account/42',
})).toEqual({
...tab,
title: 'Account · 42',
url: 'https://example.test/account/42',
});
expect(mergeFloatingTabUpdate(tab, { tabId: 9, title: 'Another tab' })).toBe(tab);
expect(mergeFloatingTabUpdate(tab, { tabId: tab.id, url: 'chrome://settings' })).toBe(tab);
});
it('applies all, allowlist, denylist and active-task visibility policies', () => {
expect(floatingPanelVisible(state(), tab, 'https://example.test')).toBe(true);
expect(floatingPanelVisible(state({ floatingPanel: { ...preferences, siteMode: 'allowlist', siteOrigins: [] } }), tab, 'https://example.test')).toBe(false);
expect(floatingPanelVisible(state({ floatingPanel: { ...preferences, siteMode: 'allowlist', siteOrigins: ['https://example.test'] } }), tab, 'https://example.test')).toBe(true);
expect(floatingPanelVisible(state({ floatingPanel: { ...preferences, siteMode: 'denylist', siteOrigins: ['https://example.test'] } }), tab, 'https://example.test')).toBe(false);
expect(floatingPanelVisible(state({ floatingPanel: { ...preferences, displayMode: 'active-task' } }), tab, 'https://example.test')).toBe(false);
expect(floatingPanelVisible(state({
floatingPanel: { ...preferences, displayMode: 'active-task' },
activeGrant: {
id: 'grant-1', taskId: 'task-1', createdAt: 1, expiresAt: 20_000,
scopes: [], targets: [{
tabId: tab.id, frameId: 0, isolationContextId: 'browser-profile:default',
origin: 'https://example.test', grantedUrl: tab.url, title: 'Fixture',
}],
},
}), tab, 'https://example.test', 10_000)).toBe(true);
});
it('clamps drag placement and snaps to the nearest side', () => {
expect(resolvePanelPlacement(10, -100, 1_000, 800)).toEqual({ side: 'left', y: 0.08 });
expect(resolvePanelPlacement(999, 2_000, 1_000, 800)).toEqual({ side: 'right', y: 0.92 });
expect(resolvePanelPlacement(400, 320, 1_000, 800)).toEqual({ side: 'left', y: 0.4 });
});
it('does not steal the keyboard shortcut from editable controls or key repeat', () => {
const event = { altKey: true, shiftKey: true, code: 'KeyY', repeat: false };
expect(isFloatingPanelShortcut(preferences, event, false)).toBe(true);
expect(isFloatingPanelShortcut(preferences, event, true)).toBe(false);
expect(isFloatingPanelShortcut(preferences, { ...event, repeat: true }, false)).toBe(false);
expect(shouldCollapseForFullscreen(preferences, true)).toBe(true);
expect(shouldCollapseForFullscreen({ ...preferences, autoCollapseFullscreen: false }, true)).toBe(false);
});
it('cancels and replaces lazy iframe unload timers deterministically', () => {
vi.useFakeTimers();
const unload = vi.fn();
const controller = createLazyUnloadController(60_000, unload);
controller.schedule();
vi.advanceTimersByTime(30_000);
controller.schedule();
vi.advanceTimersByTime(30_001);
expect(unload).not.toHaveBeenCalled();
controller.cancel();
vi.advanceTimersByTime(60_000);
expect(unload).not.toHaveBeenCalled();
controller.schedule();
vi.advanceTimersByTime(60_000);
expect(unload).toHaveBeenCalledTimes(1);
controller.dispose();
});
});
@@ -0,0 +1,105 @@
import type { ActiveTabInfo, ExtensionState, FloatingPanelPreferences } from '@/types/models';
export interface FloatingTabUpdate {
tabId: number;
title?: string;
url?: string;
}
export function mergeFloatingTabUpdate(
current: ActiveTabInfo | undefined,
update: FloatingTabUpdate,
): ActiveTabInfo | undefined {
if (!current || current.id !== update.tabId) return current;
const nextUrl = typeof update.url === 'string' && /^https?:/i.test(update.url)
? update.url.slice(0, 8_192)
: current.url;
const nextTitle = typeof update.title === 'string' && update.title.trim()
? update.title.trim().slice(0, 1_024)
: current.title;
if (nextUrl === current.url && nextTitle === current.title) return current;
return { ...current, url: nextUrl, title: nextTitle };
}
export function floatingPanelVisible(
state: ExtensionState,
tab: ActiveTabInfo | undefined,
pageOrigin: string,
now = Date.now(),
): boolean {
if (!state.floatingPanel.enabled) return false;
const siteAllowed = state.floatingPanel.siteMode === 'allowlist'
? state.floatingPanel.siteOrigins.includes(pageOrigin)
: state.floatingPanel.siteMode === 'denylist'
? !state.floatingPanel.siteOrigins.includes(pageOrigin)
: true;
if (!siteAllowed) return false;
if (state.floatingPanel.displayMode === 'always') return true;
const activeGrant = Boolean(state.activeGrant && state.activeGrant.expiresAt > now && tab
&& state.activeGrant.targets.some((target) => target.tabId === tab.id));
const waitingHandoff = Boolean(state.handoff?.state === 'waiting_for_user' && tab
&& state.handoff.target.tabId === tab.id);
return activeGrant || waitingHandoff;
}
export function resolvePanelPlacement(
clientX: number,
clientY: number,
viewportWidth: number,
viewportHeight: number,
): { side: 'left' | 'right'; y: number } {
const safeWidth = Math.max(1, viewportWidth);
const safeHeight = Math.max(1, viewportHeight);
return {
side: clientX < safeWidth / 2 ? 'left' : 'right',
y: Math.min(Math.max(clientY / safeHeight, 0.08), 0.92),
};
}
export function isFloatingPanelShortcut(
preferences: FloatingPanelPreferences,
input: Pick<KeyboardEvent, 'altKey' | 'shiftKey' | 'code' | 'repeat'>,
editableTarget: boolean,
): boolean {
return preferences.shortcutEnabled
&& input.altKey
&& input.shiftKey
&& input.code === 'KeyY'
&& !input.repeat
&& !editableTarget;
}
export function shouldCollapseForFullscreen(
preferences: FloatingPanelPreferences,
fullscreenActive: boolean,
): boolean {
return preferences.autoCollapseFullscreen && fullscreenActive;
}
export interface LazyUnloadController {
schedule(): void;
cancel(): void;
dispose(): void;
}
export function createLazyUnloadController(
delayMs: number,
unload: () => void,
): LazyUnloadController {
let timer: ReturnType<typeof globalThis.setTimeout> | undefined;
const cancel = () => {
if (timer !== undefined) globalThis.clearTimeout(timer);
timer = undefined;
};
return {
schedule() {
cancel();
timer = globalThis.setTimeout(() => {
timer = undefined;
unload();
}, Math.max(0, delayMs));
},
cancel,
dispose: cancel,
};
}
@@ -0,0 +1,63 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
const fixture = vi.hoisted(() => ({
updatedListeners: [] as Array<(tabId: number, change: { url?: string; title?: string; status?: string }) => void>,
historyListeners: [] as Array<(details: { tabId: number; frameId: number }) => void>,
fragmentListeners: [] as Array<(details: { tabId: number; frameId: number }) => void>,
removedListeners: [] as Array<(tabId: number) => void>,
tabs: new Map<number, { id: number; url: string; title: string }>(),
sendMessage: vi.fn(async () => undefined),
}));
vi.mock('wxt/browser', () => ({
browser: {
tabs: {
get: vi.fn(async (tabId: number) => {
const tab = fixture.tabs.get(tabId);
if (!tab) throw new Error('missing tab');
return tab;
}),
sendMessage: fixture.sendMessage,
onUpdated: { addListener: vi.fn((listener: typeof fixture.updatedListeners[number]) => fixture.updatedListeners.push(listener)) },
onRemoved: { addListener: vi.fn((listener: typeof fixture.removedListeners[number]) => fixture.removedListeners.push(listener)) },
},
webNavigation: {
onHistoryStateUpdated: { addListener: vi.fn((listener: typeof fixture.historyListeners[number]) => fixture.historyListeners.push(listener)) },
onReferenceFragmentUpdated: { addListener: vi.fn((listener: typeof fixture.fragmentListeners[number]) => fixture.fragmentListeners.push(listener)) },
},
},
}));
import { initializeFloatingPanelLifecycle } from './lifecycle';
describe('floating panel lifecycle relay', () => {
beforeEach(() => {
vi.useFakeTimers();
fixture.tabs.clear();
fixture.sendMessage.mockClear();
});
it('coalesces SPA URL and title changes into the current main-frame snapshot', async () => {
initializeFloatingPanelLifecycle();
fixture.tabs.set(8, { id: 8, url: 'https://example.test/account/2', title: 'Account 2' });
for (const listener of fixture.historyListeners) listener({ tabId: 8, frameId: 0 });
for (const listener of fixture.updatedListeners) listener(8, { title: 'Account 2' });
await vi.advanceTimersByTimeAsync(41);
expect(fixture.sendMessage).toHaveBeenCalledTimes(1);
expect(fixture.sendMessage).toHaveBeenCalledWith(8, {
action: 'floating.tab.changed',
payload: { tabId: 8, title: 'Account 2', url: 'https://example.test/account/2' },
}, { frameId: 0 });
});
it('ignores sub-frame navigation and cancels pending work when a tab closes', async () => {
initializeFloatingPanelLifecycle();
fixture.tabs.set(9, { id: 9, url: 'https://example.test/', title: 'Fixture' });
for (const listener of fixture.fragmentListeners) listener({ tabId: 9, frameId: 2 });
for (const listener of fixture.updatedListeners) listener(9, { url: 'https://example.test/#new' });
for (const listener of fixture.removedListeners) listener(9);
await vi.advanceTimersByTimeAsync(100);
expect(fixture.sendMessage).not.toHaveBeenCalled();
});
});
+58
View File
@@ -0,0 +1,58 @@
import { browser } from 'wxt/browser';
interface FloatingTabChangedMessage {
action: 'floating.tab.changed';
payload: { tabId: number; title?: string; url?: string };
}
let initialized = false;
const pendingTabs = new Map<number, ReturnType<typeof globalThis.setTimeout>>();
async function notifyTab(tabId: number): Promise<void> {
try {
const tab = await browser.tabs.get(tabId);
if (!tab.id || !tab.url || !/^https?:/i.test(tab.url)) return;
const message: FloatingTabChangedMessage = {
action: 'floating.tab.changed',
payload: {
tabId,
title: tab.title || undefined,
url: tab.url,
},
};
await browser.tabs.sendMessage(tabId, message, { frameId: 0 });
} catch {
// The content script may have been destroyed by a committed navigation.
// The new document initializes from tab.active, so this is not an error.
}
}
function scheduleTabNotification(tabId: number): void {
const previous = pendingTabs.get(tabId);
if (previous !== undefined) globalThis.clearTimeout(previous);
pendingTabs.set(tabId, globalThis.setTimeout(() => {
pendingTabs.delete(tabId);
void notifyTab(tabId);
}, 40));
}
export function initializeFloatingPanelLifecycle(): void {
if (initialized) return;
initialized = true;
browser.tabs.onUpdated.addListener((tabId, change) => {
if (change.url !== undefined || change.title !== undefined || change.status === 'complete') {
scheduleTabNotification(tabId);
}
});
browser.webNavigation.onHistoryStateUpdated.addListener((details) => {
if (details.frameId === 0) scheduleTabNotification(details.tabId);
});
browser.webNavigation.onReferenceFragmentUpdated.addListener((details) => {
if (details.frameId === 0) scheduleTabNotification(details.tabId);
});
browser.tabs.onRemoved.addListener((tabId) => {
const timer = pendingTabs.get(tabId);
if (timer !== undefined) globalThis.clearTimeout(timer);
pendingTabs.delete(tabId);
});
}