From aec5486c51ebcc7119bb7cd4a8168f751caa9514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A2=93=E6=81=92?= <160735726+3361559784@users.noreply.github.com> Date: Wed, 13 May 2026 12:08:38 +0800 Subject: [PATCH] test(computer-use-mcp): add desktop v3 smoke coverage (#1780) --- apps/stage-tamagotchi/package.json | 1 + .../desktop-overlay-live-window-smoke.test.ts | 40 ++ .../desktop-overlay-live-window-smoke.ts | 563 ++++++++++++++++++ apps/stage-tamagotchi/src/main/index.ts | 5 + .../src/main/windows/desktop-overlay/index.ts | 22 +- .../rpc/index.electron.test.ts | 98 +++ .../renderer/pages/desktop-overlay-polling.ts | 45 ++ .../src/renderer/pages/desktop-overlay.vue | 49 +- .../src/shared/desktop-overlay-heartbeat.ts | 5 + .../desktop-overlay-live-window-smoke.test.ts | 53 ++ .../desktop-overlay-live-window-smoke.ts | 24 + docs/desktop-lane-status.md | 78 ++- services/computer-use-mcp/package.json | 1 + .../src/bin/smoke-chrome-grounding.test.ts | 209 +++++++ .../src/bin/smoke-chrome-grounding.ts | 498 ++++++++++++++++ .../src/browser-action-router.test.ts | 12 + .../src/browser-dom/extension-bridge.test.ts | 23 + .../src/chrome-session-manager.test.ts | 489 ++++++++------- .../src/chrome-session-manager.ts | 321 +++++++--- .../src/desktop-grounding.test.ts | 89 ++- .../computer-use-mcp/src/desktop-grounding.ts | 20 +- .../server/register-chrome-session.test.ts | 62 +- .../src/server/register-chrome-session.ts | 8 +- .../src/support-matrix.test.ts | 19 + .../computer-use-mcp/src/support-matrix.ts | 24 + 25 files changed, 2352 insertions(+), 406 deletions(-) create mode 100644 apps/stage-tamagotchi/scripts/desktop-overlay-live-window-smoke.test.ts create mode 100644 apps/stage-tamagotchi/scripts/desktop-overlay-live-window-smoke.ts create mode 100644 apps/stage-tamagotchi/src/main/windows/desktop-overlay/rpc/index.electron.test.ts create mode 100644 apps/stage-tamagotchi/src/shared/desktop-overlay-heartbeat.ts create mode 100644 apps/stage-tamagotchi/src/shared/desktop-overlay-live-window-smoke.test.ts create mode 100644 apps/stage-tamagotchi/src/shared/desktop-overlay-live-window-smoke.ts create mode 100644 services/computer-use-mcp/src/bin/smoke-chrome-grounding.test.ts create mode 100644 services/computer-use-mcp/src/bin/smoke-chrome-grounding.ts diff --git a/apps/stage-tamagotchi/package.json b/apps/stage-tamagotchi/package.json index 3a74a6ba1..852517de0 100644 --- a/apps/stage-tamagotchi/package.json +++ b/apps/stage-tamagotchi/package.json @@ -30,6 +30,7 @@ "merge-latest-mac": "tsx scripts/merge-latest-mac.ts", "regenerate-windows-latest": "tsx scripts/regenerate-windows-latest.ts", "artifacts-metadata": "tsx scripts/artifacts-metadata.ts", + "smoke:desktop-overlay-live-window": "NODE_OPTIONS='--experimental-websocket' tsx scripts/desktop-overlay-live-window-smoke.ts", "update-test:generate": "tsx scripts/update-test/generate-manifest.ts", "update-test:server": "tsx scripts/update-test/start-server.ts", "update-test:matrix": "bash scripts/update-test/run-matrix.sh" diff --git a/apps/stage-tamagotchi/scripts/desktop-overlay-live-window-smoke.test.ts b/apps/stage-tamagotchi/scripts/desktop-overlay-live-window-smoke.test.ts new file mode 100644 index 000000000..aedaa4ae3 --- /dev/null +++ b/apps/stage-tamagotchi/scripts/desktop-overlay-live-window-smoke.test.ts @@ -0,0 +1,40 @@ +import { EventEmitter } from 'node:events' + +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { CdpClient } from './desktop-overlay-live-window-smoke' + +afterEach(() => { + vi.restoreAllMocks() +}) + +function createMockSocket() { + const socket = new EventEmitter() as EventEmitter & { + send: ReturnType + close: ReturnType + addEventListener: (event: string, listener: (...args: any[]) => void) => void + } + socket.send = vi.fn() + socket.close = vi.fn(() => { + socket.emit('close') + }) + socket.addEventListener = (event, listener) => { + socket.on(event, listener) + } + return socket +} + +describe('cdpClient', () => { + it('rejects pending requests when the socket closes', async () => { + const socket = createMockSocket() + const client = new CdpClient(socket as never) + + const pending = client.send('Runtime.evaluate', { expression: '1 + 1' }) + expect(socket.send).toHaveBeenCalledTimes(1) + + client.close() + + await expect(pending).rejects.toThrow('CDP socket closed before completing request 1') + expect(socket.close).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/stage-tamagotchi/scripts/desktop-overlay-live-window-smoke.ts b/apps/stage-tamagotchi/scripts/desktop-overlay-live-window-smoke.ts new file mode 100644 index 000000000..8712fd441 --- /dev/null +++ b/apps/stage-tamagotchi/scripts/desktop-overlay-live-window-smoke.ts @@ -0,0 +1,563 @@ +import type { ChildProcessWithoutNullStreams } from 'node:child_process' + +import { spawn } from 'node:child_process' +import { createWriteStream } from 'node:fs' +import { access, mkdir, writeFile } from 'node:fs/promises' +import { createServer } from 'node:net' +import { dirname, resolve } from 'node:path' +import { env, exit, kill as killProcess } from 'node:process' +import { fileURLToPath } from 'node:url' + +import { desktopOverlayPollHeartbeatMarker } from '../src/shared/desktop-overlay-heartbeat' +import { selectDesktopOverlaySmokeCandidateId } from '../src/shared/desktop-overlay-live-window-smoke' + +interface DebugTarget { + id: string + title: string + type: string + url: string + webSocketDebuggerUrl?: string +} + +interface McpResult { + content?: unknown[] + structuredContent?: Record + isError?: boolean +} + +interface McpApplyResult { + started: Array<{ name: string }> + failed: Array<{ name: string, error: string }> + skipped: Array<{ name: string, reason: string }> +} + +interface McpToolDescriptor { + name: string + serverName: string + toolName: string +} + +interface McpRuntimeStatus { + servers: Array<{ + name: string + state: 'running' | 'stopped' | 'error' + lastError?: string + }> +} + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const repoDir = resolve(packageDir, '../..') +const runId = new Date().toISOString().replaceAll(':', '-').replaceAll('.', '-') +const reportDir = resolve(repoDir, '.temp', `desktop-overlay-live-window-smoke-${runId}`) +const userDataDir = resolve(reportDir, 'stage-user-data') +const mcpSessionRoot = resolve(reportDir, 'computer-use-session') +const stageLogPath = resolve(reportDir, 'stage-tamagotchi.log') +const mcpConfigPath = resolve(userDataDir, 'mcp.json') +const requiredWorkspaceBuildOutputs = [ + 'packages/electron-screen-capture/dist/main.mjs', + 'packages/electron-vueuse/dist/main/index.mjs', + 'packages/server-runtime/dist/server.mjs', +] + +const smokeHtml = ` + + + AIRI Desktop Overlay Live Window Smoke + + + +

AIRI Desktop Overlay Live Window Smoke

+ + +` + +const smokeUrl = `data:text/html;charset=utf-8,${encodeURIComponent(smokeHtml)}` + +function assert(condition: boolean, message: string): asserts condition { + if (!condition) + throw new Error(message) +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)) +} + +async function findAvailablePort(): Promise { + return await new Promise((resolvePort, reject) => { + const server = createServer() + server.listen(0, '127.0.0.1', () => { + const address = server.address() + server.close(() => { + if (typeof address === 'object' && address?.port) { + resolvePort(address.port) + } + else { + reject(new Error('failed to allocate debug port')) + } + }) + }) + server.on('error', reject) + }) +} + +async function waitFor( + label: string, + probe: () => Promise | T | undefined, + timeoutMs: number, + intervalMs: number, +): Promise { + const start = Date.now() + let lastError: unknown + + while ((Date.now() - start) < timeoutMs) { + try { + const value = await probe() + if (value !== undefined) + return value + } + catch (error) { + lastError = error + } + await sleep(intervalMs) + } + + const suffix = lastError instanceof Error ? `: ${lastError.message}` : '' + throw new Error(`${label} timed out after ${timeoutMs}ms${suffix}`) +} + +export class CdpClient { + private socket?: WebSocket + private nextId = 1 + private pending = new Map) => void + reject: (error: Error) => void + }>() + + constructor(socket: WebSocket) { + this.socket = socket + this.socket.addEventListener('message', (event) => { + const payload = JSON.parse(String(event.data)) as Record + const id = typeof payload.id === 'number' ? payload.id : undefined + if (id === undefined) + return + + const pending = this.pending.get(id) + if (!pending) + return + + this.pending.delete(id) + if (payload.error) { + pending.reject(new Error(JSON.stringify(payload.error))) + } + else { + pending.resolve(payload) + } + }) + this.socket.addEventListener('close', () => { + this.failPending('CDP socket closed') + }) + this.socket.addEventListener('error', () => { + this.failPending('CDP socket errored') + }) + } + + static async connect(url: string): Promise { + const socket = new WebSocket(url) + await new Promise((resolveOpen, reject) => { + socket.addEventListener('open', () => resolveOpen(), { once: true }) + socket.addEventListener('error', () => reject(new Error(`failed to connect CDP target: ${url}`)), { once: true }) + }) + return new CdpClient(socket) + } + + async send(method: string, params?: Record): Promise> { + if (!this.socket) { + throw new Error('CDP socket is closed') + } + + const id = this.nextId++ + const promise = new Promise>((resolveMessage, reject) => { + this.pending.set(id, { resolve: resolveMessage, reject }) + }) + + this.socket.send(JSON.stringify({ id, method, params: params ?? {} })) + return await promise + } + + async evaluate(expression: string): Promise { + const response = await this.send('Runtime.evaluate', { + expression, + awaitPromise: true, + returnByValue: true, + }) + const result = response.result + if (!isRecord(result)) + throw new Error('Runtime.evaluate missing result') + + const exceptionDetails = result.exceptionDetails + if (exceptionDetails) { + throw new Error(JSON.stringify(exceptionDetails)) + } + + const remoteObject = result.result + if (!isRecord(remoteObject)) + throw new Error('Runtime.evaluate missing remote object') + + return remoteObject.value as T + } + + close() { + this.failPending('CDP socket closed') + this.socket?.close() + this.socket = undefined + } + + private failPending(reason: string) { + if (this.pending.size === 0) { + return + } + + for (const [id, pending] of this.pending.entries()) { + this.pending.delete(id) + pending.reject(new Error(`${reason} before completing request ${id}`)) + } + } +} + +async function fetchJson(url: string): Promise { + const response = await fetch(url) + if (!response.ok) + throw new Error(`${url} returned ${response.status}`) + return await response.json() as T +} + +async function prepareMcpConfig() { + await mkdir(userDataDir, { recursive: true }) + await mkdir(mcpSessionRoot, { recursive: true }) + + const mcpEnv: Record = { + PATH: env.PATH || '', + HOME: env.HOME || '', + SHELL: env.SHELL || '', + LANG: env.LANG || 'en_US.UTF-8', + TMPDIR: env.TMPDIR || '', + COMPUTER_USE_EXECUTOR: env.COMPUTER_USE_SMOKE_EXECUTOR || env.COMPUTER_USE_EXECUTOR || 'macos-local', + COMPUTER_USE_APPROVAL_MODE: env.COMPUTER_USE_SMOKE_APPROVAL_MODE || env.COMPUTER_USE_APPROVAL_MODE || 'never', + COMPUTER_USE_OPENABLE_APPS: env.COMPUTER_USE_OPENABLE_APPS || 'Terminal,Cursor,Google Chrome', + COMPUTER_USE_SESSION_TAG: `desktop-overlay-live-window-smoke-${runId}`, + COMPUTER_USE_SESSION_ROOT: mcpSessionRoot, + } + + for (const optionalEnvName of ['PNPM_HOME', 'COREPACK_HOME']) { + const value = env[optionalEnvName]?.trim() + if (value) { + mcpEnv[optionalEnvName] = value + } + } + + const config = { + mcpServers: { + computer_use: { + command: 'pnpm', + args: ['-F', '@proj-airi/computer-use-mcp', 'start'], + cwd: repoDir, + enabled: true, + env: mcpEnv, + }, + }, + } + + await writeFile(mcpConfigPath, `${JSON.stringify(config, null, 2)}\n`, 'utf-8') +} + +async function ensureSmokePrerequisites() { + if (typeof WebSocket !== 'function') { + throw new TypeError('APP_START_FAILED: WebSocket is unavailable in this Node runtime. Run through the package script or set NODE_OPTIONS=--experimental-websocket.') + } + + const missingOutputs: string[] = [] + for (const relativePath of requiredWorkspaceBuildOutputs) { + try { + await access(resolve(repoDir, relativePath)) + } + catch { + missingOutputs.push(relativePath) + } + } + + if (missingOutputs.length === 0) + return + + throw new Error([ + 'APP_START_FAILED: required workspace build outputs are missing.', + `Missing: ${missingOutputs.join(', ')}`, + 'Build stage-tamagotchi dependencies manually before this smoke. The smoke command does not auto-build them to avoid saturating the local machine.', + 'Suggested command: pnpm -F \'@proj-airi/stage-tamagotchi^...\' --if-present build', + ].join(' ')) +} + +async function waitForRemoteDebug(debugPort: number): Promise { + const version = await waitFor('Electron remote debug endpoint', async () => { + const data = await fetchJson<{ webSocketDebuggerUrl?: string }>(`http://127.0.0.1:${debugPort}/json/version`) + return data.webSocketDebuggerUrl + }, 120_000, 500) + + return version +} + +async function findOverlayTarget(debugPort: number): Promise { + return await waitFor('desktop overlay debug target', async () => { + const targets = await fetchJson(`http://127.0.0.1:${debugPort}/json/list`) + return targets.find(target => target.type === 'page' && target.url.includes('#/desktop-overlay')) + }, 120_000, 500) +} + +async function connectOverlayClient(debugPort: number): Promise { + const overlayTarget = await findOverlayTarget(debugPort) + if (!overlayTarget.webSocketDebuggerUrl) + throw new Error('APP_START_FAILED: overlay target missing webSocketDebuggerUrl') + + const client = await CdpClient.connect(overlayTarget.webSocketDebuggerUrl) + + await waitFor('overlay smoke bridge', async () => { + return await client.evaluate('Boolean(window.__AIRI_DESKTOP_OVERLAY_SMOKE__?.callMcpTool)') + ? true + : undefined + }, 60_000, 500) + + return client +} + +async function callOverlayMcpTool(client: CdpClient, name: string, args: Record = {}): Promise { + const result = await client.evaluate(`window.__AIRI_DESKTOP_OVERLAY_SMOKE__.callMcpTool(${JSON.stringify({ name, arguments: args })})`) + if (result.isError) { + throw new Error(`${name} returned isError=true`) + } + return result +} + +async function ensureOverlayMcpServerReady(client: CdpClient): Promise { + const applyResult = await client.evaluate('window.__AIRI_DESKTOP_OVERLAY_SMOKE__.applyAndRestartMcp()') + const failedComputerUse = applyResult.failed.find(item => item.name === 'computer_use') + if (failedComputerUse) { + throw new Error(`computer_use failed to start: ${failedComputerUse.error}`) + } + + await waitFor('computer_use MCP runtime', async () => { + const status = await client.evaluate('window.__AIRI_DESKTOP_OVERLAY_SMOKE__.getMcpRuntimeStatus()') + const computerUse = status.servers.find(server => server.name === 'computer_use') + if (computerUse?.state === 'error') { + throw new Error(`computer_use runtime error: ${computerUse.lastError ?? 'unknown error'}`) + } + return computerUse?.state === 'running' ? true : undefined + }, 30_000, 500) + + await waitFor('computer_use desktop tools', async () => { + const tools = await client.evaluate('window.__AIRI_DESKTOP_OVERLAY_SMOKE__.listMcpTools()') + const names = new Set(tools.map(tool => tool.name)) + return names.has('computer_use::desktop_get_state') + && names.has('computer_use::desktop_observe') + && names.has('computer_use::desktop_click_target') + ? true + : undefined + }, 30_000, 500) +} + +function requireStructuredContent(result: McpResult, label: string): Record { + if (!isRecord(result.structuredContent)) + throw new Error(`${label} missing structuredContent`) + + if (result.structuredContent.status && result.structuredContent.status !== 'ok') + throw new Error(`${label} expected status=ok, got ${String(result.structuredContent.status)}`) + + return result.structuredContent +} + +function requireRunState(result: McpResult, label: string): Record { + const structuredContent = requireStructuredContent(result, label) + if (!isRecord(structuredContent.runState)) + throw new Error(`${label} missing runState`) + return structuredContent.runState +} + +function startStage(debugPort: number, heartbeatLines: string[]): ChildProcessWithoutNullStreams { + const stageProcess = spawn('pnpm', ['-F', '@proj-airi/stage-tamagotchi', 'dev'], { + cwd: repoDir, + detached: true, + env: { + ...env, + APP_REMOTE_DEBUG: 'true', + APP_REMOTE_DEBUG_PORT: String(debugPort), + APP_REMOTE_DEBUG_NO_OPEN: 'true', + APP_USER_DATA_PATH: userDataDir, + AIRI_DESKTOP_OVERLAY: '1', + AIRI_DESKTOP_OVERLAY_POLL_HEARTBEAT: '1', + }, + stdio: 'pipe', + }) + + const stageLogStream = createWriteStream(stageLogPath, { flags: 'a' }) + const capture = (chunk: Buffer) => { + const text = chunk.toString('utf-8') + stageLogStream.write(text) + for (const line of text.split(/\r?\n/u)) { + if (line.includes(desktopOverlayPollHeartbeatMarker)) { + heartbeatLines.push(line) + } + } + } + stageProcess.stdout.on('data', capture) + stageProcess.stderr.on('data', capture) + stageProcess.on('close', () => stageLogStream.end()) + + return stageProcess +} + +async function stopStage(stageProcess: ChildProcessWithoutNullStreams | undefined) { + if (!stageProcess || stageProcess.exitCode !== null) + return + + const signalStageProcessGroup = (signal: NodeJS.Signals) => { + try { + if (stageProcess.pid) { + killProcess(-stageProcess.pid, signal) + return + } + } + catch { + // Fall back to the pnpm wrapper process if process-group signalling is + // unavailable. The smoke starts a detached group to make this reliable on + // macOS, but the fallback keeps the helper safe on other local setups. + } + + stageProcess.kill(signal) + } + + signalStageProcessGroup('SIGTERM') + await Promise.race([ + new Promise(resolve => stageProcess.once('exit', resolve)), + sleep(5_000).then(() => signalStageProcessGroup('SIGKILL')), + ]) +} + +function rejectWhenStageExits(stageProcess: ChildProcessWithoutNullStreams): Promise { + return new Promise((_, reject) => { + stageProcess.once('exit', (code, signal) => { + reject(new Error(`stage-tamagotchi exited with code=${String(code)} signal=${String(signal)}`)) + }) + }) +} + +async function main() { + let stageProcess: ChildProcessWithoutNullStreams | undefined + let overlayClient: CdpClient | undefined + let stoppingStage = false + const heartbeatLines: string[] = [] + + try { + await ensureSmokePrerequisites() + await mkdir(reportDir, { recursive: true }) + await prepareMcpConfig() + + const debugPort = await findAvailablePort() + stageProcess = startStage(debugPort, heartbeatLines) + const stageExited = rejectWhenStageExits(stageProcess) + stageProcess.once('exit', (code, signal) => { + if (!stoppingStage && code !== null && code !== 0) + console.error(`APP_START_FAILED: stage-tamagotchi exited with code=${code} signal=${String(signal)}`) + }) + + await Promise.race([ + waitForRemoteDebug(debugPort), + stageExited, + ]).catch((error) => { + throw new Error(`APP_START_FAILED: ${error instanceof Error ? error.message : String(error)}`) + }) + + overlayClient = await Promise.race([ + connectOverlayClient(debugPort), + stageExited, + ]).catch((error) => { + throw new Error(`APP_START_FAILED: ${error instanceof Error ? error.message : String(error)}`) + }) + + // NOTICE: + // Vite's dev optimizer can trigger one renderer reload shortly after the + // Electron window first exposes the smoke bridge. Reconnect once after a + // short settle window so the following MCP calls do not race a closing CDP + // target. This is local smoke harness discipline, not product runtime. + await sleep(5_000) + overlayClient.close() + overlayClient = await Promise.race([ + connectOverlayClient(debugPort), + stageExited, + ]).catch((error) => { + throw new Error(`APP_START_FAILED: ${error instanceof Error ? error.message : String(error)}`) + }) + + const readiness = await overlayClient.evaluate<{ state: 'booting' | 'ready' | 'degraded', error?: string }>('window.__AIRI_DESKTOP_OVERLAY_SMOKE__.getReadiness()') + if (readiness.state !== 'ready') { + throw new Error(`OVERLAY_READINESS_DEGRADED: state=${readiness.state}${readiness.error ? ` error=${readiness.error}` : ''}`) + } + + try { + await ensureOverlayMcpServerReady(overlayClient) + await callOverlayMcpTool(overlayClient, 'computer_use::desktop_ensure_chrome', { url: smokeUrl }) + await sleep(750) + await callOverlayMcpTool(overlayClient, 'computer_use::desktop_observe', { includeChrome: true }) + const preClickRunState = requireRunState( + await callOverlayMcpTool(overlayClient, 'computer_use::desktop_get_state'), + 'computer_use::desktop_get_state before click', + ) + const candidateId = selectDesktopOverlaySmokeCandidateId(preClickRunState) + await callOverlayMcpTool(overlayClient, 'computer_use::desktop_click_target', { + candidateId, + button: 'left', + clickCount: 1, + }) + const postClickRunState = requireRunState( + await callOverlayMcpTool(overlayClient, 'computer_use::desktop_get_state'), + 'computer_use::desktop_get_state after click', + ) + const pointerIntent = postClickRunState.lastPointerIntent + assert(isRecord(pointerIntent), 'computer_use::desktop_get_state missing lastPointerIntent after click') + assert(pointerIntent.candidateId === candidateId, `lastPointerIntent candidate mismatch: expected ${candidateId}, got ${String(pointerIntent.candidateId)}`) + } + catch (error) { + throw new Error(`MCP_CALL_FAILED: ${error instanceof Error ? error.message : String(error)}`) + } + + const heartbeat = await waitFor('overlay poll heartbeat', () => { + return heartbeatLines.find(line => line.includes('snapshotId=') && line.includes('pointerIntent=yes')) + }, 30_000, 250).catch((error) => { + throw new Error(`HEARTBEAT_TIMEOUT: ${error instanceof Error ? error.message : String(error)}`) + }) + + console.info(JSON.stringify({ + ok: true, + reportDir, + stageLogPath, + heartbeat, + }, null, 2)) + } + finally { + overlayClient?.close() + stoppingStage = true + await stopStage(stageProcess) + } +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)) + console.error(`stage log: ${stageLogPath}`) + exit(1) + }) +} diff --git a/apps/stage-tamagotchi/src/main/index.ts b/apps/stage-tamagotchi/src/main/index.ts index 0d4c2679e..0310a131c 100644 --- a/apps/stage-tamagotchi/src/main/index.ts +++ b/apps/stage-tamagotchi/src/main/index.ts @@ -59,6 +59,11 @@ setupDebugger() const log = useLogg('main').useGlobalConfig() +const appUserDataPath = env.APP_USER_DATA_PATH?.trim() +if (appUserDataPath) { + app.setPath('userData', appUserDataPath) +} + // Thanks to [@blurymind](https://github.com/blurymind), // // When running Electron on Linux, navigator.gpu.requestAdapter() fails. diff --git a/apps/stage-tamagotchi/src/main/windows/desktop-overlay/index.ts b/apps/stage-tamagotchi/src/main/windows/desktop-overlay/index.ts index 19a05d246..708407085 100644 --- a/apps/stage-tamagotchi/src/main/windows/desktop-overlay/index.ts +++ b/apps/stage-tamagotchi/src/main/windows/desktop-overlay/index.ts @@ -27,6 +27,7 @@ import { join, resolve } from 'node:path' import { BrowserWindow, screen } from 'electron' +import { desktopOverlayPollHeartbeatMarker, desktopOverlayPollHeartbeatQueryParam } from '../../../shared/desktop-overlay-heartbeat' import { baseUrl, getElectronMainDirname, load, withHashRoute } from '../../libs/electron/location' import { setupDesktopOverlayElectronInvokes } from './rpc/index.electron' import { @@ -40,6 +41,15 @@ export function isDesktopOverlayEnabled(): boolean { return process.env.AIRI_DESKTOP_OVERLAY === '1' } +/** + * Smoke-only overlay heartbeat mode. + * The recut desktop smoke uses this to surface renderer console lines and + * mount the in-page smoke bridge. + */ +export function isDesktopOverlayPollHeartbeatEnabled(): boolean { + return process.env.AIRI_DESKTOP_OVERLAY_POLL_HEARTBEAT === '1' +} + let overlayWindow: BrowserWindow | null = null /** @@ -81,6 +91,14 @@ export async function setupDesktopOverlayWindow(params: { overlayWindow = null }) + if (isDesktopOverlayPollHeartbeatEnabled()) { + overlayWindow.webContents.on('console-message', (_event, _level, message) => { + if (message.includes(desktopOverlayPollHeartbeatMarker)) { + console.info(message) + } + }) + } + // NOTICE: Wire eventa RPC BEFORE loading the renderer page. // The overlay's onMounted fires during load() and immediately starts // polling via callTool. If the handlers aren't registered yet, the @@ -99,7 +117,9 @@ export async function setupDesktopOverlayWindow(params: { overlayWindow, withHashRoute( baseUrl(resolve(getElectronMainDirname(), '..', 'renderer')), - '/desktop-overlay', + isDesktopOverlayPollHeartbeatEnabled() + ? `/desktop-overlay?${desktopOverlayPollHeartbeatQueryParam}=1` + : '/desktop-overlay', ), ) diff --git a/apps/stage-tamagotchi/src/main/windows/desktop-overlay/rpc/index.electron.test.ts b/apps/stage-tamagotchi/src/main/windows/desktop-overlay/rpc/index.electron.test.ts new file mode 100644 index 000000000..349fb199d --- /dev/null +++ b/apps/stage-tamagotchi/src/main/windows/desktop-overlay/rpc/index.electron.test.ts @@ -0,0 +1,98 @@ +import type { BrowserWindow } from 'electron' + +import type { I18n } from '../../../libs/i18n' +import type { ServerChannel } from '../../../services/airi/channel-server' +import type { McpStdioManager } from '../../../services/airi/mcp-servers' + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { setupDesktopOverlayElectronInvokes } from './index.electron' + +const defineInvokeHandlerMock = vi.hoisted(() => vi.fn()) +const createContextMock = vi.hoisted(() => vi.fn(() => ({ context: { id: 'desktop-overlay-test' } }))) +const setupBaseWindowElectronInvokesMock = vi.hoisted(() => vi.fn()) +const createMcpServersServiceMock = vi.hoisted(() => vi.fn()) +const ipcMainMock = vi.hoisted(() => ({ setMaxListeners: vi.fn() })) + +vi.mock('@moeru/eventa', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + defineInvokeHandler: defineInvokeHandlerMock, + } +}) + +vi.mock('@moeru/eventa/adapters/electron/main', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + createContext: createContextMock, + } +}) + +vi.mock('electron', () => ({ + ipcMain: ipcMainMock, +})) + +vi.mock('../../shared/window', () => ({ + setupBaseWindowElectronInvokes: setupBaseWindowElectronInvokesMock, +})) + +vi.mock('../../../services/airi/mcp-servers', () => ({ + createMcpServersService: createMcpServersServiceMock, +})) + +describe('setupDesktopOverlayElectronInvokes', () => { + const window = {} as BrowserWindow + const mcpStdioManager = {} as McpStdioManager + const serverChannel = {} as ServerChannel + const i18n = {} as I18n + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('publishes ready after the base window invokes and MCP services are wired', async () => { + let readinessHandler: (() => Promise<{ state: 'booting' | 'ready' | 'degraded', error?: string }>) | undefined + + defineInvokeHandlerMock.mockImplementation((_context, _contract, handler) => { + readinessHandler = handler + }) + setupBaseWindowElectronInvokesMock.mockResolvedValue(undefined) + createMcpServersServiceMock.mockReturnValue(undefined) + + await setupDesktopOverlayElectronInvokes({ + window, + mcpStdioManager, + serverChannel, + i18n, + }) + + expect(ipcMainMock.setMaxListeners).toHaveBeenCalledWith(0) + expect(createContextMock).toHaveBeenCalledTimes(1) + expect(setupBaseWindowElectronInvokesMock).toHaveBeenCalledTimes(1) + expect(createMcpServersServiceMock).toHaveBeenCalledTimes(1) + expect(readinessHandler).toBeDefined() + await expect(readinessHandler!()).resolves.toEqual({ state: 'ready' }) + }) + + it('publishes degraded when the base window invokes fail', async () => { + let readinessHandler: (() => Promise<{ state: 'booting' | 'ready' | 'degraded', error?: string }>) | undefined + + defineInvokeHandlerMock.mockImplementation((_context, _contract, handler) => { + readinessHandler = handler + }) + setupBaseWindowElectronInvokesMock.mockRejectedValueOnce(new Error('boom')) + + await setupDesktopOverlayElectronInvokes({ + window, + mcpStdioManager, + serverChannel, + i18n, + }) + + expect(createMcpServersServiceMock).not.toHaveBeenCalled() + expect(readinessHandler).toBeDefined() + await expect(readinessHandler!()).resolves.toEqual({ state: 'degraded', error: 'boom' }) + }) +}) diff --git a/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay-polling.ts b/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay-polling.ts index a44b2b018..fd5b38180 100644 --- a/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay-polling.ts +++ b/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay-polling.ts @@ -7,6 +7,8 @@ import type { McpCallToolResult } from '@proj-airi/stage-ui/stores/mcp-tool-bridge' +import { desktopOverlayPollHeartbeatMarker, desktopOverlayPollHeartbeatQueryParam } from '../../shared/desktop-overlay-heartbeat' + // --------------------------------------------------------------------------- // Types — minimal shapes matching RunState fields the overlay consumes // --------------------------------------------------------------------------- @@ -46,6 +48,12 @@ export interface OverlayState { lastBootstrapError?: string } +export interface OverlayPollHeartbeat { + snapshotId: string + candidateCount: number + hasPointerIntent: boolean +} + // --------------------------------------------------------------------------- // State extraction // --------------------------------------------------------------------------- @@ -111,6 +119,37 @@ export function extractRunStateFromResult(result: McpCallToolResult): Record } +export function createOverlayPollHeartbeat(state: OverlayState): OverlayPollHeartbeat | undefined { + if (!state.hasSnapshot || !state.snapshotId) + return undefined + + return { + snapshotId: state.snapshotId, + candidateCount: state.candidates.length, + hasPointerIntent: state.pointerIntent !== null, + } +} + +export function formatOverlayPollHeartbeat(heartbeat: OverlayPollHeartbeat): string { + return [ + desktopOverlayPollHeartbeatMarker, + `snapshotId=${heartbeat.snapshotId}`, + `candidates=${heartbeat.candidateCount}`, + `pointerIntent=${heartbeat.hasPointerIntent ? 'yes' : 'no'}`, + ].join(' ') +} + +export function isOverlayPollHeartbeatEnabled(locationLike: Pick = window.location): boolean { + const hashQuery = locationLike.hash.includes('?') + ? locationLike.hash.slice(locationLike.hash.indexOf('?') + 1) + : '' + const hashParams = new URLSearchParams(hashQuery) + const searchParams = new URLSearchParams(locationLike.search) + + return hashParams.get(desktopOverlayPollHeartbeatQueryParam) === '1' + || searchParams.get(desktopOverlayPollHeartbeatQueryParam) === '1' +} + // --------------------------------------------------------------------------- // Polling controller (framework-agnostic) // --------------------------------------------------------------------------- @@ -129,6 +168,8 @@ export interface OverlayPollConfig { callTool: (name: string) => Promise /** Callback with extracted state on each successful poll. */ onState: (state: OverlayState) => void + /** Optional debug-only callback with a small heartbeat marker. */ + onHeartbeat?: (heartbeat: OverlayPollHeartbeat) => void /** Function to ping main process readiness contract via Eventa. */ getReadiness: () => Promise<{ state: 'booting' | 'ready' | 'degraded', error?: string }> /** Normal poll interval in ms. Default: 250. */ @@ -299,6 +340,10 @@ export function createOverlayPollController(config: OverlayPollConfig): OverlayP state.bootstrapState = currentBootstrapState state.lastBootstrapError = currentBootstrapError config.onState(state) + const heartbeat = createOverlayPollHeartbeat(state) + if (heartbeat && config.onHeartbeat) { + config.onHeartbeat(heartbeat) + } } else { nextInterval = fallbackInterval diff --git a/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay.vue b/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay.vue index 87259d241..e06e3d32c 100644 --- a/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay.vue +++ b/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay.vue @@ -18,12 +18,24 @@ import type { OverlayState } from './desktop-overlay-polling' import { electron } from '@proj-airi/electron-eventa' import { useElectronEventaInvoke } from '@proj-airi/electron-vueuse' -import { getMcpToolBridge } from '@proj-airi/stage-ui/stores/mcp-tool-bridge' import { computed, onMounted, onUnmounted, ref, watch } from 'vue' -import { getDesktopOverlayReadinessContract } from '../../shared/eventa' +import { electronMcpApplyAndRestart, electronMcpCallTool, electronMcpGetRuntimeStatus, electronMcpListTools, getDesktopOverlayReadinessContract } from '../../shared/eventa' import { pointInOverlay, rectIntersectsOverlay, screenRectToLocal, screenToLocal } from './desktop-overlay-coordinates' -import { createEmptyOverlayState, createOverlayPollController } from './desktop-overlay-polling' +import { createEmptyOverlayState, createOverlayPollController, formatOverlayPollHeartbeat, isOverlayPollHeartbeatEnabled } from './desktop-overlay-polling' + +declare global { + interface Window { + __AIRI_DESKTOP_OVERLAY_SMOKE__?: { + applyAndRestartMcp: () => Promise + callMcpTool: (payload: { name: string, arguments?: Record }) => Promise + getMcpRuntimeStatus: () => Promise + getOverlayState: () => OverlayState + getReadiness: () => Promise<{ state: 'booting' | 'ready' | 'degraded', error?: string }> + listMcpTools: () => Promise + } + } +} // --------------------------------------------------------------------------- // Overlay window bounds — read once on mount from main process @@ -31,7 +43,12 @@ import { createEmptyOverlayState, createOverlayPollController } from './desktop- const getWindowBounds = useElectronEventaInvoke(electron.window.getBounds) const getReadiness = useElectronEventaInvoke(getDesktopOverlayReadinessContract) +const applyAndRestartMcp = useElectronEventaInvoke(electronMcpApplyAndRestart) +const callMcpTool = useElectronEventaInvoke(electronMcpCallTool) +const getMcpRuntimeStatus = useElectronEventaInvoke(electronMcpGetRuntimeStatus) +const listMcpTools = useElectronEventaInvoke(electronMcpListTools) const overlayBounds = ref(null) +const pollHeartbeatEnabled = isOverlayPollHeartbeatEnabled() // --------------------------------------------------------------------------- // Reactive state — single ref driven by poll controller @@ -71,21 +88,15 @@ const matchedCandidate = computed(() => { // Polling controller // --------------------------------------------------------------------------- -let bridgeAvailable = false - const controller = createOverlayPollController({ - callTool: async (name) => { - // Probe bridge availability lazily - if (!bridgeAvailable) { - getMcpToolBridge() // Throws if not set - bridgeAvailable = true - } - return getMcpToolBridge().callTool({ name }) - }, + callTool: name => callMcpTool({ name }), getReadiness: async () => getReadiness(), onState: (newState) => { state.value = newState }, + onHeartbeat: pollHeartbeatEnabled + ? heartbeat => console.info(formatOverlayPollHeartbeat(heartbeat)) + : undefined, }) // --------------------------------------------------------------------------- @@ -199,11 +210,23 @@ onMounted(async () => { } } + if (pollHeartbeatEnabled) { + window.__AIRI_DESKTOP_OVERLAY_SMOKE__ = { + applyAndRestartMcp, + callMcpTool, + getMcpRuntimeStatus, + getOverlayState: () => state.value, + getReadiness, + listMcpTools, + } + } + controller.start() }) onUnmounted(() => { controller.stop() + delete window.__AIRI_DESKTOP_OVERLAY_SMOKE__ }) diff --git a/apps/stage-tamagotchi/src/shared/desktop-overlay-heartbeat.ts b/apps/stage-tamagotchi/src/shared/desktop-overlay-heartbeat.ts new file mode 100644 index 000000000..831f9220e --- /dev/null +++ b/apps/stage-tamagotchi/src/shared/desktop-overlay-heartbeat.ts @@ -0,0 +1,5 @@ +/** Debug-only marker used by the local overlay live-window smoke. */ +export const desktopOverlayPollHeartbeatMarker = '[AIRI_DESKTOP_OVERLAY_POLL_HEARTBEAT]' + +/** Query parameter used to enable overlay polling heartbeat logs. */ +export const desktopOverlayPollHeartbeatQueryParam = 'pollHeartbeat' diff --git a/apps/stage-tamagotchi/src/shared/desktop-overlay-live-window-smoke.test.ts b/apps/stage-tamagotchi/src/shared/desktop-overlay-live-window-smoke.test.ts new file mode 100644 index 000000000..b053d1ae3 --- /dev/null +++ b/apps/stage-tamagotchi/src/shared/desktop-overlay-live-window-smoke.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest' + +import { selectDesktopOverlaySmokeCandidateId } from './desktop-overlay-live-window-smoke' + +describe('selectDesktopOverlaySmokeCandidateId', () => { + it('prefers the smoke button label', () => { + const candidateId = selectDesktopOverlaySmokeCandidateId({ + lastGroundingSnapshot: { + targetCandidates: [ + { id: 'first', label: 'Something else', role: 'link', source: 'chrome_dom' }, + { id: 'smoke', label: 'AIRI Desktop Overlay Smoke Button', role: 'button', source: 'chrome_dom' }, + ], + }, + }) + + expect(candidateId).toBe('smoke') + }) + + it('requires the smoke button to come from chrome_dom', () => { + expect(() => { + selectDesktopOverlaySmokeCandidateId({ + lastGroundingSnapshot: { + targetCandidates: [ + { id: 'ax-smoke', label: 'AIRI Desktop Overlay Smoke Button', role: 'button', source: 'ax' }, + ], + }, + }) + }).toThrow('desktop_observe did not return the AIRI Desktop Overlay Smoke Button chrome_dom candidate') + }) + + it('fails when the smoke label is missing', () => { + expect(() => { + selectDesktopOverlaySmokeCandidateId({ + lastGroundingSnapshot: { + targetCandidates: [ + { id: 'first', label: 'Alpha', role: 'link' }, + { id: 'second', label: 'Beta', role: 'button' }, + ], + }, + }) + }).toThrow('desktop_observe did not return the AIRI Desktop Overlay Smoke Button chrome_dom candidate') + }) + + it('throws when no candidate ids exist', () => { + expect(() => { + selectDesktopOverlaySmokeCandidateId({ + lastGroundingSnapshot: { + targetCandidates: [{ label: 'Missing id', role: 'button' }], + }, + }) + }).toThrow('desktop_observe did not return the AIRI Desktop Overlay Smoke Button chrome_dom candidate') + }) +}) diff --git a/apps/stage-tamagotchi/src/shared/desktop-overlay-live-window-smoke.ts b/apps/stage-tamagotchi/src/shared/desktop-overlay-live-window-smoke.ts new file mode 100644 index 000000000..ca87337c3 --- /dev/null +++ b/apps/stage-tamagotchi/src/shared/desktop-overlay-live-window-smoke.ts @@ -0,0 +1,24 @@ +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null +} + +export function selectDesktopOverlaySmokeCandidateId(runState: Record): string { + const snapshot = runState.lastGroundingSnapshot + if (!isRecord(snapshot)) + throw new Error('desktop_get_state missing lastGroundingSnapshot after desktop_observe') + + const candidates = Array.isArray(snapshot.targetCandidates) + ? snapshot.targetCandidates.filter(isRecord) + : [] + + const selected = candidates.find((candidate) => { + return candidate.source === 'chrome_dom' + && String(candidate.label || '').includes('AIRI Desktop Overlay Smoke Button') + }) + + const id = typeof selected?.id === 'string' ? selected.id : '' + if (!id) + throw new Error('desktop_observe did not return the AIRI Desktop Overlay Smoke Button chrome_dom candidate') + + return id +} diff --git a/docs/desktop-lane-status.md b/docs/desktop-lane-status.md index 228567969..b7b918e5d 100644 --- a/docs/desktop-lane-status.md +++ b/docs/desktop-lane-status.md @@ -1,8 +1,8 @@ # Desktop Lane Status -Updated: 2026-04-14 +Updated: 2026-05-08 -This note is a factual status memo for the current desktop lane work around PR #1649. It is intentionally narrow: only current state, actual blockers, and what should happen now vs later. +This note is a factual status memo for the current desktop lane work in this recut branch. It is intentionally narrow: only current state, actual blockers, and what should happen now vs later. ## What is already true @@ -18,53 +18,47 @@ This note is a factual status memo for the current desktop lane work around PR # - `makeWindowPassThrough()` uses ignore-mouse-events + non-focusable overlay behavior - `/Users/liuziheng/airi/services/computer-use-mcp/src/browser-dom/cdp-bridge.ts` - 5-second heartbeat with teardown after 3 consecutive failures + - `/Users/liuziheng/airi/services/computer-use-mcp/src/bin/smoke-chrome-grounding.ts` + - desktop v3 smoke that proves the ensure / observe / click / state chain - The Chrome extension bridge and iframe offset work are no longer hypothetical: - - PR #1649 already contains a real extension-side WebSocket client bridge - - PR #1649 already contains frame offset propagation for iframe DOM candidates + - the recut branch already contains a real extension-side WebSocket client bridge + - the recut branch already contains frame offset propagation for iframe DOM candidates +- The browser-dom route contract is now fail-closed: + - non-left clicks and multi-clicks stay on OS input + - `BrowserDomExtensionBridge` rejects `ok: false` responses instead of treating them as success ## What is actually still blocking These are the remaining real issues, ordered by severity. -### 1. Extension unknown actions still return `ok: true` +### 1. Live desktop v3 smoke exists, but it is baseline coverage, not product support. -- File: - - `/Users/liuziheng/airi-pr1649/services/computer-use-mcp/chrome-extension/background.js` -- Current behavior: - - unsupported actions fall into `result = { error: ... }` - - but the response still returns `{ ok: true, result }` -- Why this matters: - - upper layers can interpret unsupported DOM actions as successful bridge execution - - that can suppress OS-input fallback even though nothing actually happened -- This is still a real unresolved review blocker. +- Current smoke proves: + `desktop_ensure_chrome -> desktop_observe -> desktop_click_target -> desktop_get_state` +- It does not prove Chrome semantic DOM routing, live overlay-window rendering, + or user-input isolation. -### 2. Browser-dom click routing still ignores non-default click semantics - -- File: - - `/Users/liuziheng/airi-pr1649/services/computer-use-mcp/src/browser-action-router.ts` - - called from `/Users/liuziheng/airi-pr1649/services/computer-use-mcp/src/server/register-desktop-grounding.ts` -- Current behavior: - - `chrome_dom` candidates route to browser-dom if selector + bridge are available - - routing does not currently incorporate `button` / `clickCount` -- Why this matters: - - right-click or double-click can still be routed to a DOM path that only performs a standard primary click -- This is not as severe as the first issue, but it is still a real correctness gap. - -### 3. Overlay lifecycle / RPC readiness is not fully closed yet +### 2. Overlay lifecycle / RPC readiness still needs a live-window pass on this recut branch. - Files currently being worked on: - - `/Users/liuziheng/airi-pr1649/apps/stage-tamagotchi/src/main/windows/desktop-overlay/rpc/contracts.ts` - - `/Users/liuziheng/airi-pr1649/apps/stage-tamagotchi/src/main/windows/desktop-overlay/rpc/index.electron.ts` - - `/Users/liuziheng/airi-pr1649/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay-polling.ts` - - `/Users/liuziheng/airi-pr1649/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay-polling.test.ts` + - `/Users/liuziheng/airi-desktop-recut/apps/stage-tamagotchi/src/main/windows/desktop-overlay/rpc/contracts.ts` + - `/Users/liuziheng/airi-desktop-recut/apps/stage-tamagotchi/src/main/windows/desktop-overlay/rpc/index.electron.ts` + - `/Users/liuziheng/airi-desktop-recut/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay-polling.ts` + - `/Users/liuziheng/airi-desktop-recut/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay-polling.test.ts` - Current state: - - there is already a preload-order mitigation in `desktop-overlay/index.ts` - - there is already a per-call timeout in `desktop-overlay-polling.ts` - - there is now work-in-progress code for an explicit readiness contract -- Why this is not yet "done": - - the readiness flow is still uncommitted work - - the live window context still needs one narrow verification pass -- This is not proven broken today, but it is the most likely remaining runtime risk on the overlay path. + - the readiness contract is already wired in `desktop-overlay/rpc/index.electron.ts` + - the renderer poll controller already waits on readiness and handles degraded state + - the live window still needs one fresh pass on this recut branch to confirm the runtime proof is current +- This is the remaining runtime risk on the overlay path. + +### 3. Local overlay live-window smoke exists in the recut branch, but it still needs a live run on this branch. + +- The smoke is now wired in `apps/stage-tamagotchi/package.json` as + `smoke:desktop-overlay-live-window`. +- The shared candidate-selection helper is unit-tested. +- What is still missing here is a fresh pass on this recut branch with the real + Electron overlay window, to confirm the heartbeat marker and MCP polling still + behave the same as the older line. ## What is not a current blocker @@ -91,11 +85,9 @@ In short: m13v gave good runtime advice. That does not mean every suggestion is ## What should happen now -1. Fix the extension unknown-action response contract so unsupported actions return `ok: false`. -2. Restrict browser-dom click routing to left single-click only; force OS-input for right-click or multi-click. -3. Finish or explicitly shelve the overlay readiness contract work: - - if kept, validate it in a live overlay window context before merging - - if not finished now, do not half-merge it +1. No action needed for the extension unknown-action contract; it is already fail-closed. +2. Keep browser-dom routing fail-closed for non-left clicks and bridge errors; this stays covered, not product-supported. +3. Rerun the local overlay live-window smoke on this recut branch before calling it current proof. ## What should happen later diff --git a/services/computer-use-mcp/package.json b/services/computer-use-mcp/package.json index 553d390d5..2da29e4e9 100644 --- a/services/computer-use-mcp/package.json +++ b/services/computer-use-mcp/package.json @@ -42,6 +42,7 @@ "smoke:remote": "tsx ./src/bin/smoke-remote.ts", "smoke:stdio": "tsx ./src/bin/smoke-stdio.ts", "smoke:macos": "tsx ./src/bin/smoke-macos.ts", + "smoke:desktop-v3": "tsx ./src/bin/smoke-chrome-grounding.ts", "smoke:workflow": "tsx ./src/bin/smoke-workflow.ts", "mcp:inspector": "pnpx @modelcontextprotocol/inspector pnpm -F @proj-airi/computer-use-mcp start", "build": "tsdown", diff --git a/services/computer-use-mcp/src/bin/smoke-chrome-grounding.test.ts b/services/computer-use-mcp/src/bin/smoke-chrome-grounding.test.ts new file mode 100644 index 000000000..4dd53467b --- /dev/null +++ b/services/computer-use-mcp/src/bin/smoke-chrome-grounding.test.ts @@ -0,0 +1,209 @@ +import { describe, expect, it } from 'vitest' + +import { + extractOverlaySmokeState, + parseCommandArgs, + parseNumber, + parseOptionalString, + requireChromeDomSmokeCandidate, + requirePostClickOverlayState, + requireRunState, + requireStructuredContent, + requireTextContent, + selectDesktopV3SmokeCandidate, + selectPendingActionForTool, +} from './smoke-chrome-grounding' + +describe('smoke-chrome-grounding helpers', () => { + it('parses server command args with fallback', () => { + expect(parseCommandArgs(undefined, ['start'])).toEqual(['start']) + expect(parseCommandArgs(' start -- --flag ', ['fallback'])).toEqual(['start', '--', '--flag']) + }) + + it('parses numbers and optional string sentinels', () => { + expect(parseNumber(undefined, 12)).toBe(12) + expect(parseNumber('9.8', 12)).toBe(9) + expect(parseNumber('foo', 12)).toBe(12) + expect(parseOptionalString(undefined)).toBeUndefined() + expect(parseOptionalString(' undefined ')).toBeUndefined() + expect(parseOptionalString(' null ')).toBeUndefined() + expect(parseOptionalString(' t_1 ')).toBe('t_1') + }) + + it('requires structured content and text content', () => { + expect(requireStructuredContent({ + structuredContent: { + status: 'ok', + }, + }, 'tool')).toEqual({ status: 'ok' }) + + expect(requireTextContent({ + content: [ + { type: 'text', text: 'hello' }, + { type: 'image', data: 'ignored' }, + { type: 'text', text: 'world' }, + ], + }, 'tool')).toBe('hello\nworld') + + expect(() => requireStructuredContent({}, 'tool')).toThrow('tool missing structuredContent') + expect(() => requireTextContent({ content: [] }, 'tool')).toThrow('tool missing text content') + }) + + it('extracts runState from desktop_get_state structured content', () => { + expect(requireRunState({ + structuredContent: { + status: 'ok', + runState: { + lastClickedCandidateId: 't_0', + }, + }, + }, 'desktop_get_state')).toEqual({ + lastClickedCandidateId: 't_0', + }) + + expect(() => requireRunState({ + structuredContent: { + status: 'error', + }, + }, 'desktop_get_state')).toThrow('desktop_get_state expected status=ok') + }) + + it('selects explicit candidate first and otherwise requires the chrome_dom smoke target label', () => { + const runState = { + lastGroundingSnapshot: { + snapshotId: 'dg_1', + targetCandidates: [ + { + id: 't_0', + source: 'ax', + role: 'button', + label: 'Disabled', + interactable: false, + }, + { + id: 't_1', + source: 'chrome_dom', + role: 'AXToolbar', + label: 'Toolbar', + interactable: true, + }, + { + id: 't_2', + source: 'chrome_dom', + role: 'AXButton', + label: 'AIRI Desktop V3 Smoke Button', + interactable: true, + }, + ], + }, + } + + expect(selectDesktopV3SmokeCandidate(runState).id).toBe('t_2') + expect(selectDesktopV3SmokeCandidate(runState, 't_0').id).toBe('t_0') + expect(() => selectDesktopV3SmokeCandidate(runState, 'missing')).toThrow('did not return requested candidate') + }) + + it('does not fall back to a generic chrome_dom candidate when the smoke label is missing', () => { + const runState = { + lastGroundingSnapshot: { + snapshotId: 'dg_1', + targetCandidates: [ + { + id: 't_0', + role: 'AXToolbar', + label: 'Toolbar', + interactable: true, + }, + { + id: 't_1', + source: 'ax', + role: 'AXButton', + label: 'Submit', + interactable: true, + }, + { + id: 't_2', + source: 'chrome_dom', + role: 'AXButton', + label: 'Submit', + interactable: true, + }, + ], + }, + } + + expect(() => selectDesktopV3SmokeCandidate(runState)).toThrow('desktop_observe did not return the AIRI Desktop V3 Smoke Button chrome_dom candidate') + }) + + it('locks pre-click and post-click overlay state shape', () => { + const runState = { + lastGroundingSnapshot: { + snapshotId: 'dg_1', + targetCandidates: [ + { id: 't_0' }, + ], + staleFlags: { + screenshot: false, + ax: false, + chromeSemantic: false, + }, + }, + } + + expect(extractOverlaySmokeState(runState)).toMatchObject({ + hasSnapshot: true, + snapshotId: 'dg_1', + candidateCount: 1, + }) + + expect(requirePostClickOverlayState({ + ...runState, + lastPointerIntent: { + candidateId: 't_0', + phase: 'completed', + }, + lastClickedCandidateId: 't_0', + }, 't_0')).toMatchObject({ + pointerIntent: { + candidateId: 't_0', + }, + lastClickedCandidateId: 't_0', + }) + + expect(() => requirePostClickOverlayState(runState, 't_0')).toThrow('missing lastPointerIntent') + }) + + it('reads chrome_dom routing evidence from desktop_click_target structured content', () => { + const clickResult = { + structuredContent: { + status: 'executed', + backendResult: { + executionRoute: 'browser_dom (chrome_dom candidate with selector "#login-btn" routed to browser-dom bridge)', + routeReason: 'chrome_dom candidate with selector "#login-btn" routed to browser-dom bridge', + }, + }, + } + + const structured = requireStructuredContent(clickResult, 'desktop_click_target') + expect(typeof structured.backendResult).toBe('object') + expect((structured.backendResult as Record).executionRoute).toContain('browser_dom') + }) + + it('selects pending actions by tool name and fails with a useful message when missing', () => { + const pendingActions = [ + { toolName: 'desktop_open_app', id: 'p_0' }, + { toolName: 'desktop_click_target', id: 'p_1' }, + ] + + expect(selectPendingActionForTool(pendingActions, 'desktop_click_target').id).toBe('p_1') + expect(() => selectPendingActionForTool(pendingActions, 'desktop_ensure_chrome')).toThrow('no pending action for desktop_ensure_chrome (found: desktop_open_app, desktop_click_target)') + }) + + it('requires the smoke target to come from chrome_dom', () => { + expect(() => requireChromeDomSmokeCandidate({ + id: 't_0', + source: 'ax', + label: 'Smoke', + })).toThrow('smoke target button was not captured as a chrome_dom candidate') + }) +}) diff --git a/services/computer-use-mcp/src/bin/smoke-chrome-grounding.ts b/services/computer-use-mcp/src/bin/smoke-chrome-grounding.ts new file mode 100644 index 000000000..1fb863333 --- /dev/null +++ b/services/computer-use-mcp/src/bin/smoke-chrome-grounding.ts @@ -0,0 +1,498 @@ +/** + * End-to-end smoke test for the desktop v3 Chrome grounding pipeline. + * + * Verifies that: + * 1. `desktop_ensure_chrome` starts or joins an agent Chrome window. + * 2. `desktop_observe` captures a valid grounding snapshot. + * 3. `desktop_get_state` exposes overlay-consumable grounding state. + * 4. `desktop_click_target` updates pointer intent and clicked-candidate state. + * + * Usage: + * pnpm -F @proj-airi/computer-use-mcp smoke:desktop-v3 + */ + +import { dirname, resolve } from 'node:path' +import { argv, env, exit } from 'node:process' +import { fileURLToPath, pathToFileURL } from 'node:url' + +import { Client } from '@modelcontextprotocol/sdk/client/index.js' +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +const WHITESPACE_SPLIT_RE = /\s+/u +const SMOKE_TARGET_LABEL = 'AIRI Desktop V3 Smoke Button' + +const DEFAULT_SMOKE_URL = `data:text/html;charset=utf-8,${encodeURIComponent(` + + + AIRI Desktop V3 Smoke + + + +

AIRI Desktop V3 Smoke

+ + +`)}` + +export interface DesktopV3SmokeCandidate { + id: string + source?: string + role?: string + label?: string + interactable?: boolean +} + +interface CandidateRecord extends Record { + id: string +} + +export interface OverlaySmokeState { + hasSnapshot: boolean + snapshotId: string + candidateCount: number + staleFlags: unknown + pointerIntent?: Record + lastClickedCandidateId?: string +} + +export interface ApprovalEvent { + toolName: string + pendingActionId: string +} + +export interface PendingActionRecord extends Record { + toolName?: string + id?: string +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null +} + +export function parseCommandArgs(raw: string | undefined, fallback: string[]): string[] { + if (!raw?.trim()) + return fallback + + return raw + .split(WHITESPACE_SPLIT_RE) + .map(item => item.trim()) + .filter(Boolean) +} + +export function requireStructuredContent(result: unknown, label: string): Record { + if (!isRecord(result)) + throw new Error(`${label} did not return an object result`) + + const structuredContent = result.structuredContent + if (!isRecord(structuredContent)) + throw new Error(`${label} missing structuredContent`) + + return structuredContent +} + +export function requireTextContent(result: unknown, label: string): string { + if (!isRecord(result) || !Array.isArray(result.content)) + throw new Error(`${label} missing content`) + + const text = result.content + .filter(isRecord) + .map(item => typeof item.text === 'string' ? item.text : '') + .filter(Boolean) + .join('\n') + + if (!text.trim()) + throw new Error(`${label} missing text content`) + + return text +} + +export function requireRunState(result: unknown, label: string): Record { + const structuredContent = requireStructuredContent(result, label) + if (structuredContent.status !== 'ok') + throw new Error(`${label} expected status=ok, got ${String(structuredContent.status)}`) + + if (!isRecord(structuredContent.runState)) + throw new Error(`${label} missing runState`) + + return structuredContent.runState +} + +export function selectDesktopV3SmokeCandidate( + runState: Record, + requestedCandidateId?: string, +): DesktopV3SmokeCandidate { + const snapshot = runState.lastGroundingSnapshot + if (!isRecord(snapshot)) + throw new Error('desktop_get_state missing lastGroundingSnapshot after desktop_observe') + + const candidates = Array.isArray(snapshot.targetCandidates) + ? snapshot.targetCandidates.filter(isRecord) + : [] + + if (candidates.length === 0) + throw new Error('desktop_observe produced no target candidates') + + const requested = requestedCandidateId?.trim() + const candidatesWithIds = candidates.filter((candidate): candidate is CandidateRecord => { + return typeof candidate.id === 'string' && candidate.id.length > 0 + }) + const chromeDomCandidates = candidatesWithIds.filter(candidate => candidate.source === 'chrome_dom') + const selected = requested + ? candidatesWithIds.find(candidate => candidate.id === requested) + : selectDefaultChromeDomCandidate(chromeDomCandidates) + + if (!selected) { + if (!requested) { + throw new Error('desktop_observe did not return the AIRI Desktop V3 Smoke Button chrome_dom candidate') + } + throw new Error(`desktop_observe did not return requested candidate "${requested}"`) + } + + return { + id: selected.id, + source: typeof selected.source === 'string' ? selected.source : undefined, + role: typeof selected.role === 'string' ? selected.role : undefined, + label: typeof selected.label === 'string' ? selected.label : undefined, + interactable: typeof selected.interactable === 'boolean' ? selected.interactable : undefined, + } +} + +function candidateText(candidate: Record): string { + return [ + candidate.id, + candidate.label, + candidate.role, + candidate.source, + ] + .filter(item => typeof item === 'string') + .join(' ') + .toLowerCase() +} + +function selectDefaultChromeDomCandidate(candidates: CandidateRecord[]): CandidateRecord | undefined { + return candidates.find(candidate => + candidateText(candidate).includes(SMOKE_TARGET_LABEL.toLowerCase()), + ) +} + +export function extractOverlaySmokeState(runState: Record): OverlaySmokeState { + const snapshot = runState.lastGroundingSnapshot + if (!isRecord(snapshot)) + throw new Error('desktop_get_state missing lastGroundingSnapshot') + + const snapshotId = typeof snapshot.snapshotId === 'string' ? snapshot.snapshotId : '' + if (!snapshotId) + throw new Error('lastGroundingSnapshot missing snapshotId') + + const candidates = Array.isArray(snapshot.targetCandidates) ? snapshot.targetCandidates : [] + const pointerIntent = isRecord(runState.lastPointerIntent) + ? runState.lastPointerIntent + : undefined + const lastClickedCandidateId = typeof runState.lastClickedCandidateId === 'string' + ? runState.lastClickedCandidateId + : undefined + + return { + hasSnapshot: true, + snapshotId, + candidateCount: candidates.length, + staleFlags: snapshot.staleFlags, + pointerIntent, + lastClickedCandidateId, + } +} + +export function requirePostClickOverlayState( + runState: Record, + selectedCandidateId: string, +): OverlaySmokeState { + const overlayState = extractOverlaySmokeState(runState) + + if (!overlayState.pointerIntent) + throw new Error('desktop_get_state missing lastPointerIntent after desktop_click_target') + + if (overlayState.pointerIntent.candidateId !== selectedCandidateId) { + throw new Error(`lastPointerIntent candidate mismatch: expected ${selectedCandidateId}, got ${String(overlayState.pointerIntent.candidateId)}`) + } + + if (overlayState.lastClickedCandidateId !== selectedCandidateId) { + throw new Error(`lastClickedCandidateId mismatch: expected ${selectedCandidateId}, got ${String(overlayState.lastClickedCandidateId)}`) + } + + return overlayState +} + +export function requireChromeDomSmokeCandidate(candidate: DesktopV3SmokeCandidate): void { + if (candidate.source !== 'chrome_dom') { + throw new Error(`smoke target button was not captured as a chrome_dom candidate (got: ${candidate.source ?? 'unknown'}). Verify extension is connected.`) + } +} + +export function selectPendingActionForTool( + pendingActions: PendingActionRecord[], + expectedToolName: string, +): PendingActionRecord { + const matchingPending = pendingActions.find(action => action.toolName === expectedToolName) + + if (!matchingPending) { + const found = pendingActions + .map(action => action.toolName) + .filter((toolName): toolName is string => typeof toolName === 'string' && toolName.length > 0) + .join(', ') || 'none' + throw new Error(`no pending action for ${expectedToolName} (found: ${found})`) + } + + return matchingPending +} + +async function approveFirstPending( + client: Client, + expectedToolName: string, +): Promise<{ result: unknown, approvalEvent: ApprovalEvent }> { + const pending = await client.callTool({ + name: 'desktop_list_pending_actions', + arguments: {}, + }) + const pendingData = requireStructuredContent(pending, 'desktop_list_pending_actions') + const pendingActions = Array.isArray(pendingData.pendingActions) + ? pendingData.pendingActions.filter(isRecord) as PendingActionRecord[] + : [] + const matchingPending = selectPendingActionForTool(pendingActions, expectedToolName) + + const pendingActionId = typeof matchingPending.id === 'string' ? matchingPending.id : '' + if (!pendingActionId) + throw new Error(`pending action missing id after ${expectedToolName}`) + + const result = await client.callTool({ + name: 'desktop_approve_pending_action', + arguments: { id: pendingActionId }, + }) + + return { + result, + approvalEvent: { + toolName: expectedToolName, + pendingActionId, + }, + } +} + +async function resolveApprovalIfNeeded( + client: Client, + toolName: string, + result: unknown, + approvalEvents: ApprovalEvent[], +): Promise { + const structuredContent = isRecord(result) + ? result.structuredContent + : undefined + + if (!isRecord(structuredContent) || structuredContent.status !== 'approval_required') + return result + + const approved = await approveFirstPending(client, toolName) + approvalEvents.push(approved.approvalEvent) + return approved.result +} + +function assertToolRegistered(toolNames: Set, toolName: string) { + if (!toolNames.has(toolName)) + throw new Error(`required desktop v3 tool is not registered: ${toolName}`) +} + +export function parseNumber(value: string | undefined, fallback: number): number { + if (!value?.trim()) + return fallback + + const parsed = Number(value) + return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : fallback +} + +export function parseOptionalString(value: string | undefined): string | undefined { + const trimmed = value?.trim() + if (!trimmed || trimmed === 'undefined' || trimmed === 'null') + return undefined + + return trimmed +} + +function delay(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)) +} + +export async function runDesktopV3Smoke(): Promise> { + const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm' + const args = parseCommandArgs(env.COMPUTER_USE_SMOKE_SERVER_ARGS, ['start']) + const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir + const smokeUrl = env.COMPUTER_USE_DESKTOP_V3_SMOKE_URL?.trim() || DEFAULT_SMOKE_URL + const requestedCandidateId = parseOptionalString(env.COMPUTER_USE_DESKTOP_V3_SMOKE_CANDIDATE_ID) + const settleMs = parseNumber(env.COMPUTER_USE_DESKTOP_V3_SMOKE_SETTLE_MS, 750) + + const transport = new StdioClientTransport({ + command, + args, + cwd, + env: { + ...env, + COMPUTER_USE_EXECUTOR: env.COMPUTER_USE_SMOKE_EXECUTOR || env.COMPUTER_USE_EXECUTOR || 'macos-local', + COMPUTER_USE_APPROVAL_MODE: env.COMPUTER_USE_SMOKE_APPROVAL_MODE || env.COMPUTER_USE_APPROVAL_MODE || 'actions', + COMPUTER_USE_OPENABLE_APPS: env.COMPUTER_USE_OPENABLE_APPS || 'Terminal,Cursor,Google Chrome', + }, + stderr: 'pipe', + }) + const client = new Client({ + name: '@proj-airi/computer-use-mcp-smoke-desktop-v3', + version: '0.1.0', + }) + + transport.stderr?.on('data', (chunk) => { + const text = chunk.toString('utf-8').trim() + if (text) + console.error(`[computer-use-mcp stderr] ${text}`) + }) + + const approvalEvents: ApprovalEvent[] = [] + + try { + await client.connect(transport) + + const tools = await client.listTools() + const toolNames = new Set(tools.tools.map(tool => tool.name)) + const requiredTools = [ + 'desktop_ensure_chrome', + 'desktop_observe', + 'desktop_click_target', + 'desktop_get_state', + 'desktop_list_pending_actions', + 'desktop_approve_pending_action', + ] + + for (const toolName of requiredTools) { + assertToolRegistered(toolNames, toolName) + } + + console.info('=== Phase 1: desktop_ensure_chrome ===') + const ensureChrome = await resolveApprovalIfNeeded( + client, + 'desktop_ensure_chrome', + await client.callTool({ + name: 'desktop_ensure_chrome', + arguments: { url: smokeUrl }, + }), + approvalEvents, + ) + const ensureChromeData = requireStructuredContent(ensureChrome, 'desktop_ensure_chrome') + + if (settleMs > 0) + await delay(settleMs) + + console.info('=== Phase 2: desktop_observe ===') + const observation = await client.callTool({ + name: 'desktop_observe', + arguments: { includeChrome: true }, + }) + const observeText = requireTextContent(observation, 'desktop_observe') + + console.info('=== Phase 3: desktop_get_state before click ===') + const preClickState = requireRunState(await client.callTool({ + name: 'desktop_get_state', + arguments: {}, + }), 'desktop_get_state') + const preClickOverlayState = extractOverlaySmokeState(preClickState) + const selectedCandidate = selectDesktopV3SmokeCandidate(preClickState, requestedCandidateId) + requireChromeDomSmokeCandidate(selectedCandidate) + + console.info('=== Phase 4: desktop_click_target ===') + const clickTarget = await resolveApprovalIfNeeded( + client, + 'desktop_click_target', + await client.callTool({ + name: 'desktop_click_target', + arguments: { + candidateId: selectedCandidate.id, + button: 'left', + clickCount: 1, + }, + }), + approvalEvents, + ) + requireTextContent(clickTarget, 'desktop_click_target') + + console.info('=== Phase 5: desktop_get_state after click ===') + const postClickState = requireRunState(await client.callTool({ + name: 'desktop_get_state', + arguments: {}, + }), 'desktop_get_state') + const postClickOverlayState = requirePostClickOverlayState(postClickState, selectedCandidate.id) + const clickStructured = requireStructuredContent(clickTarget, 'desktop_click_target') + const clickBackendResult = isRecord(clickStructured.backendResult) + ? clickStructured.backendResult as Record + : undefined + const clickExecutionRoute = typeof clickBackendResult?.executionRoute === 'string' + ? clickBackendResult.executionRoute + : undefined + const clickRouteReason = typeof clickBackendResult?.routeReason === 'string' + ? clickBackendResult.routeReason + : undefined + + if (!clickExecutionRoute) { + throw new Error('desktop_click_target missing backendResult.executionRoute') + } + + if (selectedCandidate.source === 'chrome_dom' && !clickExecutionRoute.startsWith('browser_dom')) { + throw new Error(`expected chrome_dom candidate to route through browser_dom, got ${clickExecutionRoute}`) + } + + return { + ok: true, + toolChain: [ + 'desktop_ensure_chrome', + 'desktop_observe', + 'desktop_get_state', + 'desktop_click_target', + 'desktop_get_state', + ], + ensureChrome: ensureChromeData, + observeSummary: observeText.split('\n').slice(0, 8), + selectedCandidate, + preClickOverlayState, + postClickOverlayState, + clickExecutionRoute, + clickRouteReason, + approvalEvents, + } + } + finally { + await client.close().catch(() => {}) + } +} + +const invokedPath = argv[1] ? pathToFileURL(argv[1]).href : undefined + +if (invokedPath === import.meta.url) { + if (argv.includes('--help') || argv.includes('-h')) { + console.info(`Usage: + pnpm -F @proj-airi/computer-use-mcp smoke:desktop-v3 + +Environment: + COMPUTER_USE_DESKTOP_V3_SMOKE_URL Target URL. Defaults to an inline data: smoke page. + COMPUTER_USE_DESKTOP_V3_SMOKE_CANDIDATE_ID Optional candidate id override from desktop_observe. + COMPUTER_USE_DESKTOP_V3_SMOKE_SETTLE_MS Delay after desktop_ensure_chrome. Default: 750. + COMPUTER_USE_SMOKE_EXECUTOR Executor override. Default: macos-local. + COMPUTER_USE_SMOKE_APPROVAL_MODE Approval mode override. Default: actions. +`) + exit(0) + } + + runDesktopV3Smoke() + .then((result) => { + console.info(JSON.stringify(result, null, 2)) + }) + .catch((error) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)) + exit(1) + }) +} diff --git a/services/computer-use-mcp/src/browser-action-router.test.ts b/services/computer-use-mcp/src/browser-action-router.test.ts index baa15d75d..738af25ae 100644 --- a/services/computer-use-mcp/src/browser-action-router.test.ts +++ b/services/computer-use-mcp/src/browser-action-router.test.ts @@ -57,6 +57,18 @@ describe('decideBrowserAction', () => { expect(decision.reason).toContain('not connected') }) + it('falls back to os_input for right clicks even when chrome_dom is available', () => { + const decision = decideBrowserAction(makeCandidate(), true, 'right') + expect(decision.route).toBe('os_input') + expect(decision.reason).toContain('left single-click') + }) + + it('falls back to os_input for multi-click chrome_dom actions', () => { + const decision = decideBrowserAction(makeCandidate(), true, 'left', 2) + expect(decision.route).toBe('os_input') + expect(decision.reason).toContain('count 2') + }) + it('preserves non-zero frameId for sub-frame candidates', () => { const decision = decideBrowserAction(makeCandidate({ frameId: 3 }), true) expect(decision.route).toBe('browser_dom') diff --git a/services/computer-use-mcp/src/browser-dom/extension-bridge.test.ts b/services/computer-use-mcp/src/browser-dom/extension-bridge.test.ts index 20b7b0950..0b94734f7 100644 --- a/services/computer-use-mcp/src/browser-dom/extension-bridge.test.ts +++ b/services/computer-use-mcp/src/browser-dom/extension-bridge.test.ts @@ -287,6 +287,29 @@ describe('browserDomExtensionBridge', () => { expect(bridge.getStatus().pendingRequests).toBe(0) }) + it('rejects bridge calls when the extension responds with ok:false', async () => { + const result = await createConnectedBridge() + bridge = result.bridge + client = result.client + + client.on('message', (raw) => { + const data = JSON.parse(String(raw)) as Record + if (typeof data.id !== 'string') + return + if (data.action !== 'getActiveTab') + return + + client.send(JSON.stringify({ + id: data.id, + ok: false, + error: 'unknown action: getActiveTab', + })) + }) + + await expect(bridge.getActiveTab()).rejects.toThrow('unknown action: getActiveTab') + expect(bridge.getStatus().pendingRequests).toBe(0) + }) + it('can retry startup after an initial bind failure', async () => { blocker = new WebSocketServer({ host: '127.0.0.1', diff --git a/services/computer-use-mcp/src/chrome-session-manager.test.ts b/services/computer-use-mcp/src/chrome-session-manager.test.ts index 85421255e..699942c8d 100644 --- a/services/computer-use-mcp/src/chrome-session-manager.test.ts +++ b/services/computer-use-mcp/src/chrome-session-manager.test.ts @@ -8,20 +8,35 @@ import type { ChromeSessionManager } from './chrome-session-manager' import type { ComputerUseConfig } from './types' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { createServer } from 'node:net' + import { beforeEach, describe, expect, it, vi } from 'vitest' import { createChromeSessionManager } from './chrome-session-manager' import { runProcess } from './utils/process' -// Mock runProcess and sleep before importing the module under test +vi.mock('node:fs/promises', () => ({ + mkdir: vi.fn().mockResolvedValue(undefined), + mkdtemp: vi.fn().mockResolvedValue('/tmp/test/chrome-profile-abc123'), + rm: vi.fn().mockResolvedValue(undefined), + writeFile: vi.fn().mockResolvedValue(undefined), +})) vi.mock('./utils/process', () => ({ runProcess: vi.fn(), })) vi.mock('./utils/sleep', () => ({ sleep: vi.fn().mockResolvedValue(undefined), })) - +vi.mock('node:net', () => ({ + createServer: vi.fn(), +})) const mockedRunProcess = vi.mocked(runProcess) +const mockedMkdir = vi.mocked(mkdir) +const mockedMkdtemp = vi.mocked(mkdtemp) +const mockedRm = vi.mocked(rm) +const mockedWriteFile = vi.mocked(writeFile) +const mockedCreateServer = vi.mocked(createServer) function makeConfig(): ComputerUseConfig { return { @@ -46,42 +61,83 @@ function ok(stdout = ''): any { return { stdout, stderr: '' } } -/** - * Mock for "chrome not running → launch" flow (first call, no existing session). - * - * Call sequence when `session` is null: - * 1. osascript (getCurrentForegroundApp) - * 2. pgrep (isChromeRunning for wasAlreadyRunning) → reject (not running) - * 3. open (launchChromeWithCdp) - * 4. osascript (activateChrome) - * 5. pgrep (getChromeMainPid) → ok with pid - */ -function mockLaunchFlow(pid: number, userApp = '') { - mockedRunProcess - .mockResolvedValueOnce(ok(userApp)) // 1. getCurrentForegroundApp - .mockRejectedValueOnce(new Error('no match')) // 2. isChromeRunning → false - .mockResolvedValueOnce(ok()) // 3. open command - .mockResolvedValueOnce(ok()) // 4. activateChrome - .mockResolvedValueOnce(ok(`${pid}\n`)) // 5. getChromeMainPid +function mockPortAvailability(...results: boolean[]) { + for (const result of results) { + mockedCreateServer.mockImplementationOnce(() => { + const handlers = new Map void>() + return { + once(event: string, handler: (...args: any[]) => void) { + handlers.set(event, handler) + return this + }, + listen(_port: number, _host: string, callback: () => void) { + if (result) { + callback() + } + else { + handlers.get('error')?.(new Error('EADDRINUSE')) + } + return this + }, + close(callback: () => void) { + callback() + return this + }, + } as any + }) + } } -/** - * Mock for "chrome already running → new window" flow (first call, no existing session). - * - * Call sequence when `session` is null: - * 1. osascript (getCurrentForegroundApp) - * 2. pgrep (isChromeRunning for wasAlreadyRunning) → ok (running) - * 3. osascript (createNewWindow) - * 4. osascript (activateChrome) - * 5. pgrep (getChromeMainPid) - */ -function mockJoinFlow(pid: number, userApp = 'Terminal') { +function mockLaunchFlow(pid: number, userApp = 'Terminal', cdpPort = 9222, pidLookupAttempts = 4) { + const chromeProcessListing = ok( + `${pid} /Applications/Google Chrome.app/Contents/MacOS/Google Chrome --user-data-dir=/tmp/test/chrome-profile-abc123 --remote-debugging-port=${cdpPort}\n`, + ) + mockedRunProcess - .mockResolvedValueOnce(ok(userApp)) // 1. getCurrentForegroundApp - .mockResolvedValueOnce(ok(`${pid}\n`)) // 2. isChromeRunning → true - .mockResolvedValueOnce(ok()) // 3. createNewWindow - .mockResolvedValueOnce(ok()) // 4. activateChrome - .mockResolvedValueOnce(ok(`${pid}\n`)) // 5. getChromeMainPid + .mockResolvedValueOnce(ok(userApp)) // foreground app + .mockRejectedValueOnce(new Error('no match')) // wasAlreadyRunning → false + .mockResolvedValueOnce(ok()) // open + .mockResolvedValueOnce(ok()) // activateChrome + + for (let index = 0; index < pidLookupAttempts; index += 1) { + mockedRunProcess.mockResolvedValueOnce(chromeProcessListing) // getChromePidForProfile + } +} + +function primeDefaultPortAvailability() { + if (mockedCreateServer.mock.calls.length === 0 && mockedCreateServer.mock.results.length === 0) { + mockPortAvailability(true) + } +} + +function resetLaunchMocks() { + mockedRunProcess.mockReset() + mockedCreateServer.mockReset() +} + +function mockReuseFlow(pid: number) { + mockedRunProcess + .mockResolvedValueOnce(ok(`${pid} /Applications/Google Chrome.app/Contents/MacOS/Google Chrome --user-data-dir=/tmp/test/chrome-profile-abc123 --remote-debugging-port=9222\n`)) // getTrackedChromePid + .mockResolvedValueOnce(ok(`${pid}\n`)) // isProcessAlive + .mockResolvedValueOnce(ok('1\n')) // hasChromeWindow +} + +function mockWindowMissingFlow(pid: number) { + mockedRunProcess + .mockResolvedValueOnce(ok(`${pid}\n`)) // isProcessAlive + .mockResolvedValueOnce(ok('0\n')) // hasChromeWindow +} + +function mockEnsureWindowMissingFlow(pid: number) { + mockedRunProcess + .mockResolvedValueOnce(ok(`${pid} /Applications/Google Chrome.app/Contents/MacOS/Google Chrome --user-data-dir=/tmp/test/chrome-profile-abc123 --remote-debugging-port=9222\n`)) // getTrackedChromePid + .mockResolvedValueOnce(ok(`${pid}\n`)) // isProcessAlive + .mockResolvedValueOnce(ok('0\n')) // hasChromeWindow +} + +function mockStalePidFlow(pid: number) { + mockedRunProcess + .mockResolvedValueOnce(ok(`${pid} /Applications/Google Chrome.app/Contents/MacOS/Google Chrome --user-data-dir=/tmp/test/chrome-profile-abc123 --remote-debugging-port=9222\n`)) // getTrackedChromePid } describe('chromeSessionManager', () => { @@ -89,251 +145,234 @@ describe('chromeSessionManager', () => { beforeEach(() => { vi.clearAllMocks() + mockedRunProcess.mockReset() + mockedCreateServer.mockReset() + mockPortAvailability(true) manager = createChromeSessionManager(makeConfig()) }) - // ----------------------------------------------------------------------- - // ensureAgentWindow - // ----------------------------------------------------------------------- - describe('ensureAgentWindow', () => { - it('should launch Chrome when not running', async () => { + it('launches a dedicated Chrome profile with CDP', async () => { mockLaunchFlow(12345) const info = await manager.ensureAgentWindow() + expect(info.ensureOutcome).toBe('launched') expect(info.wasAlreadyRunning).toBe(false) expect(info.agentOwned).toBe(true) expect(info.pid).toBe(12345) expect(info.cdpUrl).toBe('http://127.0.0.1:9222') expect(info.windowId).toBe('12345:0:Google Chrome') - expect(info.createdAt).toBeTruthy() + expect(mockedMkdir).toHaveBeenCalledWith('/tmp/test', { recursive: true }) + expect(mockedMkdtemp).toHaveBeenCalledWith('/tmp/test/chrome-profile-') + expect(mockedWriteFile).toHaveBeenCalledWith('/tmp/test/chrome-profile-abc123/First Run', '') + expect(mockedRunProcess.mock.calls[2]).toEqual([ + '/usr/bin/open', + [ + '-na', + 'Google Chrome', + '--args', + '--new-window', + '--no-first-run', + '--no-default-browser-check', + '--disable-default-apps', + '--disable-features=ChromeWhatsNewUI', + '--remote-debugging-port=9222', + '--user-data-dir=/tmp/test/chrome-profile-abc123', + ], + expect.any(Object), + ]) }) - it('should create new window when Chrome is already running', async () => { - mockJoinFlow(99999, 'Terminal') - - const info = await manager.ensureAgentWindow() - - expect(info.wasAlreadyRunning).toBe(true) - expect(info.agentOwned).toBe(false) - expect(info.pid).toBe(99999) - // No CDP URL when joining existing Chrome - expect(info.cdpUrl).toBeUndefined() - }) - - it('should reuse an existing session when the agent launched a dedicated Chrome instance', async () => { + it('reuses an alive agent session', async () => { mockLaunchFlow(11111) const first = await manager.ensureAgentWindow() - // Second call: session exists → isChromeRunning check (1 call) - mockedRunProcess.mockResolvedValueOnce(ok('11111\n')) // isChromeRunning → still alive + resetLaunchMocks() + mockReuseFlow(11111) + const second = await manager.ensureAgentWindow() + expect(second.ensureOutcome).toBe('reused') expect(second).toBe(first) }) - it('should create a fresh agent window on repeated calls when joining an existing Chrome instance', async () => { - mockJoinFlow(99999, 'Terminal') - const first = await manager.ensureAgentWindow() - - vi.clearAllMocks() - - mockedRunProcess - .mockResolvedValueOnce(ok('99999\n')) // session reuse check → Chrome still running - mockJoinFlow(99999, 'Terminal') - - const second = await manager.ensureAgentWindow() - - expect(second).not.toBe(first) - expect(second.wasAlreadyRunning).toBe(true) - expect(second.pid).toBe(99999) - expect(mockedRunProcess.mock.calls[3]?.[0]).toBe('/usr/bin/osascript') - expect(mockedRunProcess.mock.calls[3]?.[1]).toEqual(['-e', 'tell application "Google Chrome" to make new window']) - }) - - it('should recreate session if Chrome crashed between calls', async () => { + it('recreates the session if the tracked process is gone', async () => { mockLaunchFlow(11111) - const first = await manager.ensureAgentWindow() - expect(first.pid).toBe(11111) + await manager.ensureAgentWindow() - // Second call: session exists → isChromeRunning fails (Chrome crashed) + resetLaunchMocks() + primeDefaultPortAvailability() mockedRunProcess.mockRejectedValueOnce(new Error('no match')) - // session=null now, goes through full launch flow (5 calls) mockLaunchFlow(22222) const second = await manager.ensureAgentWindow() + + expect(second.ensureOutcome).toBe('recreated_after_process_exit') expect(second.pid).toBe(22222) - expect(second).not.toBe(first) + expect(second).not.toBeNull() }) - it('should pass custom URL to Chrome', async () => { - mockLaunchFlow(33333) - - const info = await manager.ensureAgentWindow({ url: 'https://example.com' }) - expect(info.initialUrl).toBe('https://example.com') - }) - - it('should pass URL to osascript as argv instead of interpolating it into script source', async () => { - const maliciousUrl = 'https://example.com/" & do shell script "touch /tmp/pwned" & "' - mockJoinFlow(33333, 'Terminal') - - await manager.ensureAgentWindow({ url: maliciousUrl }) - - const createWindowCall = mockedRunProcess.mock.calls[2] - expect(createWindowCall?.[0]).toBe('/usr/bin/osascript') - expect(createWindowCall?.[1]).toEqual([ - '-e', - expect.stringContaining('item 1 of argv'), - '--', - maliciousUrl, - ]) - expect(createWindowCall?.[1]?.[1]).not.toContain(maliciousUrl) - }) - - it('should use custom CDP port', async () => { - mockLaunchFlow(44444) - - const info = await manager.ensureAgentWindow({ cdpPort: 9333 }) - expect(info.cdpUrl).toBe('http://127.0.0.1:9333') - }) - - it('should throw if Chrome PID cannot be obtained after launch', async () => { - mockedRunProcess - .mockResolvedValueOnce(ok()) // 1. foreground - .mockRejectedValueOnce(new Error('no match')) // 2. wasAlreadyRunning → false - .mockResolvedValueOnce(ok()) // 3. launch - .mockResolvedValueOnce(ok()) // 4. activate - .mockRejectedValueOnce(new Error('no match')) // 5. getChromeMainPid → fails - - await expect(manager.ensureAgentWindow()).rejects.toThrow('Failed to get Chrome PID') - }) - - it('should record the user\'s previous foreground app', async () => { - mockLaunchFlow(55555, 'Finder') - + it('does not terminate a reused pid after verifying it no longer belongs to the tracked Chrome profile', async () => { + mockLaunchFlow(11111) await manager.ensureAgentWindow() - // First call should have been getCurrentForegroundApp - const firstCall = mockedRunProcess.mock.calls[0] - expect(firstCall[1]).toContainEqual(expect.stringContaining('first application process')) + resetLaunchMocks() + primeDefaultPortAvailability() + mockStalePidFlow(33333) + mockLaunchFlow(22222) + + const second = await manager.ensureAgentWindow() + + expect(second.ensureOutcome).toBe('recreated_after_process_exit') + expect(second.pid).toBe(22222) + expect(mockedRunProcess).not.toHaveBeenCalledWith('kill', ['-TERM', '11111'], expect.any(Object)) + expect(mockedRunProcess).not.toHaveBeenCalledWith('kill', ['-KILL', '11111'], expect.any(Object)) + }) + + it('recreates the session if the Chrome process is alive but the agent window is gone', async () => { + mockLaunchFlow(11111) + await manager.ensureAgentWindow() + + resetLaunchMocks() + primeDefaultPortAvailability() + mockEnsureWindowMissingFlow(11111) + mockedRunProcess.mockResolvedValueOnce(ok()) // terminateChromeProcess: kill -TERM + mockedRunProcess.mockResolvedValueOnce(ok()) // terminateChromeProcess: post-TERM liveness check + mockLaunchFlow(22222) + + const second = await manager.ensureAgentWindow() + + expect(second.ensureOutcome).toBe('recreated_after_missing_window') + expect(second.pid).toBe(22222) + expect(mockedRunProcess).toHaveBeenCalledWith('kill', ['-TERM', '11111'], expect.any(Object)) + expect(mockedRunProcess).not.toHaveBeenCalledWith('kill', ['-KILL', '11111'], expect.any(Object)) + expect(mockedRunProcess).toHaveBeenCalledWith('/usr/bin/osascript', [ + '-e', + 'tell application "System Events" to get count of windows of (first application process whose unix id is 11111)', + ], expect.any(Object)) + }) + + it('passes a custom CDP port and URL through', async () => { + mockLaunchFlow(33333, 'Terminal', 9333) + + const info = await manager.ensureAgentWindow({ + cdpPort: 9333, + url: 'https://example.com', + }) + + expect(info.cdpUrl).toBe('http://127.0.0.1:9333') + expect(info.initialUrl).toBe('https://example.com') + expect(mockedRunProcess.mock.calls[2]?.[1]).toContain('--user-data-dir=/tmp/test/chrome-profile-abc123') + expect(mockedRunProcess.mock.calls[2]?.[1]).toContain('https://example.com') + }) + + it('falls forward to the next available default CDP port when 9222 is occupied', async () => { + resetLaunchMocks() + mockPortAvailability(false, true) + mockLaunchFlow(12345, 'Terminal', 9223) + + const info = await manager.ensureAgentWindow() + + expect(info.cdpUrl).toBe('http://127.0.0.1:9223') + expect(mockedRunProcess.mock.calls[2]?.[1]).toContain('--remote-debugging-port=9223') + }) + + it('cleans up the active chrome profile directory on endSession', async () => { + mockLaunchFlow(11111) + await manager.ensureAgentWindow() + + resetLaunchMocks() + manager.endSession() + + expect(mockedRm).toHaveBeenCalledWith('/tmp/test/chrome-profile-abc123', { + recursive: true, + force: true, + }) + expect(manager.getSessionInfo()).toBeNull() + }) + + it('cleans up the active chrome profile when relaunching after a missing window', async () => { + mockLaunchFlow(11111) + await manager.ensureAgentWindow() + + resetLaunchMocks() + primeDefaultPortAvailability() + mockWindowMissingFlow(11111) + mockedRunProcess.mockResolvedValueOnce(ok()) // terminateChromeProcess: kill -TERM + mockedRunProcess.mockResolvedValueOnce(ok()) // terminateChromeProcess: post-TERM liveness check + mockLaunchFlow(22222) + + const second = await manager.ensureAgentWindow() + + expect(second.pid).toBe(22222) + expect(mockedRm).toHaveBeenCalledWith('/tmp/test/chrome-profile-abc123', { + recursive: true, + force: true, + }) + }) + + it('terminates a launched Chrome process and cleans up the profile if launch fails before PID lookup completes', async () => { + mockedRunProcess + .mockResolvedValueOnce(ok('Terminal')) // foreground app + .mockRejectedValueOnce(new Error('no match')) // wasAlreadyRunning → false + .mockResolvedValueOnce(ok()) // open + .mockRejectedValueOnce(new Error('activate failed')) // activateChrome + .mockResolvedValueOnce(ok( + '11111 /Applications/Google Chrome.app/Contents/MacOS/Google Chrome --user-data-dir=/tmp/test/chrome-profile-abc123 --remote-debugging-port=9222\n', + )) // findAndTerminateChromeByProfile + .mockResolvedValueOnce(ok()) // findAndTerminateChromeByProfile: kill -TERM + .mockResolvedValueOnce(ok()) // findAndTerminateChromeByProfile: post-TERM liveness check + + await expect(manager.ensureAgentWindow()).rejects.toThrow('activate failed') + expect(mockedRunProcess).toHaveBeenCalledWith('kill', ['-TERM', '11111'], expect.any(Object)) + expect(mockedRm).toHaveBeenCalledWith('/tmp/test/chrome-profile-abc123', { + recursive: true, + force: true, + }) }) }) - // ----------------------------------------------------------------------- - // bringToFront - // ----------------------------------------------------------------------- - describe('bringToFront', () => { - it('should activate Chrome when session exists', async () => { + it('activates Chrome when session exists', async () => { mockLaunchFlow(11111) await manager.ensureAgentWindow() - vi.clearAllMocks() - mockedRunProcess - .mockResolvedValueOnce(ok('11111\n')) // isChromeRunning - .mockResolvedValueOnce(ok()) // activateChrome + resetLaunchMocks() + mockedRunProcess.mockResolvedValueOnce(ok('11111\n')) + mockedRunProcess.mockResolvedValueOnce(ok('1\n')) + mockedRunProcess.mockResolvedValueOnce(ok()) const result = await manager.bringToFront() expect(result).toBe(true) - expect(mockedRunProcess).toHaveBeenCalledTimes(2) - const activateCall = mockedRunProcess.mock.calls[1] - expect(activateCall[1]).toEqual(['-e', 'tell application "Google Chrome" to activate']) + expect(mockedRunProcess).toHaveBeenCalledTimes(3) }) - it('should be no-op when no session exists', async () => { - const result = await manager.bringToFront() - expect(result).toBe(false) - expect(mockedRunProcess).not.toHaveBeenCalled() - }) - - it('should clear session if Chrome crashed', async () => { + it('returns false when session is gone', async () => { mockLaunchFlow(11111) await manager.ensureAgentWindow() - vi.clearAllMocks() - // Chrome crashed + resetLaunchMocks() mockedRunProcess.mockRejectedValueOnce(new Error('no match')) const result = await manager.bringToFront() + + expect(result).toBe(false) + expect(manager.getSessionInfo()).toBeNull() + }) + + it('returns false when the agent window is gone even if the process still exists', async () => { + mockLaunchFlow(11111) + await manager.ensureAgentWindow() + + resetLaunchMocks() + mockWindowMissingFlow(11111) + + const result = await manager.bringToFront() + expect(result).toBe(false) expect(manager.getSessionInfo()).toBeNull() }) }) - - // ----------------------------------------------------------------------- - // restorePreviousForeground - // ----------------------------------------------------------------------- - - describe('restorePreviousForeground', () => { - it('should activate the user\'s previous app', async () => { - mockLaunchFlow(11111, 'Terminal') - await manager.ensureAgentWindow() - vi.clearAllMocks() - - mockedRunProcess.mockResolvedValueOnce(ok()) - - await manager.restorePreviousForeground() - - expect(mockedRunProcess).toHaveBeenCalledWith( - '/usr/bin/osascript', - ['-e', 'tell application "Terminal" to activate'], - expect.any(Object), - ) - }) - - it('should be no-op if user was already in Chrome', async () => { - mockLaunchFlow(11111, 'Google Chrome') - await manager.ensureAgentWindow() - vi.clearAllMocks() - - await manager.restorePreviousForeground() - expect(mockedRunProcess).not.toHaveBeenCalled() - }) - - it('should be no-op if no session was created', async () => { - await manager.restorePreviousForeground() - expect(mockedRunProcess).not.toHaveBeenCalled() - }) - }) - - // ----------------------------------------------------------------------- - // session lifecycle - // ----------------------------------------------------------------------- - - describe('session lifecycle', () => { - it('should return null before any session', () => { - expect(manager.getSessionInfo()).toBeNull() - }) - - it('should return session info after ensureAgentWindow', async () => { - mockLaunchFlow(55555) - - await manager.ensureAgentWindow() - const info = manager.getSessionInfo() - expect(info).not.toBeNull() - expect(info!.pid).toBe(55555) - expect(info!.agentOwned).toBe(true) - }) - - it('should clear session on endSession', async () => { - mockLaunchFlow(55555) - await manager.ensureAgentWindow() - expect(manager.getSessionInfo()).not.toBeNull() - - manager.endSession() - expect(manager.getSessionInfo()).toBeNull() - }) - - it('should restore fully after endSession and re-ensureAgentWindow', async () => { - mockLaunchFlow(11111) - await manager.ensureAgentWindow() - manager.endSession() - - mockLaunchFlow(22222) - const info = await manager.ensureAgentWindow() - expect(info.pid).toBe(22222) - }) - }) }) diff --git a/services/computer-use-mcp/src/chrome-session-manager.ts b/services/computer-use-mcp/src/chrome-session-manager.ts index fa3d7cdee..6f15bad15 100644 --- a/services/computer-use-mcp/src/chrome-session-manager.ts +++ b/services/computer-use-mcp/src/chrome-session-manager.ts @@ -2,10 +2,8 @@ * Chrome Session Manager — agent-owned Chrome window lifecycle. * * Responsibilities: - * - Detect whether Chrome is already running - * - Launch Chrome with CDP if not running - * - Create a new window in existing Chrome - * - Track window identity via PID + * - Launch a dedicated Chrome profile with CDP + * - Track the launched browser PID * - Bring agent window to front / restore user's previous foreground * * macOS only. Uses AppleScript and `open` CLI for Chrome lifecycle control. @@ -13,6 +11,10 @@ import type { ChromeSessionInfo, ComputerUseConfig } from './types' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { createServer } from 'node:net' +import { join } from 'node:path' + import { runProcess } from './utils/process' import { sleep } from './utils/sleep' @@ -22,6 +24,7 @@ import { sleep } from './utils/sleep' const CHROME_APP_NAME = 'Google Chrome' const DEFAULT_CDP_PORT = 9222 +const DEFAULT_CDP_PORT_SCAN_ATTEMPTS = 20 // --------------------------------------------------------------------------- // Public interface @@ -31,19 +34,18 @@ export interface ChromeSessionManager { /** * Ensure the agent has a usable Chrome window. * - * - Chrome not running → launch with CDP flag + new window - * - Chrome running → create new window in existing instance + * - No active agent session → launch a dedicated Chrome profile with CDP + * - Existing agent session still has a live Chrome window → reuse it * - * Reuses the cached session only when the agent launched a dedicated Chrome - * instance itself. When joining an existing user-owned Chrome process, a - * fresh window must be created on every ensure because this module does not - * track a verifiable OS window handle for the agent tab/window. + * Human-owned Chrome instances are not reused. The agent always launches its + * own profile so browser-dom/CDP capture has a stable endpoint. */ ensureAgentWindow: (options?: { url?: string, cdpPort?: number }) => Promise /** * Bring the agent's Chrome window to the foreground. - * Returns false if the tracked session is missing or Chrome is no longer running. + * Returns false if the tracked session is missing, Chrome is no longer + * running, or the tracked window is gone. */ bringToFront: () => Promise @@ -74,6 +76,7 @@ export function createChromeSessionManager( let session: ChromeSessionInfo | null = null let previousForegroundApp: string | undefined const onSessionLost = options?.onSessionLost + let activeProfileDir: string | undefined // -- Helpers ------------------------------------------------------------ @@ -90,14 +93,108 @@ export function createChromeSessionManager( } } - async function getChromeMainPid(): Promise { + async function isProcessAlive(pid: number): Promise { try { - const { stdout } = await runProcess('pgrep', ['-x', 'Google Chrome'], { + const { stdout } = await runProcess('ps', ['-p', String(pid), '-o', 'pid='], { timeoutMs: config.timeoutMs, }) - const pids = stdout.trim().split('\n').map(Number).filter(n => !Number.isNaN(n)) - // The lowest PID is typically the main Chrome process - return pids.length > 0 ? Math.min(...pids) : undefined + return stdout.trim().length > 0 + } + catch { + return false + } + } + + async function hasChromeWindow(pid: number): Promise { + try { + const { stdout } = await runProcess(config.binaries.osascript, [ + '-e', + `tell application "System Events" to get count of windows of (first application process whose unix id is ${pid})`, + ], { timeoutMs: config.timeoutMs }) + return Number.parseInt(stdout.trim(), 10) > 0 + } + catch { + return false + } + } + + async function terminateChromeProcess(pid: number): Promise { + await runProcess('kill', ['-TERM', String(pid)], { timeoutMs: config.timeoutMs }).catch(() => {}) + await sleep(250) + + if (!await isProcessAlive(pid)) { + return + } + + await runProcess('kill', ['-KILL', String(pid)], { timeoutMs: config.timeoutMs }).catch(() => {}) + await sleep(250) + + if (await isProcessAlive(pid)) { + throw new Error(`Failed to terminate stale Chrome process ${pid}`) + } + } + + async function findAndTerminateChromeByProfile(profileDir: string, cdpPort: number): Promise { + const pid = await getChromePidForProfile(profileDir, cdpPort) + if (!pid) { + return + } + + await terminateChromeProcess(pid) + } + + async function clearSessionState(): Promise { + session = null + previousForegroundApp = undefined + if (activeProfileDir) { + const profileDir = activeProfileDir + activeProfileDir = undefined + await rm(profileDir, { recursive: true, force: true }).catch(() => {}) + return + } + activeProfileDir = undefined + } + + async function getChromePidForProfile(profileDir: string, cdpPort: number): Promise { + try { + const { stdout } = await runProcess('ps', ['-axww', '-o', 'pid=,command='], { + timeoutMs: config.timeoutMs, + }) + const matchingLine = stdout + .split('\n') + .map(line => line.trim()) + .find(line => + line.includes('/Contents/MacOS/Google Chrome') + && !line.includes('Helper') + && line.includes(`--user-data-dir=${profileDir}`) + && line.includes(`--remote-debugging-port=${cdpPort}`), + ) + + if (!matchingLine) { + return undefined + } + + const pidText = matchingLine.split(/\s+/u)[0] + const pid = Number(pidText) + return Number.isFinite(pid) ? pid : undefined + } + catch { + return undefined + } + } + + async function getTrackedChromePid(trackedSession: ChromeSessionInfo): Promise { + if (!activeProfileDir || !trackedSession.cdpUrl) { + return undefined + } + + try { + const cdpPort = Number.parseInt(new URL(trackedSession.cdpUrl).port, 10) + if (!Number.isFinite(cdpPort)) { + return undefined + } + + return await getChromePidForProfile(activeProfileDir, cdpPort) } catch { return undefined @@ -117,13 +214,49 @@ export function createChromeSessionManager( } } - async function launchChromeWithCdp(cdpPort: number, url?: string): Promise { + async function canListenOnPort(port: number): Promise { + return await new Promise((resolvePromise) => { + const server = createServer() + server.once('error', () => { + resolvePromise(false) + }) + server.listen(port, '127.0.0.1', () => { + server.close(() => resolvePromise(true)) + }) + }) + } + + async function resolveLaunchCdpPort(requestedPort: number, explicit: boolean): Promise { + if (explicit) { + return requestedPort + } + + for (let index = 0; index < DEFAULT_CDP_PORT_SCAN_ATTEMPTS; index += 1) { + const candidate = requestedPort + index + if (await canListenOnPort(candidate)) { + return candidate + } + } + + throw new Error(`Could not find an available Chrome CDP port starting from ${requestedPort}`) + } + + async function launchChromeWithCdp(cdpPort: number, profileDir: string, url?: string): Promise { + // Chrome uses the user-data-dir root "First Run" sentinel to decide + // whether the branded first-run dialog should appear. + await writeFile(join(profileDir, 'First Run'), '').catch(() => {}) + const args = [ '-na', CHROME_APP_NAME, '--args', '--new-window', + '--no-first-run', + '--no-default-browser-check', + '--disable-default-apps', + '--disable-features=ChromeWhatsNewUI', `--remote-debugging-port=${cdpPort}`, + `--user-data-dir=${profileDir}`, ] if (url) { args.push(url) @@ -137,31 +270,6 @@ export function createChromeSessionManager( await sleep(2000) } - async function createNewWindow(url?: string): Promise { - // NOTICE: Pass the URL as an `osascript` argv value instead of interpolating - // it into the AppleScript source. `desktop_ensure_chrome` accepts arbitrary - // strings, so direct interpolation creates an AppleScript injection path when - // the URL contains quotes. The review report on PR #1649 correctly called this - // out as a P1 command-injection issue. - const script = url - ? `on run argv -tell application "${CHROME_APP_NAME}" to make new window with properties {mode:"normal"} -tell application "${CHROME_APP_NAME}" to set URL of active tab of front window to item 1 of argv -end run` - : `tell application "${CHROME_APP_NAME}" to make new window` - - const args = url - ? ['-e', script, '--', url] - : ['-e', script] - - await runProcess(config.binaries.osascript, args, { - timeoutMs: config.timeoutMs, - }) - - // Brief delay for the window to appear - await sleep(500) - } - async function activateChrome(): Promise { await runProcess(config.binaries.osascript, [ '-e', @@ -185,68 +293,104 @@ end run` return { async ensureAgentWindow(options) { - // If we already have a session, only reuse it when the agent launched a - // dedicated Chrome instance. Joined sessions only cache process metadata, - // not a verifiable OS window handle, so blindly reusing them after the - // user closes that window would redirect later focus/click flows onto a - // different Chrome window. + let ensureOutcome: ChromeSessionInfo['ensureOutcome'] = 'launched' + if (session) { - const stillRunning = await isChromeRunning() - if (stillRunning && !session.wasAlreadyRunning) { - return session + const trackedSession = session + const trackedChromePid = await getTrackedChromePid(trackedSession) + + if (trackedChromePid === trackedSession.pid) { + const stillRunning = await isProcessAlive(trackedSession.pid) + const stillHasWindow = stillRunning && await hasChromeWindow(trackedSession.pid) + if (stillHasWindow) { + trackedSession.ensureOutcome = 'reused' + return trackedSession + } + + try { + if (stillRunning) { + ensureOutcome = 'recreated_after_missing_window' + await terminateChromeProcess(trackedSession.pid) + } + else { + ensureOutcome = 'recreated_after_process_exit' + } + } + finally { + // Chrome died or the tracked window disappeared — clear stale session. + await clearSessionState() + onSessionLost?.() + } } - if (!stillRunning) { - // Chrome died — clear stale session. + else { + // The tracked PID no longer resolves to the expected Chrome profile. + // Treat it as stale and relaunch without touching that PID. + ensureOutcome = 'recreated_after_process_exit' + await clearSessionState() onSessionLost?.() } - session = null } - // Record the user's current foreground app before we steal focus + // Record the user's current foreground app before we steal focus. previousForegroundApp = await getCurrentForegroundApp() - const cdpPort = options?.cdpPort ?? DEFAULT_CDP_PORT + const cdpPort = await resolveLaunchCdpPort( + options?.cdpPort ?? DEFAULT_CDP_PORT, + options?.cdpPort !== undefined, + ) const wasAlreadyRunning = await isChromeRunning() + await mkdir(config.sessionRoot, { recursive: true }) + activeProfileDir = await mkdtemp(join(config.sessionRoot, 'chrome-profile-')) - if (wasAlreadyRunning) { - // Chrome is running — create a new window in the existing instance - await createNewWindow(options?.url) + try { + // Always launch a dedicated profile so CDP is stable even when Chrome is already running. + await launchChromeWithCdp(cdpPort, activeProfileDir, options?.url) + + // Bring Chrome to front. + await activateChrome() + // Brief wait for activation. + await sleep(300) + + const deadline = Date.now() + config.timeoutMs + let pid: number | undefined + while (Date.now() < deadline) { + pid = await getChromePidForProfile(activeProfileDir, cdpPort) + if (pid) { + break + } + await sleep(250) + } + if (!pid) { + throw new Error('Failed to get Chrome PID after launch') + } + + session = { + ensureOutcome, + wasAlreadyRunning, + windowId: `${pid}:0:${CHROME_APP_NAME}`, + cdpUrl: `http://127.0.0.1:${cdpPort}`, + pid, + agentOwned: true, + initialUrl: options?.url, + createdAt: new Date().toISOString(), + } + + return session } - else { - // Chrome not running — launch with CDP - await launchChromeWithCdp(cdpPort, options?.url) + catch (error) { + await findAndTerminateChromeByProfile(activeProfileDir, cdpPort) + await clearSessionState() + throw error } - - // Bring Chrome to front - await activateChrome() - // Brief wait for activation - await sleep(300) - - // Get the Chrome PID - const pid = await getChromeMainPid() - if (!pid) { - throw new Error('Failed to get Chrome PID after launch') - } - - session = { - wasAlreadyRunning, - windowId: `${pid}:0:${CHROME_APP_NAME}`, - cdpUrl: wasAlreadyRunning ? undefined : `http://127.0.0.1:${cdpPort}`, - pid, - agentOwned: !wasAlreadyRunning, - initialUrl: options?.url, - createdAt: new Date().toISOString(), - } - - return session }, async bringToFront() { if (!session) return false - const stillRunning = await isChromeRunning() - if (!stillRunning) { - session = null + const stillRunning = await isProcessAlive(session.pid) + const stillHasWindow = stillRunning && await hasChromeWindow(session.pid) + if (!stillHasWindow) { + await clearSessionState() onSessionLost?.() return false } @@ -266,8 +410,7 @@ end run` endSession() { const hadSession = session !== null - session = null - previousForegroundApp = undefined + void clearSessionState() if (hadSession) { onSessionLost?.() } diff --git a/services/computer-use-mcp/src/desktop-grounding.test.ts b/services/computer-use-mcp/src/desktop-grounding.test.ts index 03d60e73c..26966b260 100644 --- a/services/computer-use-mcp/src/desktop-grounding.test.ts +++ b/services/computer-use-mcp/src/desktop-grounding.test.ts @@ -1,9 +1,9 @@ import type { AXNode, AXSnapshot } from './accessibility/types' import type { ChromeSemanticSnapshot, DesktopGroundingSnapshot } from './desktop-grounding-types' -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' -import { buildTargetCandidates, formatGroundingForAgent } from './desktop-grounding' +import { buildTargetCandidates, captureDesktopGrounding, formatGroundingForAgent } from './desktop-grounding' // --------------------------------------------------------------------------- // Helpers @@ -170,6 +170,91 @@ describe('buildTargetCandidates', () => { }) }) +describe('captureDesktopGrounding', () => { + it('retries window observation with app filter when Chrome is foreground and the generic list misses it', async () => { + const chromeWindow = { x: 0, y: 0, width: 1920, height: 1080 } + const chromeElements = [ + { tag: 'button', text: 'Submit', rect: { x: 10, y: 10, w: 80, h: 30 } }, + ] + + const executor = { + takeScreenshot: vi.fn().mockResolvedValue({ + dataBase64: '', + mimeType: 'image/png', + path: '/tmp/screenshot.png', + capturedAt: new Date().toISOString(), + }), + observeWindows: vi.fn() + .mockResolvedValueOnce({ + frontmostAppName: 'Google Chrome', + frontmostWindowTitle: 'Chrome', + windows: [ + { + appName: 'Control Center', + title: 'Clock', + bounds: { x: 0, y: 0, width: 100, height: 30 }, + }, + ], + observedAt: new Date().toISOString(), + }) + .mockResolvedValueOnce({ + frontmostAppName: 'Google Chrome', + frontmostWindowTitle: 'Chrome', + windows: [ + { + appName: 'Google Chrome', + title: 'Chrome', + bounds: chromeWindow, + ownerPid: 1234, + id: '1234:0:Chrome', + layer: 0, + isOnScreen: true, + }, + ], + observedAt: new Date().toISOString(), + }), + focusApp: vi.fn(), + openApp: vi.fn(), + click: vi.fn(), + typeText: vi.fn(), + pressKeys: vi.fn(), + scroll: vi.fn(), + getForegroundContext: vi.fn().mockResolvedValue({ + available: true, + appName: 'Google Chrome', + platform: 'darwin', + }), + getDisplayInfo: vi.fn(), + getExecutionTarget: vi.fn(), + describe: vi.fn(), + } as any + + const cdpBridge = { + getStatus: vi.fn().mockReturnValue({ + connected: true, + pageUrl: 'https://example.com', + pageTitle: 'Example Page', + }), + collectInteractiveElements: vi.fn().mockResolvedValue(chromeElements), + } as any + + const config = { + timeoutMs: 5000, + } as any + + const snapshot = await captureDesktopGrounding({ + config, + executor, + input: { includeChrome: true }, + cdpBridge, + }) + + expect(executor.observeWindows).toHaveBeenNthCalledWith(1, { limit: 12 }) + expect(executor.observeWindows).toHaveBeenNthCalledWith(2, { app: 'Google Chrome', limit: 12 }) + expect(snapshot.targetCandidates.some(candidate => candidate.source === 'chrome_dom')).toBe(true) + }) +}) + // --------------------------------------------------------------------------- // formatGroundingForAgent // --------------------------------------------------------------------------- diff --git a/services/computer-use-mcp/src/desktop-grounding.ts b/services/computer-use-mcp/src/desktop-grounding.ts index 7931c7da3..0f60a634d 100644 --- a/services/computer-use-mcp/src/desktop-grounding.ts +++ b/services/computer-use-mcp/src/desktop-grounding.ts @@ -81,6 +81,25 @@ export async function captureDesktopGrounding(params: { const foregroundApp = windowObs.frontmostAppName || axSnapshot?.appName || 'unknown' const isChromeInFront = isChromeApp(foregroundApp) + // If Chrome is foreground, ask the executor for a Chrome-filtered window list. + // The generic top-N window snapshot is often dominated by system UI and can + // miss Chrome entirely, which would prevent chrome_dom candidates from being + // mapped to screen coordinates. + let chromeWindowBounds = findChromeWindowBounds(windowObs, foregroundApp) + if (isChromeInFront && !chromeWindowBounds) { + try { + const chromeWindows = await executor.observeWindows({ + app: foregroundApp, + limit: 12, + }) + chromeWindowBounds = findChromeWindowBounds(chromeWindows, foregroundApp) + } + catch { + // Best-effort only. Fall back to AX-only candidates if filtered window + // enumeration fails. + } + } + // Phase 2: Chrome semantic data (only if Chrome is foreground and allowed) let chromeSemanticSnapshot: ChromeSemanticSnapshot | null = null if (isChromeInFront && input?.includeChrome !== false) { @@ -88,7 +107,6 @@ export async function captureDesktopGrounding(params: { } // Phase 3: Build target candidates - const chromeWindowBounds = findChromeWindowBounds(windowObs, foregroundApp) const candidates = buildTargetCandidates({ axSnapshot, chromeSnapshot: chromeSemanticSnapshot ?? undefined, diff --git a/services/computer-use-mcp/src/server/register-chrome-session.test.ts b/services/computer-use-mcp/src/server/register-chrome-session.test.ts index 2e8a6f455..5341b4a3a 100644 --- a/services/computer-use-mcp/src/server/register-chrome-session.test.ts +++ b/services/computer-use-mcp/src/server/register-chrome-session.test.ts @@ -150,34 +150,6 @@ describe('registerChromeSessionTools', () => { expect(runtime.stateManager.getState().pendingApprovalCount).toBe(1) }) - it('audits joined Chrome sessions as open_app because ensure can create a new window', async () => { - runtime.config = createTestConfig({ - executor: 'macos-local', - approvalMode: 'all', - }) - vi.mocked(runtime.chromeSessionManager.getSessionInfo).mockReturnValue({ - wasAlreadyRunning: true, - windowId: 'chrome-window-existing', - pid: 9999, - agentOwned: false, - createdAt: new Date().toISOString(), - }) - - const { server, invoke } = createMockServer() - registerChromeSessionTools({ server, runtime }) - - const result = await invoke('desktop_ensure_chrome') - - const structured = result.structuredContent as Record - expect(structured.status).toBe('approval_required') - expect(structured.action).toEqual({ - kind: 'desktop_ensure_chrome', - input: {}, - }) - expect(structured.transparency.intent).toBe('Open an agent Chrome window with CDP support') - expect(runtime.chromeSessionManager.ensureAgentWindow).not.toHaveBeenCalled() - }) - it('consumes operation budget and persists chrome session when approvals are disabled', async () => { vi.mocked(runtime.chromeSessionManager.ensureAgentWindow).mockResolvedValue({ wasAlreadyRunning: false, @@ -207,4 +179,38 @@ describe('registerChromeSessionTools', () => { expect((runtime.session.record as any).mock.calls[0][0].event).toBe('requested') expect((runtime.session.record as any).mock.calls[1][0].event).toBe('executed') }) + + it('uses focus_app when a chrome session already exists', async () => { + runtime.config = createTestConfig({ + executor: 'macos-local', + approvalMode: 'all', + }) + vi.mocked(runtime.chromeSessionManager.getSessionInfo).mockReturnValue({ + wasAlreadyRunning: false, + windowId: 'chrome-window-existing', + pid: 9999, + agentOwned: true, + createdAt: new Date().toISOString(), + }) + + const { server, invoke } = createMockServer() + registerChromeSessionTools({ server, runtime }) + + const result = await invoke('desktop_ensure_chrome') + + const structured = result.structuredContent as Record + expect(structured.status).toBe('approval_required') + expect(structured.action).toEqual({ + kind: 'desktop_ensure_chrome', + input: {}, + }) + expect(structured.transparency.intent).toBe('Bring the agent Chrome window to the foreground') + expect(runtime.chromeSessionManager.ensureAgentWindow).not.toHaveBeenCalled() + expect((runtime.session.record as any).mock.calls[0][0].result.approvalAction).toEqual({ + kind: 'focus_app', + input: { + app: 'Google Chrome', + }, + }) + }) }) diff --git a/services/computer-use-mcp/src/server/register-chrome-session.ts b/services/computer-use-mcp/src/server/register-chrome-session.ts index 067cfacc4..e9b13753f 100644 --- a/services/computer-use-mcp/src/server/register-chrome-session.ts +++ b/services/computer-use-mcp/src/server/register-chrome-session.ts @@ -30,7 +30,7 @@ const CHROME_APP_NAME = 'Google Chrome' function getChromeSessionAction(runtime: ComputerUseServerRuntime): ActionInvocation { const sessionInfo = runtime.chromeSessionManager.getSessionInfo() return { - kind: sessionInfo && !sessionInfo.wasAlreadyRunning ? 'focus_app' : 'open_app', + kind: sessionInfo ? 'focus_app' : 'open_app', input: { app: CHROME_APP_NAME, }, @@ -63,7 +63,7 @@ export async function executeChromeEnsure( appName: 'Google Chrome', windowId: sessionInfo.windowId, pid: sessionInfo.pid, - agentLaunched: !sessionInfo.wasAlreadyRunning, + agentLaunched: sessionInfo.agentOwned, }) } @@ -76,7 +76,7 @@ export async function executeChromeEnsure( } } - // Auto-connect CDP bridge when the agent launched Chrome with CDP. + // Auto-connect CDP bridge when the agent owns Chrome and CDP is available. // Best-effort only: Chrome may need a moment before the DevTools server answers. let cdpStatus = 'not applicable' if (sessionInfo.cdpUrl) { @@ -97,7 +97,7 @@ export async function executeChromeEnsure( } const lines = [ - `Chrome session ${sessionInfo.wasAlreadyRunning ? 'joined' : 'launched'}:`, + 'Chrome session launched:', ` PID: ${sessionInfo.pid}`, ` Window: ${sessionInfo.windowId}`, ` Agent-owned: ${sessionInfo.agentOwned}`, diff --git a/services/computer-use-mcp/src/support-matrix.test.ts b/services/computer-use-mcp/src/support-matrix.test.ts index f0d62cf1a..86debe116 100644 --- a/services/computer-use-mcp/src/support-matrix.test.ts +++ b/services/computer-use-mcp/src/support-matrix.test.ts @@ -53,4 +53,23 @@ describe('support matrix', () => { const ps = getProductSupported() expect(ps.length).toBeGreaterThanOrEqual(4) }) + + it('includes desktop v3 smoke coverage as covered, not product-supported', () => { + const entry = supportMatrix.find(item => item.id === 'desktop_v3_chrome_grounding') + expect(entry).toBeDefined() + expect(entry?.lane).toBe('desktop-native') + expect(entry?.level).toBe('covered') + expect(entry?.smokeCommand).toBe('pnpm -F @proj-airi/computer-use-mcp smoke:desktop-v3') + }) + + it('includes browser-dom route contract as covered, not product-supported', () => { + const entry = supportMatrix.find(item => item.id === 'desktop_browser_dom_route_contract') + expect(entry).toBeDefined() + expect(entry?.lane).toBe('desktop-native') + expect(entry?.level).toBe('covered') + expect(entry?.unitTests).toEqual([ + 'src/browser-action-router.test.ts', + 'src/browser-dom/extension-bridge.test.ts', + ]) + }) }) diff --git a/services/computer-use-mcp/src/support-matrix.ts b/services/computer-use-mcp/src/support-matrix.ts index f6abe4363..de3e96ad0 100644 --- a/services/computer-use-mcp/src/support-matrix.ts +++ b/services/computer-use-mcp/src/support-matrix.ts @@ -157,6 +157,30 @@ export const supportMatrix: SupportMatrixEntry[] = [ smokeCommand: 'pnpm -F @proj-airi/computer-use-mcp smoke:stdio', happyPath: 'focus app → screenshot → accessibility_snapshot basic loop', }, + { + lane: 'desktop-native', + id: 'desktop_v3_chrome_grounding', + label: 'Desktop v3 Chrome grounding smoke (ensure / observe / click / state)', + level: 'covered', + unitTests: [ + 'src/bin/smoke-chrome-grounding.test.ts', + 'src/server/register-chrome-session.test.ts', + 'src/server/register-desktop-grounding.test.ts', + 'src/server/register-desktop-grounding-tools.test.ts', + ], + smokeCommand: 'pnpm -F @proj-airi/computer-use-mcp smoke:desktop-v3', + happyPath: 'desktop_ensure_chrome → desktop_observe → desktop_click_target → desktop_get_state updates grounding and pointer state', + }, + { + lane: 'desktop-native', + id: 'desktop_browser_dom_route_contract', + label: 'Browser-dom route contract (left single-click, fail-closed bridge responses)', + level: 'covered', + unitTests: [ + 'src/browser-action-router.test.ts', + 'src/browser-dom/extension-bridge.test.ts', + ], + }, { lane: 'desktop-native', id: 'desktop_click_type_press',