mirror of
https://github.com/moeru-ai/airi.git
synced 2026-08-14 00:48:06 +00:00
test(computer-use-mcp): add desktop v3 smoke coverage (#1780)
This commit is contained in:
刘梓恒
@@ -30,6 +30,7 @@
|
||||
"merge-latest-mac": "tsx scripts/merge-latest-mac.ts",
|
||||
"regenerate-windows-latest": "tsx scripts/regenerate-windows-latest.ts",
|
||||
"artifacts-metadata": "tsx scripts/artifacts-metadata.ts",
|
||||
"smoke:desktop-overlay-live-window": "NODE_OPTIONS='--experimental-websocket' tsx scripts/desktop-overlay-live-window-smoke.ts",
|
||||
"update-test:generate": "tsx scripts/update-test/generate-manifest.ts",
|
||||
"update-test:server": "tsx scripts/update-test/start-server.ts",
|
||||
"update-test:matrix": "bash scripts/update-test/run-matrix.sh"
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { CdpClient } from './desktop-overlay-live-window-smoke'
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
function createMockSocket() {
|
||||
const socket = new EventEmitter() as EventEmitter & {
|
||||
send: ReturnType<typeof vi.fn>
|
||||
close: ReturnType<typeof vi.fn>
|
||||
addEventListener: (event: string, listener: (...args: any[]) => void) => void
|
||||
}
|
||||
socket.send = vi.fn()
|
||||
socket.close = vi.fn(() => {
|
||||
socket.emit('close')
|
||||
})
|
||||
socket.addEventListener = (event, listener) => {
|
||||
socket.on(event, listener)
|
||||
}
|
||||
return socket
|
||||
}
|
||||
|
||||
describe('cdpClient', () => {
|
||||
it('rejects pending requests when the socket closes', async () => {
|
||||
const socket = createMockSocket()
|
||||
const client = new CdpClient(socket as never)
|
||||
|
||||
const pending = client.send('Runtime.evaluate', { expression: '1 + 1' })
|
||||
expect(socket.send).toHaveBeenCalledTimes(1)
|
||||
|
||||
client.close()
|
||||
|
||||
await expect(pending).rejects.toThrow('CDP socket closed before completing request 1')
|
||||
expect(socket.close).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,563 @@
|
||||
import type { ChildProcessWithoutNullStreams } from 'node:child_process'
|
||||
|
||||
import { spawn } from 'node:child_process'
|
||||
import { createWriteStream } from 'node:fs'
|
||||
import { access, mkdir, writeFile } from 'node:fs/promises'
|
||||
import { createServer } from 'node:net'
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { env, exit, kill as killProcess } from 'node:process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import { desktopOverlayPollHeartbeatMarker } from '../src/shared/desktop-overlay-heartbeat'
|
||||
import { selectDesktopOverlaySmokeCandidateId } from '../src/shared/desktop-overlay-live-window-smoke'
|
||||
|
||||
interface DebugTarget {
|
||||
id: string
|
||||
title: string
|
||||
type: string
|
||||
url: string
|
||||
webSocketDebuggerUrl?: string
|
||||
}
|
||||
|
||||
interface McpResult {
|
||||
content?: unknown[]
|
||||
structuredContent?: Record<string, unknown>
|
||||
isError?: boolean
|
||||
}
|
||||
|
||||
interface McpApplyResult {
|
||||
started: Array<{ name: string }>
|
||||
failed: Array<{ name: string, error: string }>
|
||||
skipped: Array<{ name: string, reason: string }>
|
||||
}
|
||||
|
||||
interface McpToolDescriptor {
|
||||
name: string
|
||||
serverName: string
|
||||
toolName: string
|
||||
}
|
||||
|
||||
interface McpRuntimeStatus {
|
||||
servers: Array<{
|
||||
name: string
|
||||
state: 'running' | 'stopped' | 'error'
|
||||
lastError?: string
|
||||
}>
|
||||
}
|
||||
|
||||
const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
||||
const repoDir = resolve(packageDir, '../..')
|
||||
const runId = new Date().toISOString().replaceAll(':', '-').replaceAll('.', '-')
|
||||
const reportDir = resolve(repoDir, '.temp', `desktop-overlay-live-window-smoke-${runId}`)
|
||||
const userDataDir = resolve(reportDir, 'stage-user-data')
|
||||
const mcpSessionRoot = resolve(reportDir, 'computer-use-session')
|
||||
const stageLogPath = resolve(reportDir, 'stage-tamagotchi.log')
|
||||
const mcpConfigPath = resolve(userDataDir, 'mcp.json')
|
||||
const requiredWorkspaceBuildOutputs = [
|
||||
'packages/electron-screen-capture/dist/main.mjs',
|
||||
'packages/electron-vueuse/dist/main/index.mjs',
|
||||
'packages/server-runtime/dist/server.mjs',
|
||||
]
|
||||
|
||||
const smokeHtml = `<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>AIRI Desktop Overlay Live Window Smoke</title>
|
||||
<style>
|
||||
body { font-family: sans-serif; padding: 48px; }
|
||||
button { font-size: 18px; padding: 12px 18px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>AIRI Desktop Overlay Live Window Smoke</h1>
|
||||
<button id="airi-desktop-overlay-smoke-button">AIRI Desktop Overlay Smoke Button</button>
|
||||
</body>
|
||||
</html>`
|
||||
|
||||
const smokeUrl = `data:text/html;charset=utf-8,${encodeURIComponent(smokeHtml)}`
|
||||
|
||||
function assert(condition: boolean, message: string): asserts condition {
|
||||
if (!condition)
|
||||
throw new Error(message)
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
async function findAvailablePort(): Promise<number> {
|
||||
return await new Promise((resolvePort, reject) => {
|
||||
const server = createServer()
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const address = server.address()
|
||||
server.close(() => {
|
||||
if (typeof address === 'object' && address?.port) {
|
||||
resolvePort(address.port)
|
||||
}
|
||||
else {
|
||||
reject(new Error('failed to allocate debug port'))
|
||||
}
|
||||
})
|
||||
})
|
||||
server.on('error', reject)
|
||||
})
|
||||
}
|
||||
|
||||
async function waitFor<T>(
|
||||
label: string,
|
||||
probe: () => Promise<T | undefined> | T | undefined,
|
||||
timeoutMs: number,
|
||||
intervalMs: number,
|
||||
): Promise<T> {
|
||||
const start = Date.now()
|
||||
let lastError: unknown
|
||||
|
||||
while ((Date.now() - start) < timeoutMs) {
|
||||
try {
|
||||
const value = await probe()
|
||||
if (value !== undefined)
|
||||
return value
|
||||
}
|
||||
catch (error) {
|
||||
lastError = error
|
||||
}
|
||||
await sleep(intervalMs)
|
||||
}
|
||||
|
||||
const suffix = lastError instanceof Error ? `: ${lastError.message}` : ''
|
||||
throw new Error(`${label} timed out after ${timeoutMs}ms${suffix}`)
|
||||
}
|
||||
|
||||
export class CdpClient {
|
||||
private socket?: WebSocket
|
||||
private nextId = 1
|
||||
private pending = new Map<number, {
|
||||
resolve: (value: Record<string, unknown>) => void
|
||||
reject: (error: Error) => void
|
||||
}>()
|
||||
|
||||
constructor(socket: WebSocket) {
|
||||
this.socket = socket
|
||||
this.socket.addEventListener('message', (event) => {
|
||||
const payload = JSON.parse(String(event.data)) as Record<string, unknown>
|
||||
const id = typeof payload.id === 'number' ? payload.id : undefined
|
||||
if (id === undefined)
|
||||
return
|
||||
|
||||
const pending = this.pending.get(id)
|
||||
if (!pending)
|
||||
return
|
||||
|
||||
this.pending.delete(id)
|
||||
if (payload.error) {
|
||||
pending.reject(new Error(JSON.stringify(payload.error)))
|
||||
}
|
||||
else {
|
||||
pending.resolve(payload)
|
||||
}
|
||||
})
|
||||
this.socket.addEventListener('close', () => {
|
||||
this.failPending('CDP socket closed')
|
||||
})
|
||||
this.socket.addEventListener('error', () => {
|
||||
this.failPending('CDP socket errored')
|
||||
})
|
||||
}
|
||||
|
||||
static async connect(url: string): Promise<CdpClient> {
|
||||
const socket = new WebSocket(url)
|
||||
await new Promise<void>((resolveOpen, reject) => {
|
||||
socket.addEventListener('open', () => resolveOpen(), { once: true })
|
||||
socket.addEventListener('error', () => reject(new Error(`failed to connect CDP target: ${url}`)), { once: true })
|
||||
})
|
||||
return new CdpClient(socket)
|
||||
}
|
||||
|
||||
async send(method: string, params?: Record<string, unknown>): Promise<Record<string, unknown>> {
|
||||
if (!this.socket) {
|
||||
throw new Error('CDP socket is closed')
|
||||
}
|
||||
|
||||
const id = this.nextId++
|
||||
const promise = new Promise<Record<string, unknown>>((resolveMessage, reject) => {
|
||||
this.pending.set(id, { resolve: resolveMessage, reject })
|
||||
})
|
||||
|
||||
this.socket.send(JSON.stringify({ id, method, params: params ?? {} }))
|
||||
return await promise
|
||||
}
|
||||
|
||||
async evaluate<T>(expression: string): Promise<T> {
|
||||
const response = await this.send('Runtime.evaluate', {
|
||||
expression,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
})
|
||||
const result = response.result
|
||||
if (!isRecord(result))
|
||||
throw new Error('Runtime.evaluate missing result')
|
||||
|
||||
const exceptionDetails = result.exceptionDetails
|
||||
if (exceptionDetails) {
|
||||
throw new Error(JSON.stringify(exceptionDetails))
|
||||
}
|
||||
|
||||
const remoteObject = result.result
|
||||
if (!isRecord(remoteObject))
|
||||
throw new Error('Runtime.evaluate missing remote object')
|
||||
|
||||
return remoteObject.value as T
|
||||
}
|
||||
|
||||
close() {
|
||||
this.failPending('CDP socket closed')
|
||||
this.socket?.close()
|
||||
this.socket = undefined
|
||||
}
|
||||
|
||||
private failPending(reason: string) {
|
||||
if (this.pending.size === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
for (const [id, pending] of this.pending.entries()) {
|
||||
this.pending.delete(id)
|
||||
pending.reject(new Error(`${reason} before completing request ${id}`))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchJson<T>(url: string): Promise<T> {
|
||||
const response = await fetch(url)
|
||||
if (!response.ok)
|
||||
throw new Error(`${url} returned ${response.status}`)
|
||||
return await response.json() as T
|
||||
}
|
||||
|
||||
async function prepareMcpConfig() {
|
||||
await mkdir(userDataDir, { recursive: true })
|
||||
await mkdir(mcpSessionRoot, { recursive: true })
|
||||
|
||||
const mcpEnv: Record<string, string> = {
|
||||
PATH: env.PATH || '',
|
||||
HOME: env.HOME || '',
|
||||
SHELL: env.SHELL || '',
|
||||
LANG: env.LANG || 'en_US.UTF-8',
|
||||
TMPDIR: env.TMPDIR || '',
|
||||
COMPUTER_USE_EXECUTOR: env.COMPUTER_USE_SMOKE_EXECUTOR || env.COMPUTER_USE_EXECUTOR || 'macos-local',
|
||||
COMPUTER_USE_APPROVAL_MODE: env.COMPUTER_USE_SMOKE_APPROVAL_MODE || env.COMPUTER_USE_APPROVAL_MODE || 'never',
|
||||
COMPUTER_USE_OPENABLE_APPS: env.COMPUTER_USE_OPENABLE_APPS || 'Terminal,Cursor,Google Chrome',
|
||||
COMPUTER_USE_SESSION_TAG: `desktop-overlay-live-window-smoke-${runId}`,
|
||||
COMPUTER_USE_SESSION_ROOT: mcpSessionRoot,
|
||||
}
|
||||
|
||||
for (const optionalEnvName of ['PNPM_HOME', 'COREPACK_HOME']) {
|
||||
const value = env[optionalEnvName]?.trim()
|
||||
if (value) {
|
||||
mcpEnv[optionalEnvName] = value
|
||||
}
|
||||
}
|
||||
|
||||
const config = {
|
||||
mcpServers: {
|
||||
computer_use: {
|
||||
command: 'pnpm',
|
||||
args: ['-F', '@proj-airi/computer-use-mcp', 'start'],
|
||||
cwd: repoDir,
|
||||
enabled: true,
|
||||
env: mcpEnv,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
await writeFile(mcpConfigPath, `${JSON.stringify(config, null, 2)}\n`, 'utf-8')
|
||||
}
|
||||
|
||||
async function ensureSmokePrerequisites() {
|
||||
if (typeof WebSocket !== 'function') {
|
||||
throw new TypeError('APP_START_FAILED: WebSocket is unavailable in this Node runtime. Run through the package script or set NODE_OPTIONS=--experimental-websocket.')
|
||||
}
|
||||
|
||||
const missingOutputs: string[] = []
|
||||
for (const relativePath of requiredWorkspaceBuildOutputs) {
|
||||
try {
|
||||
await access(resolve(repoDir, relativePath))
|
||||
}
|
||||
catch {
|
||||
missingOutputs.push(relativePath)
|
||||
}
|
||||
}
|
||||
|
||||
if (missingOutputs.length === 0)
|
||||
return
|
||||
|
||||
throw new Error([
|
||||
'APP_START_FAILED: required workspace build outputs are missing.',
|
||||
`Missing: ${missingOutputs.join(', ')}`,
|
||||
'Build stage-tamagotchi dependencies manually before this smoke. The smoke command does not auto-build them to avoid saturating the local machine.',
|
||||
'Suggested command: pnpm -F \'@proj-airi/stage-tamagotchi^...\' --if-present build',
|
||||
].join(' '))
|
||||
}
|
||||
|
||||
async function waitForRemoteDebug(debugPort: number): Promise<string> {
|
||||
const version = await waitFor('Electron remote debug endpoint', async () => {
|
||||
const data = await fetchJson<{ webSocketDebuggerUrl?: string }>(`http://127.0.0.1:${debugPort}/json/version`)
|
||||
return data.webSocketDebuggerUrl
|
||||
}, 120_000, 500)
|
||||
|
||||
return version
|
||||
}
|
||||
|
||||
async function findOverlayTarget(debugPort: number): Promise<DebugTarget> {
|
||||
return await waitFor('desktop overlay debug target', async () => {
|
||||
const targets = await fetchJson<DebugTarget[]>(`http://127.0.0.1:${debugPort}/json/list`)
|
||||
return targets.find(target => target.type === 'page' && target.url.includes('#/desktop-overlay'))
|
||||
}, 120_000, 500)
|
||||
}
|
||||
|
||||
async function connectOverlayClient(debugPort: number): Promise<CdpClient> {
|
||||
const overlayTarget = await findOverlayTarget(debugPort)
|
||||
if (!overlayTarget.webSocketDebuggerUrl)
|
||||
throw new Error('APP_START_FAILED: overlay target missing webSocketDebuggerUrl')
|
||||
|
||||
const client = await CdpClient.connect(overlayTarget.webSocketDebuggerUrl)
|
||||
|
||||
await waitFor('overlay smoke bridge', async () => {
|
||||
return await client.evaluate<boolean>('Boolean(window.__AIRI_DESKTOP_OVERLAY_SMOKE__?.callMcpTool)')
|
||||
? true
|
||||
: undefined
|
||||
}, 60_000, 500)
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
async function callOverlayMcpTool(client: CdpClient, name: string, args: Record<string, unknown> = {}): Promise<McpResult> {
|
||||
const result = await client.evaluate<McpResult>(`window.__AIRI_DESKTOP_OVERLAY_SMOKE__.callMcpTool(${JSON.stringify({ name, arguments: args })})`)
|
||||
if (result.isError) {
|
||||
throw new Error(`${name} returned isError=true`)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
async function ensureOverlayMcpServerReady(client: CdpClient): Promise<void> {
|
||||
const applyResult = await client.evaluate<McpApplyResult>('window.__AIRI_DESKTOP_OVERLAY_SMOKE__.applyAndRestartMcp()')
|
||||
const failedComputerUse = applyResult.failed.find(item => item.name === 'computer_use')
|
||||
if (failedComputerUse) {
|
||||
throw new Error(`computer_use failed to start: ${failedComputerUse.error}`)
|
||||
}
|
||||
|
||||
await waitFor('computer_use MCP runtime', async () => {
|
||||
const status = await client.evaluate<McpRuntimeStatus>('window.__AIRI_DESKTOP_OVERLAY_SMOKE__.getMcpRuntimeStatus()')
|
||||
const computerUse = status.servers.find(server => server.name === 'computer_use')
|
||||
if (computerUse?.state === 'error') {
|
||||
throw new Error(`computer_use runtime error: ${computerUse.lastError ?? 'unknown error'}`)
|
||||
}
|
||||
return computerUse?.state === 'running' ? true : undefined
|
||||
}, 30_000, 500)
|
||||
|
||||
await waitFor('computer_use desktop tools', async () => {
|
||||
const tools = await client.evaluate<McpToolDescriptor[]>('window.__AIRI_DESKTOP_OVERLAY_SMOKE__.listMcpTools()')
|
||||
const names = new Set(tools.map(tool => tool.name))
|
||||
return names.has('computer_use::desktop_get_state')
|
||||
&& names.has('computer_use::desktop_observe')
|
||||
&& names.has('computer_use::desktop_click_target')
|
||||
? true
|
||||
: undefined
|
||||
}, 30_000, 500)
|
||||
}
|
||||
|
||||
function requireStructuredContent(result: McpResult, label: string): Record<string, unknown> {
|
||||
if (!isRecord(result.structuredContent))
|
||||
throw new Error(`${label} missing structuredContent`)
|
||||
|
||||
if (result.structuredContent.status && result.structuredContent.status !== 'ok')
|
||||
throw new Error(`${label} expected status=ok, got ${String(result.structuredContent.status)}`)
|
||||
|
||||
return result.structuredContent
|
||||
}
|
||||
|
||||
function requireRunState(result: McpResult, label: string): Record<string, unknown> {
|
||||
const structuredContent = requireStructuredContent(result, label)
|
||||
if (!isRecord(structuredContent.runState))
|
||||
throw new Error(`${label} missing runState`)
|
||||
return structuredContent.runState
|
||||
}
|
||||
|
||||
function startStage(debugPort: number, heartbeatLines: string[]): ChildProcessWithoutNullStreams {
|
||||
const stageProcess = spawn('pnpm', ['-F', '@proj-airi/stage-tamagotchi', 'dev'], {
|
||||
cwd: repoDir,
|
||||
detached: true,
|
||||
env: {
|
||||
...env,
|
||||
APP_REMOTE_DEBUG: 'true',
|
||||
APP_REMOTE_DEBUG_PORT: String(debugPort),
|
||||
APP_REMOTE_DEBUG_NO_OPEN: 'true',
|
||||
APP_USER_DATA_PATH: userDataDir,
|
||||
AIRI_DESKTOP_OVERLAY: '1',
|
||||
AIRI_DESKTOP_OVERLAY_POLL_HEARTBEAT: '1',
|
||||
},
|
||||
stdio: 'pipe',
|
||||
})
|
||||
|
||||
const stageLogStream = createWriteStream(stageLogPath, { flags: 'a' })
|
||||
const capture = (chunk: Buffer) => {
|
||||
const text = chunk.toString('utf-8')
|
||||
stageLogStream.write(text)
|
||||
for (const line of text.split(/\r?\n/u)) {
|
||||
if (line.includes(desktopOverlayPollHeartbeatMarker)) {
|
||||
heartbeatLines.push(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
stageProcess.stdout.on('data', capture)
|
||||
stageProcess.stderr.on('data', capture)
|
||||
stageProcess.on('close', () => stageLogStream.end())
|
||||
|
||||
return stageProcess
|
||||
}
|
||||
|
||||
async function stopStage(stageProcess: ChildProcessWithoutNullStreams | undefined) {
|
||||
if (!stageProcess || stageProcess.exitCode !== null)
|
||||
return
|
||||
|
||||
const signalStageProcessGroup = (signal: NodeJS.Signals) => {
|
||||
try {
|
||||
if (stageProcess.pid) {
|
||||
killProcess(-stageProcess.pid, signal)
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// Fall back to the pnpm wrapper process if process-group signalling is
|
||||
// unavailable. The smoke starts a detached group to make this reliable on
|
||||
// macOS, but the fallback keeps the helper safe on other local setups.
|
||||
}
|
||||
|
||||
stageProcess.kill(signal)
|
||||
}
|
||||
|
||||
signalStageProcessGroup('SIGTERM')
|
||||
await Promise.race([
|
||||
new Promise(resolve => stageProcess.once('exit', resolve)),
|
||||
sleep(5_000).then(() => signalStageProcessGroup('SIGKILL')),
|
||||
])
|
||||
}
|
||||
|
||||
function rejectWhenStageExits(stageProcess: ChildProcessWithoutNullStreams): Promise<never> {
|
||||
return new Promise((_, reject) => {
|
||||
stageProcess.once('exit', (code, signal) => {
|
||||
reject(new Error(`stage-tamagotchi exited with code=${String(code)} signal=${String(signal)}`))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async function main() {
|
||||
let stageProcess: ChildProcessWithoutNullStreams | undefined
|
||||
let overlayClient: CdpClient | undefined
|
||||
let stoppingStage = false
|
||||
const heartbeatLines: string[] = []
|
||||
|
||||
try {
|
||||
await ensureSmokePrerequisites()
|
||||
await mkdir(reportDir, { recursive: true })
|
||||
await prepareMcpConfig()
|
||||
|
||||
const debugPort = await findAvailablePort()
|
||||
stageProcess = startStage(debugPort, heartbeatLines)
|
||||
const stageExited = rejectWhenStageExits(stageProcess)
|
||||
stageProcess.once('exit', (code, signal) => {
|
||||
if (!stoppingStage && code !== null && code !== 0)
|
||||
console.error(`APP_START_FAILED: stage-tamagotchi exited with code=${code} signal=${String(signal)}`)
|
||||
})
|
||||
|
||||
await Promise.race([
|
||||
waitForRemoteDebug(debugPort),
|
||||
stageExited,
|
||||
]).catch((error) => {
|
||||
throw new Error(`APP_START_FAILED: ${error instanceof Error ? error.message : String(error)}`)
|
||||
})
|
||||
|
||||
overlayClient = await Promise.race([
|
||||
connectOverlayClient(debugPort),
|
||||
stageExited,
|
||||
]).catch((error) => {
|
||||
throw new Error(`APP_START_FAILED: ${error instanceof Error ? error.message : String(error)}`)
|
||||
})
|
||||
|
||||
// NOTICE:
|
||||
// Vite's dev optimizer can trigger one renderer reload shortly after the
|
||||
// Electron window first exposes the smoke bridge. Reconnect once after a
|
||||
// short settle window so the following MCP calls do not race a closing CDP
|
||||
// target. This is local smoke harness discipline, not product runtime.
|
||||
await sleep(5_000)
|
||||
overlayClient.close()
|
||||
overlayClient = await Promise.race([
|
||||
connectOverlayClient(debugPort),
|
||||
stageExited,
|
||||
]).catch((error) => {
|
||||
throw new Error(`APP_START_FAILED: ${error instanceof Error ? error.message : String(error)}`)
|
||||
})
|
||||
|
||||
const readiness = await overlayClient.evaluate<{ state: 'booting' | 'ready' | 'degraded', error?: string }>('window.__AIRI_DESKTOP_OVERLAY_SMOKE__.getReadiness()')
|
||||
if (readiness.state !== 'ready') {
|
||||
throw new Error(`OVERLAY_READINESS_DEGRADED: state=${readiness.state}${readiness.error ? ` error=${readiness.error}` : ''}`)
|
||||
}
|
||||
|
||||
try {
|
||||
await ensureOverlayMcpServerReady(overlayClient)
|
||||
await callOverlayMcpTool(overlayClient, 'computer_use::desktop_ensure_chrome', { url: smokeUrl })
|
||||
await sleep(750)
|
||||
await callOverlayMcpTool(overlayClient, 'computer_use::desktop_observe', { includeChrome: true })
|
||||
const preClickRunState = requireRunState(
|
||||
await callOverlayMcpTool(overlayClient, 'computer_use::desktop_get_state'),
|
||||
'computer_use::desktop_get_state before click',
|
||||
)
|
||||
const candidateId = selectDesktopOverlaySmokeCandidateId(preClickRunState)
|
||||
await callOverlayMcpTool(overlayClient, 'computer_use::desktop_click_target', {
|
||||
candidateId,
|
||||
button: 'left',
|
||||
clickCount: 1,
|
||||
})
|
||||
const postClickRunState = requireRunState(
|
||||
await callOverlayMcpTool(overlayClient, 'computer_use::desktop_get_state'),
|
||||
'computer_use::desktop_get_state after click',
|
||||
)
|
||||
const pointerIntent = postClickRunState.lastPointerIntent
|
||||
assert(isRecord(pointerIntent), 'computer_use::desktop_get_state missing lastPointerIntent after click')
|
||||
assert(pointerIntent.candidateId === candidateId, `lastPointerIntent candidate mismatch: expected ${candidateId}, got ${String(pointerIntent.candidateId)}`)
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`MCP_CALL_FAILED: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
|
||||
const heartbeat = await waitFor('overlay poll heartbeat', () => {
|
||||
return heartbeatLines.find(line => line.includes('snapshotId=') && line.includes('pointerIntent=yes'))
|
||||
}, 30_000, 250).catch((error) => {
|
||||
throw new Error(`HEARTBEAT_TIMEOUT: ${error instanceof Error ? error.message : String(error)}`)
|
||||
})
|
||||
|
||||
console.info(JSON.stringify({
|
||||
ok: true,
|
||||
reportDir,
|
||||
stageLogPath,
|
||||
heartbeat,
|
||||
}, null, 2))
|
||||
}
|
||||
finally {
|
||||
overlayClient?.close()
|
||||
stoppingStage = true
|
||||
await stopStage(stageProcess)
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error))
|
||||
console.error(`stage log: ${stageLogPath}`)
|
||||
exit(1)
|
||||
})
|
||||
}
|
||||
@@ -59,6 +59,11 @@ setupDebugger()
|
||||
|
||||
const log = useLogg('main').useGlobalConfig()
|
||||
|
||||
const appUserDataPath = env.APP_USER_DATA_PATH?.trim()
|
||||
if (appUserDataPath) {
|
||||
app.setPath('userData', appUserDataPath)
|
||||
}
|
||||
|
||||
// Thanks to [@blurymind](https://github.com/blurymind),
|
||||
//
|
||||
// When running Electron on Linux, navigator.gpu.requestAdapter() fails.
|
||||
|
||||
@@ -27,6 +27,7 @@ import { join, resolve } from 'node:path'
|
||||
|
||||
import { BrowserWindow, screen } from 'electron'
|
||||
|
||||
import { desktopOverlayPollHeartbeatMarker, desktopOverlayPollHeartbeatQueryParam } from '../../../shared/desktop-overlay-heartbeat'
|
||||
import { baseUrl, getElectronMainDirname, load, withHashRoute } from '../../libs/electron/location'
|
||||
import { setupDesktopOverlayElectronInvokes } from './rpc/index.electron'
|
||||
import {
|
||||
@@ -40,6 +41,15 @@ export function isDesktopOverlayEnabled(): boolean {
|
||||
return process.env.AIRI_DESKTOP_OVERLAY === '1'
|
||||
}
|
||||
|
||||
/**
|
||||
* Smoke-only overlay heartbeat mode.
|
||||
* The recut desktop smoke uses this to surface renderer console lines and
|
||||
* mount the in-page smoke bridge.
|
||||
*/
|
||||
export function isDesktopOverlayPollHeartbeatEnabled(): boolean {
|
||||
return process.env.AIRI_DESKTOP_OVERLAY_POLL_HEARTBEAT === '1'
|
||||
}
|
||||
|
||||
let overlayWindow: BrowserWindow | null = null
|
||||
|
||||
/**
|
||||
@@ -81,6 +91,14 @@ export async function setupDesktopOverlayWindow(params: {
|
||||
overlayWindow = null
|
||||
})
|
||||
|
||||
if (isDesktopOverlayPollHeartbeatEnabled()) {
|
||||
overlayWindow.webContents.on('console-message', (_event, _level, message) => {
|
||||
if (message.includes(desktopOverlayPollHeartbeatMarker)) {
|
||||
console.info(message)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// NOTICE: Wire eventa RPC BEFORE loading the renderer page.
|
||||
// The overlay's onMounted fires during load() and immediately starts
|
||||
// polling via callTool. If the handlers aren't registered yet, the
|
||||
@@ -99,7 +117,9 @@ export async function setupDesktopOverlayWindow(params: {
|
||||
overlayWindow,
|
||||
withHashRoute(
|
||||
baseUrl(resolve(getElectronMainDirname(), '..', 'renderer')),
|
||||
'/desktop-overlay',
|
||||
isDesktopOverlayPollHeartbeatEnabled()
|
||||
? `/desktop-overlay?${desktopOverlayPollHeartbeatQueryParam}=1`
|
||||
: '/desktop-overlay',
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import type { BrowserWindow } from 'electron'
|
||||
|
||||
import type { I18n } from '../../../libs/i18n'
|
||||
import type { ServerChannel } from '../../../services/airi/channel-server'
|
||||
import type { McpStdioManager } from '../../../services/airi/mcp-servers'
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { setupDesktopOverlayElectronInvokes } from './index.electron'
|
||||
|
||||
const defineInvokeHandlerMock = vi.hoisted(() => vi.fn())
|
||||
const createContextMock = vi.hoisted(() => vi.fn(() => ({ context: { id: 'desktop-overlay-test' } })))
|
||||
const setupBaseWindowElectronInvokesMock = vi.hoisted(() => vi.fn())
|
||||
const createMcpServersServiceMock = vi.hoisted(() => vi.fn())
|
||||
const ipcMainMock = vi.hoisted(() => ({ setMaxListeners: vi.fn() }))
|
||||
|
||||
vi.mock('@moeru/eventa', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@moeru/eventa')>()
|
||||
return {
|
||||
...actual,
|
||||
defineInvokeHandler: defineInvokeHandlerMock,
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@moeru/eventa/adapters/electron/main', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@moeru/eventa/adapters/electron/main')>()
|
||||
return {
|
||||
...actual,
|
||||
createContext: createContextMock,
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
ipcMain: ipcMainMock,
|
||||
}))
|
||||
|
||||
vi.mock('../../shared/window', () => ({
|
||||
setupBaseWindowElectronInvokes: setupBaseWindowElectronInvokesMock,
|
||||
}))
|
||||
|
||||
vi.mock('../../../services/airi/mcp-servers', () => ({
|
||||
createMcpServersService: createMcpServersServiceMock,
|
||||
}))
|
||||
|
||||
describe('setupDesktopOverlayElectronInvokes', () => {
|
||||
const window = {} as BrowserWindow
|
||||
const mcpStdioManager = {} as McpStdioManager
|
||||
const serverChannel = {} as ServerChannel
|
||||
const i18n = {} as I18n
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('publishes ready after the base window invokes and MCP services are wired', async () => {
|
||||
let readinessHandler: (() => Promise<{ state: 'booting' | 'ready' | 'degraded', error?: string }>) | undefined
|
||||
|
||||
defineInvokeHandlerMock.mockImplementation((_context, _contract, handler) => {
|
||||
readinessHandler = handler
|
||||
})
|
||||
setupBaseWindowElectronInvokesMock.mockResolvedValue(undefined)
|
||||
createMcpServersServiceMock.mockReturnValue(undefined)
|
||||
|
||||
await setupDesktopOverlayElectronInvokes({
|
||||
window,
|
||||
mcpStdioManager,
|
||||
serverChannel,
|
||||
i18n,
|
||||
})
|
||||
|
||||
expect(ipcMainMock.setMaxListeners).toHaveBeenCalledWith(0)
|
||||
expect(createContextMock).toHaveBeenCalledTimes(1)
|
||||
expect(setupBaseWindowElectronInvokesMock).toHaveBeenCalledTimes(1)
|
||||
expect(createMcpServersServiceMock).toHaveBeenCalledTimes(1)
|
||||
expect(readinessHandler).toBeDefined()
|
||||
await expect(readinessHandler!()).resolves.toEqual({ state: 'ready' })
|
||||
})
|
||||
|
||||
it('publishes degraded when the base window invokes fail', async () => {
|
||||
let readinessHandler: (() => Promise<{ state: 'booting' | 'ready' | 'degraded', error?: string }>) | undefined
|
||||
|
||||
defineInvokeHandlerMock.mockImplementation((_context, _contract, handler) => {
|
||||
readinessHandler = handler
|
||||
})
|
||||
setupBaseWindowElectronInvokesMock.mockRejectedValueOnce(new Error('boom'))
|
||||
|
||||
await setupDesktopOverlayElectronInvokes({
|
||||
window,
|
||||
mcpStdioManager,
|
||||
serverChannel,
|
||||
i18n,
|
||||
})
|
||||
|
||||
expect(createMcpServersServiceMock).not.toHaveBeenCalled()
|
||||
expect(readinessHandler).toBeDefined()
|
||||
await expect(readinessHandler!()).resolves.toEqual({ state: 'degraded', error: 'boom' })
|
||||
})
|
||||
})
|
||||
@@ -7,6 +7,8 @@
|
||||
|
||||
import type { McpCallToolResult } from '@proj-airi/stage-ui/stores/mcp-tool-bridge'
|
||||
|
||||
import { desktopOverlayPollHeartbeatMarker, desktopOverlayPollHeartbeatQueryParam } from '../../shared/desktop-overlay-heartbeat'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types — minimal shapes matching RunState fields the overlay consumes
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -46,6 +48,12 @@ export interface OverlayState {
|
||||
lastBootstrapError?: string
|
||||
}
|
||||
|
||||
export interface OverlayPollHeartbeat {
|
||||
snapshotId: string
|
||||
candidateCount: number
|
||||
hasPointerIntent: boolean
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// State extraction
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -111,6 +119,37 @@ export function extractRunStateFromResult(result: McpCallToolResult): Record<str
|
||||
return sc as Record<string, unknown>
|
||||
}
|
||||
|
||||
export function createOverlayPollHeartbeat(state: OverlayState): OverlayPollHeartbeat | undefined {
|
||||
if (!state.hasSnapshot || !state.snapshotId)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
snapshotId: state.snapshotId,
|
||||
candidateCount: state.candidates.length,
|
||||
hasPointerIntent: state.pointerIntent !== null,
|
||||
}
|
||||
}
|
||||
|
||||
export function formatOverlayPollHeartbeat(heartbeat: OverlayPollHeartbeat): string {
|
||||
return [
|
||||
desktopOverlayPollHeartbeatMarker,
|
||||
`snapshotId=${heartbeat.snapshotId}`,
|
||||
`candidates=${heartbeat.candidateCount}`,
|
||||
`pointerIntent=${heartbeat.hasPointerIntent ? 'yes' : 'no'}`,
|
||||
].join(' ')
|
||||
}
|
||||
|
||||
export function isOverlayPollHeartbeatEnabled(locationLike: Pick<Location, 'hash' | 'search'> = window.location): boolean {
|
||||
const hashQuery = locationLike.hash.includes('?')
|
||||
? locationLike.hash.slice(locationLike.hash.indexOf('?') + 1)
|
||||
: ''
|
||||
const hashParams = new URLSearchParams(hashQuery)
|
||||
const searchParams = new URLSearchParams(locationLike.search)
|
||||
|
||||
return hashParams.get(desktopOverlayPollHeartbeatQueryParam) === '1'
|
||||
|| searchParams.get(desktopOverlayPollHeartbeatQueryParam) === '1'
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Polling controller (framework-agnostic)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -129,6 +168,8 @@ export interface OverlayPollConfig {
|
||||
callTool: (name: string) => Promise<McpCallToolResult>
|
||||
/** Callback with extracted state on each successful poll. */
|
||||
onState: (state: OverlayState) => void
|
||||
/** Optional debug-only callback with a small heartbeat marker. */
|
||||
onHeartbeat?: (heartbeat: OverlayPollHeartbeat) => void
|
||||
/** Function to ping main process readiness contract via Eventa. */
|
||||
getReadiness: () => Promise<{ state: 'booting' | 'ready' | 'degraded', error?: string }>
|
||||
/** Normal poll interval in ms. Default: 250. */
|
||||
@@ -299,6 +340,10 @@ export function createOverlayPollController(config: OverlayPollConfig): OverlayP
|
||||
state.bootstrapState = currentBootstrapState
|
||||
state.lastBootstrapError = currentBootstrapError
|
||||
config.onState(state)
|
||||
const heartbeat = createOverlayPollHeartbeat(state)
|
||||
if (heartbeat && config.onHeartbeat) {
|
||||
config.onHeartbeat(heartbeat)
|
||||
}
|
||||
}
|
||||
else {
|
||||
nextInterval = fallbackInterval
|
||||
|
||||
@@ -18,12 +18,24 @@ import type { OverlayState } from './desktop-overlay-polling'
|
||||
|
||||
import { electron } from '@proj-airi/electron-eventa'
|
||||
import { useElectronEventaInvoke } from '@proj-airi/electron-vueuse'
|
||||
import { getMcpToolBridge } from '@proj-airi/stage-ui/stores/mcp-tool-bridge'
|
||||
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
|
||||
|
||||
import { getDesktopOverlayReadinessContract } from '../../shared/eventa'
|
||||
import { electronMcpApplyAndRestart, electronMcpCallTool, electronMcpGetRuntimeStatus, electronMcpListTools, getDesktopOverlayReadinessContract } from '../../shared/eventa'
|
||||
import { pointInOverlay, rectIntersectsOverlay, screenRectToLocal, screenToLocal } from './desktop-overlay-coordinates'
|
||||
import { createEmptyOverlayState, createOverlayPollController } from './desktop-overlay-polling'
|
||||
import { createEmptyOverlayState, createOverlayPollController, formatOverlayPollHeartbeat, isOverlayPollHeartbeatEnabled } from './desktop-overlay-polling'
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
__AIRI_DESKTOP_OVERLAY_SMOKE__?: {
|
||||
applyAndRestartMcp: () => Promise<unknown>
|
||||
callMcpTool: (payload: { name: string, arguments?: Record<string, unknown> }) => Promise<unknown>
|
||||
getMcpRuntimeStatus: () => Promise<unknown>
|
||||
getOverlayState: () => OverlayState
|
||||
getReadiness: () => Promise<{ state: 'booting' | 'ready' | 'degraded', error?: string }>
|
||||
listMcpTools: () => Promise<unknown>
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Overlay window bounds — read once on mount from main process
|
||||
@@ -31,7 +43,12 @@ import { createEmptyOverlayState, createOverlayPollController } from './desktop-
|
||||
|
||||
const getWindowBounds = useElectronEventaInvoke(electron.window.getBounds)
|
||||
const getReadiness = useElectronEventaInvoke(getDesktopOverlayReadinessContract)
|
||||
const applyAndRestartMcp = useElectronEventaInvoke(electronMcpApplyAndRestart)
|
||||
const callMcpTool = useElectronEventaInvoke(electronMcpCallTool)
|
||||
const getMcpRuntimeStatus = useElectronEventaInvoke(electronMcpGetRuntimeStatus)
|
||||
const listMcpTools = useElectronEventaInvoke(electronMcpListTools)
|
||||
const overlayBounds = ref<Rect | null>(null)
|
||||
const pollHeartbeatEnabled = isOverlayPollHeartbeatEnabled()
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Reactive state — single ref driven by poll controller
|
||||
@@ -71,21 +88,15 @@ const matchedCandidate = computed(() => {
|
||||
// Polling controller
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let bridgeAvailable = false
|
||||
|
||||
const controller = createOverlayPollController({
|
||||
callTool: async (name) => {
|
||||
// Probe bridge availability lazily
|
||||
if (!bridgeAvailable) {
|
||||
getMcpToolBridge() // Throws if not set
|
||||
bridgeAvailable = true
|
||||
}
|
||||
return getMcpToolBridge().callTool({ name })
|
||||
},
|
||||
callTool: name => callMcpTool({ name }),
|
||||
getReadiness: async () => getReadiness(),
|
||||
onState: (newState) => {
|
||||
state.value = newState
|
||||
},
|
||||
onHeartbeat: pollHeartbeatEnabled
|
||||
? heartbeat => console.info(formatOverlayPollHeartbeat(heartbeat))
|
||||
: undefined,
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -199,11 +210,23 @@ onMounted(async () => {
|
||||
}
|
||||
}
|
||||
|
||||
if (pollHeartbeatEnabled) {
|
||||
window.__AIRI_DESKTOP_OVERLAY_SMOKE__ = {
|
||||
applyAndRestartMcp,
|
||||
callMcpTool,
|
||||
getMcpRuntimeStatus,
|
||||
getOverlayState: () => state.value,
|
||||
getReadiness,
|
||||
listMcpTools,
|
||||
}
|
||||
}
|
||||
|
||||
controller.start()
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
controller.stop()
|
||||
delete window.__AIRI_DESKTOP_OVERLAY_SMOKE__
|
||||
})
|
||||
</script>
|
||||
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
/** Debug-only marker used by the local overlay live-window smoke. */
|
||||
export const desktopOverlayPollHeartbeatMarker = '[AIRI_DESKTOP_OVERLAY_POLL_HEARTBEAT]'
|
||||
|
||||
/** Query parameter used to enable overlay polling heartbeat logs. */
|
||||
export const desktopOverlayPollHeartbeatQueryParam = 'pollHeartbeat'
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { selectDesktopOverlaySmokeCandidateId } from './desktop-overlay-live-window-smoke'
|
||||
|
||||
describe('selectDesktopOverlaySmokeCandidateId', () => {
|
||||
it('prefers the smoke button label', () => {
|
||||
const candidateId = selectDesktopOverlaySmokeCandidateId({
|
||||
lastGroundingSnapshot: {
|
||||
targetCandidates: [
|
||||
{ id: 'first', label: 'Something else', role: 'link', source: 'chrome_dom' },
|
||||
{ id: 'smoke', label: 'AIRI Desktop Overlay Smoke Button', role: 'button', source: 'chrome_dom' },
|
||||
],
|
||||
},
|
||||
})
|
||||
|
||||
expect(candidateId).toBe('smoke')
|
||||
})
|
||||
|
||||
it('requires the smoke button to come from chrome_dom', () => {
|
||||
expect(() => {
|
||||
selectDesktopOverlaySmokeCandidateId({
|
||||
lastGroundingSnapshot: {
|
||||
targetCandidates: [
|
||||
{ id: 'ax-smoke', label: 'AIRI Desktop Overlay Smoke Button', role: 'button', source: 'ax' },
|
||||
],
|
||||
},
|
||||
})
|
||||
}).toThrow('desktop_observe did not return the AIRI Desktop Overlay Smoke Button chrome_dom candidate')
|
||||
})
|
||||
|
||||
it('fails when the smoke label is missing', () => {
|
||||
expect(() => {
|
||||
selectDesktopOverlaySmokeCandidateId({
|
||||
lastGroundingSnapshot: {
|
||||
targetCandidates: [
|
||||
{ id: 'first', label: 'Alpha', role: 'link' },
|
||||
{ id: 'second', label: 'Beta', role: 'button' },
|
||||
],
|
||||
},
|
||||
})
|
||||
}).toThrow('desktop_observe did not return the AIRI Desktop Overlay Smoke Button chrome_dom candidate')
|
||||
})
|
||||
|
||||
it('throws when no candidate ids exist', () => {
|
||||
expect(() => {
|
||||
selectDesktopOverlaySmokeCandidateId({
|
||||
lastGroundingSnapshot: {
|
||||
targetCandidates: [{ label: 'Missing id', role: 'button' }],
|
||||
},
|
||||
})
|
||||
}).toThrow('desktop_observe did not return the AIRI Desktop Overlay Smoke Button chrome_dom candidate')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,24 @@
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null
|
||||
}
|
||||
|
||||
export function selectDesktopOverlaySmokeCandidateId(runState: Record<string, unknown>): string {
|
||||
const snapshot = runState.lastGroundingSnapshot
|
||||
if (!isRecord(snapshot))
|
||||
throw new Error('desktop_get_state missing lastGroundingSnapshot after desktop_observe')
|
||||
|
||||
const candidates = Array.isArray(snapshot.targetCandidates)
|
||||
? snapshot.targetCandidates.filter(isRecord)
|
||||
: []
|
||||
|
||||
const selected = candidates.find((candidate) => {
|
||||
return candidate.source === 'chrome_dom'
|
||||
&& String(candidate.label || '').includes('AIRI Desktop Overlay Smoke Button')
|
||||
})
|
||||
|
||||
const id = typeof selected?.id === 'string' ? selected.id : ''
|
||||
if (!id)
|
||||
throw new Error('desktop_observe did not return the AIRI Desktop Overlay Smoke Button chrome_dom candidate')
|
||||
|
||||
return id
|
||||
}
|
||||
+35
-43
@@ -1,8 +1,8 @@
|
||||
# Desktop Lane Status
|
||||
|
||||
Updated: 2026-04-14
|
||||
Updated: 2026-05-08
|
||||
|
||||
This note is a factual status memo for the current desktop lane work around PR #1649. It is intentionally narrow: only current state, actual blockers, and what should happen now vs later.
|
||||
This note is a factual status memo for the current desktop lane work in this recut branch. It is intentionally narrow: only current state, actual blockers, and what should happen now vs later.
|
||||
|
||||
## What is already true
|
||||
|
||||
@@ -18,53 +18,47 @@ This note is a factual status memo for the current desktop lane work around PR #
|
||||
- `makeWindowPassThrough()` uses ignore-mouse-events + non-focusable overlay behavior
|
||||
- `/Users/liuziheng/airi/services/computer-use-mcp/src/browser-dom/cdp-bridge.ts`
|
||||
- 5-second heartbeat with teardown after 3 consecutive failures
|
||||
- `/Users/liuziheng/airi/services/computer-use-mcp/src/bin/smoke-chrome-grounding.ts`
|
||||
- desktop v3 smoke that proves the ensure / observe / click / state chain
|
||||
- The Chrome extension bridge and iframe offset work are no longer hypothetical:
|
||||
- PR #1649 already contains a real extension-side WebSocket client bridge
|
||||
- PR #1649 already contains frame offset propagation for iframe DOM candidates
|
||||
- the recut branch already contains a real extension-side WebSocket client bridge
|
||||
- the recut branch already contains frame offset propagation for iframe DOM candidates
|
||||
- The browser-dom route contract is now fail-closed:
|
||||
- non-left clicks and multi-clicks stay on OS input
|
||||
- `BrowserDomExtensionBridge` rejects `ok: false` responses instead of treating them as success
|
||||
|
||||
## What is actually still blocking
|
||||
|
||||
These are the remaining real issues, ordered by severity.
|
||||
|
||||
### 1. Extension unknown actions still return `ok: true`
|
||||
### 1. Live desktop v3 smoke exists, but it is baseline coverage, not product support.
|
||||
|
||||
- File:
|
||||
- `/Users/liuziheng/airi-pr1649/services/computer-use-mcp/chrome-extension/background.js`
|
||||
- Current behavior:
|
||||
- unsupported actions fall into `result = { error: ... }`
|
||||
- but the response still returns `{ ok: true, result }`
|
||||
- Why this matters:
|
||||
- upper layers can interpret unsupported DOM actions as successful bridge execution
|
||||
- that can suppress OS-input fallback even though nothing actually happened
|
||||
- This is still a real unresolved review blocker.
|
||||
- Current smoke proves:
|
||||
`desktop_ensure_chrome -> desktop_observe -> desktop_click_target -> desktop_get_state`
|
||||
- It does not prove Chrome semantic DOM routing, live overlay-window rendering,
|
||||
or user-input isolation.
|
||||
|
||||
### 2. Browser-dom click routing still ignores non-default click semantics
|
||||
|
||||
- File:
|
||||
- `/Users/liuziheng/airi-pr1649/services/computer-use-mcp/src/browser-action-router.ts`
|
||||
- called from `/Users/liuziheng/airi-pr1649/services/computer-use-mcp/src/server/register-desktop-grounding.ts`
|
||||
- Current behavior:
|
||||
- `chrome_dom` candidates route to browser-dom if selector + bridge are available
|
||||
- routing does not currently incorporate `button` / `clickCount`
|
||||
- Why this matters:
|
||||
- right-click or double-click can still be routed to a DOM path that only performs a standard primary click
|
||||
- This is not as severe as the first issue, but it is still a real correctness gap.
|
||||
|
||||
### 3. Overlay lifecycle / RPC readiness is not fully closed yet
|
||||
### 2. Overlay lifecycle / RPC readiness still needs a live-window pass on this recut branch.
|
||||
|
||||
- Files currently being worked on:
|
||||
- `/Users/liuziheng/airi-pr1649/apps/stage-tamagotchi/src/main/windows/desktop-overlay/rpc/contracts.ts`
|
||||
- `/Users/liuziheng/airi-pr1649/apps/stage-tamagotchi/src/main/windows/desktop-overlay/rpc/index.electron.ts`
|
||||
- `/Users/liuziheng/airi-pr1649/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay-polling.ts`
|
||||
- `/Users/liuziheng/airi-pr1649/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay-polling.test.ts`
|
||||
- `/Users/liuziheng/airi-desktop-recut/apps/stage-tamagotchi/src/main/windows/desktop-overlay/rpc/contracts.ts`
|
||||
- `/Users/liuziheng/airi-desktop-recut/apps/stage-tamagotchi/src/main/windows/desktop-overlay/rpc/index.electron.ts`
|
||||
- `/Users/liuziheng/airi-desktop-recut/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay-polling.ts`
|
||||
- `/Users/liuziheng/airi-desktop-recut/apps/stage-tamagotchi/src/renderer/pages/desktop-overlay-polling.test.ts`
|
||||
- Current state:
|
||||
- there is already a preload-order mitigation in `desktop-overlay/index.ts`
|
||||
- there is already a per-call timeout in `desktop-overlay-polling.ts`
|
||||
- there is now work-in-progress code for an explicit readiness contract
|
||||
- Why this is not yet "done":
|
||||
- the readiness flow is still uncommitted work
|
||||
- the live window context still needs one narrow verification pass
|
||||
- This is not proven broken today, but it is the most likely remaining runtime risk on the overlay path.
|
||||
- the readiness contract is already wired in `desktop-overlay/rpc/index.electron.ts`
|
||||
- the renderer poll controller already waits on readiness and handles degraded state
|
||||
- the live window still needs one fresh pass on this recut branch to confirm the runtime proof is current
|
||||
- This is the remaining runtime risk on the overlay path.
|
||||
|
||||
### 3. Local overlay live-window smoke exists in the recut branch, but it still needs a live run on this branch.
|
||||
|
||||
- The smoke is now wired in `apps/stage-tamagotchi/package.json` as
|
||||
`smoke:desktop-overlay-live-window`.
|
||||
- The shared candidate-selection helper is unit-tested.
|
||||
- What is still missing here is a fresh pass on this recut branch with the real
|
||||
Electron overlay window, to confirm the heartbeat marker and MCP polling still
|
||||
behave the same as the older line.
|
||||
|
||||
## What is not a current blocker
|
||||
|
||||
@@ -91,11 +85,9 @@ In short: m13v gave good runtime advice. That does not mean every suggestion is
|
||||
|
||||
## What should happen now
|
||||
|
||||
1. Fix the extension unknown-action response contract so unsupported actions return `ok: false`.
|
||||
2. Restrict browser-dom click routing to left single-click only; force OS-input for right-click or multi-click.
|
||||
3. Finish or explicitly shelve the overlay readiness contract work:
|
||||
- if kept, validate it in a live overlay window context before merging
|
||||
- if not finished now, do not half-merge it
|
||||
1. No action needed for the extension unknown-action contract; it is already fail-closed.
|
||||
2. Keep browser-dom routing fail-closed for non-left clicks and bridge errors; this stays covered, not product-supported.
|
||||
3. Rerun the local overlay live-window smoke on this recut branch before calling it current proof.
|
||||
|
||||
## What should happen later
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@
|
||||
"smoke:remote": "tsx ./src/bin/smoke-remote.ts",
|
||||
"smoke:stdio": "tsx ./src/bin/smoke-stdio.ts",
|
||||
"smoke:macos": "tsx ./src/bin/smoke-macos.ts",
|
||||
"smoke:desktop-v3": "tsx ./src/bin/smoke-chrome-grounding.ts",
|
||||
"smoke:workflow": "tsx ./src/bin/smoke-workflow.ts",
|
||||
"mcp:inspector": "pnpx @modelcontextprotocol/inspector pnpm -F @proj-airi/computer-use-mcp start",
|
||||
"build": "tsdown",
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import {
|
||||
extractOverlaySmokeState,
|
||||
parseCommandArgs,
|
||||
parseNumber,
|
||||
parseOptionalString,
|
||||
requireChromeDomSmokeCandidate,
|
||||
requirePostClickOverlayState,
|
||||
requireRunState,
|
||||
requireStructuredContent,
|
||||
requireTextContent,
|
||||
selectDesktopV3SmokeCandidate,
|
||||
selectPendingActionForTool,
|
||||
} from './smoke-chrome-grounding'
|
||||
|
||||
describe('smoke-chrome-grounding helpers', () => {
|
||||
it('parses server command args with fallback', () => {
|
||||
expect(parseCommandArgs(undefined, ['start'])).toEqual(['start'])
|
||||
expect(parseCommandArgs(' start -- --flag ', ['fallback'])).toEqual(['start', '--', '--flag'])
|
||||
})
|
||||
|
||||
it('parses numbers and optional string sentinels', () => {
|
||||
expect(parseNumber(undefined, 12)).toBe(12)
|
||||
expect(parseNumber('9.8', 12)).toBe(9)
|
||||
expect(parseNumber('foo', 12)).toBe(12)
|
||||
expect(parseOptionalString(undefined)).toBeUndefined()
|
||||
expect(parseOptionalString(' undefined ')).toBeUndefined()
|
||||
expect(parseOptionalString(' null ')).toBeUndefined()
|
||||
expect(parseOptionalString(' t_1 ')).toBe('t_1')
|
||||
})
|
||||
|
||||
it('requires structured content and text content', () => {
|
||||
expect(requireStructuredContent({
|
||||
structuredContent: {
|
||||
status: 'ok',
|
||||
},
|
||||
}, 'tool')).toEqual({ status: 'ok' })
|
||||
|
||||
expect(requireTextContent({
|
||||
content: [
|
||||
{ type: 'text', text: 'hello' },
|
||||
{ type: 'image', data: 'ignored' },
|
||||
{ type: 'text', text: 'world' },
|
||||
],
|
||||
}, 'tool')).toBe('hello\nworld')
|
||||
|
||||
expect(() => requireStructuredContent({}, 'tool')).toThrow('tool missing structuredContent')
|
||||
expect(() => requireTextContent({ content: [] }, 'tool')).toThrow('tool missing text content')
|
||||
})
|
||||
|
||||
it('extracts runState from desktop_get_state structured content', () => {
|
||||
expect(requireRunState({
|
||||
structuredContent: {
|
||||
status: 'ok',
|
||||
runState: {
|
||||
lastClickedCandidateId: 't_0',
|
||||
},
|
||||
},
|
||||
}, 'desktop_get_state')).toEqual({
|
||||
lastClickedCandidateId: 't_0',
|
||||
})
|
||||
|
||||
expect(() => requireRunState({
|
||||
structuredContent: {
|
||||
status: 'error',
|
||||
},
|
||||
}, 'desktop_get_state')).toThrow('desktop_get_state expected status=ok')
|
||||
})
|
||||
|
||||
it('selects explicit candidate first and otherwise requires the chrome_dom smoke target label', () => {
|
||||
const runState = {
|
||||
lastGroundingSnapshot: {
|
||||
snapshotId: 'dg_1',
|
||||
targetCandidates: [
|
||||
{
|
||||
id: 't_0',
|
||||
source: 'ax',
|
||||
role: 'button',
|
||||
label: 'Disabled',
|
||||
interactable: false,
|
||||
},
|
||||
{
|
||||
id: 't_1',
|
||||
source: 'chrome_dom',
|
||||
role: 'AXToolbar',
|
||||
label: 'Toolbar',
|
||||
interactable: true,
|
||||
},
|
||||
{
|
||||
id: 't_2',
|
||||
source: 'chrome_dom',
|
||||
role: 'AXButton',
|
||||
label: 'AIRI Desktop V3 Smoke Button',
|
||||
interactable: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
expect(selectDesktopV3SmokeCandidate(runState).id).toBe('t_2')
|
||||
expect(selectDesktopV3SmokeCandidate(runState, 't_0').id).toBe('t_0')
|
||||
expect(() => selectDesktopV3SmokeCandidate(runState, 'missing')).toThrow('did not return requested candidate')
|
||||
})
|
||||
|
||||
it('does not fall back to a generic chrome_dom candidate when the smoke label is missing', () => {
|
||||
const runState = {
|
||||
lastGroundingSnapshot: {
|
||||
snapshotId: 'dg_1',
|
||||
targetCandidates: [
|
||||
{
|
||||
id: 't_0',
|
||||
role: 'AXToolbar',
|
||||
label: 'Toolbar',
|
||||
interactable: true,
|
||||
},
|
||||
{
|
||||
id: 't_1',
|
||||
source: 'ax',
|
||||
role: 'AXButton',
|
||||
label: 'Submit',
|
||||
interactable: true,
|
||||
},
|
||||
{
|
||||
id: 't_2',
|
||||
source: 'chrome_dom',
|
||||
role: 'AXButton',
|
||||
label: 'Submit',
|
||||
interactable: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
expect(() => selectDesktopV3SmokeCandidate(runState)).toThrow('desktop_observe did not return the AIRI Desktop V3 Smoke Button chrome_dom candidate')
|
||||
})
|
||||
|
||||
it('locks pre-click and post-click overlay state shape', () => {
|
||||
const runState = {
|
||||
lastGroundingSnapshot: {
|
||||
snapshotId: 'dg_1',
|
||||
targetCandidates: [
|
||||
{ id: 't_0' },
|
||||
],
|
||||
staleFlags: {
|
||||
screenshot: false,
|
||||
ax: false,
|
||||
chromeSemantic: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
expect(extractOverlaySmokeState(runState)).toMatchObject({
|
||||
hasSnapshot: true,
|
||||
snapshotId: 'dg_1',
|
||||
candidateCount: 1,
|
||||
})
|
||||
|
||||
expect(requirePostClickOverlayState({
|
||||
...runState,
|
||||
lastPointerIntent: {
|
||||
candidateId: 't_0',
|
||||
phase: 'completed',
|
||||
},
|
||||
lastClickedCandidateId: 't_0',
|
||||
}, 't_0')).toMatchObject({
|
||||
pointerIntent: {
|
||||
candidateId: 't_0',
|
||||
},
|
||||
lastClickedCandidateId: 't_0',
|
||||
})
|
||||
|
||||
expect(() => requirePostClickOverlayState(runState, 't_0')).toThrow('missing lastPointerIntent')
|
||||
})
|
||||
|
||||
it('reads chrome_dom routing evidence from desktop_click_target structured content', () => {
|
||||
const clickResult = {
|
||||
structuredContent: {
|
||||
status: 'executed',
|
||||
backendResult: {
|
||||
executionRoute: 'browser_dom (chrome_dom candidate with selector "#login-btn" routed to browser-dom bridge)',
|
||||
routeReason: 'chrome_dom candidate with selector "#login-btn" routed to browser-dom bridge',
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
const structured = requireStructuredContent(clickResult, 'desktop_click_target')
|
||||
expect(typeof structured.backendResult).toBe('object')
|
||||
expect((structured.backendResult as Record<string, unknown>).executionRoute).toContain('browser_dom')
|
||||
})
|
||||
|
||||
it('selects pending actions by tool name and fails with a useful message when missing', () => {
|
||||
const pendingActions = [
|
||||
{ toolName: 'desktop_open_app', id: 'p_0' },
|
||||
{ toolName: 'desktop_click_target', id: 'p_1' },
|
||||
]
|
||||
|
||||
expect(selectPendingActionForTool(pendingActions, 'desktop_click_target').id).toBe('p_1')
|
||||
expect(() => selectPendingActionForTool(pendingActions, 'desktop_ensure_chrome')).toThrow('no pending action for desktop_ensure_chrome (found: desktop_open_app, desktop_click_target)')
|
||||
})
|
||||
|
||||
it('requires the smoke target to come from chrome_dom', () => {
|
||||
expect(() => requireChromeDomSmokeCandidate({
|
||||
id: 't_0',
|
||||
source: 'ax',
|
||||
label: 'Smoke',
|
||||
})).toThrow('smoke target button was not captured as a chrome_dom candidate')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,498 @@
|
||||
/**
|
||||
* End-to-end smoke test for the desktop v3 Chrome grounding pipeline.
|
||||
*
|
||||
* Verifies that:
|
||||
* 1. `desktop_ensure_chrome` starts or joins an agent Chrome window.
|
||||
* 2. `desktop_observe` captures a valid grounding snapshot.
|
||||
* 3. `desktop_get_state` exposes overlay-consumable grounding state.
|
||||
* 4. `desktop_click_target` updates pointer intent and clicked-candidate state.
|
||||
*
|
||||
* Usage:
|
||||
* pnpm -F @proj-airi/computer-use-mcp smoke:desktop-v3
|
||||
*/
|
||||
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { argv, env, exit } from 'node:process'
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url'
|
||||
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
|
||||
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'
|
||||
|
||||
const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), '../..')
|
||||
const WHITESPACE_SPLIT_RE = /\s+/u
|
||||
const SMOKE_TARGET_LABEL = 'AIRI Desktop V3 Smoke Button'
|
||||
|
||||
const DEFAULT_SMOKE_URL = `data:text/html;charset=utf-8,${encodeURIComponent(`<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>AIRI Desktop V3 Smoke</title>
|
||||
<style>
|
||||
body { font-family: sans-serif; padding: 48px; }
|
||||
button { font-size: 18px; padding: 12px 18px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>AIRI Desktop V3 Smoke</h1>
|
||||
<button id="airi-desktop-v3-smoke-button">AIRI Desktop V3 Smoke Button</button>
|
||||
</body>
|
||||
</html>`)}`
|
||||
|
||||
export interface DesktopV3SmokeCandidate {
|
||||
id: string
|
||||
source?: string
|
||||
role?: string
|
||||
label?: string
|
||||
interactable?: boolean
|
||||
}
|
||||
|
||||
interface CandidateRecord extends Record<string, unknown> {
|
||||
id: string
|
||||
}
|
||||
|
||||
export interface OverlaySmokeState {
|
||||
hasSnapshot: boolean
|
||||
snapshotId: string
|
||||
candidateCount: number
|
||||
staleFlags: unknown
|
||||
pointerIntent?: Record<string, unknown>
|
||||
lastClickedCandidateId?: string
|
||||
}
|
||||
|
||||
export interface ApprovalEvent {
|
||||
toolName: string
|
||||
pendingActionId: string
|
||||
}
|
||||
|
||||
export interface PendingActionRecord extends Record<string, unknown> {
|
||||
toolName?: string
|
||||
id?: string
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null
|
||||
}
|
||||
|
||||
export function parseCommandArgs(raw: string | undefined, fallback: string[]): string[] {
|
||||
if (!raw?.trim())
|
||||
return fallback
|
||||
|
||||
return raw
|
||||
.split(WHITESPACE_SPLIT_RE)
|
||||
.map(item => item.trim())
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
export function requireStructuredContent(result: unknown, label: string): Record<string, unknown> {
|
||||
if (!isRecord(result))
|
||||
throw new Error(`${label} did not return an object result`)
|
||||
|
||||
const structuredContent = result.structuredContent
|
||||
if (!isRecord(structuredContent))
|
||||
throw new Error(`${label} missing structuredContent`)
|
||||
|
||||
return structuredContent
|
||||
}
|
||||
|
||||
export function requireTextContent(result: unknown, label: string): string {
|
||||
if (!isRecord(result) || !Array.isArray(result.content))
|
||||
throw new Error(`${label} missing content`)
|
||||
|
||||
const text = result.content
|
||||
.filter(isRecord)
|
||||
.map(item => typeof item.text === 'string' ? item.text : '')
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
|
||||
if (!text.trim())
|
||||
throw new Error(`${label} missing text content`)
|
||||
|
||||
return text
|
||||
}
|
||||
|
||||
export function requireRunState(result: unknown, label: string): Record<string, unknown> {
|
||||
const structuredContent = requireStructuredContent(result, label)
|
||||
if (structuredContent.status !== 'ok')
|
||||
throw new Error(`${label} expected status=ok, got ${String(structuredContent.status)}`)
|
||||
|
||||
if (!isRecord(structuredContent.runState))
|
||||
throw new Error(`${label} missing runState`)
|
||||
|
||||
return structuredContent.runState
|
||||
}
|
||||
|
||||
export function selectDesktopV3SmokeCandidate(
|
||||
runState: Record<string, unknown>,
|
||||
requestedCandidateId?: string,
|
||||
): DesktopV3SmokeCandidate {
|
||||
const snapshot = runState.lastGroundingSnapshot
|
||||
if (!isRecord(snapshot))
|
||||
throw new Error('desktop_get_state missing lastGroundingSnapshot after desktop_observe')
|
||||
|
||||
const candidates = Array.isArray(snapshot.targetCandidates)
|
||||
? snapshot.targetCandidates.filter(isRecord)
|
||||
: []
|
||||
|
||||
if (candidates.length === 0)
|
||||
throw new Error('desktop_observe produced no target candidates')
|
||||
|
||||
const requested = requestedCandidateId?.trim()
|
||||
const candidatesWithIds = candidates.filter((candidate): candidate is CandidateRecord => {
|
||||
return typeof candidate.id === 'string' && candidate.id.length > 0
|
||||
})
|
||||
const chromeDomCandidates = candidatesWithIds.filter(candidate => candidate.source === 'chrome_dom')
|
||||
const selected = requested
|
||||
? candidatesWithIds.find(candidate => candidate.id === requested)
|
||||
: selectDefaultChromeDomCandidate(chromeDomCandidates)
|
||||
|
||||
if (!selected) {
|
||||
if (!requested) {
|
||||
throw new Error('desktop_observe did not return the AIRI Desktop V3 Smoke Button chrome_dom candidate')
|
||||
}
|
||||
throw new Error(`desktop_observe did not return requested candidate "${requested}"`)
|
||||
}
|
||||
|
||||
return {
|
||||
id: selected.id,
|
||||
source: typeof selected.source === 'string' ? selected.source : undefined,
|
||||
role: typeof selected.role === 'string' ? selected.role : undefined,
|
||||
label: typeof selected.label === 'string' ? selected.label : undefined,
|
||||
interactable: typeof selected.interactable === 'boolean' ? selected.interactable : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
function candidateText(candidate: Record<string, unknown>): string {
|
||||
return [
|
||||
candidate.id,
|
||||
candidate.label,
|
||||
candidate.role,
|
||||
candidate.source,
|
||||
]
|
||||
.filter(item => typeof item === 'string')
|
||||
.join(' ')
|
||||
.toLowerCase()
|
||||
}
|
||||
|
||||
function selectDefaultChromeDomCandidate(candidates: CandidateRecord[]): CandidateRecord | undefined {
|
||||
return candidates.find(candidate =>
|
||||
candidateText(candidate).includes(SMOKE_TARGET_LABEL.toLowerCase()),
|
||||
)
|
||||
}
|
||||
|
||||
export function extractOverlaySmokeState(runState: Record<string, unknown>): OverlaySmokeState {
|
||||
const snapshot = runState.lastGroundingSnapshot
|
||||
if (!isRecord(snapshot))
|
||||
throw new Error('desktop_get_state missing lastGroundingSnapshot')
|
||||
|
||||
const snapshotId = typeof snapshot.snapshotId === 'string' ? snapshot.snapshotId : ''
|
||||
if (!snapshotId)
|
||||
throw new Error('lastGroundingSnapshot missing snapshotId')
|
||||
|
||||
const candidates = Array.isArray(snapshot.targetCandidates) ? snapshot.targetCandidates : []
|
||||
const pointerIntent = isRecord(runState.lastPointerIntent)
|
||||
? runState.lastPointerIntent
|
||||
: undefined
|
||||
const lastClickedCandidateId = typeof runState.lastClickedCandidateId === 'string'
|
||||
? runState.lastClickedCandidateId
|
||||
: undefined
|
||||
|
||||
return {
|
||||
hasSnapshot: true,
|
||||
snapshotId,
|
||||
candidateCount: candidates.length,
|
||||
staleFlags: snapshot.staleFlags,
|
||||
pointerIntent,
|
||||
lastClickedCandidateId,
|
||||
}
|
||||
}
|
||||
|
||||
export function requirePostClickOverlayState(
|
||||
runState: Record<string, unknown>,
|
||||
selectedCandidateId: string,
|
||||
): OverlaySmokeState {
|
||||
const overlayState = extractOverlaySmokeState(runState)
|
||||
|
||||
if (!overlayState.pointerIntent)
|
||||
throw new Error('desktop_get_state missing lastPointerIntent after desktop_click_target')
|
||||
|
||||
if (overlayState.pointerIntent.candidateId !== selectedCandidateId) {
|
||||
throw new Error(`lastPointerIntent candidate mismatch: expected ${selectedCandidateId}, got ${String(overlayState.pointerIntent.candidateId)}`)
|
||||
}
|
||||
|
||||
if (overlayState.lastClickedCandidateId !== selectedCandidateId) {
|
||||
throw new Error(`lastClickedCandidateId mismatch: expected ${selectedCandidateId}, got ${String(overlayState.lastClickedCandidateId)}`)
|
||||
}
|
||||
|
||||
return overlayState
|
||||
}
|
||||
|
||||
export function requireChromeDomSmokeCandidate(candidate: DesktopV3SmokeCandidate): void {
|
||||
if (candidate.source !== 'chrome_dom') {
|
||||
throw new Error(`smoke target button was not captured as a chrome_dom candidate (got: ${candidate.source ?? 'unknown'}). Verify extension is connected.`)
|
||||
}
|
||||
}
|
||||
|
||||
export function selectPendingActionForTool(
|
||||
pendingActions: PendingActionRecord[],
|
||||
expectedToolName: string,
|
||||
): PendingActionRecord {
|
||||
const matchingPending = pendingActions.find(action => action.toolName === expectedToolName)
|
||||
|
||||
if (!matchingPending) {
|
||||
const found = pendingActions
|
||||
.map(action => action.toolName)
|
||||
.filter((toolName): toolName is string => typeof toolName === 'string' && toolName.length > 0)
|
||||
.join(', ') || 'none'
|
||||
throw new Error(`no pending action for ${expectedToolName} (found: ${found})`)
|
||||
}
|
||||
|
||||
return matchingPending
|
||||
}
|
||||
|
||||
async function approveFirstPending(
|
||||
client: Client,
|
||||
expectedToolName: string,
|
||||
): Promise<{ result: unknown, approvalEvent: ApprovalEvent }> {
|
||||
const pending = await client.callTool({
|
||||
name: 'desktop_list_pending_actions',
|
||||
arguments: {},
|
||||
})
|
||||
const pendingData = requireStructuredContent(pending, 'desktop_list_pending_actions')
|
||||
const pendingActions = Array.isArray(pendingData.pendingActions)
|
||||
? pendingData.pendingActions.filter(isRecord) as PendingActionRecord[]
|
||||
: []
|
||||
const matchingPending = selectPendingActionForTool(pendingActions, expectedToolName)
|
||||
|
||||
const pendingActionId = typeof matchingPending.id === 'string' ? matchingPending.id : ''
|
||||
if (!pendingActionId)
|
||||
throw new Error(`pending action missing id after ${expectedToolName}`)
|
||||
|
||||
const result = await client.callTool({
|
||||
name: 'desktop_approve_pending_action',
|
||||
arguments: { id: pendingActionId },
|
||||
})
|
||||
|
||||
return {
|
||||
result,
|
||||
approvalEvent: {
|
||||
toolName: expectedToolName,
|
||||
pendingActionId,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveApprovalIfNeeded(
|
||||
client: Client,
|
||||
toolName: string,
|
||||
result: unknown,
|
||||
approvalEvents: ApprovalEvent[],
|
||||
): Promise<unknown> {
|
||||
const structuredContent = isRecord(result)
|
||||
? result.structuredContent
|
||||
: undefined
|
||||
|
||||
if (!isRecord(structuredContent) || structuredContent.status !== 'approval_required')
|
||||
return result
|
||||
|
||||
const approved = await approveFirstPending(client, toolName)
|
||||
approvalEvents.push(approved.approvalEvent)
|
||||
return approved.result
|
||||
}
|
||||
|
||||
function assertToolRegistered(toolNames: Set<string>, toolName: string) {
|
||||
if (!toolNames.has(toolName))
|
||||
throw new Error(`required desktop v3 tool is not registered: ${toolName}`)
|
||||
}
|
||||
|
||||
export function parseNumber(value: string | undefined, fallback: number): number {
|
||||
if (!value?.trim())
|
||||
return fallback
|
||||
|
||||
const parsed = Number(value)
|
||||
return Number.isFinite(parsed) ? Math.max(0, Math.floor(parsed)) : fallback
|
||||
}
|
||||
|
||||
export function parseOptionalString(value: string | undefined): string | undefined {
|
||||
const trimmed = value?.trim()
|
||||
if (!trimmed || trimmed === 'undefined' || trimmed === 'null')
|
||||
return undefined
|
||||
|
||||
return trimmed
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
export async function runDesktopV3Smoke(): Promise<Record<string, unknown>> {
|
||||
const command = env.COMPUTER_USE_SMOKE_SERVER_COMMAND?.trim() || 'pnpm'
|
||||
const args = parseCommandArgs(env.COMPUTER_USE_SMOKE_SERVER_ARGS, ['start'])
|
||||
const cwd = env.COMPUTER_USE_SMOKE_SERVER_CWD?.trim() || packageDir
|
||||
const smokeUrl = env.COMPUTER_USE_DESKTOP_V3_SMOKE_URL?.trim() || DEFAULT_SMOKE_URL
|
||||
const requestedCandidateId = parseOptionalString(env.COMPUTER_USE_DESKTOP_V3_SMOKE_CANDIDATE_ID)
|
||||
const settleMs = parseNumber(env.COMPUTER_USE_DESKTOP_V3_SMOKE_SETTLE_MS, 750)
|
||||
|
||||
const transport = new StdioClientTransport({
|
||||
command,
|
||||
args,
|
||||
cwd,
|
||||
env: {
|
||||
...env,
|
||||
COMPUTER_USE_EXECUTOR: env.COMPUTER_USE_SMOKE_EXECUTOR || env.COMPUTER_USE_EXECUTOR || 'macos-local',
|
||||
COMPUTER_USE_APPROVAL_MODE: env.COMPUTER_USE_SMOKE_APPROVAL_MODE || env.COMPUTER_USE_APPROVAL_MODE || 'actions',
|
||||
COMPUTER_USE_OPENABLE_APPS: env.COMPUTER_USE_OPENABLE_APPS || 'Terminal,Cursor,Google Chrome',
|
||||
},
|
||||
stderr: 'pipe',
|
||||
})
|
||||
const client = new Client({
|
||||
name: '@proj-airi/computer-use-mcp-smoke-desktop-v3',
|
||||
version: '0.1.0',
|
||||
})
|
||||
|
||||
transport.stderr?.on('data', (chunk) => {
|
||||
const text = chunk.toString('utf-8').trim()
|
||||
if (text)
|
||||
console.error(`[computer-use-mcp stderr] ${text}`)
|
||||
})
|
||||
|
||||
const approvalEvents: ApprovalEvent[] = []
|
||||
|
||||
try {
|
||||
await client.connect(transport)
|
||||
|
||||
const tools = await client.listTools()
|
||||
const toolNames = new Set(tools.tools.map(tool => tool.name))
|
||||
const requiredTools = [
|
||||
'desktop_ensure_chrome',
|
||||
'desktop_observe',
|
||||
'desktop_click_target',
|
||||
'desktop_get_state',
|
||||
'desktop_list_pending_actions',
|
||||
'desktop_approve_pending_action',
|
||||
]
|
||||
|
||||
for (const toolName of requiredTools) {
|
||||
assertToolRegistered(toolNames, toolName)
|
||||
}
|
||||
|
||||
console.info('=== Phase 1: desktop_ensure_chrome ===')
|
||||
const ensureChrome = await resolveApprovalIfNeeded(
|
||||
client,
|
||||
'desktop_ensure_chrome',
|
||||
await client.callTool({
|
||||
name: 'desktop_ensure_chrome',
|
||||
arguments: { url: smokeUrl },
|
||||
}),
|
||||
approvalEvents,
|
||||
)
|
||||
const ensureChromeData = requireStructuredContent(ensureChrome, 'desktop_ensure_chrome')
|
||||
|
||||
if (settleMs > 0)
|
||||
await delay(settleMs)
|
||||
|
||||
console.info('=== Phase 2: desktop_observe ===')
|
||||
const observation = await client.callTool({
|
||||
name: 'desktop_observe',
|
||||
arguments: { includeChrome: true },
|
||||
})
|
||||
const observeText = requireTextContent(observation, 'desktop_observe')
|
||||
|
||||
console.info('=== Phase 3: desktop_get_state before click ===')
|
||||
const preClickState = requireRunState(await client.callTool({
|
||||
name: 'desktop_get_state',
|
||||
arguments: {},
|
||||
}), 'desktop_get_state')
|
||||
const preClickOverlayState = extractOverlaySmokeState(preClickState)
|
||||
const selectedCandidate = selectDesktopV3SmokeCandidate(preClickState, requestedCandidateId)
|
||||
requireChromeDomSmokeCandidate(selectedCandidate)
|
||||
|
||||
console.info('=== Phase 4: desktop_click_target ===')
|
||||
const clickTarget = await resolveApprovalIfNeeded(
|
||||
client,
|
||||
'desktop_click_target',
|
||||
await client.callTool({
|
||||
name: 'desktop_click_target',
|
||||
arguments: {
|
||||
candidateId: selectedCandidate.id,
|
||||
button: 'left',
|
||||
clickCount: 1,
|
||||
},
|
||||
}),
|
||||
approvalEvents,
|
||||
)
|
||||
requireTextContent(clickTarget, 'desktop_click_target')
|
||||
|
||||
console.info('=== Phase 5: desktop_get_state after click ===')
|
||||
const postClickState = requireRunState(await client.callTool({
|
||||
name: 'desktop_get_state',
|
||||
arguments: {},
|
||||
}), 'desktop_get_state')
|
||||
const postClickOverlayState = requirePostClickOverlayState(postClickState, selectedCandidate.id)
|
||||
const clickStructured = requireStructuredContent(clickTarget, 'desktop_click_target')
|
||||
const clickBackendResult = isRecord(clickStructured.backendResult)
|
||||
? clickStructured.backendResult as Record<string, unknown>
|
||||
: undefined
|
||||
const clickExecutionRoute = typeof clickBackendResult?.executionRoute === 'string'
|
||||
? clickBackendResult.executionRoute
|
||||
: undefined
|
||||
const clickRouteReason = typeof clickBackendResult?.routeReason === 'string'
|
||||
? clickBackendResult.routeReason
|
||||
: undefined
|
||||
|
||||
if (!clickExecutionRoute) {
|
||||
throw new Error('desktop_click_target missing backendResult.executionRoute')
|
||||
}
|
||||
|
||||
if (selectedCandidate.source === 'chrome_dom' && !clickExecutionRoute.startsWith('browser_dom')) {
|
||||
throw new Error(`expected chrome_dom candidate to route through browser_dom, got ${clickExecutionRoute}`)
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
toolChain: [
|
||||
'desktop_ensure_chrome',
|
||||
'desktop_observe',
|
||||
'desktop_get_state',
|
||||
'desktop_click_target',
|
||||
'desktop_get_state',
|
||||
],
|
||||
ensureChrome: ensureChromeData,
|
||||
observeSummary: observeText.split('\n').slice(0, 8),
|
||||
selectedCandidate,
|
||||
preClickOverlayState,
|
||||
postClickOverlayState,
|
||||
clickExecutionRoute,
|
||||
clickRouteReason,
|
||||
approvalEvents,
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await client.close().catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
const invokedPath = argv[1] ? pathToFileURL(argv[1]).href : undefined
|
||||
|
||||
if (invokedPath === import.meta.url) {
|
||||
if (argv.includes('--help') || argv.includes('-h')) {
|
||||
console.info(`Usage:
|
||||
pnpm -F @proj-airi/computer-use-mcp smoke:desktop-v3
|
||||
|
||||
Environment:
|
||||
COMPUTER_USE_DESKTOP_V3_SMOKE_URL Target URL. Defaults to an inline data: smoke page.
|
||||
COMPUTER_USE_DESKTOP_V3_SMOKE_CANDIDATE_ID Optional candidate id override from desktop_observe.
|
||||
COMPUTER_USE_DESKTOP_V3_SMOKE_SETTLE_MS Delay after desktop_ensure_chrome. Default: 750.
|
||||
COMPUTER_USE_SMOKE_EXECUTOR Executor override. Default: macos-local.
|
||||
COMPUTER_USE_SMOKE_APPROVAL_MODE Approval mode override. Default: actions.
|
||||
`)
|
||||
exit(0)
|
||||
}
|
||||
|
||||
runDesktopV3Smoke()
|
||||
.then((result) => {
|
||||
console.info(JSON.stringify(result, null, 2))
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error(error instanceof Error ? error.stack || error.message : String(error))
|
||||
exit(1)
|
||||
})
|
||||
}
|
||||
@@ -57,6 +57,18 @@ describe('decideBrowserAction', () => {
|
||||
expect(decision.reason).toContain('not connected')
|
||||
})
|
||||
|
||||
it('falls back to os_input for right clicks even when chrome_dom is available', () => {
|
||||
const decision = decideBrowserAction(makeCandidate(), true, 'right')
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('left single-click')
|
||||
})
|
||||
|
||||
it('falls back to os_input for multi-click chrome_dom actions', () => {
|
||||
const decision = decideBrowserAction(makeCandidate(), true, 'left', 2)
|
||||
expect(decision.route).toBe('os_input')
|
||||
expect(decision.reason).toContain('count 2')
|
||||
})
|
||||
|
||||
it('preserves non-zero frameId for sub-frame candidates', () => {
|
||||
const decision = decideBrowserAction(makeCandidate({ frameId: 3 }), true)
|
||||
expect(decision.route).toBe('browser_dom')
|
||||
|
||||
@@ -287,6 +287,29 @@ describe('browserDomExtensionBridge', () => {
|
||||
expect(bridge.getStatus().pendingRequests).toBe(0)
|
||||
})
|
||||
|
||||
it('rejects bridge calls when the extension responds with ok:false', async () => {
|
||||
const result = await createConnectedBridge()
|
||||
bridge = result.bridge
|
||||
client = result.client
|
||||
|
||||
client.on('message', (raw) => {
|
||||
const data = JSON.parse(String(raw)) as Record<string, unknown>
|
||||
if (typeof data.id !== 'string')
|
||||
return
|
||||
if (data.action !== 'getActiveTab')
|
||||
return
|
||||
|
||||
client.send(JSON.stringify({
|
||||
id: data.id,
|
||||
ok: false,
|
||||
error: 'unknown action: getActiveTab',
|
||||
}))
|
||||
})
|
||||
|
||||
await expect(bridge.getActiveTab()).rejects.toThrow('unknown action: getActiveTab')
|
||||
expect(bridge.getStatus().pendingRequests).toBe(0)
|
||||
})
|
||||
|
||||
it('can retry startup after an initial bind failure', async () => {
|
||||
blocker = new WebSocketServer({
|
||||
host: '127.0.0.1',
|
||||
|
||||
@@ -8,20 +8,35 @@
|
||||
import type { ChromeSessionManager } from './chrome-session-manager'
|
||||
import type { ComputerUseConfig } from './types'
|
||||
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { createServer } from 'node:net'
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { createChromeSessionManager } from './chrome-session-manager'
|
||||
import { runProcess } from './utils/process'
|
||||
|
||||
// Mock runProcess and sleep before importing the module under test
|
||||
vi.mock('node:fs/promises', () => ({
|
||||
mkdir: vi.fn().mockResolvedValue(undefined),
|
||||
mkdtemp: vi.fn().mockResolvedValue('/tmp/test/chrome-profile-abc123'),
|
||||
rm: vi.fn().mockResolvedValue(undefined),
|
||||
writeFile: vi.fn().mockResolvedValue(undefined),
|
||||
}))
|
||||
vi.mock('./utils/process', () => ({
|
||||
runProcess: vi.fn(),
|
||||
}))
|
||||
vi.mock('./utils/sleep', () => ({
|
||||
sleep: vi.fn().mockResolvedValue(undefined),
|
||||
}))
|
||||
|
||||
vi.mock('node:net', () => ({
|
||||
createServer: vi.fn(),
|
||||
}))
|
||||
const mockedRunProcess = vi.mocked(runProcess)
|
||||
const mockedMkdir = vi.mocked(mkdir)
|
||||
const mockedMkdtemp = vi.mocked(mkdtemp)
|
||||
const mockedRm = vi.mocked(rm)
|
||||
const mockedWriteFile = vi.mocked(writeFile)
|
||||
const mockedCreateServer = vi.mocked(createServer)
|
||||
|
||||
function makeConfig(): ComputerUseConfig {
|
||||
return {
|
||||
@@ -46,42 +61,83 @@ function ok(stdout = ''): any {
|
||||
return { stdout, stderr: '' }
|
||||
}
|
||||
|
||||
/**
|
||||
* Mock for "chrome not running → launch" flow (first call, no existing session).
|
||||
*
|
||||
* Call sequence when `session` is null:
|
||||
* 1. osascript (getCurrentForegroundApp)
|
||||
* 2. pgrep (isChromeRunning for wasAlreadyRunning) → reject (not running)
|
||||
* 3. open (launchChromeWithCdp)
|
||||
* 4. osascript (activateChrome)
|
||||
* 5. pgrep (getChromeMainPid) → ok with pid
|
||||
*/
|
||||
function mockLaunchFlow(pid: number, userApp = '') {
|
||||
mockedRunProcess
|
||||
.mockResolvedValueOnce(ok(userApp)) // 1. getCurrentForegroundApp
|
||||
.mockRejectedValueOnce(new Error('no match')) // 2. isChromeRunning → false
|
||||
.mockResolvedValueOnce(ok()) // 3. open command
|
||||
.mockResolvedValueOnce(ok()) // 4. activateChrome
|
||||
.mockResolvedValueOnce(ok(`${pid}\n`)) // 5. getChromeMainPid
|
||||
function mockPortAvailability(...results: boolean[]) {
|
||||
for (const result of results) {
|
||||
mockedCreateServer.mockImplementationOnce(() => {
|
||||
const handlers = new Map<string, (...args: any[]) => void>()
|
||||
return {
|
||||
once(event: string, handler: (...args: any[]) => void) {
|
||||
handlers.set(event, handler)
|
||||
return this
|
||||
},
|
||||
listen(_port: number, _host: string, callback: () => void) {
|
||||
if (result) {
|
||||
callback()
|
||||
}
|
||||
else {
|
||||
handlers.get('error')?.(new Error('EADDRINUSE'))
|
||||
}
|
||||
return this
|
||||
},
|
||||
close(callback: () => void) {
|
||||
callback()
|
||||
return this
|
||||
},
|
||||
} as any
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mock for "chrome already running → new window" flow (first call, no existing session).
|
||||
*
|
||||
* Call sequence when `session` is null:
|
||||
* 1. osascript (getCurrentForegroundApp)
|
||||
* 2. pgrep (isChromeRunning for wasAlreadyRunning) → ok (running)
|
||||
* 3. osascript (createNewWindow)
|
||||
* 4. osascript (activateChrome)
|
||||
* 5. pgrep (getChromeMainPid)
|
||||
*/
|
||||
function mockJoinFlow(pid: number, userApp = 'Terminal') {
|
||||
function mockLaunchFlow(pid: number, userApp = 'Terminal', cdpPort = 9222, pidLookupAttempts = 4) {
|
||||
const chromeProcessListing = ok(
|
||||
`${pid} /Applications/Google Chrome.app/Contents/MacOS/Google Chrome --user-data-dir=/tmp/test/chrome-profile-abc123 --remote-debugging-port=${cdpPort}\n`,
|
||||
)
|
||||
|
||||
mockedRunProcess
|
||||
.mockResolvedValueOnce(ok(userApp)) // 1. getCurrentForegroundApp
|
||||
.mockResolvedValueOnce(ok(`${pid}\n`)) // 2. isChromeRunning → true
|
||||
.mockResolvedValueOnce(ok()) // 3. createNewWindow
|
||||
.mockResolvedValueOnce(ok()) // 4. activateChrome
|
||||
.mockResolvedValueOnce(ok(`${pid}\n`)) // 5. getChromeMainPid
|
||||
.mockResolvedValueOnce(ok(userApp)) // foreground app
|
||||
.mockRejectedValueOnce(new Error('no match')) // wasAlreadyRunning → false
|
||||
.mockResolvedValueOnce(ok()) // open
|
||||
.mockResolvedValueOnce(ok()) // activateChrome
|
||||
|
||||
for (let index = 0; index < pidLookupAttempts; index += 1) {
|
||||
mockedRunProcess.mockResolvedValueOnce(chromeProcessListing) // getChromePidForProfile
|
||||
}
|
||||
}
|
||||
|
||||
function primeDefaultPortAvailability() {
|
||||
if (mockedCreateServer.mock.calls.length === 0 && mockedCreateServer.mock.results.length === 0) {
|
||||
mockPortAvailability(true)
|
||||
}
|
||||
}
|
||||
|
||||
function resetLaunchMocks() {
|
||||
mockedRunProcess.mockReset()
|
||||
mockedCreateServer.mockReset()
|
||||
}
|
||||
|
||||
function mockReuseFlow(pid: number) {
|
||||
mockedRunProcess
|
||||
.mockResolvedValueOnce(ok(`${pid} /Applications/Google Chrome.app/Contents/MacOS/Google Chrome --user-data-dir=/tmp/test/chrome-profile-abc123 --remote-debugging-port=9222\n`)) // getTrackedChromePid
|
||||
.mockResolvedValueOnce(ok(`${pid}\n`)) // isProcessAlive
|
||||
.mockResolvedValueOnce(ok('1\n')) // hasChromeWindow
|
||||
}
|
||||
|
||||
function mockWindowMissingFlow(pid: number) {
|
||||
mockedRunProcess
|
||||
.mockResolvedValueOnce(ok(`${pid}\n`)) // isProcessAlive
|
||||
.mockResolvedValueOnce(ok('0\n')) // hasChromeWindow
|
||||
}
|
||||
|
||||
function mockEnsureWindowMissingFlow(pid: number) {
|
||||
mockedRunProcess
|
||||
.mockResolvedValueOnce(ok(`${pid} /Applications/Google Chrome.app/Contents/MacOS/Google Chrome --user-data-dir=/tmp/test/chrome-profile-abc123 --remote-debugging-port=9222\n`)) // getTrackedChromePid
|
||||
.mockResolvedValueOnce(ok(`${pid}\n`)) // isProcessAlive
|
||||
.mockResolvedValueOnce(ok('0\n')) // hasChromeWindow
|
||||
}
|
||||
|
||||
function mockStalePidFlow(pid: number) {
|
||||
mockedRunProcess
|
||||
.mockResolvedValueOnce(ok(`${pid} /Applications/Google Chrome.app/Contents/MacOS/Google Chrome --user-data-dir=/tmp/test/chrome-profile-abc123 --remote-debugging-port=9222\n`)) // getTrackedChromePid
|
||||
}
|
||||
|
||||
describe('chromeSessionManager', () => {
|
||||
@@ -89,251 +145,234 @@ describe('chromeSessionManager', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockedRunProcess.mockReset()
|
||||
mockedCreateServer.mockReset()
|
||||
mockPortAvailability(true)
|
||||
manager = createChromeSessionManager(makeConfig())
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// ensureAgentWindow
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('ensureAgentWindow', () => {
|
||||
it('should launch Chrome when not running', async () => {
|
||||
it('launches a dedicated Chrome profile with CDP', async () => {
|
||||
mockLaunchFlow(12345)
|
||||
|
||||
const info = await manager.ensureAgentWindow()
|
||||
|
||||
expect(info.ensureOutcome).toBe('launched')
|
||||
expect(info.wasAlreadyRunning).toBe(false)
|
||||
expect(info.agentOwned).toBe(true)
|
||||
expect(info.pid).toBe(12345)
|
||||
expect(info.cdpUrl).toBe('http://127.0.0.1:9222')
|
||||
expect(info.windowId).toBe('12345:0:Google Chrome')
|
||||
expect(info.createdAt).toBeTruthy()
|
||||
expect(mockedMkdir).toHaveBeenCalledWith('/tmp/test', { recursive: true })
|
||||
expect(mockedMkdtemp).toHaveBeenCalledWith('/tmp/test/chrome-profile-')
|
||||
expect(mockedWriteFile).toHaveBeenCalledWith('/tmp/test/chrome-profile-abc123/First Run', '')
|
||||
expect(mockedRunProcess.mock.calls[2]).toEqual([
|
||||
'/usr/bin/open',
|
||||
[
|
||||
'-na',
|
||||
'Google Chrome',
|
||||
'--args',
|
||||
'--new-window',
|
||||
'--no-first-run',
|
||||
'--no-default-browser-check',
|
||||
'--disable-default-apps',
|
||||
'--disable-features=ChromeWhatsNewUI',
|
||||
'--remote-debugging-port=9222',
|
||||
'--user-data-dir=/tmp/test/chrome-profile-abc123',
|
||||
],
|
||||
expect.any(Object),
|
||||
])
|
||||
})
|
||||
|
||||
it('should create new window when Chrome is already running', async () => {
|
||||
mockJoinFlow(99999, 'Terminal')
|
||||
|
||||
const info = await manager.ensureAgentWindow()
|
||||
|
||||
expect(info.wasAlreadyRunning).toBe(true)
|
||||
expect(info.agentOwned).toBe(false)
|
||||
expect(info.pid).toBe(99999)
|
||||
// No CDP URL when joining existing Chrome
|
||||
expect(info.cdpUrl).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should reuse an existing session when the agent launched a dedicated Chrome instance', async () => {
|
||||
it('reuses an alive agent session', async () => {
|
||||
mockLaunchFlow(11111)
|
||||
const first = await manager.ensureAgentWindow()
|
||||
|
||||
// Second call: session exists → isChromeRunning check (1 call)
|
||||
mockedRunProcess.mockResolvedValueOnce(ok('11111\n')) // isChromeRunning → still alive
|
||||
resetLaunchMocks()
|
||||
mockReuseFlow(11111)
|
||||
|
||||
const second = await manager.ensureAgentWindow()
|
||||
|
||||
expect(second.ensureOutcome).toBe('reused')
|
||||
expect(second).toBe(first)
|
||||
})
|
||||
|
||||
it('should create a fresh agent window on repeated calls when joining an existing Chrome instance', async () => {
|
||||
mockJoinFlow(99999, 'Terminal')
|
||||
const first = await manager.ensureAgentWindow()
|
||||
|
||||
vi.clearAllMocks()
|
||||
|
||||
mockedRunProcess
|
||||
.mockResolvedValueOnce(ok('99999\n')) // session reuse check → Chrome still running
|
||||
mockJoinFlow(99999, 'Terminal')
|
||||
|
||||
const second = await manager.ensureAgentWindow()
|
||||
|
||||
expect(second).not.toBe(first)
|
||||
expect(second.wasAlreadyRunning).toBe(true)
|
||||
expect(second.pid).toBe(99999)
|
||||
expect(mockedRunProcess.mock.calls[3]?.[0]).toBe('/usr/bin/osascript')
|
||||
expect(mockedRunProcess.mock.calls[3]?.[1]).toEqual(['-e', 'tell application "Google Chrome" to make new window'])
|
||||
})
|
||||
|
||||
it('should recreate session if Chrome crashed between calls', async () => {
|
||||
it('recreates the session if the tracked process is gone', async () => {
|
||||
mockLaunchFlow(11111)
|
||||
const first = await manager.ensureAgentWindow()
|
||||
expect(first.pid).toBe(11111)
|
||||
await manager.ensureAgentWindow()
|
||||
|
||||
// Second call: session exists → isChromeRunning fails (Chrome crashed)
|
||||
resetLaunchMocks()
|
||||
primeDefaultPortAvailability()
|
||||
mockedRunProcess.mockRejectedValueOnce(new Error('no match'))
|
||||
// session=null now, goes through full launch flow (5 calls)
|
||||
mockLaunchFlow(22222)
|
||||
|
||||
const second = await manager.ensureAgentWindow()
|
||||
|
||||
expect(second.ensureOutcome).toBe('recreated_after_process_exit')
|
||||
expect(second.pid).toBe(22222)
|
||||
expect(second).not.toBe(first)
|
||||
expect(second).not.toBeNull()
|
||||
})
|
||||
|
||||
it('should pass custom URL to Chrome', async () => {
|
||||
mockLaunchFlow(33333)
|
||||
|
||||
const info = await manager.ensureAgentWindow({ url: 'https://example.com' })
|
||||
expect(info.initialUrl).toBe('https://example.com')
|
||||
})
|
||||
|
||||
it('should pass URL to osascript as argv instead of interpolating it into script source', async () => {
|
||||
const maliciousUrl = 'https://example.com/" & do shell script "touch /tmp/pwned" & "'
|
||||
mockJoinFlow(33333, 'Terminal')
|
||||
|
||||
await manager.ensureAgentWindow({ url: maliciousUrl })
|
||||
|
||||
const createWindowCall = mockedRunProcess.mock.calls[2]
|
||||
expect(createWindowCall?.[0]).toBe('/usr/bin/osascript')
|
||||
expect(createWindowCall?.[1]).toEqual([
|
||||
'-e',
|
||||
expect.stringContaining('item 1 of argv'),
|
||||
'--',
|
||||
maliciousUrl,
|
||||
])
|
||||
expect(createWindowCall?.[1]?.[1]).not.toContain(maliciousUrl)
|
||||
})
|
||||
|
||||
it('should use custom CDP port', async () => {
|
||||
mockLaunchFlow(44444)
|
||||
|
||||
const info = await manager.ensureAgentWindow({ cdpPort: 9333 })
|
||||
expect(info.cdpUrl).toBe('http://127.0.0.1:9333')
|
||||
})
|
||||
|
||||
it('should throw if Chrome PID cannot be obtained after launch', async () => {
|
||||
mockedRunProcess
|
||||
.mockResolvedValueOnce(ok()) // 1. foreground
|
||||
.mockRejectedValueOnce(new Error('no match')) // 2. wasAlreadyRunning → false
|
||||
.mockResolvedValueOnce(ok()) // 3. launch
|
||||
.mockResolvedValueOnce(ok()) // 4. activate
|
||||
.mockRejectedValueOnce(new Error('no match')) // 5. getChromeMainPid → fails
|
||||
|
||||
await expect(manager.ensureAgentWindow()).rejects.toThrow('Failed to get Chrome PID')
|
||||
})
|
||||
|
||||
it('should record the user\'s previous foreground app', async () => {
|
||||
mockLaunchFlow(55555, 'Finder')
|
||||
|
||||
it('does not terminate a reused pid after verifying it no longer belongs to the tracked Chrome profile', async () => {
|
||||
mockLaunchFlow(11111)
|
||||
await manager.ensureAgentWindow()
|
||||
|
||||
// First call should have been getCurrentForegroundApp
|
||||
const firstCall = mockedRunProcess.mock.calls[0]
|
||||
expect(firstCall[1]).toContainEqual(expect.stringContaining('first application process'))
|
||||
resetLaunchMocks()
|
||||
primeDefaultPortAvailability()
|
||||
mockStalePidFlow(33333)
|
||||
mockLaunchFlow(22222)
|
||||
|
||||
const second = await manager.ensureAgentWindow()
|
||||
|
||||
expect(second.ensureOutcome).toBe('recreated_after_process_exit')
|
||||
expect(second.pid).toBe(22222)
|
||||
expect(mockedRunProcess).not.toHaveBeenCalledWith('kill', ['-TERM', '11111'], expect.any(Object))
|
||||
expect(mockedRunProcess).not.toHaveBeenCalledWith('kill', ['-KILL', '11111'], expect.any(Object))
|
||||
})
|
||||
|
||||
it('recreates the session if the Chrome process is alive but the agent window is gone', async () => {
|
||||
mockLaunchFlow(11111)
|
||||
await manager.ensureAgentWindow()
|
||||
|
||||
resetLaunchMocks()
|
||||
primeDefaultPortAvailability()
|
||||
mockEnsureWindowMissingFlow(11111)
|
||||
mockedRunProcess.mockResolvedValueOnce(ok()) // terminateChromeProcess: kill -TERM
|
||||
mockedRunProcess.mockResolvedValueOnce(ok()) // terminateChromeProcess: post-TERM liveness check
|
||||
mockLaunchFlow(22222)
|
||||
|
||||
const second = await manager.ensureAgentWindow()
|
||||
|
||||
expect(second.ensureOutcome).toBe('recreated_after_missing_window')
|
||||
expect(second.pid).toBe(22222)
|
||||
expect(mockedRunProcess).toHaveBeenCalledWith('kill', ['-TERM', '11111'], expect.any(Object))
|
||||
expect(mockedRunProcess).not.toHaveBeenCalledWith('kill', ['-KILL', '11111'], expect.any(Object))
|
||||
expect(mockedRunProcess).toHaveBeenCalledWith('/usr/bin/osascript', [
|
||||
'-e',
|
||||
'tell application "System Events" to get count of windows of (first application process whose unix id is 11111)',
|
||||
], expect.any(Object))
|
||||
})
|
||||
|
||||
it('passes a custom CDP port and URL through', async () => {
|
||||
mockLaunchFlow(33333, 'Terminal', 9333)
|
||||
|
||||
const info = await manager.ensureAgentWindow({
|
||||
cdpPort: 9333,
|
||||
url: 'https://example.com',
|
||||
})
|
||||
|
||||
expect(info.cdpUrl).toBe('http://127.0.0.1:9333')
|
||||
expect(info.initialUrl).toBe('https://example.com')
|
||||
expect(mockedRunProcess.mock.calls[2]?.[1]).toContain('--user-data-dir=/tmp/test/chrome-profile-abc123')
|
||||
expect(mockedRunProcess.mock.calls[2]?.[1]).toContain('https://example.com')
|
||||
})
|
||||
|
||||
it('falls forward to the next available default CDP port when 9222 is occupied', async () => {
|
||||
resetLaunchMocks()
|
||||
mockPortAvailability(false, true)
|
||||
mockLaunchFlow(12345, 'Terminal', 9223)
|
||||
|
||||
const info = await manager.ensureAgentWindow()
|
||||
|
||||
expect(info.cdpUrl).toBe('http://127.0.0.1:9223')
|
||||
expect(mockedRunProcess.mock.calls[2]?.[1]).toContain('--remote-debugging-port=9223')
|
||||
})
|
||||
|
||||
it('cleans up the active chrome profile directory on endSession', async () => {
|
||||
mockLaunchFlow(11111)
|
||||
await manager.ensureAgentWindow()
|
||||
|
||||
resetLaunchMocks()
|
||||
manager.endSession()
|
||||
|
||||
expect(mockedRm).toHaveBeenCalledWith('/tmp/test/chrome-profile-abc123', {
|
||||
recursive: true,
|
||||
force: true,
|
||||
})
|
||||
expect(manager.getSessionInfo()).toBeNull()
|
||||
})
|
||||
|
||||
it('cleans up the active chrome profile when relaunching after a missing window', async () => {
|
||||
mockLaunchFlow(11111)
|
||||
await manager.ensureAgentWindow()
|
||||
|
||||
resetLaunchMocks()
|
||||
primeDefaultPortAvailability()
|
||||
mockWindowMissingFlow(11111)
|
||||
mockedRunProcess.mockResolvedValueOnce(ok()) // terminateChromeProcess: kill -TERM
|
||||
mockedRunProcess.mockResolvedValueOnce(ok()) // terminateChromeProcess: post-TERM liveness check
|
||||
mockLaunchFlow(22222)
|
||||
|
||||
const second = await manager.ensureAgentWindow()
|
||||
|
||||
expect(second.pid).toBe(22222)
|
||||
expect(mockedRm).toHaveBeenCalledWith('/tmp/test/chrome-profile-abc123', {
|
||||
recursive: true,
|
||||
force: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('terminates a launched Chrome process and cleans up the profile if launch fails before PID lookup completes', async () => {
|
||||
mockedRunProcess
|
||||
.mockResolvedValueOnce(ok('Terminal')) // foreground app
|
||||
.mockRejectedValueOnce(new Error('no match')) // wasAlreadyRunning → false
|
||||
.mockResolvedValueOnce(ok()) // open
|
||||
.mockRejectedValueOnce(new Error('activate failed')) // activateChrome
|
||||
.mockResolvedValueOnce(ok(
|
||||
'11111 /Applications/Google Chrome.app/Contents/MacOS/Google Chrome --user-data-dir=/tmp/test/chrome-profile-abc123 --remote-debugging-port=9222\n',
|
||||
)) // findAndTerminateChromeByProfile
|
||||
.mockResolvedValueOnce(ok()) // findAndTerminateChromeByProfile: kill -TERM
|
||||
.mockResolvedValueOnce(ok()) // findAndTerminateChromeByProfile: post-TERM liveness check
|
||||
|
||||
await expect(manager.ensureAgentWindow()).rejects.toThrow('activate failed')
|
||||
expect(mockedRunProcess).toHaveBeenCalledWith('kill', ['-TERM', '11111'], expect.any(Object))
|
||||
expect(mockedRm).toHaveBeenCalledWith('/tmp/test/chrome-profile-abc123', {
|
||||
recursive: true,
|
||||
force: true,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// bringToFront
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('bringToFront', () => {
|
||||
it('should activate Chrome when session exists', async () => {
|
||||
it('activates Chrome when session exists', async () => {
|
||||
mockLaunchFlow(11111)
|
||||
await manager.ensureAgentWindow()
|
||||
vi.clearAllMocks()
|
||||
|
||||
mockedRunProcess
|
||||
.mockResolvedValueOnce(ok('11111\n')) // isChromeRunning
|
||||
.mockResolvedValueOnce(ok()) // activateChrome
|
||||
resetLaunchMocks()
|
||||
mockedRunProcess.mockResolvedValueOnce(ok('11111\n'))
|
||||
mockedRunProcess.mockResolvedValueOnce(ok('1\n'))
|
||||
mockedRunProcess.mockResolvedValueOnce(ok())
|
||||
|
||||
const result = await manager.bringToFront()
|
||||
|
||||
expect(result).toBe(true)
|
||||
expect(mockedRunProcess).toHaveBeenCalledTimes(2)
|
||||
const activateCall = mockedRunProcess.mock.calls[1]
|
||||
expect(activateCall[1]).toEqual(['-e', 'tell application "Google Chrome" to activate'])
|
||||
expect(mockedRunProcess).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
|
||||
it('should be no-op when no session exists', async () => {
|
||||
const result = await manager.bringToFront()
|
||||
expect(result).toBe(false)
|
||||
expect(mockedRunProcess).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should clear session if Chrome crashed', async () => {
|
||||
it('returns false when session is gone', async () => {
|
||||
mockLaunchFlow(11111)
|
||||
await manager.ensureAgentWindow()
|
||||
vi.clearAllMocks()
|
||||
|
||||
// Chrome crashed
|
||||
resetLaunchMocks()
|
||||
mockedRunProcess.mockRejectedValueOnce(new Error('no match'))
|
||||
|
||||
const result = await manager.bringToFront()
|
||||
|
||||
expect(result).toBe(false)
|
||||
expect(manager.getSessionInfo()).toBeNull()
|
||||
})
|
||||
|
||||
it('returns false when the agent window is gone even if the process still exists', async () => {
|
||||
mockLaunchFlow(11111)
|
||||
await manager.ensureAgentWindow()
|
||||
|
||||
resetLaunchMocks()
|
||||
mockWindowMissingFlow(11111)
|
||||
|
||||
const result = await manager.bringToFront()
|
||||
|
||||
expect(result).toBe(false)
|
||||
expect(manager.getSessionInfo()).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// restorePreviousForeground
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('restorePreviousForeground', () => {
|
||||
it('should activate the user\'s previous app', async () => {
|
||||
mockLaunchFlow(11111, 'Terminal')
|
||||
await manager.ensureAgentWindow()
|
||||
vi.clearAllMocks()
|
||||
|
||||
mockedRunProcess.mockResolvedValueOnce(ok())
|
||||
|
||||
await manager.restorePreviousForeground()
|
||||
|
||||
expect(mockedRunProcess).toHaveBeenCalledWith(
|
||||
'/usr/bin/osascript',
|
||||
['-e', 'tell application "Terminal" to activate'],
|
||||
expect.any(Object),
|
||||
)
|
||||
})
|
||||
|
||||
it('should be no-op if user was already in Chrome', async () => {
|
||||
mockLaunchFlow(11111, 'Google Chrome')
|
||||
await manager.ensureAgentWindow()
|
||||
vi.clearAllMocks()
|
||||
|
||||
await manager.restorePreviousForeground()
|
||||
expect(mockedRunProcess).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should be no-op if no session was created', async () => {
|
||||
await manager.restorePreviousForeground()
|
||||
expect(mockedRunProcess).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// session lifecycle
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('session lifecycle', () => {
|
||||
it('should return null before any session', () => {
|
||||
expect(manager.getSessionInfo()).toBeNull()
|
||||
})
|
||||
|
||||
it('should return session info after ensureAgentWindow', async () => {
|
||||
mockLaunchFlow(55555)
|
||||
|
||||
await manager.ensureAgentWindow()
|
||||
const info = manager.getSessionInfo()
|
||||
expect(info).not.toBeNull()
|
||||
expect(info!.pid).toBe(55555)
|
||||
expect(info!.agentOwned).toBe(true)
|
||||
})
|
||||
|
||||
it('should clear session on endSession', async () => {
|
||||
mockLaunchFlow(55555)
|
||||
await manager.ensureAgentWindow()
|
||||
expect(manager.getSessionInfo()).not.toBeNull()
|
||||
|
||||
manager.endSession()
|
||||
expect(manager.getSessionInfo()).toBeNull()
|
||||
})
|
||||
|
||||
it('should restore fully after endSession and re-ensureAgentWindow', async () => {
|
||||
mockLaunchFlow(11111)
|
||||
await manager.ensureAgentWindow()
|
||||
manager.endSession()
|
||||
|
||||
mockLaunchFlow(22222)
|
||||
const info = await manager.ensureAgentWindow()
|
||||
expect(info.pid).toBe(22222)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -2,10 +2,8 @@
|
||||
* Chrome Session Manager — agent-owned Chrome window lifecycle.
|
||||
*
|
||||
* Responsibilities:
|
||||
* - Detect whether Chrome is already running
|
||||
* - Launch Chrome with CDP if not running
|
||||
* - Create a new window in existing Chrome
|
||||
* - Track window identity via PID
|
||||
* - Launch a dedicated Chrome profile with CDP
|
||||
* - Track the launched browser PID
|
||||
* - Bring agent window to front / restore user's previous foreground
|
||||
*
|
||||
* macOS only. Uses AppleScript and `open` CLI for Chrome lifecycle control.
|
||||
@@ -13,6 +11,10 @@
|
||||
|
||||
import type { ChromeSessionInfo, ComputerUseConfig } from './types'
|
||||
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { createServer } from 'node:net'
|
||||
import { join } from 'node:path'
|
||||
|
||||
import { runProcess } from './utils/process'
|
||||
import { sleep } from './utils/sleep'
|
||||
|
||||
@@ -22,6 +24,7 @@ import { sleep } from './utils/sleep'
|
||||
|
||||
const CHROME_APP_NAME = 'Google Chrome'
|
||||
const DEFAULT_CDP_PORT = 9222
|
||||
const DEFAULT_CDP_PORT_SCAN_ATTEMPTS = 20
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public interface
|
||||
@@ -31,19 +34,18 @@ export interface ChromeSessionManager {
|
||||
/**
|
||||
* Ensure the agent has a usable Chrome window.
|
||||
*
|
||||
* - Chrome not running → launch with CDP flag + new window
|
||||
* - Chrome running → create new window in existing instance
|
||||
* - No active agent session → launch a dedicated Chrome profile with CDP
|
||||
* - Existing agent session still has a live Chrome window → reuse it
|
||||
*
|
||||
* Reuses the cached session only when the agent launched a dedicated Chrome
|
||||
* instance itself. When joining an existing user-owned Chrome process, a
|
||||
* fresh window must be created on every ensure because this module does not
|
||||
* track a verifiable OS window handle for the agent tab/window.
|
||||
* Human-owned Chrome instances are not reused. The agent always launches its
|
||||
* own profile so browser-dom/CDP capture has a stable endpoint.
|
||||
*/
|
||||
ensureAgentWindow: (options?: { url?: string, cdpPort?: number }) => Promise<ChromeSessionInfo>
|
||||
|
||||
/**
|
||||
* Bring the agent's Chrome window to the foreground.
|
||||
* Returns false if the tracked session is missing or Chrome is no longer running.
|
||||
* Returns false if the tracked session is missing, Chrome is no longer
|
||||
* running, or the tracked window is gone.
|
||||
*/
|
||||
bringToFront: () => Promise<boolean>
|
||||
|
||||
@@ -74,6 +76,7 @@ export function createChromeSessionManager(
|
||||
let session: ChromeSessionInfo | null = null
|
||||
let previousForegroundApp: string | undefined
|
||||
const onSessionLost = options?.onSessionLost
|
||||
let activeProfileDir: string | undefined
|
||||
|
||||
// -- Helpers ------------------------------------------------------------
|
||||
|
||||
@@ -90,14 +93,108 @@ export function createChromeSessionManager(
|
||||
}
|
||||
}
|
||||
|
||||
async function getChromeMainPid(): Promise<number | undefined> {
|
||||
async function isProcessAlive(pid: number): Promise<boolean> {
|
||||
try {
|
||||
const { stdout } = await runProcess('pgrep', ['-x', 'Google Chrome'], {
|
||||
const { stdout } = await runProcess('ps', ['-p', String(pid), '-o', 'pid='], {
|
||||
timeoutMs: config.timeoutMs,
|
||||
})
|
||||
const pids = stdout.trim().split('\n').map(Number).filter(n => !Number.isNaN(n))
|
||||
// The lowest PID is typically the main Chrome process
|
||||
return pids.length > 0 ? Math.min(...pids) : undefined
|
||||
return stdout.trim().length > 0
|
||||
}
|
||||
catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function hasChromeWindow(pid: number): Promise<boolean> {
|
||||
try {
|
||||
const { stdout } = await runProcess(config.binaries.osascript, [
|
||||
'-e',
|
||||
`tell application "System Events" to get count of windows of (first application process whose unix id is ${pid})`,
|
||||
], { timeoutMs: config.timeoutMs })
|
||||
return Number.parseInt(stdout.trim(), 10) > 0
|
||||
}
|
||||
catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function terminateChromeProcess(pid: number): Promise<void> {
|
||||
await runProcess('kill', ['-TERM', String(pid)], { timeoutMs: config.timeoutMs }).catch(() => {})
|
||||
await sleep(250)
|
||||
|
||||
if (!await isProcessAlive(pid)) {
|
||||
return
|
||||
}
|
||||
|
||||
await runProcess('kill', ['-KILL', String(pid)], { timeoutMs: config.timeoutMs }).catch(() => {})
|
||||
await sleep(250)
|
||||
|
||||
if (await isProcessAlive(pid)) {
|
||||
throw new Error(`Failed to terminate stale Chrome process ${pid}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function findAndTerminateChromeByProfile(profileDir: string, cdpPort: number): Promise<void> {
|
||||
const pid = await getChromePidForProfile(profileDir, cdpPort)
|
||||
if (!pid) {
|
||||
return
|
||||
}
|
||||
|
||||
await terminateChromeProcess(pid)
|
||||
}
|
||||
|
||||
async function clearSessionState(): Promise<void> {
|
||||
session = null
|
||||
previousForegroundApp = undefined
|
||||
if (activeProfileDir) {
|
||||
const profileDir = activeProfileDir
|
||||
activeProfileDir = undefined
|
||||
await rm(profileDir, { recursive: true, force: true }).catch(() => {})
|
||||
return
|
||||
}
|
||||
activeProfileDir = undefined
|
||||
}
|
||||
|
||||
async function getChromePidForProfile(profileDir: string, cdpPort: number): Promise<number | undefined> {
|
||||
try {
|
||||
const { stdout } = await runProcess('ps', ['-axww', '-o', 'pid=,command='], {
|
||||
timeoutMs: config.timeoutMs,
|
||||
})
|
||||
const matchingLine = stdout
|
||||
.split('\n')
|
||||
.map(line => line.trim())
|
||||
.find(line =>
|
||||
line.includes('/Contents/MacOS/Google Chrome')
|
||||
&& !line.includes('Helper')
|
||||
&& line.includes(`--user-data-dir=${profileDir}`)
|
||||
&& line.includes(`--remote-debugging-port=${cdpPort}`),
|
||||
)
|
||||
|
||||
if (!matchingLine) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const pidText = matchingLine.split(/\s+/u)[0]
|
||||
const pid = Number(pidText)
|
||||
return Number.isFinite(pid) ? pid : undefined
|
||||
}
|
||||
catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
async function getTrackedChromePid(trackedSession: ChromeSessionInfo): Promise<number | undefined> {
|
||||
if (!activeProfileDir || !trackedSession.cdpUrl) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
try {
|
||||
const cdpPort = Number.parseInt(new URL(trackedSession.cdpUrl).port, 10)
|
||||
if (!Number.isFinite(cdpPort)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return await getChromePidForProfile(activeProfileDir, cdpPort)
|
||||
}
|
||||
catch {
|
||||
return undefined
|
||||
@@ -117,13 +214,49 @@ export function createChromeSessionManager(
|
||||
}
|
||||
}
|
||||
|
||||
async function launchChromeWithCdp(cdpPort: number, url?: string): Promise<void> {
|
||||
async function canListenOnPort(port: number): Promise<boolean> {
|
||||
return await new Promise<boolean>((resolvePromise) => {
|
||||
const server = createServer()
|
||||
server.once('error', () => {
|
||||
resolvePromise(false)
|
||||
})
|
||||
server.listen(port, '127.0.0.1', () => {
|
||||
server.close(() => resolvePromise(true))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async function resolveLaunchCdpPort(requestedPort: number, explicit: boolean): Promise<number> {
|
||||
if (explicit) {
|
||||
return requestedPort
|
||||
}
|
||||
|
||||
for (let index = 0; index < DEFAULT_CDP_PORT_SCAN_ATTEMPTS; index += 1) {
|
||||
const candidate = requestedPort + index
|
||||
if (await canListenOnPort(candidate)) {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Could not find an available Chrome CDP port starting from ${requestedPort}`)
|
||||
}
|
||||
|
||||
async function launchChromeWithCdp(cdpPort: number, profileDir: string, url?: string): Promise<void> {
|
||||
// Chrome uses the user-data-dir root "First Run" sentinel to decide
|
||||
// whether the branded first-run dialog should appear.
|
||||
await writeFile(join(profileDir, 'First Run'), '').catch(() => {})
|
||||
|
||||
const args = [
|
||||
'-na',
|
||||
CHROME_APP_NAME,
|
||||
'--args',
|
||||
'--new-window',
|
||||
'--no-first-run',
|
||||
'--no-default-browser-check',
|
||||
'--disable-default-apps',
|
||||
'--disable-features=ChromeWhatsNewUI',
|
||||
`--remote-debugging-port=${cdpPort}`,
|
||||
`--user-data-dir=${profileDir}`,
|
||||
]
|
||||
if (url) {
|
||||
args.push(url)
|
||||
@@ -137,31 +270,6 @@ export function createChromeSessionManager(
|
||||
await sleep(2000)
|
||||
}
|
||||
|
||||
async function createNewWindow(url?: string): Promise<void> {
|
||||
// NOTICE: Pass the URL as an `osascript` argv value instead of interpolating
|
||||
// it into the AppleScript source. `desktop_ensure_chrome` accepts arbitrary
|
||||
// strings, so direct interpolation creates an AppleScript injection path when
|
||||
// the URL contains quotes. The review report on PR #1649 correctly called this
|
||||
// out as a P1 command-injection issue.
|
||||
const script = url
|
||||
? `on run argv
|
||||
tell application "${CHROME_APP_NAME}" to make new window with properties {mode:"normal"}
|
||||
tell application "${CHROME_APP_NAME}" to set URL of active tab of front window to item 1 of argv
|
||||
end run`
|
||||
: `tell application "${CHROME_APP_NAME}" to make new window`
|
||||
|
||||
const args = url
|
||||
? ['-e', script, '--', url]
|
||||
: ['-e', script]
|
||||
|
||||
await runProcess(config.binaries.osascript, args, {
|
||||
timeoutMs: config.timeoutMs,
|
||||
})
|
||||
|
||||
// Brief delay for the window to appear
|
||||
await sleep(500)
|
||||
}
|
||||
|
||||
async function activateChrome(): Promise<void> {
|
||||
await runProcess(config.binaries.osascript, [
|
||||
'-e',
|
||||
@@ -185,68 +293,104 @@ end run`
|
||||
|
||||
return {
|
||||
async ensureAgentWindow(options) {
|
||||
// If we already have a session, only reuse it when the agent launched a
|
||||
// dedicated Chrome instance. Joined sessions only cache process metadata,
|
||||
// not a verifiable OS window handle, so blindly reusing them after the
|
||||
// user closes that window would redirect later focus/click flows onto a
|
||||
// different Chrome window.
|
||||
let ensureOutcome: ChromeSessionInfo['ensureOutcome'] = 'launched'
|
||||
|
||||
if (session) {
|
||||
const stillRunning = await isChromeRunning()
|
||||
if (stillRunning && !session.wasAlreadyRunning) {
|
||||
return session
|
||||
const trackedSession = session
|
||||
const trackedChromePid = await getTrackedChromePid(trackedSession)
|
||||
|
||||
if (trackedChromePid === trackedSession.pid) {
|
||||
const stillRunning = await isProcessAlive(trackedSession.pid)
|
||||
const stillHasWindow = stillRunning && await hasChromeWindow(trackedSession.pid)
|
||||
if (stillHasWindow) {
|
||||
trackedSession.ensureOutcome = 'reused'
|
||||
return trackedSession
|
||||
}
|
||||
|
||||
try {
|
||||
if (stillRunning) {
|
||||
ensureOutcome = 'recreated_after_missing_window'
|
||||
await terminateChromeProcess(trackedSession.pid)
|
||||
}
|
||||
else {
|
||||
ensureOutcome = 'recreated_after_process_exit'
|
||||
}
|
||||
}
|
||||
finally {
|
||||
// Chrome died or the tracked window disappeared — clear stale session.
|
||||
await clearSessionState()
|
||||
onSessionLost?.()
|
||||
}
|
||||
}
|
||||
if (!stillRunning) {
|
||||
// Chrome died — clear stale session.
|
||||
else {
|
||||
// The tracked PID no longer resolves to the expected Chrome profile.
|
||||
// Treat it as stale and relaunch without touching that PID.
|
||||
ensureOutcome = 'recreated_after_process_exit'
|
||||
await clearSessionState()
|
||||
onSessionLost?.()
|
||||
}
|
||||
session = null
|
||||
}
|
||||
|
||||
// Record the user's current foreground app before we steal focus
|
||||
// Record the user's current foreground app before we steal focus.
|
||||
previousForegroundApp = await getCurrentForegroundApp()
|
||||
|
||||
const cdpPort = options?.cdpPort ?? DEFAULT_CDP_PORT
|
||||
const cdpPort = await resolveLaunchCdpPort(
|
||||
options?.cdpPort ?? DEFAULT_CDP_PORT,
|
||||
options?.cdpPort !== undefined,
|
||||
)
|
||||
const wasAlreadyRunning = await isChromeRunning()
|
||||
await mkdir(config.sessionRoot, { recursive: true })
|
||||
activeProfileDir = await mkdtemp(join(config.sessionRoot, 'chrome-profile-'))
|
||||
|
||||
if (wasAlreadyRunning) {
|
||||
// Chrome is running — create a new window in the existing instance
|
||||
await createNewWindow(options?.url)
|
||||
try {
|
||||
// Always launch a dedicated profile so CDP is stable even when Chrome is already running.
|
||||
await launchChromeWithCdp(cdpPort, activeProfileDir, options?.url)
|
||||
|
||||
// Bring Chrome to front.
|
||||
await activateChrome()
|
||||
// Brief wait for activation.
|
||||
await sleep(300)
|
||||
|
||||
const deadline = Date.now() + config.timeoutMs
|
||||
let pid: number | undefined
|
||||
while (Date.now() < deadline) {
|
||||
pid = await getChromePidForProfile(activeProfileDir, cdpPort)
|
||||
if (pid) {
|
||||
break
|
||||
}
|
||||
await sleep(250)
|
||||
}
|
||||
if (!pid) {
|
||||
throw new Error('Failed to get Chrome PID after launch')
|
||||
}
|
||||
|
||||
session = {
|
||||
ensureOutcome,
|
||||
wasAlreadyRunning,
|
||||
windowId: `${pid}:0:${CHROME_APP_NAME}`,
|
||||
cdpUrl: `http://127.0.0.1:${cdpPort}`,
|
||||
pid,
|
||||
agentOwned: true,
|
||||
initialUrl: options?.url,
|
||||
createdAt: new Date().toISOString(),
|
||||
}
|
||||
|
||||
return session
|
||||
}
|
||||
else {
|
||||
// Chrome not running — launch with CDP
|
||||
await launchChromeWithCdp(cdpPort, options?.url)
|
||||
catch (error) {
|
||||
await findAndTerminateChromeByProfile(activeProfileDir, cdpPort)
|
||||
await clearSessionState()
|
||||
throw error
|
||||
}
|
||||
|
||||
// Bring Chrome to front
|
||||
await activateChrome()
|
||||
// Brief wait for activation
|
||||
await sleep(300)
|
||||
|
||||
// Get the Chrome PID
|
||||
const pid = await getChromeMainPid()
|
||||
if (!pid) {
|
||||
throw new Error('Failed to get Chrome PID after launch')
|
||||
}
|
||||
|
||||
session = {
|
||||
wasAlreadyRunning,
|
||||
windowId: `${pid}:0:${CHROME_APP_NAME}`,
|
||||
cdpUrl: wasAlreadyRunning ? undefined : `http://127.0.0.1:${cdpPort}`,
|
||||
pid,
|
||||
agentOwned: !wasAlreadyRunning,
|
||||
initialUrl: options?.url,
|
||||
createdAt: new Date().toISOString(),
|
||||
}
|
||||
|
||||
return session
|
||||
},
|
||||
|
||||
async bringToFront() {
|
||||
if (!session)
|
||||
return false
|
||||
const stillRunning = await isChromeRunning()
|
||||
if (!stillRunning) {
|
||||
session = null
|
||||
const stillRunning = await isProcessAlive(session.pid)
|
||||
const stillHasWindow = stillRunning && await hasChromeWindow(session.pid)
|
||||
if (!stillHasWindow) {
|
||||
await clearSessionState()
|
||||
onSessionLost?.()
|
||||
return false
|
||||
}
|
||||
@@ -266,8 +410,7 @@ end run`
|
||||
|
||||
endSession() {
|
||||
const hadSession = session !== null
|
||||
session = null
|
||||
previousForegroundApp = undefined
|
||||
void clearSessionState()
|
||||
if (hadSession) {
|
||||
onSessionLost?.()
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import type { AXNode, AXSnapshot } from './accessibility/types'
|
||||
import type { ChromeSemanticSnapshot, DesktopGroundingSnapshot } from './desktop-grounding-types'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { buildTargetCandidates, formatGroundingForAgent } from './desktop-grounding'
|
||||
import { buildTargetCandidates, captureDesktopGrounding, formatGroundingForAgent } from './desktop-grounding'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
@@ -170,6 +170,91 @@ describe('buildTargetCandidates', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('captureDesktopGrounding', () => {
|
||||
it('retries window observation with app filter when Chrome is foreground and the generic list misses it', async () => {
|
||||
const chromeWindow = { x: 0, y: 0, width: 1920, height: 1080 }
|
||||
const chromeElements = [
|
||||
{ tag: 'button', text: 'Submit', rect: { x: 10, y: 10, w: 80, h: 30 } },
|
||||
]
|
||||
|
||||
const executor = {
|
||||
takeScreenshot: vi.fn().mockResolvedValue({
|
||||
dataBase64: '',
|
||||
mimeType: 'image/png',
|
||||
path: '/tmp/screenshot.png',
|
||||
capturedAt: new Date().toISOString(),
|
||||
}),
|
||||
observeWindows: vi.fn()
|
||||
.mockResolvedValueOnce({
|
||||
frontmostAppName: 'Google Chrome',
|
||||
frontmostWindowTitle: 'Chrome',
|
||||
windows: [
|
||||
{
|
||||
appName: 'Control Center',
|
||||
title: 'Clock',
|
||||
bounds: { x: 0, y: 0, width: 100, height: 30 },
|
||||
},
|
||||
],
|
||||
observedAt: new Date().toISOString(),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
frontmostAppName: 'Google Chrome',
|
||||
frontmostWindowTitle: 'Chrome',
|
||||
windows: [
|
||||
{
|
||||
appName: 'Google Chrome',
|
||||
title: 'Chrome',
|
||||
bounds: chromeWindow,
|
||||
ownerPid: 1234,
|
||||
id: '1234:0:Chrome',
|
||||
layer: 0,
|
||||
isOnScreen: true,
|
||||
},
|
||||
],
|
||||
observedAt: new Date().toISOString(),
|
||||
}),
|
||||
focusApp: vi.fn(),
|
||||
openApp: vi.fn(),
|
||||
click: vi.fn(),
|
||||
typeText: vi.fn(),
|
||||
pressKeys: vi.fn(),
|
||||
scroll: vi.fn(),
|
||||
getForegroundContext: vi.fn().mockResolvedValue({
|
||||
available: true,
|
||||
appName: 'Google Chrome',
|
||||
platform: 'darwin',
|
||||
}),
|
||||
getDisplayInfo: vi.fn(),
|
||||
getExecutionTarget: vi.fn(),
|
||||
describe: vi.fn(),
|
||||
} as any
|
||||
|
||||
const cdpBridge = {
|
||||
getStatus: vi.fn().mockReturnValue({
|
||||
connected: true,
|
||||
pageUrl: 'https://example.com',
|
||||
pageTitle: 'Example Page',
|
||||
}),
|
||||
collectInteractiveElements: vi.fn().mockResolvedValue(chromeElements),
|
||||
} as any
|
||||
|
||||
const config = {
|
||||
timeoutMs: 5000,
|
||||
} as any
|
||||
|
||||
const snapshot = await captureDesktopGrounding({
|
||||
config,
|
||||
executor,
|
||||
input: { includeChrome: true },
|
||||
cdpBridge,
|
||||
})
|
||||
|
||||
expect(executor.observeWindows).toHaveBeenNthCalledWith(1, { limit: 12 })
|
||||
expect(executor.observeWindows).toHaveBeenNthCalledWith(2, { app: 'Google Chrome', limit: 12 })
|
||||
expect(snapshot.targetCandidates.some(candidate => candidate.source === 'chrome_dom')).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// formatGroundingForAgent
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -81,6 +81,25 @@ export async function captureDesktopGrounding(params: {
|
||||
const foregroundApp = windowObs.frontmostAppName || axSnapshot?.appName || 'unknown'
|
||||
const isChromeInFront = isChromeApp(foregroundApp)
|
||||
|
||||
// If Chrome is foreground, ask the executor for a Chrome-filtered window list.
|
||||
// The generic top-N window snapshot is often dominated by system UI and can
|
||||
// miss Chrome entirely, which would prevent chrome_dom candidates from being
|
||||
// mapped to screen coordinates.
|
||||
let chromeWindowBounds = findChromeWindowBounds(windowObs, foregroundApp)
|
||||
if (isChromeInFront && !chromeWindowBounds) {
|
||||
try {
|
||||
const chromeWindows = await executor.observeWindows({
|
||||
app: foregroundApp,
|
||||
limit: 12,
|
||||
})
|
||||
chromeWindowBounds = findChromeWindowBounds(chromeWindows, foregroundApp)
|
||||
}
|
||||
catch {
|
||||
// Best-effort only. Fall back to AX-only candidates if filtered window
|
||||
// enumeration fails.
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: Chrome semantic data (only if Chrome is foreground and allowed)
|
||||
let chromeSemanticSnapshot: ChromeSemanticSnapshot | null = null
|
||||
if (isChromeInFront && input?.includeChrome !== false) {
|
||||
@@ -88,7 +107,6 @@ export async function captureDesktopGrounding(params: {
|
||||
}
|
||||
|
||||
// Phase 3: Build target candidates
|
||||
const chromeWindowBounds = findChromeWindowBounds(windowObs, foregroundApp)
|
||||
const candidates = buildTargetCandidates({
|
||||
axSnapshot,
|
||||
chromeSnapshot: chromeSemanticSnapshot ?? undefined,
|
||||
|
||||
@@ -150,34 +150,6 @@ describe('registerChromeSessionTools', () => {
|
||||
expect(runtime.stateManager.getState().pendingApprovalCount).toBe(1)
|
||||
})
|
||||
|
||||
it('audits joined Chrome sessions as open_app because ensure can create a new window', async () => {
|
||||
runtime.config = createTestConfig({
|
||||
executor: 'macos-local',
|
||||
approvalMode: 'all',
|
||||
})
|
||||
vi.mocked(runtime.chromeSessionManager.getSessionInfo).mockReturnValue({
|
||||
wasAlreadyRunning: true,
|
||||
windowId: 'chrome-window-existing',
|
||||
pid: 9999,
|
||||
agentOwned: false,
|
||||
createdAt: new Date().toISOString(),
|
||||
})
|
||||
|
||||
const { server, invoke } = createMockServer()
|
||||
registerChromeSessionTools({ server, runtime })
|
||||
|
||||
const result = await invoke('desktop_ensure_chrome')
|
||||
|
||||
const structured = result.structuredContent as Record<string, any>
|
||||
expect(structured.status).toBe('approval_required')
|
||||
expect(structured.action).toEqual({
|
||||
kind: 'desktop_ensure_chrome',
|
||||
input: {},
|
||||
})
|
||||
expect(structured.transparency.intent).toBe('Open an agent Chrome window with CDP support')
|
||||
expect(runtime.chromeSessionManager.ensureAgentWindow).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('consumes operation budget and persists chrome session when approvals are disabled', async () => {
|
||||
vi.mocked(runtime.chromeSessionManager.ensureAgentWindow).mockResolvedValue({
|
||||
wasAlreadyRunning: false,
|
||||
@@ -207,4 +179,38 @@ describe('registerChromeSessionTools', () => {
|
||||
expect((runtime.session.record as any).mock.calls[0][0].event).toBe('requested')
|
||||
expect((runtime.session.record as any).mock.calls[1][0].event).toBe('executed')
|
||||
})
|
||||
|
||||
it('uses focus_app when a chrome session already exists', async () => {
|
||||
runtime.config = createTestConfig({
|
||||
executor: 'macos-local',
|
||||
approvalMode: 'all',
|
||||
})
|
||||
vi.mocked(runtime.chromeSessionManager.getSessionInfo).mockReturnValue({
|
||||
wasAlreadyRunning: false,
|
||||
windowId: 'chrome-window-existing',
|
||||
pid: 9999,
|
||||
agentOwned: true,
|
||||
createdAt: new Date().toISOString(),
|
||||
})
|
||||
|
||||
const { server, invoke } = createMockServer()
|
||||
registerChromeSessionTools({ server, runtime })
|
||||
|
||||
const result = await invoke('desktop_ensure_chrome')
|
||||
|
||||
const structured = result.structuredContent as Record<string, any>
|
||||
expect(structured.status).toBe('approval_required')
|
||||
expect(structured.action).toEqual({
|
||||
kind: 'desktop_ensure_chrome',
|
||||
input: {},
|
||||
})
|
||||
expect(structured.transparency.intent).toBe('Bring the agent Chrome window to the foreground')
|
||||
expect(runtime.chromeSessionManager.ensureAgentWindow).not.toHaveBeenCalled()
|
||||
expect((runtime.session.record as any).mock.calls[0][0].result.approvalAction).toEqual({
|
||||
kind: 'focus_app',
|
||||
input: {
|
||||
app: 'Google Chrome',
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -30,7 +30,7 @@ const CHROME_APP_NAME = 'Google Chrome'
|
||||
function getChromeSessionAction(runtime: ComputerUseServerRuntime): ActionInvocation {
|
||||
const sessionInfo = runtime.chromeSessionManager.getSessionInfo()
|
||||
return {
|
||||
kind: sessionInfo && !sessionInfo.wasAlreadyRunning ? 'focus_app' : 'open_app',
|
||||
kind: sessionInfo ? 'focus_app' : 'open_app',
|
||||
input: {
|
||||
app: CHROME_APP_NAME,
|
||||
},
|
||||
@@ -63,7 +63,7 @@ export async function executeChromeEnsure(
|
||||
appName: 'Google Chrome',
|
||||
windowId: sessionInfo.windowId,
|
||||
pid: sessionInfo.pid,
|
||||
agentLaunched: !sessionInfo.wasAlreadyRunning,
|
||||
agentLaunched: sessionInfo.agentOwned,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -76,7 +76,7 @@ export async function executeChromeEnsure(
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-connect CDP bridge when the agent launched Chrome with CDP.
|
||||
// Auto-connect CDP bridge when the agent owns Chrome and CDP is available.
|
||||
// Best-effort only: Chrome may need a moment before the DevTools server answers.
|
||||
let cdpStatus = 'not applicable'
|
||||
if (sessionInfo.cdpUrl) {
|
||||
@@ -97,7 +97,7 @@ export async function executeChromeEnsure(
|
||||
}
|
||||
|
||||
const lines = [
|
||||
`Chrome session ${sessionInfo.wasAlreadyRunning ? 'joined' : 'launched'}:`,
|
||||
'Chrome session launched:',
|
||||
` PID: ${sessionInfo.pid}`,
|
||||
` Window: ${sessionInfo.windowId}`,
|
||||
` Agent-owned: ${sessionInfo.agentOwned}`,
|
||||
|
||||
@@ -53,4 +53,23 @@ describe('support matrix', () => {
|
||||
const ps = getProductSupported()
|
||||
expect(ps.length).toBeGreaterThanOrEqual(4)
|
||||
})
|
||||
|
||||
it('includes desktop v3 smoke coverage as covered, not product-supported', () => {
|
||||
const entry = supportMatrix.find(item => item.id === 'desktop_v3_chrome_grounding')
|
||||
expect(entry).toBeDefined()
|
||||
expect(entry?.lane).toBe('desktop-native')
|
||||
expect(entry?.level).toBe('covered')
|
||||
expect(entry?.smokeCommand).toBe('pnpm -F @proj-airi/computer-use-mcp smoke:desktop-v3')
|
||||
})
|
||||
|
||||
it('includes browser-dom route contract as covered, not product-supported', () => {
|
||||
const entry = supportMatrix.find(item => item.id === 'desktop_browser_dom_route_contract')
|
||||
expect(entry).toBeDefined()
|
||||
expect(entry?.lane).toBe('desktop-native')
|
||||
expect(entry?.level).toBe('covered')
|
||||
expect(entry?.unitTests).toEqual([
|
||||
'src/browser-action-router.test.ts',
|
||||
'src/browser-dom/extension-bridge.test.ts',
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -157,6 +157,30 @@ export const supportMatrix: SupportMatrixEntry[] = [
|
||||
smokeCommand: 'pnpm -F @proj-airi/computer-use-mcp smoke:stdio',
|
||||
happyPath: 'focus app → screenshot → accessibility_snapshot basic loop',
|
||||
},
|
||||
{
|
||||
lane: 'desktop-native',
|
||||
id: 'desktop_v3_chrome_grounding',
|
||||
label: 'Desktop v3 Chrome grounding smoke (ensure / observe / click / state)',
|
||||
level: 'covered',
|
||||
unitTests: [
|
||||
'src/bin/smoke-chrome-grounding.test.ts',
|
||||
'src/server/register-chrome-session.test.ts',
|
||||
'src/server/register-desktop-grounding.test.ts',
|
||||
'src/server/register-desktop-grounding-tools.test.ts',
|
||||
],
|
||||
smokeCommand: 'pnpm -F @proj-airi/computer-use-mcp smoke:desktop-v3',
|
||||
happyPath: 'desktop_ensure_chrome → desktop_observe → desktop_click_target → desktop_get_state updates grounding and pointer state',
|
||||
},
|
||||
{
|
||||
lane: 'desktop-native',
|
||||
id: 'desktop_browser_dom_route_contract',
|
||||
label: 'Browser-dom route contract (left single-click, fail-closed bridge responses)',
|
||||
level: 'covered',
|
||||
unitTests: [
|
||||
'src/browser-action-router.test.ts',
|
||||
'src/browser-dom/extension-bridge.test.ts',
|
||||
],
|
||||
},
|
||||
{
|
||||
lane: 'desktop-native',
|
||||
id: 'desktop_click_type_press',
|
||||
|
||||
Reference in New Issue
Block a user