diff --git a/apps/stage-tamagotchi/src/main/windows/desktop-overlay/index.ts b/apps/stage-tamagotchi/src/main/windows/desktop-overlay/index.ts index 4945593de..19a05d246 100644 --- a/apps/stage-tamagotchi/src/main/windows/desktop-overlay/index.ts +++ b/apps/stage-tamagotchi/src/main/windows/desktop-overlay/index.ts @@ -29,6 +29,11 @@ import { BrowserWindow, screen } from 'electron' import { baseUrl, getElectronMainDirname, load, withHashRoute } from '../../libs/electron/location' import { setupDesktopOverlayElectronInvokes } from './rpc/index.electron' +import { + applyDesktopOverlayInputIsolation, + createDesktopOverlayWindowOptions, + showDesktopOverlayWithoutFocus, +} from './window-contract' /** Whether the desktop overlay feature is enabled */ export function isDesktopOverlayEnabled(): boolean { @@ -59,46 +64,17 @@ export async function setupDesktopOverlayWindow(params: { // Use primary display bounds (not just size) — the origin may be non-zero // when multiple displays are arranged in macOS Display Preferences. const primaryDisplay = screen.getPrimaryDisplay() - const { x, y, width, height } = primaryDisplay.bounds + const preloadPath = join(getElectronMainDirname(), '../preload/index.mjs') - overlayWindow = new BrowserWindow({ - title: 'AIRI Desktop Overlay', - width, - height, - x, - y, - show: false, - frame: false, - transparent: true, - alwaysOnTop: true, - skipTaskbar: true, - hasShadow: false, - // Round corners off for pixel-accurate overlay - roundedCorners: false, - // Prevent the overlay from stealing focus - focusable: false, - webPreferences: { - preload: join(getElectronMainDirname(), '../preload/index.mjs'), - sandbox: false, - // Disable background throttling so animations stay smooth - backgroundThrottling: false, - }, - }) - - // Make click-through: all mouse events pass through to the desktop - overlayWindow.setIgnoreMouseEvents(true, { forward: true }) - - // Set to screen level (above all other windows) - overlayWindow.setAlwaysOnTop(true, 'screen-saver') - - // Prevent the window from appearing in screenshots/recordings if possible - overlayWindow.setContentProtection(true) - - // Hide from Mission Control / Exposé on macOS - overlayWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true }) + overlayWindow = new BrowserWindow(createDesktopOverlayWindowOptions({ + bounds: primaryDisplay.bounds, + preloadPath, + })) + applyDesktopOverlayInputIsolation(overlayWindow) overlayWindow.on('ready-to-show', () => { - overlayWindow?.show() + if (overlayWindow) + showDesktopOverlayWithoutFocus(overlayWindow) }) overlayWindow.on('closed', () => { diff --git a/apps/stage-tamagotchi/src/main/windows/desktop-overlay/window-contract.test.ts b/apps/stage-tamagotchi/src/main/windows/desktop-overlay/window-contract.test.ts new file mode 100644 index 000000000..fa6d19f37 --- /dev/null +++ b/apps/stage-tamagotchi/src/main/windows/desktop-overlay/window-contract.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it, vi } from 'vitest' + +import { + applyDesktopOverlayInputIsolation, + createDesktopOverlayWindowOptions, + showDesktopOverlayWithoutFocus, +} from './window-contract' + +describe('createDesktopOverlayWindowOptions', () => { + it('creates non-focusable transparent overlay window options for display bounds', () => { + const options = createDesktopOverlayWindowOptions({ + bounds: { x: -222, y: -1080, width: 1920, height: 1080 }, + preloadPath: '/tmp/airi-overlay-preload.js', + }) + + expect(options.title).toBe('AIRI Desktop Overlay') + expect(options.x).toBe(-222) + expect(options.y).toBe(-1080) + expect(options.width).toBe(1920) + expect(options.height).toBe(1080) + expect(options.show).toBe(false) + expect(options.frame).toBe(false) + expect(options.transparent).toBe(true) + expect(options.alwaysOnTop).toBe(true) + expect(options.skipTaskbar).toBe(true) + expect(options.hasShadow).toBe(false) + expect(options.roundedCorners).toBe(false) + expect(options.focusable).toBe(false) + expect(options.webPreferences?.preload).toBe('/tmp/airi-overlay-preload.js') + expect(options.webPreferences?.sandbox).toBe(false) + expect(options.webPreferences?.backgroundThrottling).toBe(false) + }) +}) + +describe('applyDesktopOverlayInputIsolation', () => { + it('applies click-through and non-interactive overlay window flags', () => { + const window = { + setAlwaysOnTop: vi.fn(), + setContentProtection: vi.fn(), + setIgnoreMouseEvents: vi.fn(), + setVisibleOnAllWorkspaces: vi.fn(), + } + + applyDesktopOverlayInputIsolation(window) + + expect(window.setIgnoreMouseEvents).toHaveBeenCalledWith(true, { forward: true }) + expect(window.setAlwaysOnTop).toHaveBeenCalledWith(true, 'screen-saver') + expect(window.setContentProtection).toHaveBeenCalledWith(true) + expect(window.setVisibleOnAllWorkspaces).toHaveBeenCalledWith(true, { visibleOnFullScreen: true }) + }) +}) + +describe('showDesktopOverlayWithoutFocus', () => { + it('uses showInactive and never calls active show or focus paths', () => { + const window = { + focus: vi.fn(), + show: vi.fn(), + showInactive: vi.fn(), + } + + showDesktopOverlayWithoutFocus(window) + + expect(window.showInactive).toHaveBeenCalledTimes(1) + expect(window.show).not.toHaveBeenCalled() + expect(window.focus).not.toHaveBeenCalled() + }) +}) diff --git a/apps/stage-tamagotchi/src/main/windows/desktop-overlay/window-contract.ts b/apps/stage-tamagotchi/src/main/windows/desktop-overlay/window-contract.ts new file mode 100644 index 000000000..17532fcca --- /dev/null +++ b/apps/stage-tamagotchi/src/main/windows/desktop-overlay/window-contract.ts @@ -0,0 +1,82 @@ +import type { BrowserWindow, BrowserWindowConstructorOptions, Rectangle } from 'electron' + +/** + * Build BrowserWindow options for the desktop grounding overlay. + * + * Use when: + * - Creating the transparent desktop overlay BrowserWindow + * - Testing overlay input-isolation without starting Electron + * + * Expects: + * - `bounds` are Electron screen logical coordinates for the display being covered + * - `preloadPath` is an absolute path to the renderer preload script + * + * Returns: + * - BrowserWindow options that keep the overlay visual-only and non-focusable + */ +export function createDesktopOverlayWindowOptions(params: { + bounds: Rectangle + preloadPath: string +}): BrowserWindowConstructorOptions { + return { + title: 'AIRI Desktop Overlay', + width: params.bounds.width, + height: params.bounds.height, + x: params.bounds.x, + y: params.bounds.y, + show: false, + frame: false, + transparent: true, + alwaysOnTop: true, + skipTaskbar: true, + hasShadow: false, + roundedCorners: false, + focusable: false, + webPreferences: { + preload: params.preloadPath, + sandbox: false, + backgroundThrottling: false, + }, + } +} + +/** + * Apply input-isolation flags to the desktop grounding overlay. + * + * Use when: + * - The overlay window has been created and must become click-through + * - The overlay should render above apps without stealing mouse or focus + * + * Expects: + * - The window is the dedicated desktop overlay window + * + * Returns: + * - Nothing; mutates Electron window flags in place + */ +export function applyDesktopOverlayInputIsolation( + window: Pick, +): void { + window.setIgnoreMouseEvents(true, { forward: true }) + window.setAlwaysOnTop(true, 'screen-saver') + window.setContentProtection(true) + window.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true }) +} + +/** + * Show the overlay without activating or focusing it. + * + * Use when: + * - The overlay renderer is ready and should become visible + * - User focus must remain on the controlled application + * + * Expects: + * - The BrowserWindow supports Electron's `showInactive()` + * + * Returns: + * - Nothing; shows the window without stealing focus + */ +export function showDesktopOverlayWithoutFocus( + window: Pick, +): void { + window.showInactive() +} diff --git a/services/computer-use-mcp/src/chrome-semantic-adapter.test.ts b/services/computer-use-mcp/src/chrome-semantic-adapter.test.ts index 16e754fba..2fde83ee9 100644 --- a/services/computer-use-mcp/src/chrome-semantic-adapter.test.ts +++ b/services/computer-use-mcp/src/chrome-semantic-adapter.test.ts @@ -332,6 +332,36 @@ describe('chromeElementsToTargetCandidates', () => { expect(candidates[0].bounds.y).toBe(50 + 88 + 140 + 24) }) + it('uses cumulative nested iframe offsets before converting to screen coordinates', () => { + const parentFrameOffset = { x: 320, y: 180 } + const childFrameOffset = { x: 24, y: 48 } + const nestedFrameOffset = { + x: parentFrameOffset.x + childFrameOffset.x, + y: parentFrameOffset.y + childFrameOffset.y, + } + const taggedEl = { + tag: 'button', + text: 'Nested iframe CTA', + rect: { x: 12, y: 24, w: 90, h: 32 }, + _frameId: 9, + _frameOffsetX: nestedFrameOffset.x, + _frameOffsetY: nestedFrameOffset.y, + } as any + + const candidates = chromeElementsToTargetCandidates( + [taggedEl], + windowBounds, + 88, + 0, + ) + + expect(candidates[0].frameId).toBe(9) + expect(candidates[0].bounds.x).toBe(100 + 320 + 24 + 12) + expect(candidates[0].bounds.y).toBe(50 + 88 + 180 + 48 + 24) + expect(candidates[0].bounds.width).toBe(90) + expect(candidates[0].bounds.height).toBe(32) + }) + it('falls back to function-level frameId when _frameId is absent', () => { const el = { tag: 'button', diff --git a/services/computer-use-mcp/src/server/register-desktop-grounding.test.ts b/services/computer-use-mcp/src/server/register-desktop-grounding.test.ts index 1af1e8fb6..6f3cccb46 100644 --- a/services/computer-use-mcp/src/server/register-desktop-grounding.test.ts +++ b/services/computer-use-mcp/src/server/register-desktop-grounding.test.ts @@ -470,12 +470,14 @@ describe('desktop_click_target handler integration', () => { it('falls back to OS click when the connected extension transport is read-only', async () => { const sm = new RunStateManager() + const iframeAbsoluteBounds = { x: 456, y: 390, width: 90, height: 32 } const candidate = makeCandidate({ id: 't_0', source: 'chrome_dom', selector: '#login-btn', - frameId: 0, + frameId: 7, isPageContent: true, + bounds: iframeAbsoluteBounds, }) sm.updateGroundingSnapshot(freshSnapshot([candidate])) @@ -494,7 +496,13 @@ describe('desktop_click_target handler integration', () => { expect(result.executionRoute).toBe('os_input') expect(result.routeReason).toContain('does not support getClickTarget + clickAt') expect(bridge.clickSelector).not.toHaveBeenCalled() - expect(executor.click).toHaveBeenCalledOnce() + expect(executor.click).toHaveBeenCalledWith({ + x: 501, + y: 406, + button: 'left', + clickCount: 1, + }) + expect(result.text).toContain('Point: (501, 406)') }) // -----------------------------------------------------------------------