feat(computer-use-mcp): add browser repair suggestions (#1920)

## Summary Adds browser DOM repair suggestions inside `services/computer-use-mcp`. Known browser DOM action failures now get structured, advisory-only repair suggestions for selector misses, hidden elements, timeouts, detached frames, and stale elements. The registration path uses the suggestions for `browser_dom_click` and `browser_dom_wait_for_element` thrown-error responses. ## Boundary - Only touches `services/computer-use-mcp`. - Does not touch app, renderer, overlay, extension assets, desktop v3 files, or root project governance files. - Does not import preview changes that delete current upstream Chrome/desktop files. - Does not include local AI config, screenshots, tokens, account identifiers, or raw environment dumps. ## Validation Sanitized evidence is committed in `services/computer-use-mcp/validation/browser-repair-contract.md`. Ran: - `pnpm install --ignore-scripts --frozen-lockfile` - `pnpm -F @proj-airi/computer-use-mcp exec vitest run src/browser-dom/browser-repair-contract.test.ts src/server/register-tools-pty-approval.test.ts --config ./vitest.config.ts` - `pnpm exec moeru-lint --fix services/computer-use-mcp/validation/browser-repair-contract.md services/computer-use-mcp/src/browser-dom/browser-repair-contract.ts services/computer-use-mcp/src/browser-dom/browser-repair-contract.test.ts services/computer-use-mcp/src/server/register-tools.ts services/computer-use-mcp/src/server/register-tools-pty-approval.test.ts` - `git diff --check` `pnpm -F @proj-airi/computer-use-mcp typecheck` is currently blocked by existing baseline errors in `chrome-session-manager` and `desktop-grounding`, outside this patch. The validation note lists the error classes without local paths or raw logs.
2026-08-14 00:48:06 +00:00 · 2026-06-02 10:50:34 +08:00
parent 7e3d39c5dd
commit d86c502f04
5 changed files with 358 additions and 11 deletions
@@ -0,0 +1,74 @@
+import { describe, expect, it } from 'vitest'
+
+import { diagnoseBrowserActionError } from './browser-repair-contract'
+
+describe('browser-repair-contract', () => {
+  it('suggests reading the page when a selector is not found', () => {
+    const suggestion = diagnoseBrowserActionError(
+      new Error('selector "#submit" did not match any element'),
+      '#submit',
+      'browser_dom_click',
+    )
+
+    expect(suggestion).toMatchObject({
+      pattern: 'element_not_found',
+      suggestedTool: 'browser_dom_read_page',
+    })
+    expect(suggestion?.reactionText).toContain('#submit')
+  })
+
+  it('suggests checking computed styles when an element is hidden', () => {
+    const suggestion = diagnoseBrowserActionError(
+      new Error('element is not visible or is covered'),
+      '#menu',
+      'browser_dom_click',
+    )
+
+    expect(suggestion).toMatchObject({
+      pattern: 'element_not_visible',
+      suggestedTool: 'browser_dom_get_computed_styles',
+      suggestedParams: {
+        selector: '#menu',
+      },
+    })
+  })
+
+  it('suggests waiting for the selector when an action times out', () => {
+    const suggestion = diagnoseBrowserActionError(
+      new Error('timed out waiting for selector'),
+      '.toast',
+      'browser_dom_wait_for_element',
+    )
+
+    expect(suggestion).toMatchObject({
+      pattern: 'action_timeout',
+      suggestedTool: 'browser_dom_wait_for_element',
+      suggestedParams: {
+        selector: '.toast',
+      },
+    })
+  })
+
+  it('suggests rediscovering the active tab when a frame is detached', () => {
+    const suggestion = diagnoseBrowserActionError(
+      new Error('frame was detached before dispatch'),
+      'button',
+      'browser_dom_click',
+    )
+
+    expect(suggestion).toMatchObject({
+      pattern: 'frame_detached',
+      suggestedTool: 'browser_dom_get_active_tab',
+    })
+  })
+
+  it('returns null for unrecognized errors', () => {
+    const suggestion = diagnoseBrowserActionError(
+      new Error('extension returned a custom opaque error'),
+      '#submit',
+      'browser_dom_click',
+    )
+
+    expect(suggestion).toBeNull()
+  })
+})
@@ -0,0 +1,86 @@
+import { errorMessageFrom } from '@moeru/std'
+
+export interface BrowserRepairSuggestion {
+  /** The matched error pattern. */
+  pattern: string
+  /** Human-readable explanation of the failure. */
+  reason: string
+  /** Existing MCP tool that can help recover from the failure. */
+  suggestedTool: string
+  /** Suggested parameters for the recovery tool. */
+  suggestedParams: Record<string, unknown>
+  /** Short instruction that can be appended to the tool response. */
+  reactionText: string
+}
+
+const ERROR_PATTERNS: Array<{
+  pattern: RegExp
+  build: (selector: string, actionKind: string) => BrowserRepairSuggestion
+}> = [
+  {
+    pattern: /not found|no .* match|could not find|cannot find|selector .* did not match/i,
+    build: selector => ({
+      pattern: 'element_not_found',
+      reason: `Selector "${selector}" did not match any element in the page.`,
+      suggestedTool: 'browser_dom_read_page',
+      suggestedParams: {},
+      reactionText: `Re-read the page DOM before retrying "${selector}". The selector may be stale, too specific, or not loaded yet.`,
+    }),
+  },
+  {
+    pattern: /not visible|not interactable|element .* hidden|element .* obscured|element .* covered|zero.*(width|height)/i,
+    build: selector => ({
+      pattern: 'element_not_visible',
+      reason: `Element "${selector}" exists but is not visibly interactable.`,
+      suggestedTool: 'browser_dom_get_computed_styles',
+      suggestedParams: { selector },
+      reactionText: `Inspect computed styles for "${selector}" and check whether an overlay, hidden state, or off-screen position is blocking interaction.`,
+    }),
+  },
+  {
+    pattern: /timed? ?out|exceeded.*deadline/i,
+    build: selector => ({
+      pattern: 'action_timeout',
+      reason: `The action timed out while waiting for "${selector}".`,
+      suggestedTool: 'browser_dom_wait_for_element',
+      suggestedParams: { selector },
+      reactionText: `Wait for "${selector}" with browser_dom_wait_for_element, then retry the action after the page settles.`,
+    }),
+  },
+  {
+    pattern: /frame .* (detached|removed|not available)|tab .* (closed|not found)/i,
+    build: selector => ({
+      pattern: 'frame_detached',
+      reason: `The frame or tab containing "${selector}" is no longer available.`,
+      suggestedTool: 'browser_dom_get_active_tab',
+      suggestedParams: {},
+      reactionText: 'Re-discover the active tab and frames before retrying the browser DOM action.',
+    }),
+  },
+  {
+    pattern: /stale .* reference|element .* (changed|replaced|removed|no longer)/i,
+    build: selector => ({
+      pattern: 'stale_element',
+      reason: `Element "${selector}" changed after it was discovered.`,
+      suggestedTool: 'browser_dom_find_elements',
+      suggestedParams: { selector },
+      reactionText: `Re-query "${selector}" with browser_dom_find_elements and retry immediately with the refreshed match.`,
+    }),
+  },
+]
+
+export function diagnoseBrowserActionError(
+  error: unknown,
+  selector: string,
+  actionKind: string,
+): BrowserRepairSuggestion | null {
+  const message = errorMessageFrom(error) ?? String(error)
+
+  for (const { pattern, build } of ERROR_PATTERNS) {
+    if (pattern.test(message)) {
+      return build(selector, actionKind)
+    }
+  }
+
+  return null
+}
@@ -71,6 +71,7 @@ describe('registerComputerUseTools: PTY approval bridge', () => {
      browserDomBridge: {
        triggerEvent: vi.fn(),
        clickSelector: vi.fn(),
+        waitForElement: vi.fn(),
        getStatus: vi.fn(() => ({ enabled: false, connected: false })),
        supportsAction: vi.fn(() => true),
      },
@@ -288,6 +289,77 @@ describe('registerComputerUseTools: PTY approval bridge', () => {
    expect((runtime.browserDomBridge.clickSelector as any)).not.toHaveBeenCalled()
  })

+  it('returns a browser repair suggestion when browser_dom_click throws a known selector error', async () => {
+    ;(runtime.browserDomBridge.getStatus as any).mockReturnValue({
+      enabled: true,
+      connected: true,
+      pendingRequests: 0,
+    })
+    ;(runtime.browserDomBridge.clickSelector as any).mockRejectedValue(
+      new Error('selector "#submit" did not match any element'),
+    )
+
+    const { server, invoke } = createMockServer()
+    registerComputerUseTools({
+      server,
+      runtime,
+      executeAction: vi.fn(),
+      enableTestTools: false,
+    })
+
+    const result = await invoke('browser_dom_click', {
+      selector: '#submit',
+    })
+
+    expect(result.isError).toBe(true)
+    expect(result.content[0].text).toContain('Re-read the page DOM')
+    expect(result.structuredContent).toMatchObject({
+      status: 'error',
+      selector: '#submit',
+      actionKind: 'browser_dom_click',
+      repairSuggestion: {
+        pattern: 'element_not_found',
+        suggestedTool: 'browser_dom_read_page',
+      },
+    })
+  })
+
+  it('returns a browser repair suggestion when browser_dom_wait_for_element times out', async () => {
+    ;(runtime.browserDomBridge.getStatus as any).mockReturnValue({
+      enabled: true,
+      connected: true,
+      pendingRequests: 0,
+    })
+    ;(runtime.browserDomBridge.waitForElement as any).mockRejectedValue(
+      new Error('timed out waiting for selector'),
+    )
+
+    const { server, invoke } = createMockServer()
+    registerComputerUseTools({
+      server,
+      runtime,
+      executeAction: vi.fn(),
+      enableTestTools: false,
+    })
+
+    const result = await invoke('browser_dom_wait_for_element', {
+      selector: '.toast',
+      timeoutMs: 500,
+    })
+
+    expect(result.isError).toBe(true)
+    expect(result.content[0].text).toContain('browser_dom_wait_for_element')
+    expect(result.structuredContent).toMatchObject({
+      status: 'error',
+      selector: '.toast',
+      actionKind: 'browser_dom_wait_for_element',
+      repairSuggestion: {
+        pattern: 'action_timeout',
+        suggestedTool: 'browser_dom_wait_for_element',
+      },
+    })
+  })
+
  it('rejects browser_dom_trigger_event when the connected extension transport does not support writes', async () => {
    ;(runtime.browserDomBridge.getStatus as any).mockReturnValue({
      enabled: true,
@@ -16,6 +16,7 @@ import type { ComputerUseServerRuntime } from './runtime'
 import { errorMessageFrom } from '@moeru/std'
 import { z } from 'zod'

+import { diagnoseBrowserActionError } from '../browser-dom/browser-repair-contract'
 import { getUnsupportedBrowserDomActions, isBrowserDomActionSupported } from '../browser-dom/capabilities'
 import { getRuntimePreflight } from '../preflight'
 import { summarizeRunState } from '../transparency'
@@ -104,6 +105,36 @@ function buildBrowserDomUnavailableResponse(runtime: ComputerUseServerRuntime, u
  }
 }

+function buildBrowserDomActionErrorResponse(params: {
+  runtime: ComputerUseServerRuntime
+  error: unknown
+  selector: string
+  actionKind: string
+}) {
+  const { runtime, error, selector, actionKind } = params
+  const message = errorMessageFrom(error) ?? 'unknown error'
+  const repairSuggestion = diagnoseBrowserActionError(error, selector, actionKind)
+
+  return {
+    isError: true,
+    content: [
+      textContent(
+        repairSuggestion
+          ? `${actionKind} failed for "${selector}": ${message}\n\n${repairSuggestion.reactionText}`
+          : `${actionKind} failed for "${selector}": ${message}`,
+      ),
+    ],
+    structuredContent: {
+      status: 'error',
+      selector,
+      actionKind,
+      error: message,
+      repairSuggestion: repairSuggestion ?? undefined,
+      bridge: runtime.browserDomBridge.getStatus(),
+    },
+  }
+}
+
 export function registerComputerUseTools(params: RegisterComputerUseToolsOptions) {
  const { runtime, executeAction, enableTestTools } = params
  const server = createToolLaneHygieneServer(params.server, runtime.stateManager)
@@ -569,11 +600,22 @@ export function registerComputerUseTools(params: RegisterComputerUseToolsOptions
      if (!isBrowserDomActionSupported(runtime.browserDomBridge, ...requiredActions))
        return buildBrowserDomUnavailableResponse(runtime, getUnsupportedBrowserDomActions(runtime.browserDomBridge, ...requiredActions))

-      const result = await runtime.browserDomBridge.clickSelector({
-        selector,
-        tabId,
-        frameIds,
-      })
+      let result: Awaited<ReturnType<typeof runtime.browserDomBridge.clickSelector>>
+      try {
+        result = await runtime.browserDomBridge.clickSelector({
+          selector,
+          tabId,
+          frameIds,
+        })
+      }
+      catch (error) {
+        return buildBrowserDomActionErrorResponse({
+          runtime,
+          error,
+          selector,
+          actionKind: 'browser_dom_click',
+        })
+      }

      // NOTICE: clickSelector resolves even when the clickAt step misses
      // (e.g. reflow between target lookup and click dispatch). Inspect
@@ -771,12 +813,23 @@ export function registerComputerUseTools(params: RegisterComputerUseToolsOptions
      if (!isBrowserDomActionSupported(runtime.browserDomBridge, ...requiredActions))
        return buildBrowserDomUnavailableResponse(runtime, getUnsupportedBrowserDomActions(runtime.browserDomBridge, ...requiredActions))

-      const results = await runtime.browserDomBridge.waitForElement({
-        selector,
-        timeoutMs,
-        tabId,
-        frameIds,
-      })
+      let results: Awaited<ReturnType<typeof runtime.browserDomBridge.waitForElement>>
+      try {
+        results = await runtime.browserDomBridge.waitForElement({
+          selector,
+          timeoutMs,
+          tabId,
+          frameIds,
+        })
+      }
+      catch (error) {
+        return buildBrowserDomActionErrorResponse({
+          runtime,
+          error,
+          selector,
+          actionKind: 'browser_dom_wait_for_element',
+        })
+      }
      return {
        content: [
          textContent(summarizeBrowserDomFrameResults(`wait_for_element for "${selector}"`, results)),
@@ -0,0 +1,62 @@
+# Browser Repair Contract Validation
+
+Date: 2026-06-01
+
+Scope:
+- `services/computer-use-mcp/src/browser-dom/browser-repair-contract.ts`
+- `services/computer-use-mcp/src/browser-dom/browser-repair-contract.test.ts`
+- `services/computer-use-mcp/src/server/register-tools.ts`
+- `services/computer-use-mcp/src/server/register-tools-pty-approval.test.ts`
+
+Privacy note:
+- Evidence is sanitized for a public repository.
+- No local absolute paths, tokens, account identifiers, screenshots, or raw environment dumps are included.
+
+Commands run:
+
+```sh
+pnpm install --ignore-scripts --frozen-lockfile
+```
+
+Result: passed. Lockfile stayed unchanged; lifecycle scripts were intentionally skipped for local verification setup.
+
+```sh
+pnpm -F @proj-airi/computer-use-mcp exec vitest run \
+  src/browser-dom/browser-repair-contract.test.ts \
+  src/server/register-tools-pty-approval.test.ts \
+  --config ./vitest.config.ts
+```
+
+Result: passed. 2 test files, 12 tests.
+
+```sh
+pnpm exec moeru-lint --fix \
+  services/computer-use-mcp/validation/browser-repair-contract.md \
+  services/computer-use-mcp/src/browser-dom/browser-repair-contract.ts \
+  services/computer-use-mcp/src/browser-dom/browser-repair-contract.test.ts \
+  services/computer-use-mcp/src/server/register-tools.ts \
+  services/computer-use-mcp/src/server/register-tools-pty-approval.test.ts
+```
+
+Result: passed with 0 warnings and 0 errors when run under Node 24.
+
+```sh
+git diff --check
+```
+
+Result: passed.
+
+```sh
+pnpm -F @proj-airi/computer-use-mcp typecheck
+```
+
+Result: failed on existing baseline files outside this change:
+- `src/chrome-session-manager.ts`
+- `src/chrome-session-manager.test.ts`
+- `src/desktop-grounding.ts`
+
+Observed baseline error classes:
+- `TS2339` and `TS2353` around `ChromeSessionInfo.ensureOutcome`
+- `TS2451` / `TS2304` around duplicated `chromeWindowBounds` and missing `isChromeInFront`
+
+No typecheck errors were reported for the files changed by this patch.