From 813746f5cc66db42e2a30eb176e81630eed2e430 Mon Sep 17 00:00:00 2001 From: paisley <8197966+su8su@users.noreply.github.com> Date: Thu, 6 Aug 2026 10:43:44 +0800 Subject: [PATCH 1/3] fix(gateway): make heartbeat observability-only (#1227) --- README.ja-JP.md | 2 +- README.md | 2 +- README.zh-CN.md | 2 +- electron/gateway/manager.ts | 58 +------------- .../specs/rules/gateway-heartbeat-safety.md | 18 +++++ .../gateway-backend-communication.md | 3 + .../scenarios/gateway-startup-diagnostics.md | 6 +- ...ke-gateway-heartbeat-observability-only.md | 47 ++++++++++++ .../unit/gateway-manager-diagnostics.test.ts | 10 +-- tests/unit/gateway-manager-heartbeat.test.ts | 76 ++++++------------- 10 files changed, 105 insertions(+), 119 deletions(-) create mode 100644 harness/specs/rules/gateway-heartbeat-safety.md create mode 100644 harness/specs/tasks/make-gateway-heartbeat-observability-only.md diff --git a/README.ja-JP.md b/README.ja-JP.md index 0c3b845c..286dfe7c 100644 --- a/README.ja-JP.md +++ b/README.ja-JP.md @@ -226,7 +226,7 @@ ClawXには、Electron、OpenClaw Gateway、またはTelegramなどのチャネ ClawXは、**デュアルプロセス + Host API 統一アクセス**構成を採用しています。Renderer は単一クライアント抽象を呼び出し、プロトコル選択とライフサイクルは Main が管理します: -OpenClaw の設定配信も Electron Main が一元管理します。Gateway の実行中は `config.get` の正規スナップショットを基準にし、変更を `config.set` でコミットします。Gateway が停止中または起動中の場合は、同じコーディネーターが解決済みの JSON5 設定ファイルだけを更新し、Gateway を起動しません。そのため、通常の Provider、Agent、Channel、バインディング、Skill、モデル変更では Gateway プロセスを置き換えません。完全な再起動は、プロキシなどのプロセス起動環境の変更、ユーザーによる明示的な操作、ヘルスチェックやクラッシュ復旧に限定されます。認証プロファイルを SQLite に書き込んだ後は OpenClaw の `secrets.reload` を呼び出し、実行中の Agent がプロセス再起動なしで新しい認証情報を読み取れるようにします。 +OpenClaw の設定配信も Electron Main が一元管理します。Gateway の実行中は `config.get` の正規スナップショットを基準にし、変更を `config.set` でコミットします。Gateway が停止中または起動中の場合は、同じコーディネーターが解決済みの JSON5 設定ファイルだけを更新し、Gateway を起動しません。そのため、通常の Provider、Agent、Channel、バインディング、Skill、モデル変更では Gateway プロセスを置き換えません。完全な再起動は、プロキシなどのプロセス起動環境の変更とユーザーによる明示的な操作に限定されます。確認済みのプロセス終了と WebSocket 切断では、既存の自動再接続経路が引き続き使用されます。WebSocket のハートビート欠落は診断とヘルス状態だけを更新し、Gateway プロセスを置き換えないため、pong 処理の遅延によって長時間実行中の処理が中断されることはありません。認証プロファイルを SQLite に書き込んだ後は OpenClaw の `secrets.reload` を呼び出し、実行中の Agent がプロセス再起動なしで新しい認証情報を読み取れるようにします。 Chat は Electron Main が所有する ACP stdio bridge を使用します。Renderer は型付き host event を受け取り、メモリ上の ACP timeline を描画します。Gateway は providers、models、skills、workspace、settings、diagnostics、media configuration などの非 Chat 機能を引き続き担当します。 diff --git a/README.md b/README.md index 6614810e..79d8557a 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ Notes: ClawX employs a **dual-process architecture** with a unified host API layer. The renderer talks to a single client abstraction, while Electron Main owns protocol selection and process lifecycle: -Electron Main also owns OpenClaw configuration delivery. While the Gateway is running, ClawX reads the authoritative `config.get` snapshot and commits changes with `config.set`; while it is stopped or starting, the same coordinator updates the resolved JSON5 config file without starting the Gateway. Ordinary provider, agent, channel, binding, skill, and model changes therefore do not replace the Gateway process. Full restarts remain for process-launch environment changes such as proxy settings, explicit user actions, and health or crash recovery. Auth-profile SQLite updates use OpenClaw's `secrets.reload` RPC so running agents see new credentials without a process restart. +Electron Main also owns OpenClaw configuration delivery. While the Gateway is running, ClawX reads the authoritative `config.get` snapshot and commits changes with `config.set`; while it is stopped or starting, the same coordinator updates the resolved JSON5 config file without starting the Gateway. Ordinary provider, agent, channel, binding, skill, and model changes therefore do not replace the Gateway process. Full restarts remain for process-launch environment changes such as proxy settings and explicit user actions. Confirmed process exits and WebSocket closes retain their existing automatic reconnect paths. WebSocket heartbeat misses update diagnostics and health state but do not replace the Gateway process, so delayed pong handling cannot interrupt long-running work. Auth-profile SQLite updates use OpenClaw's `secrets.reload` RPC so running agents see new credentials without a process restart. Chat uses an ACP stdio bridge owned by Electron Main. Renderer receives typed host events and renders an in-memory ACP timeline. Gateway remains responsible for non-Chat capabilities such as providers, models, skills, workspace, settings, diagnostics, and media configuration. diff --git a/README.zh-CN.md b/README.zh-CN.md index 5ec5ac04..148c505e 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -227,7 +227,7 @@ ClawX 内置了代理设置,适用于需要通过本地代理客户端访问 ClawX 采用 **双进程 + Host API 统一接入架构**。渲染进程只调用统一客户端抽象,协议选择与进程生命周期由 Electron 主进程统一管理: -OpenClaw 配置交付也统一由 Electron Main 管理。Gateway 运行时,ClawX 以 `config.get` 返回的权威快照为基线,并通过 `config.set` 提交修改;Gateway 停止或启动中时,同一个协调器只更新解析后的 JSON5 配置文件,不会因此启动 Gateway。因此,普通的 Provider、Agent、Channel、绑定、Skill 和模型修改不会替换 Gateway 进程。完整重启仅保留给代理等进程启动环境变化、用户显式操作以及健康检查或崩溃恢复。认证配置写入 SQLite 后,ClawX 会调用 OpenClaw 的 `secrets.reload`,让运行中的 Agent 无需重启即可读取新凭据。 +OpenClaw 配置交付也统一由 Electron Main 管理。Gateway 运行时,ClawX 以 `config.get` 返回的权威快照为基线,并通过 `config.set` 提交修改;Gateway 停止或启动中时,同一个协调器只更新解析后的 JSON5 配置文件,不会因此启动 Gateway。因此,普通的 Provider、Agent、Channel、绑定、Skill 和模型修改不会替换 Gateway 进程。完整重启仅保留给代理等进程启动环境变化和用户显式操作。已确认的进程退出与 WebSocket 关闭继续使用现有的自动重连路径。WebSocket 心跳缺失只更新诊断和健康状态,不会替换 Gateway 进程,因此延迟处理 pong 不会中断长时间运行的任务。认证配置写入 SQLite 后,ClawX 会调用 OpenClaw 的 `secrets.reload`,让运行中的 Agent 无需重启即可读取新凭据。 Chat 使用由 Electron Main 持有的 ACP stdio bridge。Renderer 接收类型化 host events,并渲染内存中的 ACP timeline。Gateway 仍负责 providers、models、skills、workspace、settings、diagnostics 和 media configuration 等非 Chat 能力。 diff --git a/electron/gateway/manager.ts b/electron/gateway/manager.ts index 6029a824..a30818ad 100644 --- a/electron/gateway/manager.ts +++ b/electron/gateway/manager.ts @@ -190,7 +190,6 @@ export class GatewayManager extends EventEmitter { private readonly lifecycleController = new GatewayLifecycleController(); private readonly restartController = new GatewayRestartController(); private readonly restartGovernor = new GatewayRestartGovernor(); - private initialReadyHeartbeatRecoveryTimer: NodeJS.Timeout | null = null; private upgradeSnapshotCleanupAttempted = false; private externalShutdownSupported: boolean | null = null; private reconnectAttemptsTotal = 0; @@ -200,7 +199,6 @@ export class GatewayManager extends EventEmitter { private static readonly HEARTBEAT_MAX_MISSES = 4; public static readonly RESTART_COOLDOWN_MS = 5_000; private static readonly GATEWAY_READY_FALLBACK_PROBE_DELAYS_MS = [1_500, 3_000, 5_000, 8_000, 12_000, 30_000] as const; - private static readonly INITIAL_READY_HEARTBEAT_RECOVERY_GRACE_MS = 5 * 60_000; private lastRestartAt = 0; /** Set by scheduleReconnect() before calling start() to signal auto-reconnect. */ private isAutoReconnectStart = false; @@ -244,7 +242,6 @@ export class GatewayManager extends EventEmitter { this.on('gateway:ready', () => { this.resetGatewayReadyFallback(); - this.clearInitialReadyHeartbeatRecoveryTimer(); if (this.status.state === 'running' && !this.status.gatewayReady) { logger.info('Gateway subsystems ready (event received)'); this.setStatus({ gatewayReady: true }); @@ -686,7 +683,6 @@ export class GatewayManager extends EventEmitter { this.connectionMonitor.clear(); this.restartController.clearDebounceTimer(); this.resetGatewayReadyFallback(); - this.clearInitialReadyHeartbeatRecoveryTimer(); } private clearGatewayReadyFallbackTimer(): void { @@ -893,7 +889,6 @@ export class GatewayManager extends EventEmitter { } private recordGatewayAlive(): void { - this.clearInitialReadyHeartbeatRecoveryTimer(); this.diagnostics.lastAliveAt = Date.now(); this.diagnostics.consecutiveHeartbeatMisses = 0; } @@ -1140,7 +1135,7 @@ export class GatewayManager extends EventEmitter { } /** - * Start ping interval to keep connection alive + * Observe Gateway control-plane responsiveness without owning process recovery. */ private startPing(): void { this.connectionMonitor.startPing({ @@ -1155,62 +1150,15 @@ export class GatewayManager extends EventEmitter { onHeartbeatTimeout: ({ consecutiveMisses, timeoutMs }) => { this.recordHeartbeatTimeout(consecutiveMisses); const pid = this.process?.pid ?? 'unknown'; - const shouldAttemptRecovery = this.shouldReconnect && this.status.state === 'running'; logger.warn( `Gateway heartbeat: ${consecutiveMisses} consecutive pong misses ` + - `(timeout=${timeoutMs}ms, pid=${pid}, state=${this.status.state}, autoReconnect=${this.shouldReconnect}).`, + `(timeout=${timeoutMs}ms, pid=${pid}, state=${this.status.state}, autoReconnect=${this.shouldReconnect}). ` + + 'No restart requested; relying on process exit and socket close recovery.', ); - if (!shouldAttemptRecovery) { - logger.warn('Gateway heartbeat recovery skipped (lifecycle is not in auto-recoverable running state)'); - return; - } - const initialReadyRecoveryDelayMs = this.getInitialReadyHeartbeatRecoveryDelayMs(); - if (initialReadyRecoveryDelayMs > 0) { - logger.warn( - `Gateway heartbeat recovery deferred while waiting for initial gateway.ready ` + - `(retryAfterMs=${initialReadyRecoveryDelayMs})`, - ); - this.scheduleInitialReadyHeartbeatRecovery(initialReadyRecoveryDelayMs); - return; - } - logger.warn('Gateway heartbeat recovery: restarting unresponsive gateway process'); - void this.restart().catch((error) => { - logger.warn('Gateway heartbeat recovery failed:', error); - }); }, }); } - private getInitialReadyHeartbeatRecoveryDelayMs(now = Date.now()): number { - if (this.status.gatewayReady || !this.status.connectedAt) return 0; - const connectedForMs = Math.max(0, now - this.status.connectedAt); - return Math.max(0, GatewayManager.INITIAL_READY_HEARTBEAT_RECOVERY_GRACE_MS - connectedForMs); - } - - private scheduleInitialReadyHeartbeatRecovery(delayMs: number): void { - if (this.initialReadyHeartbeatRecoveryTimer) return; - this.initialReadyHeartbeatRecoveryTimer = setTimeout(() => { - this.initialReadyHeartbeatRecoveryTimer = null; - if ( - !this.shouldReconnect - || this.status.state !== 'running' - || this.status.gatewayReady - ) { - return; - } - logger.warn('Gateway heartbeat recovery: initial gateway.ready grace expired, restarting unresponsive gateway process'); - void this.restart().catch((error) => { - logger.warn('Gateway heartbeat recovery failed:', error); - }); - }, delayMs); - } - - private clearInitialReadyHeartbeatRecoveryTimer(): void { - if (!this.initialReadyHeartbeatRecoveryTimer) return; - clearTimeout(this.initialReadyHeartbeatRecoveryTimer); - this.initialReadyHeartbeatRecoveryTimer = null; - } - private async cleanupOpenClawUpgradeSnapshot(): Promise { if (this.upgradeSnapshotCleanupAttempted) return; this.upgradeSnapshotCleanupAttempted = true; diff --git a/harness/specs/rules/gateway-heartbeat-safety.md b/harness/specs/rules/gateway-heartbeat-safety.md new file mode 100644 index 00000000..38d5d919 --- /dev/null +++ b/harness/specs/rules/gateway-heartbeat-safety.md @@ -0,0 +1,18 @@ +--- +id: gateway-heartbeat-safety +title: Gateway Heartbeat Safety +type: ai-coding-rule +appliesTo: + - gateway-backend-communication +requiredTests: + - tests/unit/gateway-manager-heartbeat.test.ts + - tests/unit/gateway-manager-diagnostics.test.ts +--- + +WebSocket heartbeat misses are availability and diagnostic evidence, not proof that the local Gateway process is dead. + +Reaching the heartbeat miss threshold must update diagnostics and health state, but must not by itself terminate the socket, kill the owned Gateway process, or request `GatewayManager.restart`. Long-running model, tool, compaction, and scheduled work may temporarily block Gateway control-plane responses while remaining valid. + +Automatic lifecycle recovery remains owned by authoritative transport and process signals such as child-process exit, WebSocket close, and Gateway restart close code 1012. Explicit user restart remains available. + +Do not weaken this rule by only increasing heartbeat intervals or miss thresholds. A timeout change delays false recovery but does not make missing pong frames proof of process death. diff --git a/harness/specs/scenarios/gateway-backend-communication.md b/harness/specs/scenarios/gateway-backend-communication.md index c9430aac..972c38e5 100644 --- a/harness/specs/scenarios/gateway-backend-communication.md +++ b/harness/specs/scenarios/gateway-backend-communication.md @@ -51,6 +51,7 @@ requiredRules: - host-api-fallback-policy - host-events-fallback-policy - gateway-readiness-policy + - gateway-heartbeat-safety - channel-plugin-migration-guards - capability-owner-resolution - active-config-guards @@ -96,3 +97,5 @@ Scheduled-task history is Main-owned backend data. Current OpenClaw versions mus The local HTML Preview privileged bridge is also Main-owned: Renderer may load a validated local HTML file or open that current file externally through the typed Host API. The guest is an implementation detail of the existing `preview` tab; there is no `web-browser` artifact tab or general address navigation. The durable guest contract is `harness/reference/web-browser.md`. Gateway session-catalog subscription, normalization, ordered list/event replay, attention transitions, and reconnect recovery are documented in `harness/reference/sidebar-session-attention.md`. + +Gateway WebSocket heartbeat misses are diagnostic availability signals only. They may mark health unresponsive, but must not terminate the socket or replace the Gateway process; authoritative process-exit and socket-close signals retain automatic lifecycle recovery ownership. diff --git a/harness/specs/scenarios/gateway-startup-diagnostics.md b/harness/specs/scenarios/gateway-startup-diagnostics.md index d97c5855..c871aa4e 100644 --- a/harness/specs/scenarios/gateway-startup-diagnostics.md +++ b/harness/specs/scenarios/gateway-startup-diagnostics.md @@ -12,6 +12,7 @@ requiredProfiles: - comms requiredRules: - gateway-readiness-policy + - gateway-heartbeat-safety - renderer-main-boundary - backend-communication-boundary - api-client-transport-policy @@ -32,6 +33,8 @@ ClawX should prefer OpenClaw-native signals over stderr string matching: stderr is supporting evidence only. It should not be the primary source for deciding whether the Gateway is ready, blocked, or should be restarted. +WebSocket heartbeat misses likewise prove only that the Gateway control plane did not answer within the observation window. They update heartbeat diagnostics and may mark health unresponsive, but do not terminate the socket or restart the process. Process exit and socket close remain the authoritative automatic recovery signals, so long-running work is not killed solely because pong handling is delayed. + ## Failure Shape Treat these as the same incident family until proven otherwise: @@ -195,7 +198,7 @@ Expected behavior: - Do not mark Gateway fully ready from a pure timer fallback. - The fallback must probe `system-presence` before emitting ready. -- Heartbeat recovery may defer restart during the initial grace window, but it should not loop restart while the Gateway is still performing startup work. +- Heartbeat misses remain observable during startup work, but do not trigger process restart. ### Capability Degraded But Core Alive @@ -275,6 +278,7 @@ pnpm exec openclaw gateway call status >/tmp/clawx-status.json - `health` and `status` are captured in Gateway diagnostics when available. - Memory doctor calls return when the memory capability is available. - `doctor.memory.*` and `channels.status` failures degrade their capability only and do not trigger Gateway restart. +- Consecutive heartbeat misses update diagnostics and health state without terminating the socket or replacing the Gateway process. - Logs no longer repeat stale runtime cache or escaped managed-skill symlink warnings for entries ClawX can safely clean. ## Required Regression Coverage diff --git a/harness/specs/tasks/make-gateway-heartbeat-observability-only.md b/harness/specs/tasks/make-gateway-heartbeat-observability-only.md new file mode 100644 index 00000000..6bcf76b5 --- /dev/null +++ b/harness/specs/tasks/make-gateway-heartbeat-observability-only.md @@ -0,0 +1,47 @@ +--- +id: make-gateway-heartbeat-observability-only +title: Make Gateway heartbeat misses observability-only +scenario: gateway-backend-communication +taskType: runtime-bridge +intent: Prevent long-running OpenClaw work from being interrupted when the Gateway control plane temporarily stops answering WebSocket ping frames. +touchedAreas: + - harness/specs/tasks/make-gateway-heartbeat-observability-only.md + - harness/specs/rules/gateway-heartbeat-safety.md + - harness/specs/scenarios/gateway-backend-communication.md + - harness/specs/scenarios/gateway-startup-diagnostics.md + - electron/gateway/manager.ts + - tests/unit/gateway-manager-heartbeat.test.ts + - tests/unit/gateway-manager-diagnostics.test.ts + - README.md + - README.zh-CN.md + - README.ja-JP.md +expectedUserBehavior: + - Long-running model, tool, and scheduled tasks are not interrupted solely because Gateway pong frames are delayed. + - Consecutive heartbeat misses still mark Gateway health as unresponsive and remain visible in diagnostics. + - A real Gateway process exit or WebSocket close continues to use the existing reconnect and crash-recovery paths. + - Users can still explicitly restart Gateway when they decide recovery is necessary. +requiredProfiles: + - fast + - comms +requiredRules: + - gateway-heartbeat-safety + - gateway-readiness-policy + - backend-communication-boundary + - comms-regression + - docs-sync +requiredTests: + - tests/unit/gateway-manager-heartbeat.test.ts + - tests/unit/gateway-manager-diagnostics.test.ts + - tests/unit/gateway-connection-monitor.test.ts +acceptance: + - Reaching the consecutive heartbeat miss threshold records heartbeat timeout diagnostics but does not terminate the socket or call GatewayManager.restart on any platform. + - Heartbeat recovery through a pong or any incoming Gateway message resets the miss counter as before. + - Gateway health becomes unresponsive at the existing miss threshold. + - Process exit, socket close, explicit restart, and code-1012 reconnect behavior are unchanged. + - The initial gateway.ready heartbeat restart timer is removed because heartbeat misses no longer own process recovery. + - README translations describe heartbeat misses as diagnostic evidence rather than a process-restart trigger. +docs: + required: true +--- + +Use this task spec for the first-stage heartbeat safety change. It deliberately does not add active-work tracking or a new automatic hard-restart policy. diff --git a/tests/unit/gateway-manager-diagnostics.test.ts b/tests/unit/gateway-manager-diagnostics.test.ts index 8fc6a3c7..2554d6c6 100644 --- a/tests/unit/gateway-manager-diagnostics.test.ts +++ b/tests/unit/gateway-manager-diagnostics.test.ts @@ -215,7 +215,7 @@ describe('GatewayManager diagnostics', () => { expect(snapshot.core.rpcRouter).toBe('blocked'); }); - it('restarts windows gateway on heartbeat misses and marks health unresponsive', async () => { + it('marks windows gateway health unresponsive without restarting on heartbeat misses', async () => { Object.defineProperty(process, 'platform', { value: 'win32' }); const { GatewayManager } = await import('@electron/gateway/manager'); @@ -241,14 +241,12 @@ describe('GatewayManager diagnostics', () => { (manager as unknown as { startPing: () => void }).startPing(); vi.advanceTimersByTime(300_000); - expect(restartSpy).toHaveBeenCalledTimes(1); + expect(restartSpy).not.toHaveBeenCalled(); + expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(4); const health = buildGatewayHealthSummary({ status: { state: 'running', port: 18789 }, - diagnostics: { - ...manager.getDiagnostics(), - consecutiveHeartbeatMisses: 4, - }, + diagnostics: manager.getDiagnostics(), }); expect(health.state).toBe('unresponsive'); expect(health.reasons).toContain('gateway_unresponsive'); diff --git a/tests/unit/gateway-manager-heartbeat.test.ts b/tests/unit/gateway-manager-heartbeat.test.ts index 4038d93c..8500c43a 100644 --- a/tests/unit/gateway-manager-heartbeat.test.ts +++ b/tests/unit/gateway-manager-heartbeat.test.ts @@ -1,6 +1,6 @@ // @vitest-environment node -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('electron', () => ({ app: { @@ -12,7 +12,7 @@ vi.mock('electron', () => ({ }, })); -describe('GatewayManager heartbeat recovery', () => { +describe('GatewayManager heartbeat observability', () => { const originalPlatform = process.platform; beforeEach(() => { @@ -28,7 +28,7 @@ describe('GatewayManager heartbeat recovery', () => { Object.defineProperty(process, 'platform', { value: originalPlatform }); }); - it('restarts after consecutive heartbeat misses reach threshold', async () => { + it('records consecutive heartbeat misses without terminating or restarting Gateway', async () => { const { GatewayManager } = await import('@electron/gateway/manager'); const manager = new GatewayManager(); @@ -41,24 +41,26 @@ describe('GatewayManager heartbeat recovery', () => { (manager as unknown as { ws: typeof ws }).ws = ws; (manager as unknown as { shouldReconnect: boolean }).shouldReconnect = true; - (manager as unknown as { status: { state: string; port: number } }).status = { + (manager as unknown as { status: { state: string; port: number; gatewayReady: boolean } }).status = { state: 'running', port: 18789, + gatewayReady: true, }; const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue(); (manager as unknown as { startPing: () => void }).startPing(); - vi.advanceTimersByTime(300_000); expect(ws.ping).toHaveBeenCalledTimes(4); expect(ws.terminate).not.toHaveBeenCalled(); - expect(restartSpy).toHaveBeenCalledTimes(1); + expect(restartSpy).not.toHaveBeenCalled(); + expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(4); + expect(manager.getDiagnostics().lastHeartbeatTimeoutAt).toBe(Date.now()); (manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear(); }); - it('defers heartbeat restart while initial gateway.ready is still within grace', async () => { + it('does not schedule a delayed restart while initial gateway.ready is pending', async () => { const { GatewayManager } = await import('@electron/gateway/manager'); const manager = new GatewayManager(); @@ -69,32 +71,27 @@ describe('GatewayManager heartbeat recovery', () => { on: vi.fn(), }; - const connectedAt = Date.now(); (manager as unknown as { ws: typeof ws }).ws = ws; (manager as unknown as { shouldReconnect: boolean }).shouldReconnect = true; (manager as unknown as { status: { state: string; port: number; connectedAt: number; gatewayReady: boolean } }).status = { state: 'running', port: 18789, - connectedAt, + connectedAt: Date.now(), gatewayReady: false, }; const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue(); (manager as unknown as { startPing: () => void }).startPing(); + vi.advanceTimersByTime(600_000); - vi.advanceTimersByTime(120_000); + expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(4); + expect(ws.terminate).not.toHaveBeenCalled(); expect(restartSpy).not.toHaveBeenCalled(); - vi.advanceTimersByTime(179_999); - expect(restartSpy).not.toHaveBeenCalled(); - - vi.advanceTimersByTime(1); - expect(restartSpy).toHaveBeenCalledTimes(1); - (manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear(); }); - it('does not restart when heartbeat is recovered by incoming messages', async () => { + it('resets heartbeat diagnostics when responsiveness recovers through an incoming message', async () => { const { GatewayManager } = await import('@electron/gateway/manager'); const manager = new GatewayManager(); @@ -114,50 +111,20 @@ describe('GatewayManager heartbeat recovery', () => { const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue(); (manager as unknown as { startPing: () => void }).startPing(); + vi.advanceTimersByTime(300_000); + expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(4); - vi.advanceTimersByTime(60_000); // ping #1 - vi.advanceTimersByTime(60_000); // miss #1 + ping #2 (manager as unknown as { handleMessage: (message: unknown) => void }).handleMessage('alive'); - vi.advanceTimersByTime(60_000); // recovered, ping #3 - vi.advanceTimersByTime(60_000); // miss #1 + ping #4 - vi.advanceTimersByTime(60_000); // miss #2 + ping #5 - + expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(0); + expect(manager.getDiagnostics().lastAliveAt).toBe(Date.now()); expect(ws.terminate).not.toHaveBeenCalled(); expect(restartSpy).not.toHaveBeenCalled(); (manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear(); }); - it('skips heartbeat recovery when auto-reconnect is disabled', async () => { - const { GatewayManager } = await import('@electron/gateway/manager'); - const manager = new GatewayManager(); - - const ws = { - readyState: 1, - ping: vi.fn(), - terminate: vi.fn(), - on: vi.fn(), - }; - - (manager as unknown as { ws: typeof ws }).ws = ws; - (manager as unknown as { shouldReconnect: boolean }).shouldReconnect = false; - (manager as unknown as { status: { state: string; port: number } }).status = { - state: 'running', - port: 18789, - }; - const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue(); - - (manager as unknown as { startPing: () => void }).startPing(); - - vi.advanceTimersByTime(300_000); - - expect(restartSpy).not.toHaveBeenCalled(); - - (manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear(); - }); - - it('restarts after consecutive heartbeat misses on windows', async () => { + it('keeps heartbeat misses observability-only on windows', async () => { Object.defineProperty(process, 'platform', { value: 'win32' }); const { GatewayManager } = await import('@electron/gateway/manager'); @@ -179,11 +146,12 @@ describe('GatewayManager heartbeat recovery', () => { const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue(); (manager as unknown as { startPing: () => void }).startPing(); - vi.advanceTimersByTime(300_000); expect(ws.ping).toHaveBeenCalledTimes(4); - expect(restartSpy).toHaveBeenCalledTimes(1); + expect(ws.terminate).not.toHaveBeenCalled(); + expect(restartSpy).not.toHaveBeenCalled(); + expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(4); (manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear(); }); From 46ff98c2197563f2cd1caae60265a65a0423044f Mon Sep 17 00:00:00 2001 From: paisley <8197966+su8su@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:10:16 +0800 Subject: [PATCH 2/3] release 0.5.3 (#1229) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f9c3602c..1015ade4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clawx", - "version": "0.5.2", + "version": "0.5.3", "pnpm": { "onlyBuiltDependencies": [ "@discordjs/opus", From 0909120b03baf463bbd9e2171d8ba771914bee6a Mon Sep 17 00:00:00 2001 From: paisley <8197966+su8su@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:44:53 +0800 Subject: [PATCH 3/3] fix(gateway): restart after 10 missed heartbeats (#1231) --- README.ja-JP.md | 2 +- README.md | 2 +- README.zh-CN.md | 2 +- electron/gateway/manager.ts | 16 +++-- electron/utils/gateway-health.ts | 2 +- .../specs/rules/gateway-heartbeat-safety.md | 8 +-- .../gateway-backend-communication.md | 2 +- .../scenarios/gateway-startup-diagnostics.md | 6 +- ...ke-gateway-heartbeat-observability-only.md | 2 +- ...way-heartbeat-recovery-after-ten-misses.md | 51 +++++++++++++++ tests/unit/gateway-connection-monitor.test.ts | 2 +- .../unit/gateway-manager-diagnostics.test.ts | 8 +-- tests/unit/gateway-manager-heartbeat.test.ts | 63 ++++++++++++------- 13 files changed, 122 insertions(+), 44 deletions(-) create mode 100644 harness/specs/tasks/restore-gateway-heartbeat-recovery-after-ten-misses.md diff --git a/README.ja-JP.md b/README.ja-JP.md index 286dfe7c..6ffd554b 100644 --- a/README.ja-JP.md +++ b/README.ja-JP.md @@ -226,7 +226,7 @@ ClawXには、Electron、OpenClaw Gateway、またはTelegramなどのチャネ ClawXは、**デュアルプロセス + Host API 統一アクセス**構成を採用しています。Renderer は単一クライアント抽象を呼び出し、プロトコル選択とライフサイクルは Main が管理します: -OpenClaw の設定配信も Electron Main が一元管理します。Gateway の実行中は `config.get` の正規スナップショットを基準にし、変更を `config.set` でコミットします。Gateway が停止中または起動中の場合は、同じコーディネーターが解決済みの JSON5 設定ファイルだけを更新し、Gateway を起動しません。そのため、通常の Provider、Agent、Channel、バインディング、Skill、モデル変更では Gateway プロセスを置き換えません。完全な再起動は、プロキシなどのプロセス起動環境の変更とユーザーによる明示的な操作に限定されます。確認済みのプロセス終了と WebSocket 切断では、既存の自動再接続経路が引き続き使用されます。WebSocket のハートビート欠落は診断とヘルス状態だけを更新し、Gateway プロセスを置き換えないため、pong 処理の遅延によって長時間実行中の処理が中断されることはありません。認証プロファイルを SQLite に書き込んだ後は OpenClaw の `secrets.reload` を呼び出し、実行中の Agent がプロセス再起動なしで新しい認証情報を読み取れるようにします。 +OpenClaw の設定配信も Electron Main が一元管理します。Gateway の実行中は `config.get` の正規スナップショットを基準にし、変更を `config.set` でコミットします。Gateway が停止中または起動中の場合は、同じコーディネーターが解決済みの JSON5 設定ファイルだけを更新し、Gateway を起動しません。そのため、通常の Provider、Agent、Channel、バインディング、Skill、モデル変更では Gateway プロセスを置き換えません。完全な再起動は、プロキシなどのプロセス起動環境の変更とユーザーによる明示的な操作に限定されます。確認済みのプロセス終了と WebSocket 切断では、既存の自動再接続経路が引き続き使用されます。WebSocket ハートビートの連続 9 回目までの欠落は診断のみとし、短い pong 遅延で長時間実行中の処理を中断しません。pong または任意のメッセージを受信するとカウントをリセットし、10 回連続で応答がない場合にのみ、ライフサイクル保護付きの Gateway 自動再起動を要求します。認証プロファイルを SQLite に書き込んだ後は OpenClaw の `secrets.reload` を呼び出し、実行中の Agent がプロセス再起動なしで新しい認証情報を読み取れるようにします。 Chat は Electron Main が所有する ACP stdio bridge を使用します。Renderer は型付き host event を受け取り、メモリ上の ACP timeline を描画します。Gateway は providers、models、skills、workspace、settings、diagnostics、media configuration などの非 Chat 機能を引き続き担当します。 diff --git a/README.md b/README.md index 79d8557a..e8d49192 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ Notes: ClawX employs a **dual-process architecture** with a unified host API layer. The renderer talks to a single client abstraction, while Electron Main owns protocol selection and process lifecycle: -Electron Main also owns OpenClaw configuration delivery. While the Gateway is running, ClawX reads the authoritative `config.get` snapshot and commits changes with `config.set`; while it is stopped or starting, the same coordinator updates the resolved JSON5 config file without starting the Gateway. Ordinary provider, agent, channel, binding, skill, and model changes therefore do not replace the Gateway process. Full restarts remain for process-launch environment changes such as proxy settings and explicit user actions. Confirmed process exits and WebSocket closes retain their existing automatic reconnect paths. WebSocket heartbeat misses update diagnostics and health state but do not replace the Gateway process, so delayed pong handling cannot interrupt long-running work. Auth-profile SQLite updates use OpenClaw's `secrets.reload` RPC so running agents see new credentials without a process restart. +Electron Main also owns OpenClaw configuration delivery. While the Gateway is running, ClawX reads the authoritative `config.get` snapshot and commits changes with `config.set`; while it is stopped or starting, the same coordinator updates the resolved JSON5 config file without starting the Gateway. Ordinary provider, agent, channel, binding, skill, and model changes therefore do not replace the Gateway process. Full restarts remain for process-launch environment changes such as proxy settings and explicit user actions. Confirmed process exits and WebSocket closes retain their existing automatic reconnect paths. The first nine consecutive WebSocket heartbeat misses remain diagnostic-only so brief pong delays do not interrupt long-running work; a pong or any incoming message resets the count, while a tenth consecutive miss requests guarded automatic Gateway recovery. Auth-profile SQLite updates use OpenClaw's `secrets.reload` RPC so running agents see new credentials without a process restart. Chat uses an ACP stdio bridge owned by Electron Main. Renderer receives typed host events and renders an in-memory ACP timeline. Gateway remains responsible for non-Chat capabilities such as providers, models, skills, workspace, settings, diagnostics, and media configuration. diff --git a/README.zh-CN.md b/README.zh-CN.md index 148c505e..da47a31e 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -227,7 +227,7 @@ ClawX 内置了代理设置,适用于需要通过本地代理客户端访问 ClawX 采用 **双进程 + Host API 统一接入架构**。渲染进程只调用统一客户端抽象,协议选择与进程生命周期由 Electron 主进程统一管理: -OpenClaw 配置交付也统一由 Electron Main 管理。Gateway 运行时,ClawX 以 `config.get` 返回的权威快照为基线,并通过 `config.set` 提交修改;Gateway 停止或启动中时,同一个协调器只更新解析后的 JSON5 配置文件,不会因此启动 Gateway。因此,普通的 Provider、Agent、Channel、绑定、Skill 和模型修改不会替换 Gateway 进程。完整重启仅保留给代理等进程启动环境变化和用户显式操作。已确认的进程退出与 WebSocket 关闭继续使用现有的自动重连路径。WebSocket 心跳缺失只更新诊断和健康状态,不会替换 Gateway 进程,因此延迟处理 pong 不会中断长时间运行的任务。认证配置写入 SQLite 后,ClawX 会调用 OpenClaw 的 `secrets.reload`,让运行中的 Agent 无需重启即可读取新凭据。 +OpenClaw 配置交付也统一由 Electron Main 管理。Gateway 运行时,ClawX 以 `config.get` 返回的权威快照为基线,并通过 `config.set` 提交修改;Gateway 停止或启动中时,同一个协调器只更新解析后的 JSON5 配置文件,不会因此启动 Gateway。因此,普通的 Provider、Agent、Channel、绑定、Skill 和模型修改不会替换 Gateway 进程。完整重启仅保留给代理等进程启动环境变化和用户显式操作。已确认的进程退出与 WebSocket 关闭继续使用现有的自动重连路径。连续前 9 次 WebSocket 心跳无响应只更新诊断,不会因短暂的 pong 延迟中断长时间运行的任务;收到 pong 或任意消息会重置计数,连续第 10 次无响应时才请求受生命周期保护的 Gateway 自动重启。认证配置写入 SQLite 后,ClawX 会调用 OpenClaw 的 `secrets.reload`,让运行中的 Agent 无需重启即可读取新凭据。 Chat 使用由 Electron Main 持有的 ACP stdio bridge。Renderer 接收类型化 host events,并渲染内存中的 ACP timeline。Gateway 仍负责 providers、models、skills、workspace、settings、diagnostics 和 media configuration 等非 Chat 能力。 diff --git a/electron/gateway/manager.ts b/electron/gateway/manager.ts index a30818ad..7a5b8338 100644 --- a/electron/gateway/manager.ts +++ b/electron/gateway/manager.ts @@ -196,7 +196,7 @@ export class GatewayManager extends EventEmitter { private reconnectSuccessTotal = 0; private static readonly HEARTBEAT_INTERVAL_MS = 60_000; private static readonly HEARTBEAT_TIMEOUT_MS = 30_000; - private static readonly HEARTBEAT_MAX_MISSES = 4; + private static readonly HEARTBEAT_MAX_MISSES = 10; public static readonly RESTART_COOLDOWN_MS = 5_000; private static readonly GATEWAY_READY_FALLBACK_PROBE_DELAYS_MS = [1_500, 3_000, 5_000, 8_000, 12_000, 30_000] as const; private lastRestartAt = 0; @@ -1135,7 +1135,7 @@ export class GatewayManager extends EventEmitter { } /** - * Observe Gateway control-plane responsiveness without owning process recovery. + * Observe Gateway control-plane responsiveness and recover after a sustained outage. */ private startPing(): void { this.connectionMonitor.startPing({ @@ -1150,11 +1150,19 @@ export class GatewayManager extends EventEmitter { onHeartbeatTimeout: ({ consecutiveMisses, timeoutMs }) => { this.recordHeartbeatTimeout(consecutiveMisses); const pid = this.process?.pid ?? 'unknown'; + const shouldAttemptRecovery = this.shouldReconnect && this.status.state === 'running'; logger.warn( `Gateway heartbeat: ${consecutiveMisses} consecutive pong misses ` + - `(timeout=${timeoutMs}ms, pid=${pid}, state=${this.status.state}, autoReconnect=${this.shouldReconnect}). ` + - 'No restart requested; relying on process exit and socket close recovery.', + `(timeout=${timeoutMs}ms, pid=${pid}, state=${this.status.state}, autoReconnect=${this.shouldReconnect}).`, ); + if (!shouldAttemptRecovery) { + logger.warn('Gateway heartbeat recovery skipped (lifecycle is not in auto-recoverable running state)'); + return; + } + logger.warn('Gateway heartbeat recovery: restarting persistently unresponsive gateway process'); + void this.restart().catch((error) => { + logger.warn('Gateway heartbeat recovery failed:', error); + }); }, }); } diff --git a/electron/utils/gateway-health.ts b/electron/utils/gateway-health.ts index 4005f2d2..a6d37165 100644 --- a/electron/utils/gateway-health.ts +++ b/electron/utils/gateway-health.ts @@ -13,7 +13,7 @@ type BuildGatewayHealthSummaryOptions = { }; const CHANNEL_STATUS_FAILURE_WINDOW_MS = 2 * 60_000; -const HEARTBEAT_MISS_THRESHOLD = 4; +const HEARTBEAT_MISS_THRESHOLD = 10; export function buildGatewayHealthSummary( options: BuildGatewayHealthSummaryOptions, diff --git a/harness/specs/rules/gateway-heartbeat-safety.md b/harness/specs/rules/gateway-heartbeat-safety.md index 38d5d919..9c351329 100644 --- a/harness/specs/rules/gateway-heartbeat-safety.md +++ b/harness/specs/rules/gateway-heartbeat-safety.md @@ -9,10 +9,10 @@ requiredTests: - tests/unit/gateway-manager-diagnostics.test.ts --- -WebSocket heartbeat misses are availability and diagnostic evidence, not proof that the local Gateway process is dead. +WebSocket heartbeat misses are availability evidence. A short sequence is not proof that the local Gateway process is dead because long-running model, tool, compaction, and scheduled work may temporarily delay Gateway control-plane responses. -Reaching the heartbeat miss threshold must update diagnostics and health state, but must not by itself terminate the socket, kill the owned Gateway process, or request `GatewayManager.restart`. Long-running model, tool, compaction, and scheduled work may temporarily block Gateway control-plane responses while remaining valid. +Misses one through nine must remain diagnostic-only: they must not terminate the socket, kill the owned Gateway process, or request `GatewayManager.restart`. A pong or any incoming Gateway message resets the sequence. -Automatic lifecycle recovery remains owned by authoritative transport and process signals such as child-process exit, WebSocket close, and Gateway restart close code 1012. Explicit user restart remains available. +After ten consecutive missed responses, ClawX may treat the Gateway as persistently unresponsive and request the guarded `GatewayManager.restart` path only when auto-recovery is enabled and lifecycle state is still `running`. The heartbeat callback must not directly terminate the socket or process, and it must request recovery at most once per uninterrupted miss sequence. -Do not weaken this rule by only increasing heartbeat intervals or miss thresholds. A timeout change delays false recovery but does not make missing pong frames proof of process death. +Authoritative child-process exit, WebSocket close, and Gateway restart close code 1012 signals retain their existing automatic lifecycle paths. Explicit user restart remains available. diff --git a/harness/specs/scenarios/gateway-backend-communication.md b/harness/specs/scenarios/gateway-backend-communication.md index 972c38e5..4cebceba 100644 --- a/harness/specs/scenarios/gateway-backend-communication.md +++ b/harness/specs/scenarios/gateway-backend-communication.md @@ -98,4 +98,4 @@ The local HTML Preview privileged bridge is also Main-owned: Renderer may load a Gateway session-catalog subscription, normalization, ordered list/event replay, attention transitions, and reconnect recovery are documented in `harness/reference/sidebar-session-attention.md`. -Gateway WebSocket heartbeat misses are diagnostic availability signals only. They may mark health unresponsive, but must not terminate the socket or replace the Gateway process; authoritative process-exit and socket-close signals retain automatic lifecycle recovery ownership. +Gateway WebSocket heartbeat misses are diagnostic availability signals for the first nine consecutive misses and must not interrupt long-running work during that window. A pong or any incoming Gateway message resets the sequence. On the tenth consecutive miss, Main may request the guarded Gateway restart path when auto-recovery is enabled and lifecycle state is still running; the heartbeat callback must not directly terminate the socket or process. Authoritative process-exit and socket-close signals retain their existing automatic lifecycle paths. diff --git a/harness/specs/scenarios/gateway-startup-diagnostics.md b/harness/specs/scenarios/gateway-startup-diagnostics.md index c871aa4e..5d5fc27d 100644 --- a/harness/specs/scenarios/gateway-startup-diagnostics.md +++ b/harness/specs/scenarios/gateway-startup-diagnostics.md @@ -33,7 +33,7 @@ ClawX should prefer OpenClaw-native signals over stderr string matching: stderr is supporting evidence only. It should not be the primary source for deciding whether the Gateway is ready, blocked, or should be restarted. -WebSocket heartbeat misses likewise prove only that the Gateway control plane did not answer within the observation window. They update heartbeat diagnostics and may mark health unresponsive, but do not terminate the socket or restart the process. Process exit and socket close remain the authoritative automatic recovery signals, so long-running work is not killed solely because pong handling is delayed. +WebSocket heartbeat misses show that the Gateway control plane did not answer within the observation window. The first nine consecutive misses remain diagnostic-only so transient pong delays do not interrupt long-running work. A pong or any incoming message resets the sequence. A tenth consecutive miss marks persistent unresponsiveness and may request the guarded Gateway restart path when auto-recovery is enabled and lifecycle state is still running. Process exit and socket close retain their existing automatic recovery paths. ## Failure Shape @@ -198,7 +198,7 @@ Expected behavior: - Do not mark Gateway fully ready from a pure timer fallback. - The fallback must probe `system-presence` before emitting ready. -- Heartbeat misses remain observable during startup work, but do not trigger process restart. +- Heartbeat misses remain observable during startup work; only ten uninterrupted misses may request guarded process recovery. ### Capability Degraded But Core Alive @@ -278,7 +278,7 @@ pnpm exec openclaw gateway call status >/tmp/clawx-status.json - `health` and `status` are captured in Gateway diagnostics when available. - Memory doctor calls return when the memory capability is available. - `doctor.memory.*` and `channels.status` failures degrade their capability only and do not trigger Gateway restart. -- Consecutive heartbeat misses update diagnostics and health state without terminating the socket or replacing the Gateway process. +- The first nine consecutive heartbeat misses do not replace the Gateway process; the tenth records unresponsive diagnostics and requests one guarded restart when lifecycle auto-recovery is allowed. - Logs no longer repeat stale runtime cache or escaped managed-skill symlink warnings for entries ClawX can safely clean. ## Required Regression Coverage diff --git a/harness/specs/tasks/make-gateway-heartbeat-observability-only.md b/harness/specs/tasks/make-gateway-heartbeat-observability-only.md index 6bcf76b5..86db73bc 100644 --- a/harness/specs/tasks/make-gateway-heartbeat-observability-only.md +++ b/harness/specs/tasks/make-gateway-heartbeat-observability-only.md @@ -44,4 +44,4 @@ docs: required: true --- -Use this task spec for the first-stage heartbeat safety change. It deliberately does not add active-work tracking or a new automatic hard-restart policy. +Historical first-stage heartbeat safety task. Its observability-only recovery policy is superseded by `restore-gateway-heartbeat-recovery-after-ten-misses`, which keeps the first nine misses diagnostic-only and permits guarded recovery on the tenth consecutive miss. diff --git a/harness/specs/tasks/restore-gateway-heartbeat-recovery-after-ten-misses.md b/harness/specs/tasks/restore-gateway-heartbeat-recovery-after-ten-misses.md new file mode 100644 index 00000000..c0bc14d6 --- /dev/null +++ b/harness/specs/tasks/restore-gateway-heartbeat-recovery-after-ten-misses.md @@ -0,0 +1,51 @@ +--- +id: restore-gateway-heartbeat-recovery-after-ten-misses +title: Restore Gateway heartbeat recovery after ten misses +scenario: gateway-backend-communication +taskType: runtime-bridge +intent: Recover a persistently unresponsive local Gateway automatically while giving long-running work a longer heartbeat window before process replacement. +touchedAreas: + - harness/specs/tasks/restore-gateway-heartbeat-recovery-after-ten-misses.md + - harness/specs/tasks/make-gateway-heartbeat-observability-only.md + - harness/specs/rules/gateway-heartbeat-safety.md + - harness/specs/scenarios/gateway-backend-communication.md + - harness/specs/scenarios/gateway-startup-diagnostics.md + - electron/gateway/manager.ts + - electron/utils/gateway-health.ts + - tests/unit/gateway-manager-heartbeat.test.ts + - tests/unit/gateway-manager-diagnostics.test.ts + - tests/unit/gateway-connection-monitor.test.ts + - README.md + - README.zh-CN.md + - README.ja-JP.md +expectedUserBehavior: + - One to nine consecutive missed Gateway heartbeat responses remain diagnostic-only and do not interrupt long-running work. + - Ten consecutive missed heartbeat responses mark the Gateway unresponsive and request an automatic restart when lifecycle auto-recovery is enabled and the Gateway is still running. + - A pong or any incoming Gateway message before the tenth miss resets the consecutive-miss counter. + - Process exit, WebSocket close, explicit restart, and code-1012 reconnect behavior remain unchanged. +requiredProfiles: + - fast + - comms +requiredRules: + - gateway-heartbeat-safety + - gateway-readiness-policy + - backend-communication-boundary + - comms-regression + - docs-sync +requiredTests: + - tests/unit/gateway-manager-heartbeat.test.ts + - tests/unit/gateway-manager-diagnostics.test.ts + - tests/unit/gateway-connection-monitor.test.ts +acceptance: + - The Gateway heartbeat threshold is ten consecutive misses. + - Misses one through nine update monitor state without calling GatewayManager.restart or terminating the socket. + - The tenth consecutive miss records timeout diagnostics and calls GatewayManager.restart exactly once when auto-recovery is allowed. + - The tenth miss does not restart when auto-reconnect is disabled or the Gateway is not running. + - Recovery through a pong or any incoming Gateway message resets the sequence, so only ten new consecutive misses can trigger recovery. + - Automatic heartbeat recovery behaves consistently on Windows, macOS, and Linux. + - Documentation in all maintained README translations describes the ten-miss automatic recovery policy. +docs: + required: true +--- + +This task supersedes the recovery policy from `make-gateway-heartbeat-observability-only`: heartbeat misses remain non-authoritative during the first nine misses, while a tenth consecutive miss is treated as persistent unresponsiveness and may request guarded lifecycle recovery. diff --git a/tests/unit/gateway-connection-monitor.test.ts b/tests/unit/gateway-connection-monitor.test.ts index 00b4da49..0e65eae1 100644 --- a/tests/unit/gateway-connection-monitor.test.ts +++ b/tests/unit/gateway-connection-monitor.test.ts @@ -18,7 +18,7 @@ describe('GatewayConnectionMonitor heartbeat', () => { vi.setSystemTime(new Date('2026-03-19T00:00:00.000Z')); }); - it('terminates only after consecutive heartbeat misses reach threshold', () => { + it('invokes the timeout callback only after consecutive heartbeat misses reach threshold', () => { const monitor = new GatewayConnectionMonitor(); const sendPing = vi.fn(); const onHeartbeatTimeout = vi.fn(); diff --git a/tests/unit/gateway-manager-diagnostics.test.ts b/tests/unit/gateway-manager-diagnostics.test.ts index 2554d6c6..696fa63c 100644 --- a/tests/unit/gateway-manager-diagnostics.test.ts +++ b/tests/unit/gateway-manager-diagnostics.test.ts @@ -215,7 +215,7 @@ describe('GatewayManager diagnostics', () => { expect(snapshot.core.rpcRouter).toBe('blocked'); }); - it('marks windows gateway health unresponsive without restarting on heartbeat misses', async () => { + it('marks windows gateway health unresponsive and restarts after ten heartbeat misses', async () => { Object.defineProperty(process, 'platform', { value: 'win32' }); const { GatewayManager } = await import('@electron/gateway/manager'); @@ -239,10 +239,10 @@ describe('GatewayManager diagnostics', () => { const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue(); (manager as unknown as { startPing: () => void }).startPing(); - vi.advanceTimersByTime(300_000); + vi.advanceTimersByTime(660_000); - expect(restartSpy).not.toHaveBeenCalled(); - expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(4); + expect(restartSpy).toHaveBeenCalledTimes(1); + expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(10); const health = buildGatewayHealthSummary({ status: { state: 'running', port: 18789 }, diff --git a/tests/unit/gateway-manager-heartbeat.test.ts b/tests/unit/gateway-manager-heartbeat.test.ts index 8500c43a..4ac9d016 100644 --- a/tests/unit/gateway-manager-heartbeat.test.ts +++ b/tests/unit/gateway-manager-heartbeat.test.ts @@ -12,7 +12,7 @@ vi.mock('electron', () => ({ }, })); -describe('GatewayManager heartbeat observability', () => { +describe('GatewayManager heartbeat recovery', () => { const originalPlatform = process.platform; beforeEach(() => { @@ -28,7 +28,7 @@ describe('GatewayManager heartbeat observability', () => { Object.defineProperty(process, 'platform', { value: originalPlatform }); }); - it('records consecutive heartbeat misses without terminating or restarting Gateway', async () => { + it('restarts only after ten consecutive heartbeat misses', async () => { const { GatewayManager } = await import('@electron/gateway/manager'); const manager = new GatewayManager(); @@ -49,18 +49,30 @@ describe('GatewayManager heartbeat observability', () => { const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue(); (manager as unknown as { startPing: () => void }).startPing(); - vi.advanceTimersByTime(300_000); + vi.advanceTimersByTime(659_999); - expect(ws.ping).toHaveBeenCalledTimes(4); - expect(ws.terminate).not.toHaveBeenCalled(); + expect(ws.ping).toHaveBeenCalledTimes(10); + expect( + (manager as unknown as { connectionMonitor: { getConsecutiveMisses: () => number } }) + .connectionMonitor.getConsecutiveMisses(), + ).toBe(9); expect(restartSpy).not.toHaveBeenCalled(); - expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(4); + expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(0); + + vi.advanceTimersByTime(1); + + expect(ws.terminate).not.toHaveBeenCalled(); + expect(restartSpy).toHaveBeenCalledTimes(1); + expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(10); expect(manager.getDiagnostics().lastHeartbeatTimeoutAt).toBe(Date.now()); + vi.advanceTimersByTime(180_000); + expect(restartSpy).toHaveBeenCalledTimes(1); + (manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear(); }); - it('does not schedule a delayed restart while initial gateway.ready is pending', async () => { + it('does not restart after ten misses when auto-reconnect is disabled', async () => { const { GatewayManager } = await import('@electron/gateway/manager'); const manager = new GatewayManager(); @@ -72,26 +84,24 @@ describe('GatewayManager heartbeat observability', () => { }; (manager as unknown as { ws: typeof ws }).ws = ws; - (manager as unknown as { shouldReconnect: boolean }).shouldReconnect = true; - (manager as unknown as { status: { state: string; port: number; connectedAt: number; gatewayReady: boolean } }).status = { + (manager as unknown as { shouldReconnect: boolean }).shouldReconnect = false; + (manager as unknown as { status: { state: string; port: number } }).status = { state: 'running', port: 18789, - connectedAt: Date.now(), - gatewayReady: false, }; const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue(); (manager as unknown as { startPing: () => void }).startPing(); - vi.advanceTimersByTime(600_000); + vi.advanceTimersByTime(660_000); - expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(4); + expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(10); expect(ws.terminate).not.toHaveBeenCalled(); expect(restartSpy).not.toHaveBeenCalled(); (manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear(); }); - it('resets heartbeat diagnostics when responsiveness recovers through an incoming message', async () => { + it('requires ten new consecutive misses after responsiveness recovers', async () => { const { GatewayManager } = await import('@electron/gateway/manager'); const manager = new GatewayManager(); @@ -111,20 +121,29 @@ describe('GatewayManager heartbeat observability', () => { const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue(); (manager as unknown as { startPing: () => void }).startPing(); - vi.advanceTimersByTime(300_000); - expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(4); + vi.advanceTimersByTime(600_000); + expect( + (manager as unknown as { connectionMonitor: { getConsecutiveMisses: () => number } }) + .connectionMonitor.getConsecutiveMisses(), + ).toBe(9); (manager as unknown as { handleMessage: (message: unknown) => void }).handleMessage('alive'); expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(0); expect(manager.getDiagnostics().lastAliveAt).toBe(Date.now()); - expect(ws.terminate).not.toHaveBeenCalled(); + + vi.advanceTimersByTime(600_000); expect(restartSpy).not.toHaveBeenCalled(); + vi.advanceTimersByTime(60_000); + expect(ws.terminate).not.toHaveBeenCalled(); + expect(restartSpy).toHaveBeenCalledTimes(1); + expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(10); + (manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear(); }); - it('keeps heartbeat misses observability-only on windows', async () => { + it('restarts after ten consecutive heartbeat misses on windows', async () => { Object.defineProperty(process, 'platform', { value: 'win32' }); const { GatewayManager } = await import('@electron/gateway/manager'); @@ -146,12 +165,12 @@ describe('GatewayManager heartbeat observability', () => { const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue(); (manager as unknown as { startPing: () => void }).startPing(); - vi.advanceTimersByTime(300_000); + vi.advanceTimersByTime(660_000); - expect(ws.ping).toHaveBeenCalledTimes(4); + expect(ws.ping).toHaveBeenCalledTimes(10); expect(ws.terminate).not.toHaveBeenCalled(); - expect(restartSpy).not.toHaveBeenCalled(); - expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(4); + expect(restartSpy).toHaveBeenCalledTimes(1); + expect(manager.getDiagnostics().consecutiveHeartbeatMisses).toBe(10); (manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear(); });