fix: integrate OpenClaw Gateway health signals (#957)

This commit is contained in:
Lingxuan Zuo
2026-05-01 21:49:35 +08:00
committed by GitHub
parent d47101ab4c
commit 8c9b4ea670
28 changed files with 1152 additions and 61 deletions
+1
View File
@@ -268,6 +268,7 @@ ClawXは、**デュアルプロセス + Host API 統一アクセス**構成を
- 単一起動保護は Electron のロックに加え、ローカルのプロセスロックファイルも併用し、デスクトップ IPC / セッションバスが不安定な環境でも重複起動を防ぎます。
- ローリングアップグレード中に旧版/新版が混在すると、単一起動保護の挙動が非対称になる場合があります。安定運用のため、デスクトップクライアントは可能な限り同一バージョンへ揃えてください。
- ただし OpenClaw Gateway の待受は常に**単一**であるべきです。`127.0.0.1:18789` を Listen しているプロセスは1つだけです。
- Gateway の readiness は `system-presence`、`health`、`status` などの OpenClaw コア信号を基準にし、memory、Dreams、チャネルの失敗はグローバルな Gateway 障害ではなく capability degradation として表示します。
- Listen プロセスの確認例:
- macOS/Linux: `lsof -nP -iTCP:18789 -sTCP:LISTEN`
- Windows (PowerShell): `Get-NetTCPConnection -LocalPort 18789 -State Listen`
+1
View File
@@ -272,6 +272,7 @@ ClawX employs a **dual-process architecture** with a unified host API layer. The
- Single-instance protection uses Electron's lock plus a local process-file lock fallback, preventing duplicate app launch in environments where desktop IPC/session bus is unstable.
- During rolling upgrades, mixed old/new app versions can still have asymmetric protection behavior. For best reliability, upgrade all desktop clients to the same version.
- The OpenClaw Gateway listener should still be **single-owner**: only one process should listen on `127.0.0.1:18789`.
- Gateway readiness is based on OpenClaw core signals such as `system-presence`, `health`, and `status`; memory, Dreams, or channel failures are shown as capability degradation instead of global Gateway failure.
- To verify the active listener:
- macOS/Linux: `lsof -nP -iTCP:18789 -sTCP:LISTEN`
- Windows (PowerShell): `Get-NetTCPConnection -LocalPort 18789 -State Listen`
+1
View File
@@ -272,6 +272,7 @@ ClawX 采用 **双进程 + Host API 统一接入架构**。渲染进程只调用
- 单实例保护同时使用 Electron 自带锁与本地进程文件锁回退机制,可在桌面会话总线异常时避免重复启动。
- 滚动升级期间若新旧版本混跑,单实例保护仍可能出现不对称行为。为保证稳定性,建议桌面客户端尽量统一升级到同一版本。
- 但 OpenClaw Gateway 监听应始终保持**单实例**:`127.0.0.1:18789` 只能有一个监听者。
- Gateway readiness 以 OpenClaw 的 `system-presence`、`health`、`status` 等核心信号为准;memory、Dreams 或频道失败会显示为能力降级,而不是全局 Gateway 故障。
- 可用以下命令确认监听进程:
- macOS/Linux`lsof -nP -iTCP:18789 -sTCP:LISTEN`
- WindowsPowerShell):`Get-NetTCPConnection -LocalPort 18789 -State Listen`
+17 -9
View File
@@ -48,7 +48,10 @@ export async function handleDiagnosticsRoutes(
url: URL,
ctx: HostApiContext,
): Promise<boolean> {
if (url.pathname === '/api/diagnostics/gateway-snapshot' && req.method === 'GET') {
if (
(url.pathname === '/api/diagnostics/gateway-snapshot' || url.pathname === '/api/gateway/diagnostics')
&& req.method === 'GET'
) {
try {
const { channels } = await buildChannelAccountsView(ctx, { probe: false });
const diagnostics = ctx.gatewayManager.getDiagnostics?.() ?? {
@@ -56,15 +59,20 @@ export async function handleDiagnosticsRoutes(
consecutiveRpcFailures: 0,
};
const channelStatusDiagnostics = getChannelStatusDiagnostics();
const gatewayStatus = ctx.gatewayManager.getStatus();
const gatewaySummary = buildGatewayHealthSummary({
status: gatewayStatus,
diagnostics,
lastChannelsStatusOkAt: channelStatusDiagnostics.lastChannelsStatusOkAt,
lastChannelsStatusFailureAt: channelStatusDiagnostics.lastChannelsStatusFailureAt,
platform: process.platform,
});
const gateway = {
...ctx.gatewayManager.getStatus(),
...buildGatewayHealthSummary({
status: ctx.gatewayManager.getStatus(),
diagnostics,
lastChannelsStatusOkAt: channelStatusDiagnostics.lastChannelsStatusOkAt,
lastChannelsStatusFailureAt: channelStatusDiagnostics.lastChannelsStatusFailureAt,
platform: process.platform,
}),
...gatewayStatus,
...gatewaySummary,
capabilities: typeof ctx.gatewayManager.getCapabilitySnapshot === 'function'
? ctx.gatewayManager.getCapabilitySnapshot(gatewaySummary)
: undefined,
};
const openClawDir = getOpenClawConfigDir();
sendJson(res, 200, {
+3 -1
View File
@@ -29,7 +29,9 @@ export async function handleGatewayRoutes(
}
if (url.pathname === '/api/gateway/health' && req.method === 'GET') {
const health = await ctx.gatewayManager.checkHealth();
const health = await ctx.gatewayManager.checkHealth({
probe: url.searchParams.get('probe') === '1' || url.searchParams.get('probe') === 'true',
});
sendJson(res, 200, health);
return true;
}
+140
View File
@@ -0,0 +1,140 @@
import type {
GatewayDiagnosticsSnapshot,
GatewayHealthSummary,
GatewayStatus,
} from './manager';
export type GatewayCapabilityName = 'openclawHealth' | 'openclawStatus' | 'channels' | 'memory';
export interface GatewayCapabilityProbe {
state: 'unknown' | 'healthy' | 'degraded';
checkedAt?: number;
durationMs?: number;
error?: string;
payload?: unknown;
}
export interface GatewayCoreProbe {
ok: boolean;
checkedAt: number;
durationMs?: number;
error?: string;
}
export interface GatewayCapabilitySnapshot {
core: {
process: GatewayStatus['state'];
transport: 'connected' | 'disconnected';
rpcRouter: 'unknown' | 'ready' | 'blocked';
lastProbe?: GatewayCoreProbe;
};
openclawHealth: GatewayCapabilityProbe;
openclawStatus: GatewayCapabilityProbe;
presence: GatewayCapabilityProbe;
channels: GatewayCapabilityProbe;
memory: GatewayCapabilityProbe;
diagnostics: GatewayDiagnosticsSnapshot;
summary?: GatewayHealthSummary;
}
function formatError(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function capabilityFromPayload(payload: unknown, checkedAt = Date.now()): GatewayCapabilityProbe {
return {
state: 'healthy',
checkedAt,
payload,
};
}
function capabilityFromError(error: unknown, checkedAt = Date.now()): GatewayCapabilityProbe {
return {
state: 'degraded',
checkedAt,
error: formatError(error),
};
}
const UNKNOWN_CAPABILITY: GatewayCapabilityProbe = { state: 'unknown' };
export class GatewayCapabilityMonitor {
private openclawHealth: GatewayCapabilityProbe = UNKNOWN_CAPABILITY;
private openclawStatus: GatewayCapabilityProbe = UNKNOWN_CAPABILITY;
private presence: GatewayCapabilityProbe = UNKNOWN_CAPABILITY;
private channels: GatewayCapabilityProbe = UNKNOWN_CAPABILITY;
private memory: GatewayCapabilityProbe = UNKNOWN_CAPABILITY;
private lastCoreProbe: GatewayCoreProbe | undefined;
recordOpenClawHealth(payload: unknown): void {
this.openclawHealth = capabilityFromPayload(payload);
}
recordOpenClawStatus(payload: unknown): void {
this.openclawStatus = capabilityFromPayload(payload);
}
recordPresence(payload: unknown): void {
this.presence = capabilityFromPayload(payload);
}
recordCoreProbe(probe: GatewayCoreProbe): void {
this.lastCoreProbe = probe;
}
recordCapabilitySuccess(name: GatewayCapabilityName, payload: unknown, durationMs?: number): void {
const probe: GatewayCapabilityProbe = {
state: 'healthy',
checkedAt: Date.now(),
durationMs,
payload,
};
this.setCapability(name, probe);
}
recordCapabilityFailure(name: GatewayCapabilityName, error: unknown, durationMs?: number): void {
const probe = capabilityFromError(error);
probe.durationMs = durationMs;
this.setCapability(name, probe);
}
buildSnapshot(params: {
status: GatewayStatus;
transportConnected: boolean;
diagnostics: GatewayDiagnosticsSnapshot;
summary?: GatewayHealthSummary;
}): GatewayCapabilitySnapshot {
return {
core: {
process: params.status.state,
transport: params.transportConnected ? 'connected' : 'disconnected',
rpcRouter: this.lastCoreProbe?.ok === false
? 'blocked'
: params.status.gatewayReady === true || this.lastCoreProbe?.ok === true
? 'ready'
: 'unknown',
lastProbe: this.lastCoreProbe,
},
openclawHealth: this.openclawHealth,
openclawStatus: this.openclawStatus,
presence: this.presence,
channels: this.channels,
memory: this.memory,
diagnostics: params.diagnostics,
summary: params.summary,
};
}
private setCapability(name: GatewayCapabilityName, probe: GatewayCapabilityProbe): void {
if (name === 'openclawHealth') {
this.openclawHealth = probe;
} else if (name === 'openclawStatus') {
this.openclawStatus = probe;
} else if (name === 'channels') {
this.channels = probe;
} else if (name === 'memory') {
this.memory = probe;
}
}
}
+183 -21
View File
@@ -18,7 +18,14 @@ function fsPath(filePath: string): string {
import { getAllSettings } from '../utils/store';
import { getApiKey, getDefaultProvider, getProvider } from '../utils/secure-storage';
import { getProviderEnvVar, getKeyableProviderTypes } from '../utils/provider-registry';
import { getOpenClawDir, getOpenClawEntryPath, isOpenClawPresent } from '../utils/paths';
import {
getOpenClawConfigDir,
getOpenClawDir,
getOpenClawEntryPath,
getOpenClawResolvedDir,
getOpenClawSkillsDir,
isOpenClawPresent,
} from '../utils/paths';
import { getUvMirrorEnv } from '../utils/uv-env';
import { cleanupDanglingWeChatPluginState, listConfiguredChannelsFromConfig, readOpenClawConfig } from '../utils/channel-config';
import { sanitizeOpenClawConfig, batchSyncConfigFields } from '../utils/openclaw-auth';
@@ -29,6 +36,14 @@ import { prependPathEntry } from '../utils/env-path';
import { copyPluginFromNodeModules, fixupPluginManifest, cpSyncSafe } from '../utils/plugin-install';
import { stripSystemdSupervisorEnv } from './config-sync-env';
import { cleanupAgentsSymlinkedSkills, cleanupStalePluginRuntimeDeps } from './skills-symlink-cleanup';
import {
buildPrelaunchMaintenanceCacheKey,
directoryChildrenSignature,
pathSignature,
runCachedPrelaunchMaintenanceTask,
type PrelaunchMaintenanceRunResult,
type PrelaunchMaintenanceTaskName,
} from './prelaunch-maintenance-cache';
export interface GatewayLaunchContext {
@@ -44,6 +59,12 @@ export interface GatewayLaunchContext {
channelStartupSummary: string;
}
export interface GatewayPrelaunchSyncSummary {
timingsMs: Record<string, number>;
maintenance: Partial<Record<PrelaunchMaintenanceTaskName, PrelaunchMaintenanceRunResult>>;
configuredChannels: string[];
}
// ── Auto-upgrade bundled plugins on startup ──────────────────────
const CHANNEL_PLUGIN_MAP: Record<string, { dirName: string; npmName: string }> = {
@@ -99,12 +120,39 @@ function buildBundledPluginSources(pluginDirName: string): string[] {
];
}
function measureSync<T>(timings: Record<string, number>, key: string, fn: () => T): T {
const startedAt = Date.now();
try {
return fn();
} finally {
timings[key] = Date.now() - startedAt;
}
}
async function measureAsync<T>(timings: Record<string, number>, key: string, fn: () => Promise<T>): Promise<T> {
const startedAt = Date.now();
try {
return await fn();
} finally {
timings[key] = Date.now() - startedAt;
}
}
function appVersionForCache(): string {
try {
return app.getVersion();
} catch {
return 'unknown';
}
}
/**
* Auto-upgrade all configured channel plugins before Gateway start.
* - Packaged mode: uses bundled plugins from resources/ (includes deps)
* - Dev mode: falls back to node_modules/ with pnpm-aware dep collection
*/
function ensureConfiguredPluginsUpgraded(configuredChannels: string[]): void {
function ensureConfiguredPluginsUpgraded(configuredChannels: string[]): boolean {
let succeeded = true;
for (const channelType of configuredChannels) {
const pluginInfo = CHANNEL_PLUGIN_MAP[channelType];
if (!pluginInfo) continue;
@@ -131,6 +179,7 @@ function ensureConfiguredPluginsUpgraded(configuredChannels: string[]): void {
fixupPluginManifest(targetDir);
} catch (err) {
logger.warn(`[plugin] Failed to ${isInstalled ? 'auto-upgrade' : 'install'} ${channelType} plugin:`, err);
succeeded = false;
}
} else if (isInstalled) {
// Same version already installed — still patch manifest ID in case it was
@@ -160,9 +209,11 @@ function ensureConfiguredPluginsUpgraded(configuredChannels: string[]): void {
fixupPluginManifest(targetDir);
} catch (err) {
logger.warn(`[plugin] Failed to ${isInstalled ? 'auto-upgrade' : 'install'} ${channelType} plugin from node_modules:`, err);
succeeded = false;
}
}
}
return succeeded;
}
/**
@@ -171,7 +222,8 @@ function ensureConfiguredPluginsUpgraded(configuredChannels: string[]): void {
* from scanning residual plugin manifests that were installed by a previous
* configuration but are no longer needed.
*/
function cleanupUnconfiguredChannelPlugins(configuredChannels: string[]): void {
function cleanupUnconfiguredChannelPlugins(configuredChannels: string[]): boolean {
let succeeded = true;
const configuredSet = new Set(configuredChannels);
for (const [channelType, pluginInfo] of Object.entries(CHANNEL_PLUGIN_MAP)) {
@@ -186,8 +238,67 @@ function cleanupUnconfiguredChannelPlugins(configuredChannels: string[]): void {
rmSync(fsPath(targetDir), { recursive: true, force: true });
} catch (err) {
logger.warn(`[plugin] Failed to remove unconfigured channel plugin ${channelType}:`, err);
succeeded = false;
}
}
return succeeded;
}
function buildPluginSourceSignatures(configuredChannels: string[]): Record<string, unknown> {
const signatures: Record<string, unknown> = {};
for (const channelType of [...configuredChannels].sort()) {
const pluginInfo = CHANNEL_PLUGIN_MAP[channelType];
if (!pluginInfo) continue;
const bundledSources = buildBundledPluginSources(pluginInfo.dirName);
const bundledDir = bundledSources.find((dir) => existsSync(fsPath(join(dir, 'openclaw.plugin.json'))));
const devPkgPath = join(process.cwd(), 'node_modules', ...pluginInfo.npmName.split('/'));
const sourceDir = bundledDir || (!app.isPackaged ? devPkgPath : '');
signatures[channelType] = sourceDir
? {
sourceDir,
manifest: pathSignature(join(sourceDir, 'openclaw.plugin.json')),
packageJson: pathSignature(join(sourceDir, 'package.json')),
}
: 'missing';
}
return signatures;
}
function buildPluginMaintenanceCacheKey(openclawDir: string, configuredChannels: string[]): string {
return buildPrelaunchMaintenanceCacheKey({
task: 'plugin-maintenance',
appVersion: appVersionForCache(),
openclawDir,
cwd: process.cwd(),
configuredChannels: [...configuredChannels].sort(),
extensionsDir: directoryChildrenSignature(join(homedir(), '.openclaw', 'extensions')),
sourceSignatures: buildPluginSourceSignatures(configuredChannels),
});
}
function buildSkillsSymlinkCleanupCacheKey(openclawDir: string): string {
const workspaceSkillsDir = join(getOpenClawConfigDir(), 'workspace', 'skills');
return buildPrelaunchMaintenanceCacheKey({
task: 'skills-symlink-cleanup',
appVersion: appVersionForCache(),
openclawDir,
skillsDir: getOpenClawSkillsDir(),
skillsDirSignature: directoryChildrenSignature(getOpenClawSkillsDir()),
workspaceSkillsDir,
workspaceSkillsDirSignature: directoryChildrenSignature(workspaceSkillsDir),
});
}
function buildRuntimeDepsCleanupCacheKey(openclawDir: string): string {
const runtimeDepsDir = join(getOpenClawConfigDir(), 'plugin-runtime-deps');
return buildPrelaunchMaintenanceCacheKey({
task: 'runtime-deps-cleanup',
appVersion: appVersionForCache(),
openclawDir,
currentOpenClawDir: getOpenClawResolvedDir(),
runtimeDepsDir,
runtimeDepsDirSignature: directoryChildrenSignature(runtimeDepsDir),
});
}
/**
@@ -276,22 +387,29 @@ function ensureExtensionDepsResolvable(openclawDir: string): void {
export async function syncGatewayConfigBeforeLaunch(
appSettings: Awaited<ReturnType<typeof getAllSettings>>,
): Promise<void> {
openclawDir: string,
): Promise<GatewayPrelaunchSyncSummary> {
const timingsMs: Record<string, number> = {};
const maintenance: GatewayPrelaunchSyncSummary['maintenance'] = {};
let configuredChannels: string[] = [];
// Reset the extension-deps cache so that newly installed extensions
// (e.g. user added a channel while the app was running) get their
// node_modules linked on the next Gateway spawn.
resetExtensionDepsLinked();
await syncProxyConfigToOpenClaw(appSettings, { preserveExistingWhenDisabled: true });
await measureAsync(timingsMs, 'proxySyncMs', async () => {
await syncProxyConfigToOpenClaw(appSettings, { preserveExistingWhenDisabled: true });
});
try {
await sanitizeOpenClawConfig();
await measureAsync(timingsMs, 'sanitizeMs', sanitizeOpenClawConfig);
} catch (err) {
logger.warn('Failed to sanitize openclaw.json:', err);
}
try {
await cleanupDanglingWeChatPluginState();
await measureAsync(timingsMs, 'wechatStateCleanupMs', cleanupDanglingWeChatPluginState);
} catch (err) {
logger.warn('Failed to clean dangling WeChat plugin state before launch:', err);
}
@@ -299,7 +417,7 @@ export async function syncGatewayConfigBeforeLaunch(
// Remove stale copies of built-in extensions (Discord, Telegram) that
// override OpenClaw's working built-in plugins and break channel loading.
try {
cleanupStaleBuiltInExtensions();
measureSync(timingsMs, 'staleBuiltinExtensionCleanupMs', cleanupStaleBuiltInExtensions);
} catch (err) {
logger.warn('Failed to clean stale built-in extensions:', err);
}
@@ -310,7 +428,12 @@ export async function syncGatewayConfigBeforeLaunch(
// still discovered via the agents-skills-personal source, so the symlinks
// are pure log noise. Transitional workaround for openclaw/openclaw#59219.
try {
cleanupAgentsSymlinkedSkills();
const result = measureSync(timingsMs, 'skillsCleanupMs', () => runCachedPrelaunchMaintenanceTask(
'skills-symlink-cleanup',
() => buildSkillsSymlinkCleanupCacheKey(openclawDir),
() => (cleanupAgentsSymlinkedSkills().failed ?? 0) === 0,
));
maintenance['skills-symlink-cleanup'] = result;
} catch (err) {
logger.warn('Failed to clean .agents/skills-targeted skill symlinks:', err);
}
@@ -320,7 +443,12 @@ export async function syncGatewayConfigBeforeLaunch(
// a long time in synchronous fs.open/copy calls before the RPC router is
// responsive.
try {
cleanupStalePluginRuntimeDeps();
const result = measureSync(timingsMs, 'runtimeDepsCleanupMs', () => runCachedPrelaunchMaintenanceTask(
'runtime-deps-cleanup',
() => buildRuntimeDepsCleanupCacheKey(openclawDir),
() => (cleanupStalePluginRuntimeDeps().failed ?? 0) === 0,
));
maintenance['runtime-deps-cleanup'] = result;
} catch (err) {
logger.warn('Failed to clean stale OpenClaw plugin runtime deps:', err);
}
@@ -330,21 +458,39 @@ export async function syncGatewayConfigBeforeLaunch(
// Only install/upgrade plugins for channels that are actually configured
// in openclaw.json — do NOT expand the list from plugins.allow.
try {
const rawCfg = await readOpenClawConfig();
const configuredChannels = await listConfiguredChannelsFromConfig(rawCfg);
configuredChannels = await measureAsync(timingsMs, 'configuredChannelsMs', async () => {
const rawCfg = await readOpenClawConfig();
return await listConfiguredChannelsFromConfig(rawCfg);
});
ensureConfiguredPluginsUpgraded(configuredChannels);
cleanupUnconfiguredChannelPlugins(configuredChannels);
const result = measureSync(timingsMs, 'pluginMaintenanceMs', () => runCachedPrelaunchMaintenanceTask(
'plugin-maintenance',
() => buildPluginMaintenanceCacheKey(openclawDir, configuredChannels),
() => {
const upgradeOk = ensureConfiguredPluginsUpgraded(configuredChannels);
const cleanupOk = cleanupUnconfiguredChannelPlugins(configuredChannels);
return upgradeOk && cleanupOk;
},
));
maintenance['plugin-maintenance'] = result;
} catch (err) {
logger.warn('Failed to auto-upgrade plugins:', err);
}
// Batch gateway token, browser config, and session idle into one read+write cycle.
try {
await batchSyncConfigFields(appSettings.gatewayToken);
await measureAsync(timingsMs, 'configFieldSyncMs', async () => {
await batchSyncConfigFields(appSettings.gatewayToken);
});
} catch (err) {
logger.warn('Failed to batch-sync config fields to openclaw.json:', err);
}
return {
timingsMs,
maintenance,
configuredChannels,
};
}
async function loadProviderEnv(): Promise<{ providerEnv: Record<string, string>; loadedProviderKeyCount: number }> {
@@ -416,6 +562,8 @@ async function resolveChannelStartupPolicy(): Promise<{
}
export async function prepareGatewayLaunchContext(port: number): Promise<GatewayLaunchContext> {
const timingsMs: Record<string, number> = {};
const totalStartedAt = Date.now();
const openclawDir = getOpenClawDir();
const entryScript = getOpenClawEntryPath();
@@ -423,8 +571,10 @@ export async function prepareGatewayLaunchContext(port: number): Promise<Gateway
throw new Error(`OpenClaw package not found at: ${openclawDir}`);
}
const appSettings = await getAllSettings();
await syncGatewayConfigBeforeLaunch(appSettings);
const appSettings = await measureAsync(timingsMs, 'settingsMs', getAllSettings);
const prelaunchSummary = await measureAsync(timingsMs, 'prelaunchSyncMs', async () => (
await syncGatewayConfigBeforeLaunch(appSettings, openclawDir)
));
if (!existsSync(entryScript)) {
throw new Error(`OpenClaw entry script not found at: ${entryScript}`);
@@ -441,9 +591,13 @@ export async function prepareGatewayLaunchContext(port: number): Promise<Gateway
: path.join(process.cwd(), 'resources', 'bin', target);
const binPathExists = existsSync(binPath);
const { providerEnv, loadedProviderKeyCount } = await loadProviderEnv();
const { skipChannels, channelStartupSummary } = await resolveChannelStartupPolicy();
const uvEnv = await getUvMirrorEnv();
const { providerEnv, loadedProviderKeyCount } = await measureAsync(timingsMs, 'providerEnvMs', loadProviderEnv);
const { skipChannels, channelStartupSummary } = await measureAsync(
timingsMs,
'channelStartupPolicyMs',
resolveChannelStartupPolicy,
);
const uvEnv = await measureAsync(timingsMs, 'uvEnvMs', getUvMirrorEnv);
const proxyEnv = buildProxyEnv(appSettings);
const resolvedProxy = resolveProxySettings(appSettings);
const proxySummary = appSettings.proxyEnabled
@@ -469,7 +623,15 @@ export async function prepareGatewayLaunchContext(port: number): Promise<Gateway
// Ensure extension-specific packages (e.g. grammy from the telegram
// extension) are resolvable by shared dist/ chunks via symlinks in
// openclaw/node_modules/. NODE_PATH does NOT work for ESM imports.
ensureExtensionDepsResolvable(openclawDir);
measureSync(timingsMs, 'extensionDepsMs', () => ensureExtensionDepsResolvable(openclawDir));
timingsMs.totalMs = Date.now() - totalStartedAt;
logger.info('[metric] gateway.prelaunch', {
...prelaunchSummary.timingsMs,
...timingsMs,
maintenance: prelaunchSummary.maintenance,
configuredChannelCount: prelaunchSummary.configuredChannels.length,
});
return {
appSettings,
+6
View File
@@ -30,6 +30,12 @@ export function dispatchProtocolEvent(
case 'ready':
emitter.emit('gateway:ready', payload);
break;
case 'health':
emitter.emit('gateway:health', payload);
break;
case 'presence':
emitter.emit('gateway:presence', payload);
break;
default:
emitter.emit('notification', { method: event, params: payload });
}
+110 -5
View File
@@ -51,6 +51,11 @@ import {
} from './reload-policy';
import { classifyGatewayStderrMessage, recordGatewayStartupStderrLine } from './startup-stderr';
import { runGatewayStartupSequence } from './startup-orchestrator';
import {
GatewayCapabilityMonitor,
type GatewayCapabilityName,
type GatewayCapabilitySnapshot,
} from './capability-monitor';
export interface GatewayStatus {
state: GatewayLifecycleState;
@@ -79,6 +84,14 @@ export interface GatewayHealthSummary {
lastChannelsStatusFailureAt?: number;
}
export interface GatewayHealthReport {
ok: boolean;
error?: string;
uptime?: number;
version?: string;
capabilities: GatewayCapabilitySnapshot;
}
export interface GatewayDiagnosticsSnapshot {
lastAliveAt?: number;
lastRpcSuccessAt?: number;
@@ -91,14 +104,27 @@ export interface GatewayDiagnosticsSnapshot {
consecutiveRpcFailures: number;
}
function isTransportRpcFailure(error: unknown): boolean {
function isCoreRpcMethod(method: string): boolean {
return method === 'system-presence';
}
function isTransportRpcFailure(method: string, error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
return message.includes('RPC timeout:')
|| message.includes('Gateway not connected')
? isCoreRpcMethod(method)
: message.includes('Gateway not connected')
|| message.includes('Gateway stopped')
|| message.includes('Failed to send RPC request:');
}
function classifyCapabilityMethod(method: string): GatewayCapabilityName | null {
if (method === 'health') return 'openclawHealth';
if (method === 'status') return 'openclawStatus';
if (method === 'channels.status') return 'channels';
if (method.startsWith('doctor.memory.')) return 'memory';
return null;
}
/**
* Gateway Manager Events
*/
@@ -108,6 +134,8 @@ export interface GatewayManagerEvents {
notification: (notification: JsonRpcNotification) => void;
exit: (code: number | null) => void;
error: (error: Error) => void;
'gateway:health': (data: unknown) => void;
'gateway:presence': (data: unknown) => void;
'channel:status': (data: { channelId: string; status: string }) => void;
'chat:message': (data: { message: unknown }) => void;
}
@@ -162,6 +190,7 @@ export class GatewayManager extends EventEmitter {
/** Set by scheduleReconnect() before calling start() to signal auto-reconnect. */
private isAutoReconnectStart = false;
private gatewayReadyFallbackTimer: NodeJS.Timeout | null = null;
private readonly capabilityMonitor = new GatewayCapabilityMonitor();
private diagnostics: GatewayDiagnosticsSnapshot = {
consecutiveHeartbeatMisses: 0,
consecutiveRpcFailures: 0,
@@ -205,6 +234,12 @@ export class GatewayManager extends EventEmitter {
this.setStatus({ gatewayReady: true });
}
});
this.on('gateway:health', (payload) => {
this.capabilityMonitor.recordOpenClawHealth(payload);
});
this.on('gateway:presence', (payload) => {
this.capabilityMonitor.recordPresence(payload);
});
}
private async initDeviceIdentity(): Promise<void> {
@@ -242,6 +277,19 @@ export class GatewayManager extends EventEmitter {
return { ...this.diagnostics };
}
getCapabilitySnapshot(summary?: GatewayHealthSummary): GatewayCapabilitySnapshot {
return this.capabilityMonitor.buildSnapshot({
status: this.status,
transportConnected: this.ws?.readyState === WebSocket.OPEN,
diagnostics: this.getDiagnostics(),
summary,
});
}
recordCapabilityFailure(name: GatewayCapabilityName, error: unknown, durationMs?: number): void {
this.capabilityMonitor.recordCapabilityFailure(name, error, durationMs);
}
/**
* Check if Gateway is connected and ready
*/
@@ -757,13 +805,25 @@ export class GatewayManager extends EventEmitter {
}
logger.info('Gateway ready fallback triggered; probing RPC router before marking ready');
const startedAt = Date.now();
try {
await this.rpc('system-presence', {}, 5_000);
this.capabilityMonitor.recordCoreProbe({
ok: true,
checkedAt: Date.now(),
durationMs: Date.now() - startedAt,
});
if (this.status.state === 'running' && !this.status.gatewayReady) {
logger.info('Gateway ready fallback RPC router probe succeeded');
this.setStatus({ gatewayReady: true });
}
} catch (error) {
this.capabilityMonitor.recordCoreProbe({
ok: false,
checkedAt: Date.now(),
durationMs: Date.now() - startedAt,
error: error instanceof Error ? error.message : String(error),
});
logger.warn('Gateway ready fallback RPC router probe failed; waiting for gateway.ready event or heartbeat recovery:', error);
if (this.status.state === 'running' && !this.status.gatewayReady) {
this.scheduleGatewayReadyFallback();
@@ -776,6 +836,7 @@ export class GatewayManager extends EventEmitter {
* Uses OpenClaw protocol format: { type: "req", id: "...", method: "...", params: {...} }
*/
async rpc<T>(method: string, params?: unknown, timeoutMs = 30000): Promise<T> {
const startedAt = Date.now();
return await new Promise<T>((resolve, reject) => {
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
reject(new Error('Gateway not connected'));
@@ -811,9 +872,30 @@ export class GatewayManager extends EventEmitter {
}
}).then((result) => {
this.recordRpcSuccess();
if (isCoreRpcMethod(method)) {
this.capabilityMonitor.recordCoreProbe({
ok: true,
checkedAt: Date.now(),
durationMs: Date.now() - startedAt,
});
}
const capability = classifyCapabilityMethod(method);
if (capability) {
this.capabilityMonitor.recordCapabilitySuccess(capability, result, Date.now() - startedAt);
}
return result;
}).catch((error) => {
if (isTransportRpcFailure(error)) {
const capability = classifyCapabilityMethod(method);
if (capability) {
this.capabilityMonitor.recordCapabilityFailure(capability, error, Date.now() - startedAt);
}
if (isTransportRpcFailure(method, error)) {
this.capabilityMonitor.recordCoreProbe({
ok: false,
checkedAt: Date.now(),
durationMs: Date.now() - startedAt,
error: error instanceof Error ? error.message : String(error),
});
this.recordRpcFailure(method);
}
throw error;
@@ -826,7 +908,7 @@ export class GatewayManager extends EventEmitter {
private startHealthCheck(): void {
this.connectionMonitor.startHealthCheck({
shouldCheck: () => this.status.state === 'running',
checkHealth: () => this.checkHealth(),
checkHealth: () => this.checkTransportHealth(),
onUnhealthy: (errorMessage) => {
this.emit('error', new Error(errorMessage));
},
@@ -840,7 +922,7 @@ export class GatewayManager extends EventEmitter {
* Check Gateway health via WebSocket ping
* OpenClaw Gateway doesn't have an HTTP /health endpoint
*/
async checkHealth(): Promise<{ ok: boolean; error?: string; uptime?: number }> {
private async checkTransportHealth(): Promise<{ ok: boolean; error?: string; uptime?: number }> {
try {
if (this.ws && this.ws.readyState === WebSocket.OPEN) {
const uptime = this.status.connectedAt
@@ -854,6 +936,29 @@ export class GatewayManager extends EventEmitter {
}
}
async checkHealth(options?: { probe?: boolean }): Promise<GatewayHealthReport> {
const transport = await this.checkTransportHealth();
if (transport.ok && this.status.state === 'running' && this.status.gatewayReady !== false) {
const timeoutMs = options?.probe ? 8_000 : 3_000;
const [healthResult, statusResult] = await Promise.allSettled([
this.rpc('health', { probe: options?.probe === true }, timeoutMs),
this.rpc('status', {}, timeoutMs),
]);
if (healthResult.status === 'fulfilled') {
this.capabilityMonitor.recordOpenClawHealth(healthResult.value);
}
if (statusResult.status === 'fulfilled') {
this.capabilityMonitor.recordOpenClawStatus(statusResult.value);
}
}
return {
...transport,
capabilities: this.getCapabilitySnapshot(),
};
}
private recordGatewayAlive(): void {
this.clearInitialReadyHeartbeatRecoveryTimer();
this.diagnostics.lastAliveAt = Date.now();
@@ -0,0 +1,160 @@
import { app } from 'electron';
import {
existsSync,
mkdirSync,
readdirSync,
readFileSync,
statSync,
writeFileSync,
} from 'node:fs';
import { dirname, join } from 'node:path';
const CACHE_SCHEMA_VERSION = 1;
const CACHE_FILE_NAME = 'gateway-prelaunch-maintenance-cache.json';
export type PrelaunchMaintenanceTaskName =
| 'plugin-maintenance'
| 'runtime-deps-cleanup'
| 'skills-symlink-cleanup';
export interface PrelaunchMaintenanceRunResult {
executed: boolean;
reason: 'cache-hit' | 'cache-miss' | 'cache-unavailable' | 'task-failed';
}
type CacheKeyInput = string | (() => string);
type MaintenanceTask = () => void | boolean;
interface CacheEntry {
key: string;
updatedAt: string;
}
interface CacheFile {
schemaVersion: number;
tasks: Partial<Record<PrelaunchMaintenanceTaskName, CacheEntry>>;
}
function getDefaultCachePath(): string {
return join(app.getPath('userData'), CACHE_FILE_NAME);
}
function emptyCache(): CacheFile {
return {
schemaVersion: CACHE_SCHEMA_VERSION,
tasks: {},
};
}
function readCache(cachePath: string): CacheFile | null {
try {
if (!existsSync(cachePath)) return emptyCache();
const parsed = JSON.parse(readFileSync(cachePath, 'utf-8')) as CacheFile;
if (parsed.schemaVersion !== CACHE_SCHEMA_VERSION || !parsed.tasks) {
return emptyCache();
}
return parsed;
} catch {
return null;
}
}
function writeCache(cachePath: string, cache: CacheFile): boolean {
try {
mkdirSync(dirname(cachePath), { recursive: true });
writeFileSync(cachePath, `${JSON.stringify(cache, null, 2)}\n`, 'utf-8');
return true;
} catch {
return false;
}
}
export function stableJson(value: unknown): string {
if (value == null || typeof value !== 'object') return JSON.stringify(value);
if (Array.isArray(value)) {
return `[${value.map((item) => stableJson(item)).join(',')}]`;
}
const entries = Object.entries(value as Record<string, unknown>)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, entryValue]) => `${JSON.stringify(key)}:${stableJson(entryValue)}`);
return `{${entries.join(',')}}`;
}
export function pathSignature(path: string): string {
try {
const stat = statSync(path);
return `${stat.isDirectory() ? 'dir' : 'file'}:${Math.round(stat.mtimeMs)}:${stat.size}`;
} catch {
return 'missing';
}
}
export function directoryChildrenSignature(path: string, maxEntries = 200): string {
try {
const entries = readdirSync(path, { withFileTypes: true, encoding: 'utf8' })
.sort((left, right) => left.name.localeCompare(right.name))
.slice(0, maxEntries)
.map((entry) => {
const childPath = join(path, entry.name);
return [
entry.name,
entry.isDirectory() ? 'dir' : entry.isSymbolicLink() ? 'symlink' : 'file',
pathSignature(childPath),
].join(':');
});
return stableJson(entries);
} catch {
return 'missing';
}
}
export function buildPrelaunchMaintenanceCacheKey(parts: Record<string, unknown>): string {
return stableJson({
schemaVersion: CACHE_SCHEMA_VERSION,
...parts,
});
}
export function runCachedPrelaunchMaintenanceTask(
taskName: PrelaunchMaintenanceTaskName,
cacheKey: CacheKeyInput,
task: MaintenanceTask,
options: { cachePath?: string } = {},
): PrelaunchMaintenanceRunResult {
const readCacheKey = (): string => (typeof cacheKey === 'function' ? cacheKey() : cacheKey);
const cachePath = options.cachePath ?? getDefaultCachePath();
const cache = readCache(cachePath);
if (!cache) {
task();
return { executed: true, reason: 'cache-unavailable' };
}
let initialCacheKey: string;
try {
initialCacheKey = readCacheKey();
} catch {
task();
return { executed: true, reason: 'cache-unavailable' };
}
if (cache.tasks[taskName]?.key === initialCacheKey) {
return { executed: false, reason: 'cache-hit' };
}
const taskResult = task();
if (taskResult === false) {
return { executed: true, reason: 'task-failed' };
}
let finalCacheKey: string;
try {
finalCacheKey = readCacheKey();
} catch {
return { executed: true, reason: 'cache-unavailable' };
}
cache.tasks[taskName] = {
key: finalCacheKey,
updatedAt: new Date().toISOString(),
};
writeCache(cachePath, cache);
return { executed: true, reason: 'cache-miss' };
}
@@ -63,6 +63,8 @@ export interface CleanupResult {
removed: string[];
/** Total number of symlink entries that were inspected. */
examined: number;
/** Cleanup operations that could not be completed and should be retried later. */
failed?: number;
}
export interface PluginRuntimeDepsCleanupOptions {
@@ -76,6 +78,10 @@ function defaultSkillsDir(): string {
return getOpenClawSkillsDir();
}
function recordCleanupFailure(result: CleanupResult): void {
result.failed = (result.failed ?? 0) + 1;
}
function defaultAgentsDir(): string {
return path.join(homedir(), '.agents', 'skills');
}
@@ -186,6 +192,9 @@ export function cleanupAgentsSymlinkedSkills(opts: CleanupOptions = {}): Cleanup
const rootResult = cleanupSkillsDir(root.skillsDir, root.agentsDir);
result.removed.push(...rootResult.removed);
result.examined += rootResult.examined;
if (rootResult.failed) {
result.failed = (result.failed ?? 0) + rootResult.failed;
}
}
return result;
@@ -221,6 +230,7 @@ export function cleanupStalePluginRuntimeDeps(
entries = readdirSync(runtimeDepsDir, { withFileTypes: true, encoding: 'utf8' });
} catch (err) {
logger.warn(`[plugin-runtime-deps-cleanup] Failed to list ${runtimeDepsDir}:`, err);
recordCleanupFailure(result);
return result;
}
@@ -241,6 +251,7 @@ export function cleanupStalePluginRuntimeDeps(
result.removed.push(entry.name);
} catch (err) {
logger.warn(`[plugin-runtime-deps-cleanup] Failed to remove ${cacheRoot}:`, err);
recordCleanupFailure(result);
}
}
@@ -316,6 +327,7 @@ function cleanupSkillsDir(skillsDir: string, agentsDir: string): CleanupResult {
entries = readdirSync(skillsDir, { withFileTypes: true, encoding: 'utf8' });
} catch (err) {
logger.warn(`[skills-cleanup] Failed to list ${skillsDir}:`, err);
recordCleanupFailure(result);
return result;
}
@@ -354,6 +366,7 @@ function cleanupSkillsDir(skillsDir: string, agentsDir: string): CleanupResult {
result.removed.push(entry.name);
} catch (err) {
logger.warn(`[skills-cleanup] Failed to remove ${entryPath}:`, err);
recordCleanupFailure(result);
}
}
+8
View File
@@ -413,6 +413,14 @@ async function initialize(): Promise<void> {
hostEventBus.emit('gateway:notification', notification);
});
gatewayManager.on('gateway:health', (data) => {
hostEventBus.emit('gateway:health', data);
});
gatewayManager.on('gateway:presence', (data) => {
hostEventBus.emit('gateway:presence', data);
});
gatewayManager.on('chat:message', (data) => {
hostEventBus.emit('gateway:chat-message', data);
});
+12
View File
@@ -1417,6 +1417,18 @@ function registerGatewayHandlers(
}
});
gatewayManager.on('gateway:health', (data) => {
if (!mainWindow.isDestroyed()) {
mainWindow.webContents.send('gateway:health-changed', data);
}
});
gatewayManager.on('gateway:presence', (data) => {
if (!mainWindow.isDestroyed()) {
mainWindow.webContents.send('gateway:presence-changed', data);
}
});
gatewayManager.on('channel:status', (data) => {
if (!mainWindow.isDestroyed()) {
mainWindow.webContents.send('gateway:channel-status', data);
+4
View File
@@ -160,6 +160,8 @@ const electronAPI = {
'gateway:status-changed',
'gateway:message',
'gateway:notification',
'gateway:health-changed',
'gateway:presence-changed',
'gateway:channel-status',
'gateway:chat-message',
'channel:whatsapp-qr',
@@ -209,6 +211,8 @@ const electronAPI = {
'gateway:status-changed',
'gateway:message',
'gateway:notification',
'gateway:health-changed',
'gateway:presence-changed',
'gateway:channel-status',
'gateway:chat-message',
'channel:whatsapp-qr',
@@ -22,6 +22,17 @@ requiredRules:
Use this spec when ClawX shows the Gateway as starting/running but UI data does not refresh, Dreams cannot load, or Gateway RPC calls time out after a restart.
ClawX should prefer OpenClaw-native signals over stderr string matching:
- `system-presence` proves the core RPC router is serving requests.
- `health` provides the Gateway health snapshot; use cached `probe:false` first.
- `status` provides presence, health, stateVersion, uptime, and session defaults.
- `channels.status` is the channel capability signal.
- `doctor.memory.status` is the memory/dreams capability signal.
- `gateway.ready`, `health`, and `presence` events should update ClawX's main-process capability cache.
stderr is supporting evidence only. It should not be the primary source for deciding whether the Gateway is ready, blocked, or should be restarted.
## Failure Shape
Treat these as the same incident family until proven otherwise:
@@ -41,6 +52,12 @@ Important distinction:
UI features that depend on Gateway runtime data must prefer RPC-ready evidence over port-ready evidence.
Capability failures are not Gateway core failures:
- `doctor.memory.status` timeout means memory capability degraded until `system-presence` also fails.
- `channels.status` timeout means channel capability degraded until `system-presence` also fails.
- dreams cron unavailable, missing memory files, stale session keys, or provider credential errors do not trigger Gateway restart by themselves.
## Fast Triage
1. Confirm the process and ports:
@@ -56,15 +73,18 @@ lsof -nP -iTCP:5173 -sTCP:LISTEN || true
tail -n 160 "$HOME/Library/Application Support/clawx/logs/clawx-$(date +%F).log"
```
3. Probe a low-cost RPC. Redirect output for memory-related calls because successful responses may contain user data:
3. Probe OpenClaw-native signals in this order. Redirect output for memory-related calls because successful responses may contain user data:
```bash
pnpm exec openclaw gateway call system-presence >/tmp/clawx-system-presence.json
pnpm exec openclaw gateway call health --params '{"probe":false}' >/tmp/clawx-health.json
pnpm exec openclaw gateway call status >/tmp/clawx-status.json
pnpm exec openclaw gateway call channels.status --params '{"probe":false}' >/tmp/clawx-channels-status.json
pnpm exec openclaw gateway call doctor.memory.status >/tmp/clawx-memory-status.json
pnpm exec openclaw gateway call doctor.memory.dreamDiary >/tmp/clawx-dream-diary.json
```
4. If port is listening but RPC times out, agree on the sampling scope, then sample the Gateway process on macOS:
4. Only if port is listening and the core RPC probe (`system-presence`) times out, agree on the sampling scope, then sample the Gateway process on macOS:
```bash
sample <gateway-pid> 3 -mayDie >/tmp/clawx-gateway.sample.txt
@@ -168,6 +188,21 @@ Expected behavior:
- The fallback must probe `system-presence` before emitting ready.
- Heartbeat recovery may defer restart during the initial grace window, but it should not loop restart while the Gateway is still performing startup work.
### Capability Degraded But Core Alive
Symptoms:
- `system-presence`, `health`, or `status` succeeds.
- `doctor.memory.status`, `doctor.memory.dreamDiary`, or `channels.status` times out.
- stderr may mention dreams cron unavailable, missing memory files, stale session keys, or credentials provider errors.
Expected behavior:
- Keep global Gateway state based on process, transport, and core RPC readiness.
- Mark only the relevant capability as degraded.
- Do not restart Gateway automatically.
- Let the user retry the capability probe or fix provider/channel credentials.
### Restart Deferral By Active Work
Symptoms:
@@ -201,21 +236,30 @@ pnpm exec tsx -e "import { sanitizeOpenClawConfig } from './electron/utils/openc
}
```
5. Confirm RPC readiness:
5. Confirm core RPC readiness:
```bash
pnpm exec openclaw gateway call system-presence >/tmp/clawx-system-presence.json
```
6. Only after `system-presence` succeeds, verify feature-specific RPCs such as Dreams or memory doctor calls.
6. Confirm cached OpenClaw health before deeper probes:
```bash
pnpm exec openclaw gateway call health --params '{"probe":false}' >/tmp/clawx-health.json
pnpm exec openclaw gateway call status >/tmp/clawx-status.json
```
7. Only after `system-presence` succeeds, verify feature-specific RPCs such as Dreams, memory doctor calls, or channel probes.
## Acceptance Criteria
- Gateway starts without restart loops.
- `configSyncMs` stays small relative to total startup time.
- `system-presence` succeeds after startup settles.
- `health` and `status` are captured in Gateway diagnostics when available.
- Dreams page can refresh once the Gateway process is running and RPC-ready.
- `doctor.memory.status` and `doctor.memory.dreamDiary` return when Dreams is enabled.
- `doctor.memory.*` and `channels.status` failures degrade their capability only and do not trigger Gateway restart.
- Logs no longer repeat stale runtime cache or escaped managed-skill symlink warnings for entries ClawX can safely clean.
## Required Regression Coverage
+2 -1
View File
@@ -67,6 +67,7 @@
},
"errors": {
"openFullUi": "Unable to open the full OpenClaw Dreams UI.",
"configHashMissing": "Unable to update Dreams because the config base hash is unavailable."
"configHashMissing": "Unable to update Dreams because the config base hash is unavailable.",
"memoryInitializing": "Dreams memory is still initializing. Please retry after Gateway finishes startup."
}
}
+2 -1
View File
@@ -67,6 +67,7 @@
},
"errors": {
"openFullUi": "完全版 OpenClaw Dreams UI を開けません。",
"configHashMissing": "config base hash が利用できないため Dreams を更新できません。"
"configHashMissing": "config base hash が利用できないため Dreams を更新できません。",
"memoryInitializing": "Dreams memory はまだ初期化中です。Gateway の起動完了後に再試行してください。"
}
}
+2 -1
View File
@@ -67,6 +67,7 @@
},
"errors": {
"openFullUi": "Не удалось открыть полный OpenClaw Dreams UI.",
"configHashMissing": "Не удалось обновить Dreams: недоступен config base hash."
"configHashMissing": "Не удалось обновить Dreams: недоступен config base hash.",
"memoryInitializing": "Память Dreams еще инициализируется. Повторите попытку после завершения запуска Gateway."
}
}
+2 -1
View File
@@ -67,6 +67,7 @@
},
"errors": {
"openFullUi": "无法打开完整 OpenClaw Dreams UI。",
"configHashMissing": "无法更新梦境,因为配置 base hash 不可用。"
"configHashMissing": "无法更新梦境,因为配置 base hash 不可用。",
"memoryInitializing": "梦境记忆仍在初始化。请在网关完成启动后重试。"
}
}
+2
View File
@@ -6,6 +6,8 @@ const HOST_EVENT_TO_IPC_CHANNEL: Record<string, string> = {
'gateway:status': 'gateway:status-changed',
'gateway:error': 'gateway:error',
'gateway:notification': 'gateway:notification',
'gateway:health': 'gateway:health-changed',
'gateway:presence': 'gateway:presence-changed',
'gateway:chat-message': 'gateway:chat-message',
'gateway:channel-status': 'gateway:channel-status',
'gateway:exit': 'gateway:exit',
+17 -7
View File
@@ -206,6 +206,14 @@ function firstNumber(result: unknown, keys: string[]): number | undefined {
return undefined;
}
function isMemoryDoctorStartupError(message: string): boolean {
const lower = message.toLowerCase();
return lower.includes('rpc timeout: doctor.memory.')
|| lower.includes('service not initialized')
|| lower.includes('not yet ready')
|| lower.includes('unavailable during gateway startup');
}
export function Dreams() {
const { t } = useTranslation(['dreams', 'common']);
const gatewayStatus = useGatewayStore((state) => state.status);
@@ -223,8 +231,10 @@ export function Dreams() {
const refreshInFlightRef = useRef<Promise<void> | null>(null);
const gatewayRunning = gatewayStatus.state === 'running';
const gatewayReady = gatewayStatus.gatewayReady !== false;
const dreamsReady = gatewayRunning && gatewayReady;
const busy = runningAction != null || runningToggle != null;
const actionsDisabled = !gatewayRunning || busy;
const actionsDisabled = !dreamsReady || busy;
const diaryEntries = useMemo(() => parseDreamDiary(diary?.content).slice(0, 4), [diary?.content]);
const recentSignals = useMemo(() => {
@@ -238,7 +248,7 @@ export function Dreams() {
return refreshInFlightRef.current;
}
if (!gatewayRunning) {
if (!dreamsReady) {
setLoading(false);
setError(null);
return;
@@ -257,7 +267,7 @@ export function Dreams() {
setDiary(diaryResponse);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setError(message);
setError(isMemoryDoctorStartupError(message) ? t('errors.memoryInitializing') : message);
} finally {
setLoading(false);
if (refreshInFlightRef.current === refreshPromise) {
@@ -268,7 +278,7 @@ export function Dreams() {
refreshInFlightRef.current = refreshPromise;
return refreshPromise;
}, [gatewayRunning, rpc]);
}, [dreamsReady, rpc, t]);
useEffect(() => {
void refreshAll();
@@ -407,7 +417,7 @@ export function Dreams() {
variant={dreaming?.enabled ? 'outline' : 'default'}
size="sm"
onClick={() => void setDreamingEnabled(!dreaming?.enabled)}
disabled={!gatewayRunning || busy || loading}
disabled={!dreamsReady || busy || loading}
className={dreaming?.enabled ? QUIET_BUTTON_CLASS : undefined}
>
{runningToggle ? <Loader2 className="mr-2 h-4 w-4 animate-spin" /> : <Power className="mr-2 h-4 w-4" />}
@@ -418,7 +428,7 @@ export function Dreams() {
variant="outline"
size="sm"
onClick={() => void refreshAll({ force: true })}
disabled={!gatewayRunning}
disabled={!dreamsReady}
className={QUIET_BUTTON_CLASS}
>
{loading ? <Loader2 className="mr-2 h-4 w-4 animate-spin" /> : <RefreshCw className="mr-2 h-4 w-4" />}
@@ -439,7 +449,7 @@ export function Dreams() {
</header>
<main className="min-h-0 flex-1 overflow-auto px-10 pb-10">
{!gatewayRunning && (
{!dreamsReady && (
<div className="mb-4 rounded-lg border border-black/10 bg-surface-input px-4 py-3 text-sm text-foreground/70 dark:border-white/10">
{t('gatewayNotReady')}
</div>
+9 -7
View File
@@ -6,7 +6,7 @@ import { create } from 'zustand';
import { hostApiFetch } from '@/lib/host-api';
import { invokeIpc } from '@/lib/api-client';
import { subscribeHostEvent } from '@/lib/host-events';
import type { GatewayStatus } from '../types/gateway';
import type { GatewayHealth, GatewayStatus } from '../types/gateway';
let gatewayInitPromise: Promise<void> | null = null;
let gatewayEventUnsubscribers: Array<() => void> | null = null;
@@ -19,12 +19,6 @@ let lastLoadSessionsAt = 0;
let lastLoadHistoryAt = 0;
let cronRepairTriggeredThisSession = false;
interface GatewayHealth {
ok: boolean;
error?: string;
uptime?: number;
}
interface GatewayState {
status: GatewayStatus;
health: GatewayHealth | null;
@@ -287,6 +281,14 @@ export const useGatewayStore = create<GatewayState>((set, get) => ({
handleGatewayNotification(payload);
},
));
unsubscribers.push(subscribeHostEvent('gateway:health', (payload) => {
const current = get().health;
set({ health: { ...(current ?? { ok: true }), ok: true, openclawHealth: payload } });
}));
unsubscribers.push(subscribeHostEvent('gateway:presence', (payload) => {
const current = get().health;
set({ health: { ...(current ?? { ok: true }), presence: payload } });
}));
unsubscribers.push(subscribeHostEvent('gateway:chat-message', (payload) => {
handleGatewayChatMessage(payload);
}));
+41
View File
@@ -31,11 +31,52 @@ export interface GatewayRpcResponse<T = unknown> {
/**
* Gateway health check response
*/
export interface GatewayCapabilityProbe {
state: 'unknown' | 'healthy' | 'degraded';
checkedAt?: number;
durationMs?: number;
error?: string;
payload?: unknown;
}
export interface GatewayCapabilitySnapshot {
core: {
process: GatewayStatus['state'];
transport: 'connected' | 'disconnected';
rpcRouter: 'unknown' | 'ready' | 'blocked';
lastProbe?: {
ok: boolean;
checkedAt: number;
durationMs?: number;
error?: string;
};
};
openclawHealth: GatewayCapabilityProbe;
openclawStatus: GatewayCapabilityProbe;
presence: GatewayCapabilityProbe;
channels: GatewayCapabilityProbe;
memory: GatewayCapabilityProbe;
diagnostics: {
lastAliveAt?: number;
lastRpcSuccessAt?: number;
lastRpcFailureAt?: number;
lastRpcFailureMethod?: string;
lastHeartbeatTimeoutAt?: number;
consecutiveHeartbeatMisses: number;
lastSocketCloseAt?: number;
lastSocketCloseCode?: number;
consecutiveRpcFailures: number;
};
}
export interface GatewayHealth {
ok: boolean;
error?: string;
uptime?: number;
version?: string;
capabilities?: GatewayCapabilitySnapshot;
openclawHealth?: unknown;
presence?: unknown;
}
/**
+90
View File
@@ -0,0 +1,90 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { act, render, screen, waitFor } from '@testing-library/react';
import { Dreams } from '@/pages/Dreams';
const rpcMock = vi.fn();
const hostApiFetchMock = vi.fn();
const tMock = (key: string) => key;
const { gatewayState } = vi.hoisted(() => ({
gatewayState: {
status: { state: 'running', port: 18789, gatewayReady: true } as {
state: string;
port: number;
gatewayReady?: boolean;
},
},
}));
vi.mock('@/stores/gateway', () => ({
useGatewayStore: (selector: (state: typeof gatewayState & { rpc: typeof rpcMock }) => unknown) => selector({
...gatewayState,
rpc: rpcMock,
}),
}));
vi.mock('@/lib/host-api', () => ({
hostApiFetch: (...args: unknown[]) => hostApiFetchMock(...args),
}));
vi.mock('react-i18next', () => ({
useTranslation: () => ({
t: tMock,
}),
}));
vi.mock('sonner', () => ({
toast: {
success: vi.fn(),
error: vi.fn(),
},
}));
describe('Dreams page gateway readiness', () => {
beforeEach(() => {
vi.clearAllMocks();
gatewayState.status = { state: 'running', port: 18789, gatewayReady: true };
rpcMock.mockImplementation(async (method: string) => {
if (method === 'doctor.memory.status') {
return {
dreaming: {
enabled: true,
shortTermCount: 1,
groundedSignalCount: 0,
totalSignalCount: 1,
promotedToday: 0,
shortTermEntries: [],
promotedEntries: [],
},
};
}
if (method === 'doctor.memory.dreamDiary') {
return { found: true, content: '' };
}
return {};
});
});
it('does not call memory doctor RPCs until gatewayReady is true', async () => {
gatewayState.status = { state: 'running', port: 18789, gatewayReady: false };
const { rerender } = render(<Dreams />);
expect(screen.getByTestId('dreams-refresh')).toBeDisabled();
expect(screen.getByTestId('dreams-enable')).toBeDisabled();
expect(screen.getByText('gatewayNotReady')).toBeVisible();
await waitFor(() => {
expect(rpcMock).not.toHaveBeenCalled();
});
gatewayState.status = { state: 'running', port: 18789, gatewayReady: true };
await act(async () => {
rerender(<Dreams />);
});
await waitFor(() => {
expect(rpcMock).toHaveBeenCalledWith('doctor.memory.status', {}, 12_000);
expect(rpcMock).toHaveBeenCalledWith('doctor.memory.dreamDiary', {}, 12_000);
});
expect(screen.getByTestId('dreams-refresh')).toBeEnabled();
});
});
+11
View File
@@ -31,6 +31,17 @@ describe('dispatchProtocolEvent', () => {
expect(emitter.emit).toHaveBeenCalledWith('channel:status', { channelId: 'telegram', status: 'connected' });
});
it('dispatches native health and presence events separately from generic notifications', () => {
const emitter = createMockEmitter();
dispatchProtocolEvent(emitter, 'health', { ok: true });
dispatchProtocolEvent(emitter, 'presence', [{ mode: 'gateway', ts: 1 }]);
expect(emitter.emit).toHaveBeenCalledWith('gateway:health', { ok: true });
expect(emitter.emit).toHaveBeenCalledWith('gateway:presence', [{ mode: 'gateway', ts: 1 }]);
expect(emitter.emit).not.toHaveBeenCalledWith('notification', expect.objectContaining({ method: 'health' }));
expect(emitter.emit).not.toHaveBeenCalledWith('notification', expect.objectContaining({ method: 'presence' }));
});
it('dispatches chat to chat:message', () => {
const emitter = createMockEmitter();
dispatchProtocolEvent(emitter, 'chat', { text: 'hello' });
+8
View File
@@ -32,11 +32,19 @@ describe('gateway store event wiring', () => {
expect(subscribeHostEventMock).toHaveBeenCalledWith('gateway:status', expect.any(Function));
expect(subscribeHostEventMock).toHaveBeenCalledWith('gateway:error', expect.any(Function));
expect(subscribeHostEventMock).toHaveBeenCalledWith('gateway:notification', expect.any(Function));
expect(subscribeHostEventMock).toHaveBeenCalledWith('gateway:health', expect.any(Function));
expect(subscribeHostEventMock).toHaveBeenCalledWith('gateway:presence', expect.any(Function));
expect(subscribeHostEventMock).toHaveBeenCalledWith('gateway:chat-message', expect.any(Function));
expect(subscribeHostEventMock).toHaveBeenCalledWith('gateway:channel-status', expect.any(Function));
handlers.get('gateway:status')?.({ state: 'stopped', port: 18789 });
expect(useGatewayStore.getState().status.state).toBe('stopped');
handlers.get('gateway:health')?.({ ok: true, ts: 1 });
expect(useGatewayStore.getState().health?.openclawHealth).toEqual({ ok: true, ts: 1 });
handlers.get('gateway:presence')?.([{ mode: 'gateway', ts: 2 }]);
expect(useGatewayStore.getState().health?.presence).toEqual([{ mode: 'gateway', ts: 2 }]);
});
it('propagates gatewayReady field from status events', async () => {
+93 -3
View File
@@ -70,13 +70,13 @@ describe('GatewayManager diagnostics', () => {
expect(manager.getDiagnostics().lastRpcSuccessAt).toBe(Date.now());
expect(manager.getDiagnostics().consecutiveRpcFailures).toBe(0);
const failurePromise = manager.rpc('chat.history', {}, 1000);
const failurePromise = manager.rpc('system-presence', {}, 1000);
vi.advanceTimersByTime(1001);
await expect(failurePromise).rejects.toThrow('RPC timeout: chat.history');
await expect(failurePromise).rejects.toThrow('RPC timeout: system-presence');
const diagnostics = manager.getDiagnostics();
expect(diagnostics.lastRpcFailureAt).toBe(Date.now());
expect(diagnostics.lastRpcFailureMethod).toBe('chat.history');
expect(diagnostics.lastRpcFailureMethod).toBe('system-presence');
expect(diagnostics.consecutiveRpcFailures).toBe(1);
(manager as unknown as { recordSocketClose: (code: number) => void }).recordSocketClose(1006);
@@ -125,6 +125,96 @@ describe('GatewayManager diagnostics', () => {
expect(health.reasons).not.toContain('rpc_timeout');
});
it('records capability timeouts without counting them as core transport failures', async () => {
const { GatewayManager } = await import('@electron/gateway/manager');
const manager = new GatewayManager();
const ws = {
readyState: 1,
send: vi.fn(),
ping: vi.fn(),
terminate: vi.fn(),
on: vi.fn(),
};
(manager as unknown as { ws: typeof ws }).ws = ws;
(manager as unknown as { status: { state: string; port: number } }).status = {
state: 'running',
port: 18789,
};
const memoryPromise = manager.rpc('doctor.memory.status', {}, 1000);
vi.advanceTimersByTime(1001);
await expect(memoryPromise).rejects.toThrow('RPC timeout: doctor.memory.status');
expect(manager.getDiagnostics().consecutiveRpcFailures).toBe(0);
expect(manager.getCapabilitySnapshot().memory.state).toBe('degraded');
expect(manager.getCapabilitySnapshot().memory.error).toContain('doctor.memory.status');
});
it('does not let health polling mark core rpc degraded when OpenClaw health/status are slow', async () => {
const { GatewayManager } = await import('@electron/gateway/manager');
const { buildGatewayHealthSummary } = await import('@electron/utils/gateway-health');
const manager = new GatewayManager();
const ws = {
readyState: 1,
send: vi.fn(),
ping: vi.fn(),
terminate: vi.fn(),
on: vi.fn(),
};
(manager as unknown as { ws: typeof ws }).ws = ws;
(manager as unknown as { status: { state: string; port: number; gatewayReady: boolean } }).status = {
state: 'running',
port: 18789,
gatewayReady: true,
};
const healthPromise = manager.checkHealth();
await vi.advanceTimersByTimeAsync(3001);
const health = await healthPromise;
expect(health.ok).toBe(true);
expect(manager.getDiagnostics().consecutiveRpcFailures).toBe(0);
expect(manager.getCapabilitySnapshot().openclawHealth.state).toBe('degraded');
expect(manager.getCapabilitySnapshot().openclawStatus.state).toBe('degraded');
const summary = buildGatewayHealthSummary({
status: manager.getStatus(),
diagnostics: manager.getDiagnostics(),
platform: process.platform,
});
expect(summary.reasons).not.toContain('rpc_timeout');
});
it('reports rpc router blocked when a fresh core probe fails after gateway was ready', async () => {
const { GatewayCapabilityMonitor } = await import('@electron/gateway/capability-monitor');
const monitor = new GatewayCapabilityMonitor();
monitor.recordCoreProbe({
ok: false,
checkedAt: Date.now(),
error: 'RPC timeout: system-presence',
});
const snapshot = monitor.buildSnapshot({
status: {
state: 'running',
port: 18789,
gatewayReady: true,
},
transportConnected: true,
diagnostics: {
consecutiveHeartbeatMisses: 0,
consecutiveRpcFailures: 1,
},
});
expect(snapshot.core.rpcRouter).toBe('blocked');
});
it('keeps windows heartbeat recovery disabled while diagnostics degrade', async () => {
Object.defineProperty(process, 'platform', { value: 'win32' });
@@ -0,0 +1,166 @@
import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import {
buildPrelaunchMaintenanceCacheKey,
runCachedPrelaunchMaintenanceTask,
} from '@electron/gateway/prelaunch-maintenance-cache';
describe('prelaunch maintenance cache', () => {
let tempDir: string;
let cachePath: string;
beforeEach(() => {
tempDir = mkdtempSync(join(tmpdir(), 'clawx-prelaunch-cache-'));
cachePath = join(tempDir, 'cache.json');
});
afterEach(() => {
rmSync(tempDir, { recursive: true, force: true });
});
it('runs a task on cache miss and skips it when the cache key is unchanged', () => {
const task = vi.fn();
const cacheKey = buildPrelaunchMaintenanceCacheKey({
task: 'skills-symlink-cleanup',
appVersion: '1.0.0',
rootSignature: 'mtime-a',
});
expect(runCachedPrelaunchMaintenanceTask(
'skills-symlink-cleanup',
cacheKey,
task,
{ cachePath },
)).toEqual({ executed: true, reason: 'cache-miss' });
expect(task).toHaveBeenCalledTimes(1);
expect(runCachedPrelaunchMaintenanceTask(
'skills-symlink-cleanup',
cacheKey,
task,
{ cachePath },
)).toEqual({ executed: false, reason: 'cache-hit' });
expect(task).toHaveBeenCalledTimes(1);
});
it('stores the post-task cache key when the task mutates signed inputs', () => {
let rootSignature = 'dirty';
const cacheKey = () => buildPrelaunchMaintenanceCacheKey({
task: 'skills-symlink-cleanup',
appVersion: '1.0.0',
rootSignature,
});
const task = vi.fn(() => {
rootSignature = 'clean';
});
expect(runCachedPrelaunchMaintenanceTask(
'skills-symlink-cleanup',
cacheKey,
task,
{ cachePath },
)).toEqual({ executed: true, reason: 'cache-miss' });
expect(task).toHaveBeenCalledTimes(1);
const writtenCache = JSON.parse(readFileSync(cachePath, 'utf-8'));
expect(writtenCache.tasks['skills-symlink-cleanup'].key).toBe(cacheKey());
expect(runCachedPrelaunchMaintenanceTask(
'skills-symlink-cleanup',
cacheKey,
task,
{ cachePath },
)).toEqual({ executed: false, reason: 'cache-hit' });
expect(task).toHaveBeenCalledTimes(1);
});
it('does not cache a task that reports maintenance failure', () => {
const cacheKey = buildPrelaunchMaintenanceCacheKey({
task: 'plugin-maintenance',
appVersion: '1.0.0',
configuredChannels: ['feishu'],
});
const task = vi.fn()
.mockReturnValueOnce(false)
.mockReturnValueOnce(true);
expect(runCachedPrelaunchMaintenanceTask(
'plugin-maintenance',
cacheKey,
task,
{ cachePath },
)).toEqual({ executed: true, reason: 'task-failed' });
expect(task).toHaveBeenCalledTimes(1);
expect(runCachedPrelaunchMaintenanceTask(
'plugin-maintenance',
cacheKey,
task,
{ cachePath },
)).toEqual({ executed: true, reason: 'cache-miss' });
expect(task).toHaveBeenCalledTimes(2);
});
it('reruns a task when the cache key changes', () => {
const task = vi.fn();
const firstKey = buildPrelaunchMaintenanceCacheKey({
task: 'runtime-deps-cleanup',
openclawDir: '/old/openclaw',
});
const secondKey = buildPrelaunchMaintenanceCacheKey({
task: 'runtime-deps-cleanup',
openclawDir: '/new/openclaw',
});
runCachedPrelaunchMaintenanceTask('runtime-deps-cleanup', firstKey, task, { cachePath });
const result = runCachedPrelaunchMaintenanceTask('runtime-deps-cleanup', secondKey, task, { cachePath });
expect(result).toEqual({ executed: true, reason: 'cache-miss' });
expect(task).toHaveBeenCalledTimes(2);
});
it('treats cache schema changes as misses', () => {
const task = vi.fn();
const cacheKey = buildPrelaunchMaintenanceCacheKey({
task: 'plugin-maintenance',
configuredChannels: ['feishu'],
});
writeFileSync(cachePath, JSON.stringify({
schemaVersion: 0,
tasks: {
'plugin-maintenance': {
key: cacheKey,
updatedAt: new Date().toISOString(),
},
},
}), 'utf-8');
const result = runCachedPrelaunchMaintenanceTask('plugin-maintenance', cacheKey, task, { cachePath });
expect(result).toEqual({ executed: true, reason: 'cache-miss' });
expect(task).toHaveBeenCalledTimes(1);
expect(JSON.parse(readFileSync(cachePath, 'utf-8')).schemaVersion).toBe(1);
});
it('runs conservatively when the cache file cannot be read', () => {
const task = vi.fn();
const blockedCachePath = join(tempDir, 'blocked-cache');
mkdirSync(blockedCachePath);
const cacheKey = buildPrelaunchMaintenanceCacheKey({
task: 'plugin-maintenance',
configuredChannels: [],
});
const result = runCachedPrelaunchMaintenanceTask(
'plugin-maintenance',
cacheKey,
task,
{ cachePath: blockedCachePath },
);
expect(result).toEqual({ executed: true, reason: 'cache-unavailable' });
expect(task).toHaveBeenCalledTimes(1);
});
});