diff --git a/src/commands/autopilot.ts b/src/commands/autopilot.ts index 390305cbc..3f27cb21f 100644 --- a/src/commands/autopilot.ts +++ b/src/commands/autopilot.ts @@ -25,6 +25,11 @@ import { execSync } from 'child_process'; import type { BrainEngine } from '../core/engine.ts'; import { loadPreferences } from '../core/preferences.ts'; import { loadConfig, loadConfigFileOnly, saveConfig, gbrainPath as gbrainHomePath } from '../core/config.ts'; +import { + classifyAutopilotLockHolder, + type AutopilotLockProbeDeps, + isPidAlive, +} from '../core/autopilot-lock.ts'; import { ChildWorkerSupervisor } from '../core/minions/child-worker-supervisor.ts'; import { VERSION } from '../version.ts'; import { @@ -249,19 +254,22 @@ export function shouldSpawnAutopilotWorker(args: string[]): boolean { return !args.includes('--no-worker'); } -export function isPidAlive(pid: number): boolean { - if (!Number.isFinite(pid) || pid <= 0) return false; +export { isPidAlive }; + +export const AUTOPILOT_FOREIGN_PID_TAKEOVER_GRACE_MS = 10 * 60 * 1000; + +function autopilotLockAgeMs(lockPath: string): number | null { try { - process.kill(pid, 0); - return true; - } catch (error: unknown) { - return (error as NodeJS.ErrnoException).code === 'EPERM'; + return Date.now() - statSync(lockPath).mtimeMs; + } catch { + return null; } } export function decideLockAcquisition( lockPath: string, currentPid: number, + deps: AutopilotLockProbeDeps = {}, ): { action: 'acquire' } | { action: 'exit'; holderPid: number } | { action: 'takeover'; reason: string } { if (!existsSync(lockPath)) return { action: 'acquire' }; @@ -273,10 +281,21 @@ export function decideLockAcquisition( } const holderPid = Number.parseInt(raw, 10); - const sameProcess = Number.isFinite(holderPid) && holderPid === currentPid; - const alive = !sameProcess && isPidAlive(holderPid); + const holder = classifyAutopilotLockHolder(holderPid, currentPid, deps); - if (alive) return { action: 'exit', holderPid }; + if (holder.state === 'alive-autopilot' || holder.state === 'alive-unknown') { + return { action: 'exit', holderPid }; + } + if (holder.state === 'alive-foreign') { + const lockAgeMs = autopilotLockAgeMs(lockPath); + if (lockAgeMs !== null && lockAgeMs >= AUTOPILOT_FOREIGN_PID_TAKEOVER_GRACE_MS) { + return { action: 'takeover', reason: `foreign pid ${raw || ''} with stale lock` }; + } + return { action: 'exit', holderPid }; + } + if (holder.state === 'self') { + return { action: 'takeover', reason: `own pid ${raw || ''}` }; + } return { action: 'takeover', reason: `dead pid ${raw || ''}` }; } diff --git a/src/commands/status.ts b/src/commands/status.ts index de223cf16..cb17ef4f4 100644 --- a/src/commands/status.ts +++ b/src/commands/status.ts @@ -15,7 +15,7 @@ * - Workers — supervisor health from the audit JSONL * - Queue — live minion_jobs counts BY status (NO time window — * old stuck jobs are exactly what status surfaces) - * - Autopilot — daemon PID liveness via kill -0 probe + * - Autopilot — daemon PID liveness plus gbrain-autopilot identity probe * * Exit codes (kubectl-style): * 0 snapshot produced successfully (even if it carries warnings) @@ -40,6 +40,10 @@ import { existsSync, readFileSync } from 'node:fs'; import { gbrainPath, loadConfig, isThinClient } from '../core/config.ts'; import { callRemoteTool, unpackToolResult } from '../core/mcp-client.ts'; import { VERSION } from '../version.ts'; +import { + classifyAutopilotLockHolder, + type AutopilotLockProbeDeps, +} from '../core/autopilot-lock.ts'; import { buildSyncStatusReport, type SyncStatusReport, @@ -297,8 +301,10 @@ function buildWorkerSummary(): WorkerSummary { return { crashes_24h, clean_exits_24h, by_cause, last_event_ts }; } -function buildAutopilotStatus(): AutopilotStatus { - const lockPath = gbrainPath('autopilot.lock'); +export function buildAutopilotStatus( + lockPath: string = gbrainPath('autopilot.lock'), + deps: AutopilotLockProbeDeps = {}, +): AutopilotStatus { const lockfile_present = existsSync(lockPath); let pid: number | null = null; let running = false; @@ -308,16 +314,8 @@ function buildAutopilotStatus(): AutopilotStatus { const parsed = parseInt(raw, 10); if (Number.isFinite(parsed) && parsed > 0) { pid = parsed; - try { - // kill -0 probes liveness without sending a real signal. Throws ESRCH - // if the PID is gone, EPERM if alive but owned by another user (which - // still tells us "something with that PID exists"). - process.kill(parsed, 0); - running = true; - } catch (err) { - const code = (err as NodeJS.ErrnoException).code; - running = code === 'EPERM'; - } + const holder = classifyAutopilotLockHolder(parsed, process.pid, deps); + running = holder.state === 'alive-autopilot' || holder.state === 'alive-unknown'; } } catch { /* unreadable lockfile, leave pid=null/running=false */ @@ -592,7 +590,7 @@ function renderHuman(report: StatusReport): string { if (a.running) { lines.push(` running (PID ${a.pid})`); } else if (a.lockfile_present) { - lines.push(` stale lockfile (PID ${a.pid ?? '?'} not alive). Run \`gbrain autopilot --install\` to restart.`); + lines.push(` stale lockfile (PID ${a.pid ?? '?'} is not a live autopilot process). Run \`gbrain autopilot --install\` to restart.`); } else { lines.push(' not running. Install with `gbrain autopilot --install`.'); } diff --git a/src/core/autopilot-lock.ts b/src/core/autopilot-lock.ts new file mode 100644 index 000000000..ab5268277 --- /dev/null +++ b/src/core/autopilot-lock.ts @@ -0,0 +1,64 @@ +import { execFileSync } from 'node:child_process'; + +export type AutopilotLockHolder = + | { state: 'dead' } + | { state: 'self' } + | { state: 'alive-autopilot' } + | { state: 'alive-foreign' } + | { state: 'alive-unknown' }; + +export interface AutopilotLockProbeDeps { + isPidAlive?: (pid: number) => boolean; + readProcessCommand?: (pid: number) => string | null; +} + +export function isPidAlive(pid: number): boolean { + if (!Number.isFinite(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (error: unknown) { + return (error as NodeJS.ErrnoException).code === 'EPERM'; + } +} + +export function readProcessCommand(pid: number): string | null { + if (!Number.isFinite(pid) || pid <= 0) return null; + try { + const out = execFileSync('ps', ['-p', String(pid), '-o', 'args='], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + timeout: 1000, + }).trim(); + return out.length > 0 ? out : null; + } catch { + return null; + } +} + +export function looksLikeGbrainAutopilotCommand(command: string): boolean { + const normalized = command.replace(/\\/g, '/').trim(); + if (!/(^|\s)autopilot(\s|$)/i.test(normalized)) return false; + if (/(^|[\/\s])gbrain(?:\.exe)?(\s|$)/i.test(normalized)) return true; + return /(^|\s)(?:\S+\/)?(?:\.{1,2}\/)?(?:src\/)?cli\.(?:ts|js|mjs)(\s|$)/i.test(normalized) + || /(^|\s)\S*\/src\/cli\.(?:ts|js|mjs)(\s|$)/i.test(normalized); +} + +export function classifyAutopilotLockHolder( + pid: number, + currentPid: number = process.pid, + deps: AutopilotLockProbeDeps = {}, +): AutopilotLockHolder { + if (!Number.isFinite(pid) || pid <= 0) return { state: 'dead' }; + if (pid === currentPid) return { state: 'self' }; + + const probeAlive = deps.isPidAlive ?? isPidAlive; + if (!probeAlive(pid)) return { state: 'dead' }; + + const probeCommand = deps.readProcessCommand ?? readProcessCommand; + const command = probeCommand(pid); + if (command === null) return { state: 'alive-unknown' }; + return looksLikeGbrainAutopilotCommand(command) + ? { state: 'alive-autopilot' } + : { state: 'alive-foreign' }; +} diff --git a/test/autopilot-lock.test.ts b/test/autopilot-lock.test.ts index f3795ec5a..a48efeb92 100644 --- a/test/autopilot-lock.test.ts +++ b/test/autopilot-lock.test.ts @@ -1,8 +1,13 @@ import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; -import { mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { mkdtempSync, rmSync, utimesSync, writeFileSync } from 'fs'; import { join } from 'path'; import { tmpdir } from 'os'; -import { decideLockAcquisition, isPidAlive } from '../src/commands/autopilot.ts'; +import { + AUTOPILOT_FOREIGN_PID_TAKEOVER_GRACE_MS, + decideLockAcquisition, + isPidAlive, +} from '../src/commands/autopilot.ts'; +import { looksLikeGbrainAutopilotCommand } from '../src/core/autopilot-lock.ts'; let tmp: string; let lockPath: string; @@ -41,11 +46,49 @@ describe('decideLockAcquisition', () => { }); }); - test('keeps a lock whose holder is alive regardless of age', () => { - writeFileSync(lockPath, String(process.pid)); - expect(decideLockAcquisition(lockPath, process.pid + 100_000)).toEqual({ + test('keeps a lock whose holder is a live gbrain autopilot process', () => { + writeFileSync(lockPath, '1234'); + expect(decideLockAcquisition(lockPath, process.pid, { + isPidAlive: (pid) => pid === 1234, + readProcessCommand: () => 'gbrain autopilot --repo repo', + })).toEqual({ action: 'exit', - holderPid: process.pid, + holderPid: 1234, + }); + }); + + test('keeps a fresh lock when the live PID command is unrecognized', () => { + writeFileSync(lockPath, '1234'); + expect(decideLockAcquisition(lockPath, process.pid, { + isPidAlive: (pid) => pid === 1234, + readProcessCommand: () => '/sbin/launchd', + })).toEqual({ + action: 'exit', + holderPid: 1234, + }); + }); + + test('takes over a stale lock when the PID was reused by a foreign process', () => { + writeFileSync(lockPath, '1234'); + const stale = new Date(Date.now() - AUTOPILOT_FOREIGN_PID_TAKEOVER_GRACE_MS - 1000); + utimesSync(lockPath, stale, stale); + expect(decideLockAcquisition(lockPath, process.pid, { + isPidAlive: (pid) => pid === 1234, + readProcessCommand: () => '/sbin/launchd', + })).toEqual({ + action: 'takeover', + reason: 'foreign pid 1234 with stale lock', + }); + }); + + test('keeps a live lock when process identity cannot be inspected', () => { + writeFileSync(lockPath, '1234'); + expect(decideLockAcquisition(lockPath, process.pid, { + isPidAlive: (pid) => pid === 1234, + readProcessCommand: () => null, + })).toEqual({ + action: 'exit', + holderPid: 1234, }); }); @@ -56,3 +99,16 @@ describe('decideLockAcquisition', () => { expect(decideLockAcquisition(lockPath, process.pid).action).toBe('takeover'); }); }); + +describe('looksLikeGbrainAutopilotCommand', () => { + test('matches packaged and source-tree autopilot invocations', () => { + expect(looksLikeGbrainAutopilotCommand('gbrain autopilot --repo repo')).toBe(true); + expect(looksLikeGbrainAutopilotCommand('./gbrain/src/cli.ts autopilot')).toBe(true); + expect(looksLikeGbrainAutopilotCommand('bun src/cli.ts autopilot --repo repo')).toBe(true); + }); + + test('rejects unrelated live processes', () => { + expect(looksLikeGbrainAutopilotCommand('/sbin/launchd')).toBe(false); + expect(looksLikeGbrainAutopilotCommand('/usr/bin/python worker.py')).toBe(false); + }); +}); diff --git a/test/status-autopilot-lock.test.ts b/test/status-autopilot-lock.test.ts new file mode 100644 index 000000000..23da7ad22 --- /dev/null +++ b/test/status-autopilot-lock.test.ts @@ -0,0 +1,45 @@ +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { buildAutopilotStatus } from '../src/commands/status.ts'; + +let tmp: string; +let lockPath: string; + +beforeEach(() => { + tmp = mkdtempSync(join(tmpdir(), 'gbrain-status-autopilot-lock-')); + lockPath = join(tmp, 'autopilot.lock'); +}); + +afterEach(() => { + rmSync(tmp, { recursive: true, force: true }); +}); + +describe('buildAutopilotStatus', () => { + test('reports a reused foreign PID lock as stale, not running', () => { + writeFileSync(lockPath, '1234'); + const status = buildAutopilotStatus(lockPath, { + isPidAlive: (pid) => pid === 1234, + readProcessCommand: () => '/sbin/launchd', + }); + + expect(status).toEqual({ + installed: true, + lockfile_present: true, + pid: 1234, + running: false, + }); + }); + + test('reports a live gbrain autopilot process as running', () => { + writeFileSync(lockPath, '1234'); + const status = buildAutopilotStatus(lockPath, { + isPidAlive: (pid) => pid === 1234, + readProcessCommand: () => 'gbrain autopilot --repo repo', + }); + + expect(status.running).toBe(true); + expect(status.pid).toBe(1234); + }); +});