mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 08:53:22 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
556d184332 | ||
|
|
12a585ca52 |
+60
-15
@@ -19,7 +19,7 @@
|
||||
|
||||
import { existsSync, readFileSync, writeFileSync, mkdirSync, appendFileSync, utimesSync, unlinkSync, chmodSync } from 'fs';
|
||||
import { setCliExitVerdict } from '../core/cli-force-exit.ts';
|
||||
import { join } from 'path';
|
||||
import { dirname, join } from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { loadPreferences } from '../core/preferences.ts';
|
||||
@@ -151,6 +151,16 @@ export function shouldSpawnAutopilotWorker(args: string[]): boolean {
|
||||
return !args.includes('--no-worker');
|
||||
}
|
||||
|
||||
/**
|
||||
* #2234: exit code for the autopilot shutdown path. Give-up reasons — the
|
||||
* daemon decided it cannot make progress — must exit non-zero so systemd
|
||||
* `Restart=on-failure` (and any monitoring on the exit status) sees a
|
||||
* failure, not a clean stop. Signal-driven stops stay 0.
|
||||
*/
|
||||
export function shutdownExitCode(reason: string): number {
|
||||
return reason === 'max_crashes' || reason === 'cycle-failure-cap' ? 1 : 0;
|
||||
}
|
||||
|
||||
export function isPidAlive(pid: number): boolean {
|
||||
if (!Number.isFinite(pid) || pid <= 0) return false;
|
||||
try {
|
||||
@@ -521,7 +531,10 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
}
|
||||
}
|
||||
try { unlinkSync(lockPath); } catch { /* already gone */ }
|
||||
process.exit(0);
|
||||
// #2234: give-up paths (max_crashes, cycle-failure-cap) exit NON-zero so a
|
||||
// systemd `Restart=on-failure` unit restarts a daemon that decided it
|
||||
// can't make progress. Operator-initiated stops (SIGTERM/SIGINT) stay 0.
|
||||
process.exit(shutdownExitCode(sig));
|
||||
};
|
||||
process.on('SIGTERM', () => { void shutdown('SIGTERM'); });
|
||||
process.on('SIGINT', () => { void shutdown('SIGINT'); });
|
||||
@@ -1166,18 +1179,35 @@ function detectOpenClaw(): { detected: boolean; bootstrapCandidates: string[] }
|
||||
return { detected: signal, bootstrapCandidates: existing };
|
||||
}
|
||||
|
||||
function writeWrapperScript(repoPath: string): string {
|
||||
const home = process.env.HOME || '';
|
||||
const gbrainDir = join(home, '.gbrain');
|
||||
mkdirSync(gbrainDir, { recursive: true });
|
||||
|
||||
// Wrapper sources the user's shell profile for API keys so nothing is
|
||||
// baked into plist/crontab/systemd unit files (#2).
|
||||
const wrapperPath = join(gbrainDir, 'autopilot-run.sh');
|
||||
const gbrainPath = resolveGbrainCliPath();
|
||||
const safeRepoPath = repoPath.replace(/'/g, "'\\''");
|
||||
const safeGbrainPath = gbrainPath.replace(/'/g, "'\\''");
|
||||
const wrapper = `#!/bin/bash
|
||||
/**
|
||||
* Pure wrapper-script generator (exported for tests, #1868).
|
||||
*
|
||||
* The wrapper runs under launchd/systemd/cron with a minimal PATH (systemd
|
||||
* user units get ~ /usr/bin:/bin), and the shell profiles it sources are a
|
||||
* macOS/zsh convention that often doesn't exist on a Linux server. If the
|
||||
* resolved `gbrain` is a bun-shebang shim, `#!/usr/bin/env bun` then fails
|
||||
* with "bun: not found" and the service crash-loops. Bake the runtime dirs
|
||||
* (resolved at INSTALL time: gbrain's own dir, the current runtime's dir,
|
||||
* ~/.bun/bin) onto PATH AFTER profile sourcing so they win regardless of
|
||||
* what the profiles did.
|
||||
*/
|
||||
export function generateWrapperScript(
|
||||
gbrainPath: string,
|
||||
repoPath: string,
|
||||
opts?: { home?: string; execPath?: string },
|
||||
): string {
|
||||
const home = opts?.home ?? process.env.HOME ?? '';
|
||||
const execPath = opts?.execPath ?? process.execPath ?? '';
|
||||
const sq = (s: string) => s.replace(/'/g, "'\\''");
|
||||
const pathDirs = [...new Set(
|
||||
[dirname(gbrainPath), execPath ? dirname(execPath) : '', home ? join(home, '.bun', 'bin') : '']
|
||||
.filter((d) => d && d !== '.' && d !== '/'),
|
||||
)];
|
||||
// Empty-safe: never emit `PATH='':"$PATH"` (a leading empty entry means cwd).
|
||||
const pathLine = pathDirs.length > 0
|
||||
? `export PATH='${sq(pathDirs.join(':'))}':"$PATH"`
|
||||
: ': # no runtime dirs resolved at install time';
|
||||
return `#!/bin/bash
|
||||
# Auto-generated by gbrain autopilot --install
|
||||
# Sources shell profile for API keys, then runs autopilot.
|
||||
# zshenv is the canonical place for env vars in zsh on macOS (zshrc is for
|
||||
@@ -1186,8 +1216,23 @@ function writeWrapperScript(repoPath: string): string {
|
||||
# OPENAI/ANTHROPIC keys exported in zshenv reach autopilot.
|
||||
[ -f ~/.zshenv ] && source ~/.zshenv 2>/dev/null
|
||||
source ~/.zshrc 2>/dev/null || source ~/.bashrc 2>/dev/null || true
|
||||
exec '${safeGbrainPath}' autopilot --repo '${safeRepoPath}'
|
||||
# Runtime dirs resolved at install time (#1868): supervisors (systemd user
|
||||
# units especially) start with a minimal PATH, and a bun-shebang gbrain shim
|
||||
# needs bun's bin dir on PATH. Prepend AFTER profile sourcing so these win.
|
||||
${pathLine}
|
||||
exec '${sq(gbrainPath)}' autopilot --repo '${sq(repoPath)}'
|
||||
`;
|
||||
}
|
||||
|
||||
function writeWrapperScript(repoPath: string): string {
|
||||
const home = process.env.HOME || '';
|
||||
const gbrainDir = join(home, '.gbrain');
|
||||
mkdirSync(gbrainDir, { recursive: true });
|
||||
|
||||
// Wrapper sources the user's shell profile for API keys so nothing is
|
||||
// baked into plist/crontab/systemd unit files (#2).
|
||||
const wrapperPath = join(gbrainDir, 'autopilot-run.sh');
|
||||
const wrapper = generateWrapperScript(resolveGbrainCliPath(), repoPath);
|
||||
writeFileSync(wrapperPath, wrapper, { mode: 0o755 });
|
||||
return wrapperPath;
|
||||
}
|
||||
|
||||
@@ -7918,12 +7918,22 @@ export async function runRemediate(
|
||||
` gbrain doctor --remediate --resume ${planHash}\n`,
|
||||
);
|
||||
},
|
||||
onNoWorker: () => {
|
||||
console.error(
|
||||
`[remediate] no jobs worker running — submitted steps would time out unwatched (#2116).\n` +
|
||||
`Start one first:\n` +
|
||||
` gbrain jobs work # foreground worker\n` +
|
||||
` gbrain jobs supervisor start # managed daemon\n` +
|
||||
`(worker on another host? set GBRAIN_REMEDIATION_ASSUME_WORKER=1)`,
|
||||
);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// CLI surfaces — target unreachable / resume missed already emitted via hooks.
|
||||
// Library returns synthetic result with target_unreachable populated; exit 2.
|
||||
if (result.target_unreachable) process.exit(2);
|
||||
if (result.no_worker) process.exit(1);
|
||||
|
||||
if (dryRun && result.submitted.length > 0) {
|
||||
console.log(`[remediate --dry-run] Would submit ${result.submitted.length} jobs:`);
|
||||
|
||||
@@ -184,10 +184,20 @@ export async function runOnboard(engine: BrainEngine, args: string[]): Promise<v
|
||||
`Checkpoint saved. Resume with:\n gbrain doctor --remediate --resume ${planHash}\n`,
|
||||
);
|
||||
},
|
||||
onNoWorker: () => {
|
||||
process.stderr.write(
|
||||
`[onboard] no jobs worker running — submitted steps would time out unwatched (#2116).\n` +
|
||||
`Start one first:\n` +
|
||||
` gbrain jobs work # foreground worker\n` +
|
||||
` gbrain jobs supervisor start # managed daemon\n` +
|
||||
`(worker on another host? set GBRAIN_REMEDIATION_ASSUME_WORKER=1)\n`,
|
||||
);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if (result.target_unreachable) process.exit(2);
|
||||
if (result.no_worker) process.exit(1);
|
||||
|
||||
if (jsonOutput) {
|
||||
process.stdout.write(JSON.stringify(result, null, 2) + '\n');
|
||||
|
||||
@@ -129,6 +129,21 @@ export interface GBrainConfig {
|
||||
scrub_pii?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* #2731 — connection-pool routing knobs (file plane; read before any DB
|
||||
* connect, so it must live here, not the DB plane).
|
||||
*/
|
||||
pool?: {
|
||||
/**
|
||||
* Disable the direct (session-mode) pool and route everything through the
|
||||
* single read pool. Config-plane twin of the GBRAIN_DISABLE_DIRECT_POOL
|
||||
* env var so daemons/launchd/cron that don't inherit the shell env honor
|
||||
* the switch. Env wins when set (see resolveKillSwitch in
|
||||
* connection-manager.ts).
|
||||
*/
|
||||
disable_direct?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* v0.42 — self-upgrade settings (file plane; read on the hot path before any
|
||||
* DB connect, so it must live here, not the DB plane). `mode` is the only
|
||||
|
||||
@@ -40,6 +40,7 @@ import postgres from 'postgres';
|
||||
import { resolvePrepare, resolveSessionTimeouts, resolvePoolSize, endPoolBounded } from './db.ts';
|
||||
import { redactPgUrl } from './url-redact.ts';
|
||||
import { logConnectionEvent } from './connection-audit.ts';
|
||||
import { loadConfig } from './config.ts';
|
||||
|
||||
export type Sql = ReturnType<typeof postgres>;
|
||||
|
||||
@@ -176,6 +177,28 @@ export function readKillSwitchEnv(): boolean {
|
||||
process.env.GBRAIN_DISABLE_DIRECT_POOL === 'true';
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve kill-switch state: env ABOVE config (pace-mode precedence, #2731).
|
||||
*
|
||||
* `GBRAIN_DISABLE_DIRECT_POOL` stays the incident-time escape hatch — when the
|
||||
* env var is SET (to anything), it decides in both directions ('1'/'true' → on,
|
||||
* anything else → off, so `=0` can override a config-plane `true`). When unset,
|
||||
* fall back to the file-plane `pool.disable_direct` config key so daemons /
|
||||
* launchd / cron jobs that don't inherit the operator's shell env honor the
|
||||
* switch too. File plane (not DB plane) because this runs before any connect.
|
||||
*/
|
||||
export function resolveKillSwitch(): boolean {
|
||||
const env = process.env.GBRAIN_DISABLE_DIRECT_POOL;
|
||||
if (env !== undefined && env !== '') {
|
||||
return env === '1' || env === 'true';
|
||||
}
|
||||
try {
|
||||
return loadConfig()?.pool?.disable_direct === true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve direct pool size: explicit > env > default.
|
||||
*/
|
||||
@@ -211,7 +234,7 @@ export class ConnectionManager {
|
||||
this._readPool = opts.parent.peekReadPool();
|
||||
this._readPoolOwnedExternally = true; // never end the parent's pool
|
||||
} else {
|
||||
this._killSwitch = readKillSwitchEnv();
|
||||
this._killSwitch = resolveKillSwitch();
|
||||
this._isSupabase = isSupabasePoolerUrl(opts.url);
|
||||
// Direct URL: explicit override > env > derive > null
|
||||
const envOverride = process.env.GBRAIN_DIRECT_DATABASE_URL;
|
||||
|
||||
@@ -25,6 +25,41 @@ import type {
|
||||
StepResult,
|
||||
} from './types.ts';
|
||||
|
||||
/**
|
||||
* #2116: worker-liveness pre-flight for the Postgres submission path.
|
||||
*
|
||||
* With no jobs worker running, every submitted step just sits 'waiting'
|
||||
* until waitForCompletion burns its full (est_seconds + 60) timeout, then
|
||||
* aborts — a plan of N steps silently wastes N timeouts and remediates
|
||||
* nothing. Check BEFORE the step loop and fail fast instead.
|
||||
*
|
||||
* Liveness signals (either suffices):
|
||||
* 1. Local worker registry (ground truth on this host — every
|
||||
* `gbrain jobs work`, standalone or supervisor-spawned, registers).
|
||||
* 2. DB proxy: any active job holding a live lock (covers a busy worker
|
||||
* on ANOTHER host sharing the same Postgres).
|
||||
*
|
||||
* Known ceiling: an IDLE worker on another host shows neither signal.
|
||||
* `GBRAIN_REMEDIATION_ASSUME_WORKER=1` is the escape hatch for that
|
||||
* topology. Probe failures fail OPEN — a broken probe must never block
|
||||
* remediation that would otherwise work.
|
||||
*/
|
||||
export async function hasLiveJobsWorker(engine: BrainEngine): Promise<boolean> {
|
||||
if (process.env.GBRAIN_REMEDIATION_ASSUME_WORKER === '1') return true;
|
||||
try {
|
||||
const { readWorkers } = await import('../minions/worker-registry.ts');
|
||||
if (readWorkers().length > 0) return true;
|
||||
} catch { /* registry unavailable — fall through to the DB probe */ }
|
||||
try {
|
||||
const rows = await engine.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM minion_jobs WHERE status = 'active' AND lock_until > now()`,
|
||||
);
|
||||
return parseInt(rows[0]?.count ?? '0', 10) > 0;
|
||||
} catch {
|
||||
return true; // fail open
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Submit ordered Remediation jobs sequentially per D3, with D5 cascade
|
||||
* on failure and D7 scoped recheck between steps.
|
||||
@@ -189,6 +224,22 @@ export async function runRemediation(
|
||||
const isPGLite = engine.kind === 'pglite';
|
||||
const queue = new MinionQueue(engine);
|
||||
|
||||
// #2116: Postgres path needs a live jobs worker to drain what we submit.
|
||||
// PGLite runs inline (no durable worker), so the check doesn't apply there.
|
||||
if (!isPGLite && !(await hasLiveJobsWorker(engine))) {
|
||||
hooks.onNoWorker?.();
|
||||
return {
|
||||
doctor_run_id: crypto.randomUUID(),
|
||||
brain_score_initial: initialHealth.brain_score,
|
||||
brain_score_final: initialHealth.brain_score,
|
||||
brain_score_target: targetScore,
|
||||
target_reached: false,
|
||||
submitted: [],
|
||||
aborted_count: 0,
|
||||
no_worker: true,
|
||||
};
|
||||
}
|
||||
|
||||
// A4 amended: install a BudgetTracker scope around the plan-step loop so
|
||||
// any gateway.chat / embed / rerank inside a Minion handler (synthesize,
|
||||
// patterns, consolidate) auto-enforces the cap. On BudgetExhausted, the
|
||||
|
||||
@@ -101,6 +101,12 @@ export interface RemediationResult {
|
||||
target: number;
|
||||
ceiling: number;
|
||||
};
|
||||
/**
|
||||
* #2116: set when the Postgres pre-flight found no live jobs worker.
|
||||
* Nothing was submitted — without a worker every step would burn its full
|
||||
* timeout and abort silently. Caller decides exit code + hint.
|
||||
*/
|
||||
no_worker?: true;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -124,4 +130,6 @@ export interface RemediationHooks {
|
||||
onResumeLoaded?: (planHash: string, completedCount: number, remainingCount: number) => void;
|
||||
/** Fired on resume-checkpoint miss (resume mode only). */
|
||||
onResumeMissed?: (planHash: string, requested?: string) => void;
|
||||
/** Fired when the Postgres worker-liveness pre-flight found no worker (#2116). */
|
||||
onNoWorker?: () => void;
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ import { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, existsSync
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
|
||||
import { detectInstallTarget } from '../src/commands/autopilot.ts';
|
||||
import { detectInstallTarget, generateWrapperScript, shutdownExitCode } from '../src/commands/autopilot.ts';
|
||||
|
||||
let tmp: string;
|
||||
const envSnapshot: Record<string, string | undefined> = {};
|
||||
@@ -85,6 +85,71 @@ describe('detectInstallTarget', () => {
|
||||
// exported in zshrc never reach the LaunchAgent subprocess. Operators who
|
||||
// exported GBRAIN_DATABASE_URL or {OPENAI,ANTHROPIC}_API_KEY in zshrc and
|
||||
// expected autopilot to inherit them hit silent missing-secret failures.
|
||||
// issue #1868: the wrapper runs under systemd/cron with a minimal PATH and the
|
||||
// zsh profiles it sources are a macOS convention that often doesn't exist on a
|
||||
// Linux server. A bun-shebang gbrain shim then dies with "bun: not found" and
|
||||
// the service crash-loops. The generator must bake the install-time runtime
|
||||
// dirs (gbrain's dir, the running runtime's dir, ~/.bun/bin) onto PATH, AFTER
|
||||
// profile sourcing so they win even if a profile reset PATH.
|
||||
describe('generateWrapperScript — runtime PATH baked in (#1868)', () => {
|
||||
const script = generateWrapperScript('/usr/local/bin/gbrain', '/srv/brain', {
|
||||
home: '/home/u',
|
||||
execPath: '/home/u/.bun/bin/bun',
|
||||
});
|
||||
|
||||
test('prepends gbrain dir, runtime dir, and ~/.bun/bin to PATH', () => {
|
||||
expect(script).toContain(`export PATH='/usr/local/bin:/home/u/.bun/bin':"$PATH"`);
|
||||
});
|
||||
|
||||
test('PATH export comes AFTER profile sourcing (profiles must not clobber it)', () => {
|
||||
const pathIdx = script.indexOf('export PATH=');
|
||||
const zshrcIdx = script.indexOf('~/.zshrc');
|
||||
const bashrcIdx = script.indexOf('~/.bashrc');
|
||||
expect(pathIdx).toBeGreaterThan(zshrcIdx);
|
||||
expect(pathIdx).toBeGreaterThan(bashrcIdx);
|
||||
});
|
||||
|
||||
test('PATH export comes BEFORE the exec line', () => {
|
||||
expect(script.indexOf('export PATH=')).toBeLessThan(script.indexOf('exec '));
|
||||
});
|
||||
|
||||
test('exec line uses the resolved gbrain path + repo', () => {
|
||||
expect(script).toContain(`exec '/usr/local/bin/gbrain' autopilot --repo '/srv/brain'`);
|
||||
});
|
||||
|
||||
test('deduplicates identical dirs', () => {
|
||||
const s = generateWrapperScript('/opt/bin/gbrain', '/repo', { home: '', execPath: '/opt/bin/bun' });
|
||||
expect(s).toContain(`export PATH='/opt/bin':"$PATH"`);
|
||||
});
|
||||
|
||||
test('never emits an empty PATH entry (cwd injection) when nothing resolves', () => {
|
||||
const s = generateWrapperScript('gbrain', '/repo', { home: '', execPath: '' });
|
||||
expect(s).not.toContain(`PATH='':`);
|
||||
expect(s).not.toMatch(/export PATH=''/);
|
||||
});
|
||||
|
||||
test('single-quote-escapes hostile dirs', () => {
|
||||
const s = generateWrapperScript("/tmp/o'brien/gbrain", '/repo', { home: '', execPath: '' });
|
||||
expect(s).toContain(`'/tmp/o'\\''brien'`);
|
||||
});
|
||||
});
|
||||
|
||||
// issue #2234 (part 2): a daemon that GIVES UP (max_crashes /
|
||||
// cycle-failure-cap) must exit non-zero so systemd Restart=on-failure
|
||||
// restarts it; pre-fix it exited 0 and stayed silently dead for days.
|
||||
// Operator-initiated stops stay 0.
|
||||
describe('shutdownExitCode (#2234)', () => {
|
||||
test('give-up paths exit non-zero', () => {
|
||||
expect(shutdownExitCode('max_crashes')).toBe(1);
|
||||
expect(shutdownExitCode('cycle-failure-cap')).toBe(1);
|
||||
});
|
||||
|
||||
test('signal-driven stops exit 0', () => {
|
||||
expect(shutdownExitCode('SIGTERM')).toBe(0);
|
||||
expect(shutdownExitCode('SIGINT')).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('autopilot wrapper script — env source order (v0.36.1.x #966)', () => {
|
||||
test('wrapper sources ~/.zshenv before ~/.zshrc', async () => {
|
||||
const { readFileSync } = await import('fs');
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import { describe, expect, test, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import {
|
||||
isSupabasePoolerUrl,
|
||||
deriveDirectUrl,
|
||||
readKillSwitchEnv,
|
||||
resolveKillSwitch,
|
||||
resolveDirectPoolSize,
|
||||
ConnectionManager,
|
||||
DEFAULT_DIRECT_POOL_SIZE,
|
||||
@@ -109,6 +113,62 @@ describe('readKillSwitchEnv', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// #2731: env-above-config kill-switch. Daemons/launchd/cron that don't inherit
|
||||
// the shell env need a config.json plane (`pool.disable_direct`); the env var
|
||||
// stays the incident-time escape hatch and wins IN BOTH DIRECTIONS when set.
|
||||
describe('resolveKillSwitch (#2731, config plane)', () => {
|
||||
let tmp: string;
|
||||
let envKill: string | undefined;
|
||||
let envHome: string | undefined;
|
||||
|
||||
const writeConfig = (cfg: Record<string, unknown>) => {
|
||||
mkdirSync(join(tmp, '.gbrain'), { recursive: true });
|
||||
writeFileSync(join(tmp, '.gbrain', 'config.json'), JSON.stringify(cfg));
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
envKill = process.env.GBRAIN_DISABLE_DIRECT_POOL;
|
||||
envHome = process.env.GBRAIN_HOME;
|
||||
tmp = mkdtempSync(join(tmpdir(), 'gbrain-killswitch-'));
|
||||
process.env.GBRAIN_HOME = tmp;
|
||||
delete process.env.GBRAIN_DISABLE_DIRECT_POOL;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (envKill === undefined) delete process.env.GBRAIN_DISABLE_DIRECT_POOL;
|
||||
else process.env.GBRAIN_DISABLE_DIRECT_POOL = envKill;
|
||||
if (envHome === undefined) delete process.env.GBRAIN_HOME;
|
||||
else process.env.GBRAIN_HOME = envHome;
|
||||
try { rmSync(tmp, { recursive: true, force: true }); } catch { /* best-effort */ }
|
||||
});
|
||||
|
||||
test('config pool.disable_direct=true engages the kill-switch with env unset', () => {
|
||||
writeConfig({ engine: 'postgres', database_url: 'postgresql://u:p@h:6543/db', pool: { disable_direct: true } });
|
||||
expect(resolveKillSwitch()).toBe(true);
|
||||
});
|
||||
|
||||
test('no config, no env → false', () => {
|
||||
expect(resolveKillSwitch()).toBe(false);
|
||||
});
|
||||
|
||||
test('config without pool key → false', () => {
|
||||
writeConfig({ engine: 'postgres', database_url: 'postgresql://u:p@h:6543/db' });
|
||||
expect(resolveKillSwitch()).toBe(false);
|
||||
});
|
||||
|
||||
test('env=1 wins over config false', () => {
|
||||
writeConfig({ engine: 'postgres', database_url: 'postgresql://u:p@h:6543/db', pool: { disable_direct: false } });
|
||||
process.env.GBRAIN_DISABLE_DIRECT_POOL = '1';
|
||||
expect(resolveKillSwitch()).toBe(true);
|
||||
});
|
||||
|
||||
test('env=0 wins over config true (escape hatch works in both directions)', () => {
|
||||
writeConfig({ engine: 'postgres', database_url: 'postgresql://u:p@h:6543/db', pool: { disable_direct: true } });
|
||||
process.env.GBRAIN_DISABLE_DIRECT_POOL = '0';
|
||||
expect(resolveKillSwitch()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveDirectPoolSize', () => {
|
||||
let original: string | undefined;
|
||||
beforeEach(() => { original = process.env.GBRAIN_DIRECT_POOL_SIZE; });
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
/**
|
||||
* issue #2116 — `onboard --auto` / `doctor --remediate` submitted remediation
|
||||
* jobs with no worker-liveness pre-flight: with workers quiesced, every step
|
||||
* burned its full (est_seconds + 60) timeout and aborted silently.
|
||||
*
|
||||
* hasLiveJobsWorker is the pre-flight runRemediation now runs before the
|
||||
* Postgres step loop. Signals: local worker registry (ground truth on this
|
||||
* host) OR a DB proxy (any active job with a live lock — a busy worker on
|
||||
* another host). GBRAIN_REMEDIATION_ASSUME_WORKER=1 is the escape hatch for
|
||||
* an idle remote worker; probe failures fail OPEN.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { mkdtempSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import { hasLiveJobsWorker } from '../src/core/remediation/run.ts';
|
||||
import { registerWorker } from '../src/core/minions/worker-registry.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
|
||||
function stubEngine(activeCount: string | Error): BrainEngine {
|
||||
return {
|
||||
kind: 'postgres',
|
||||
executeRaw: async () => {
|
||||
if (activeCount instanceof Error) throw activeCount;
|
||||
return [{ count: activeCount }];
|
||||
},
|
||||
} as unknown as BrainEngine;
|
||||
}
|
||||
|
||||
/** Fresh empty GBRAIN_HOME (empty worker registry) + no assume-worker escape
|
||||
* hatch, restored via withEnv. Tmp dir is intentionally leaked (OS reaps),
|
||||
* same as test/helpers/with-env.ts emptyHome(). */
|
||||
function inEmptyHome<T>(
|
||||
fn: () => T | Promise<T>,
|
||||
extra: Record<string, string | undefined> = {},
|
||||
): Promise<T> {
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'gbrain-liveness-'));
|
||||
return withEnv(
|
||||
{ GBRAIN_HOME: tmp, GBRAIN_REMEDIATION_ASSUME_WORKER: undefined, ...extra },
|
||||
fn,
|
||||
);
|
||||
}
|
||||
|
||||
describe('hasLiveJobsWorker (#2116)', () => {
|
||||
test('false when registry empty and no active locked jobs — the fix', async () => {
|
||||
await inEmptyHome(async () => {
|
||||
expect(await hasLiveJobsWorker(stubEngine('0'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
test('true when DB shows an active job with a live lock (busy remote worker)', async () => {
|
||||
await inEmptyHome(async () => {
|
||||
expect(await hasLiveJobsWorker(stubEngine('2'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
test('true when a local worker is registered (this-host ground truth)', async () => {
|
||||
await inEmptyHome(async () => {
|
||||
const cleanup = registerWorker({
|
||||
pid: process.pid,
|
||||
queue: 'default',
|
||||
nice_requested: null,
|
||||
nice_effective: null,
|
||||
started_at: Date.now(),
|
||||
});
|
||||
try {
|
||||
expect(await hasLiveJobsWorker(stubEngine('0'))).toBe(true);
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test('GBRAIN_REMEDIATION_ASSUME_WORKER=1 skips the probe (remote idle worker)', async () => {
|
||||
await inEmptyHome(
|
||||
async () => {
|
||||
expect(await hasLiveJobsWorker(stubEngine(new Error('should not be called')))).toBe(true);
|
||||
},
|
||||
{ GBRAIN_REMEDIATION_ASSUME_WORKER: '1' },
|
||||
);
|
||||
});
|
||||
|
||||
test('fails OPEN when the DB probe throws', async () => {
|
||||
await inEmptyHome(async () => {
|
||||
expect(await hasLiveJobsWorker(stubEngine(new Error('relation minion_jobs missing')))).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user