Compare commits

...
Author SHA1 Message Date
Garry TanandClaude Fable 5 556d184332 test: use withEnv() in remediation-worker-liveness test (check:test-isolation)
CI verify failed on check:test-isolation rule R1 (direct process.env
mutation in a non-serial test file). Replace the beforeEach/afterEach
env save/restore with the canonical withEnv() helper.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 10:53:56 -07:00
Garry TanandClaude Fable 5 12a585ca52 fix(jobs/autopilot): worker liveness pre-flight, installer PATH, give-up exit codes, direct-pool config plane
Four backlog fixes in the jobs/autopilot workers, locks & installers area:

- #1868: autopilot --install wrapper script now bakes the install-time
  runtime dirs (gbrain's dir, the running runtime's dir, ~/.bun/bin) onto
  PATH after profile sourcing, so a bun-shebang gbrain no longer
  crash-loops under systemd's minimal PATH. Pure generateWrapperScript()
  exported for tests.

- #2116: runRemediation (onboard --auto / doctor --remediate / MCP
  run_onboard) runs a worker-liveness pre-flight on the Postgres path
  before submitting steps: local worker registry OR active-job live-lock
  DB proxy. No worker -> fail fast with a start-a-worker hint (exit 1)
  instead of burning every step's full timeout in silence.
  GBRAIN_REMEDIATION_ASSUME_WORKER=1 escape hatch; probe failures fail open.

- #2234 (part 2): autopilot give-up paths (max_crashes,
  cycle-failure-cap) now exit non-zero via shutdownExitCode() so systemd
  Restart=on-failure restarts a daemon that decided it can't make
  progress. SIGTERM/SIGINT stay 0.

- #2731: GBRAIN_DISABLE_DIRECT_POOL gets a config.json plane
  (pool.disable_direct) read when the env var is unset; env wins in both
  directions when set (pace-mode precedence). Daemons/launchd/cron that
  don't inherit the shell env can now persist the kill-switch.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 14:37:08 -07:00
10 changed files with 395 additions and 17 deletions
+60 -15
View File
@@ -19,7 +19,7 @@
import { existsSync, readFileSync, writeFileSync, mkdirSync, appendFileSync, utimesSync, unlinkSync, chmodSync } from 'fs';
import { setCliExitVerdict } from '../core/cli-force-exit.ts';
import { join } from 'path';
import { dirname, join } from 'path';
import { execSync } from 'child_process';
import type { BrainEngine } from '../core/engine.ts';
import { loadPreferences } from '../core/preferences.ts';
@@ -151,6 +151,16 @@ export function shouldSpawnAutopilotWorker(args: string[]): boolean {
return !args.includes('--no-worker');
}
/**
* #2234: exit code for the autopilot shutdown path. Give-up reasons — the
* daemon decided it cannot make progress — must exit non-zero so systemd
* `Restart=on-failure` (and any monitoring on the exit status) sees a
* failure, not a clean stop. Signal-driven stops stay 0.
*/
export function shutdownExitCode(reason: string): number {
return reason === 'max_crashes' || reason === 'cycle-failure-cap' ? 1 : 0;
}
export function isPidAlive(pid: number): boolean {
if (!Number.isFinite(pid) || pid <= 0) return false;
try {
@@ -521,7 +531,10 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
}
}
try { unlinkSync(lockPath); } catch { /* already gone */ }
process.exit(0);
// #2234: give-up paths (max_crashes, cycle-failure-cap) exit NON-zero so a
// systemd `Restart=on-failure` unit restarts a daemon that decided it
// can't make progress. Operator-initiated stops (SIGTERM/SIGINT) stay 0.
process.exit(shutdownExitCode(sig));
};
process.on('SIGTERM', () => { void shutdown('SIGTERM'); });
process.on('SIGINT', () => { void shutdown('SIGINT'); });
@@ -1166,18 +1179,35 @@ function detectOpenClaw(): { detected: boolean; bootstrapCandidates: string[] }
return { detected: signal, bootstrapCandidates: existing };
}
function writeWrapperScript(repoPath: string): string {
const home = process.env.HOME || '';
const gbrainDir = join(home, '.gbrain');
mkdirSync(gbrainDir, { recursive: true });
// Wrapper sources the user's shell profile for API keys so nothing is
// baked into plist/crontab/systemd unit files (#2).
const wrapperPath = join(gbrainDir, 'autopilot-run.sh');
const gbrainPath = resolveGbrainCliPath();
const safeRepoPath = repoPath.replace(/'/g, "'\\''");
const safeGbrainPath = gbrainPath.replace(/'/g, "'\\''");
const wrapper = `#!/bin/bash
/**
* Pure wrapper-script generator (exported for tests, #1868).
*
* The wrapper runs under launchd/systemd/cron with a minimal PATH (systemd
* user units get ~ /usr/bin:/bin), and the shell profiles it sources are a
* macOS/zsh convention that often doesn't exist on a Linux server. If the
* resolved `gbrain` is a bun-shebang shim, `#!/usr/bin/env bun` then fails
* with "bun: not found" and the service crash-loops. Bake the runtime dirs
* (resolved at INSTALL time: gbrain's own dir, the current runtime's dir,
* ~/.bun/bin) onto PATH AFTER profile sourcing so they win regardless of
* what the profiles did.
*/
export function generateWrapperScript(
gbrainPath: string,
repoPath: string,
opts?: { home?: string; execPath?: string },
): string {
const home = opts?.home ?? process.env.HOME ?? '';
const execPath = opts?.execPath ?? process.execPath ?? '';
const sq = (s: string) => s.replace(/'/g, "'\\''");
const pathDirs = [...new Set(
[dirname(gbrainPath), execPath ? dirname(execPath) : '', home ? join(home, '.bun', 'bin') : '']
.filter((d) => d && d !== '.' && d !== '/'),
)];
// Empty-safe: never emit `PATH='':"$PATH"` (a leading empty entry means cwd).
const pathLine = pathDirs.length > 0
? `export PATH='${sq(pathDirs.join(':'))}':"$PATH"`
: ': # no runtime dirs resolved at install time';
return `#!/bin/bash
# Auto-generated by gbrain autopilot --install
# Sources shell profile for API keys, then runs autopilot.
# zshenv is the canonical place for env vars in zsh on macOS (zshrc is for
@@ -1186,8 +1216,23 @@ function writeWrapperScript(repoPath: string): string {
# OPENAI/ANTHROPIC keys exported in zshenv reach autopilot.
[ -f ~/.zshenv ] && source ~/.zshenv 2>/dev/null
source ~/.zshrc 2>/dev/null || source ~/.bashrc 2>/dev/null || true
exec '${safeGbrainPath}' autopilot --repo '${safeRepoPath}'
# Runtime dirs resolved at install time (#1868): supervisors (systemd user
# units especially) start with a minimal PATH, and a bun-shebang gbrain shim
# needs bun's bin dir on PATH. Prepend AFTER profile sourcing so these win.
${pathLine}
exec '${sq(gbrainPath)}' autopilot --repo '${sq(repoPath)}'
`;
}
function writeWrapperScript(repoPath: string): string {
const home = process.env.HOME || '';
const gbrainDir = join(home, '.gbrain');
mkdirSync(gbrainDir, { recursive: true });
// Wrapper sources the user's shell profile for API keys so nothing is
// baked into plist/crontab/systemd unit files (#2).
const wrapperPath = join(gbrainDir, 'autopilot-run.sh');
const wrapper = generateWrapperScript(resolveGbrainCliPath(), repoPath);
writeFileSync(wrapperPath, wrapper, { mode: 0o755 });
return wrapperPath;
}
+10
View File
@@ -7918,12 +7918,22 @@ export async function runRemediate(
` gbrain doctor --remediate --resume ${planHash}\n`,
);
},
onNoWorker: () => {
console.error(
`[remediate] no jobs worker running — submitted steps would time out unwatched (#2116).\n` +
`Start one first:\n` +
` gbrain jobs work # foreground worker\n` +
` gbrain jobs supervisor start # managed daemon\n` +
`(worker on another host? set GBRAIN_REMEDIATION_ASSUME_WORKER=1)`,
);
},
},
);
// CLI surfaces — target unreachable / resume missed already emitted via hooks.
// Library returns synthetic result with target_unreachable populated; exit 2.
if (result.target_unreachable) process.exit(2);
if (result.no_worker) process.exit(1);
if (dryRun && result.submitted.length > 0) {
console.log(`[remediate --dry-run] Would submit ${result.submitted.length} jobs:`);
+10
View File
@@ -184,10 +184,20 @@ export async function runOnboard(engine: BrainEngine, args: string[]): Promise<v
`Checkpoint saved. Resume with:\n gbrain doctor --remediate --resume ${planHash}\n`,
);
},
onNoWorker: () => {
process.stderr.write(
`[onboard] no jobs worker running — submitted steps would time out unwatched (#2116).\n` +
`Start one first:\n` +
` gbrain jobs work # foreground worker\n` +
` gbrain jobs supervisor start # managed daemon\n` +
`(worker on another host? set GBRAIN_REMEDIATION_ASSUME_WORKER=1)\n`,
);
},
},
);
if (result.target_unreachable) process.exit(2);
if (result.no_worker) process.exit(1);
if (jsonOutput) {
process.stdout.write(JSON.stringify(result, null, 2) + '\n');
+15
View File
@@ -129,6 +129,21 @@ export interface GBrainConfig {
scrub_pii?: boolean;
};
/**
* #2731 — connection-pool routing knobs (file plane; read before any DB
* connect, so it must live here, not the DB plane).
*/
pool?: {
/**
* Disable the direct (session-mode) pool and route everything through the
* single read pool. Config-plane twin of the GBRAIN_DISABLE_DIRECT_POOL
* env var so daemons/launchd/cron that don't inherit the shell env honor
* the switch. Env wins when set (see resolveKillSwitch in
* connection-manager.ts).
*/
disable_direct?: boolean;
};
/**
* v0.42 — self-upgrade settings (file plane; read on the hot path before any
* DB connect, so it must live here, not the DB plane). `mode` is the only
+24 -1
View File
@@ -40,6 +40,7 @@ import postgres from 'postgres';
import { resolvePrepare, resolveSessionTimeouts, resolvePoolSize, endPoolBounded } from './db.ts';
import { redactPgUrl } from './url-redact.ts';
import { logConnectionEvent } from './connection-audit.ts';
import { loadConfig } from './config.ts';
export type Sql = ReturnType<typeof postgres>;
@@ -176,6 +177,28 @@ export function readKillSwitchEnv(): boolean {
process.env.GBRAIN_DISABLE_DIRECT_POOL === 'true';
}
/**
* Resolve kill-switch state: env ABOVE config (pace-mode precedence, #2731).
*
* `GBRAIN_DISABLE_DIRECT_POOL` stays the incident-time escape hatch — when the
* env var is SET (to anything), it decides in both directions ('1'/'true' → on,
* anything else → off, so `=0` can override a config-plane `true`). When unset,
* fall back to the file-plane `pool.disable_direct` config key so daemons /
* launchd / cron jobs that don't inherit the operator's shell env honor the
* switch too. File plane (not DB plane) because this runs before any connect.
*/
export function resolveKillSwitch(): boolean {
const env = process.env.GBRAIN_DISABLE_DIRECT_POOL;
if (env !== undefined && env !== '') {
return env === '1' || env === 'true';
}
try {
return loadConfig()?.pool?.disable_direct === true;
} catch {
return false;
}
}
/**
* Resolve direct pool size: explicit > env > default.
*/
@@ -211,7 +234,7 @@ export class ConnectionManager {
this._readPool = opts.parent.peekReadPool();
this._readPoolOwnedExternally = true; // never end the parent's pool
} else {
this._killSwitch = readKillSwitchEnv();
this._killSwitch = resolveKillSwitch();
this._isSupabase = isSupabasePoolerUrl(opts.url);
// Direct URL: explicit override > env > derive > null
const envOverride = process.env.GBRAIN_DIRECT_DATABASE_URL;
+51
View File
@@ -25,6 +25,41 @@ import type {
StepResult,
} from './types.ts';
/**
* #2116: worker-liveness pre-flight for the Postgres submission path.
*
* With no jobs worker running, every submitted step just sits 'waiting'
* until waitForCompletion burns its full (est_seconds + 60) timeout, then
* aborts a plan of N steps silently wastes N timeouts and remediates
* nothing. Check BEFORE the step loop and fail fast instead.
*
* Liveness signals (either suffices):
* 1. Local worker registry (ground truth on this host every
* `gbrain jobs work`, standalone or supervisor-spawned, registers).
* 2. DB proxy: any active job holding a live lock (covers a busy worker
* on ANOTHER host sharing the same Postgres).
*
* Known ceiling: an IDLE worker on another host shows neither signal.
* `GBRAIN_REMEDIATION_ASSUME_WORKER=1` is the escape hatch for that
* topology. Probe failures fail OPEN a broken probe must never block
* remediation that would otherwise work.
*/
export async function hasLiveJobsWorker(engine: BrainEngine): Promise<boolean> {
if (process.env.GBRAIN_REMEDIATION_ASSUME_WORKER === '1') return true;
try {
const { readWorkers } = await import('../minions/worker-registry.ts');
if (readWorkers().length > 0) return true;
} catch { /* registry unavailable — fall through to the DB probe */ }
try {
const rows = await engine.executeRaw<{ count: string }>(
`SELECT count(*)::text AS count FROM minion_jobs WHERE status = 'active' AND lock_until > now()`,
);
return parseInt(rows[0]?.count ?? '0', 10) > 0;
} catch {
return true; // fail open
}
}
/**
* Submit ordered Remediation jobs sequentially per D3, with D5 cascade
* on failure and D7 scoped recheck between steps.
@@ -189,6 +224,22 @@ export async function runRemediation(
const isPGLite = engine.kind === 'pglite';
const queue = new MinionQueue(engine);
// #2116: Postgres path needs a live jobs worker to drain what we submit.
// PGLite runs inline (no durable worker), so the check doesn't apply there.
if (!isPGLite && !(await hasLiveJobsWorker(engine))) {
hooks.onNoWorker?.();
return {
doctor_run_id: crypto.randomUUID(),
brain_score_initial: initialHealth.brain_score,
brain_score_final: initialHealth.brain_score,
brain_score_target: targetScore,
target_reached: false,
submitted: [],
aborted_count: 0,
no_worker: true,
};
}
// A4 amended: install a BudgetTracker scope around the plan-step loop so
// any gateway.chat / embed / rerank inside a Minion handler (synthesize,
// patterns, consolidate) auto-enforces the cap. On BudgetExhausted, the
+8
View File
@@ -101,6 +101,12 @@ export interface RemediationResult {
target: number;
ceiling: number;
};
/**
* #2116: set when the Postgres pre-flight found no live jobs worker.
* Nothing was submitted without a worker every step would burn its full
* timeout and abort silently. Caller decides exit code + hint.
*/
no_worker?: true;
}
/**
@@ -124,4 +130,6 @@ export interface RemediationHooks {
onResumeLoaded?: (planHash: string, completedCount: number, remainingCount: number) => void;
/** Fired on resume-checkpoint miss (resume mode only). */
onResumeMissed?: (planHash: string, requested?: string) => void;
/** Fired when the Postgres worker-liveness pre-flight found no worker (#2116). */
onNoWorker?: () => void;
}
+66 -1
View File
@@ -20,7 +20,7 @@ import { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, existsSync
import { join } from 'path';
import { tmpdir } from 'os';
import { detectInstallTarget } from '../src/commands/autopilot.ts';
import { detectInstallTarget, generateWrapperScript, shutdownExitCode } from '../src/commands/autopilot.ts';
let tmp: string;
const envSnapshot: Record<string, string | undefined> = {};
@@ -85,6 +85,71 @@ describe('detectInstallTarget', () => {
// exported in zshrc never reach the LaunchAgent subprocess. Operators who
// exported GBRAIN_DATABASE_URL or {OPENAI,ANTHROPIC}_API_KEY in zshrc and
// expected autopilot to inherit them hit silent missing-secret failures.
// issue #1868: the wrapper runs under systemd/cron with a minimal PATH and the
// zsh profiles it sources are a macOS convention that often doesn't exist on a
// Linux server. A bun-shebang gbrain shim then dies with "bun: not found" and
// the service crash-loops. The generator must bake the install-time runtime
// dirs (gbrain's dir, the running runtime's dir, ~/.bun/bin) onto PATH, AFTER
// profile sourcing so they win even if a profile reset PATH.
describe('generateWrapperScript — runtime PATH baked in (#1868)', () => {
const script = generateWrapperScript('/usr/local/bin/gbrain', '/srv/brain', {
home: '/home/u',
execPath: '/home/u/.bun/bin/bun',
});
test('prepends gbrain dir, runtime dir, and ~/.bun/bin to PATH', () => {
expect(script).toContain(`export PATH='/usr/local/bin:/home/u/.bun/bin':"$PATH"`);
});
test('PATH export comes AFTER profile sourcing (profiles must not clobber it)', () => {
const pathIdx = script.indexOf('export PATH=');
const zshrcIdx = script.indexOf('~/.zshrc');
const bashrcIdx = script.indexOf('~/.bashrc');
expect(pathIdx).toBeGreaterThan(zshrcIdx);
expect(pathIdx).toBeGreaterThan(bashrcIdx);
});
test('PATH export comes BEFORE the exec line', () => {
expect(script.indexOf('export PATH=')).toBeLessThan(script.indexOf('exec '));
});
test('exec line uses the resolved gbrain path + repo', () => {
expect(script).toContain(`exec '/usr/local/bin/gbrain' autopilot --repo '/srv/brain'`);
});
test('deduplicates identical dirs', () => {
const s = generateWrapperScript('/opt/bin/gbrain', '/repo', { home: '', execPath: '/opt/bin/bun' });
expect(s).toContain(`export PATH='/opt/bin':"$PATH"`);
});
test('never emits an empty PATH entry (cwd injection) when nothing resolves', () => {
const s = generateWrapperScript('gbrain', '/repo', { home: '', execPath: '' });
expect(s).not.toContain(`PATH='':`);
expect(s).not.toMatch(/export PATH=''/);
});
test('single-quote-escapes hostile dirs', () => {
const s = generateWrapperScript("/tmp/o'brien/gbrain", '/repo', { home: '', execPath: '' });
expect(s).toContain(`'/tmp/o'\\''brien'`);
});
});
// issue #2234 (part 2): a daemon that GIVES UP (max_crashes /
// cycle-failure-cap) must exit non-zero so systemd Restart=on-failure
// restarts it; pre-fix it exited 0 and stayed silently dead for days.
// Operator-initiated stops stay 0.
describe('shutdownExitCode (#2234)', () => {
test('give-up paths exit non-zero', () => {
expect(shutdownExitCode('max_crashes')).toBe(1);
expect(shutdownExitCode('cycle-failure-cap')).toBe(1);
});
test('signal-driven stops exit 0', () => {
expect(shutdownExitCode('SIGTERM')).toBe(0);
expect(shutdownExitCode('SIGINT')).toBe(0);
});
});
describe('autopilot wrapper script — env source order (v0.36.1.x #966)', () => {
test('wrapper sources ~/.zshenv before ~/.zshrc', async () => {
const { readFileSync } = await import('fs');
+60
View File
@@ -1,8 +1,12 @@
import { describe, expect, test, beforeEach, afterEach } from 'bun:test';
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import {
isSupabasePoolerUrl,
deriveDirectUrl,
readKillSwitchEnv,
resolveKillSwitch,
resolveDirectPoolSize,
ConnectionManager,
DEFAULT_DIRECT_POOL_SIZE,
@@ -109,6 +113,62 @@ describe('readKillSwitchEnv', () => {
});
});
// #2731: env-above-config kill-switch. Daemons/launchd/cron that don't inherit
// the shell env need a config.json plane (`pool.disable_direct`); the env var
// stays the incident-time escape hatch and wins IN BOTH DIRECTIONS when set.
describe('resolveKillSwitch (#2731, config plane)', () => {
let tmp: string;
let envKill: string | undefined;
let envHome: string | undefined;
const writeConfig = (cfg: Record<string, unknown>) => {
mkdirSync(join(tmp, '.gbrain'), { recursive: true });
writeFileSync(join(tmp, '.gbrain', 'config.json'), JSON.stringify(cfg));
};
beforeEach(() => {
envKill = process.env.GBRAIN_DISABLE_DIRECT_POOL;
envHome = process.env.GBRAIN_HOME;
tmp = mkdtempSync(join(tmpdir(), 'gbrain-killswitch-'));
process.env.GBRAIN_HOME = tmp;
delete process.env.GBRAIN_DISABLE_DIRECT_POOL;
});
afterEach(() => {
if (envKill === undefined) delete process.env.GBRAIN_DISABLE_DIRECT_POOL;
else process.env.GBRAIN_DISABLE_DIRECT_POOL = envKill;
if (envHome === undefined) delete process.env.GBRAIN_HOME;
else process.env.GBRAIN_HOME = envHome;
try { rmSync(tmp, { recursive: true, force: true }); } catch { /* best-effort */ }
});
test('config pool.disable_direct=true engages the kill-switch with env unset', () => {
writeConfig({ engine: 'postgres', database_url: 'postgresql://u:p@h:6543/db', pool: { disable_direct: true } });
expect(resolveKillSwitch()).toBe(true);
});
test('no config, no env → false', () => {
expect(resolveKillSwitch()).toBe(false);
});
test('config without pool key → false', () => {
writeConfig({ engine: 'postgres', database_url: 'postgresql://u:p@h:6543/db' });
expect(resolveKillSwitch()).toBe(false);
});
test('env=1 wins over config false', () => {
writeConfig({ engine: 'postgres', database_url: 'postgresql://u:p@h:6543/db', pool: { disable_direct: false } });
process.env.GBRAIN_DISABLE_DIRECT_POOL = '1';
expect(resolveKillSwitch()).toBe(true);
});
test('env=0 wins over config true (escape hatch works in both directions)', () => {
writeConfig({ engine: 'postgres', database_url: 'postgresql://u:p@h:6543/db', pool: { disable_direct: true } });
process.env.GBRAIN_DISABLE_DIRECT_POOL = '0';
expect(resolveKillSwitch()).toBe(false);
});
});
describe('resolveDirectPoolSize', () => {
let original: string | undefined;
beforeEach(() => { original = process.env.GBRAIN_DIRECT_POOL_SIZE; });
+91
View File
@@ -0,0 +1,91 @@
/**
* issue #2116 `onboard --auto` / `doctor --remediate` submitted remediation
* jobs with no worker-liveness pre-flight: with workers quiesced, every step
* burned its full (est_seconds + 60) timeout and aborted silently.
*
* hasLiveJobsWorker is the pre-flight runRemediation now runs before the
* Postgres step loop. Signals: local worker registry (ground truth on this
* host) OR a DB proxy (any active job with a live lock a busy worker on
* another host). GBRAIN_REMEDIATION_ASSUME_WORKER=1 is the escape hatch for
* an idle remote worker; probe failures fail OPEN.
*/
import { describe, test, expect } from 'bun:test';
import { mkdtempSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import { withEnv } from './helpers/with-env.ts';
import { hasLiveJobsWorker } from '../src/core/remediation/run.ts';
import { registerWorker } from '../src/core/minions/worker-registry.ts';
import type { BrainEngine } from '../src/core/engine.ts';
function stubEngine(activeCount: string | Error): BrainEngine {
return {
kind: 'postgres',
executeRaw: async () => {
if (activeCount instanceof Error) throw activeCount;
return [{ count: activeCount }];
},
} as unknown as BrainEngine;
}
/** Fresh empty GBRAIN_HOME (empty worker registry) + no assume-worker escape
* hatch, restored via withEnv. Tmp dir is intentionally leaked (OS reaps),
* same as test/helpers/with-env.ts emptyHome(). */
function inEmptyHome<T>(
fn: () => T | Promise<T>,
extra: Record<string, string | undefined> = {},
): Promise<T> {
const tmp = mkdtempSync(join(tmpdir(), 'gbrain-liveness-'));
return withEnv(
{ GBRAIN_HOME: tmp, GBRAIN_REMEDIATION_ASSUME_WORKER: undefined, ...extra },
fn,
);
}
describe('hasLiveJobsWorker (#2116)', () => {
test('false when registry empty and no active locked jobs — the fix', async () => {
await inEmptyHome(async () => {
expect(await hasLiveJobsWorker(stubEngine('0'))).toBe(false);
});
});
test('true when DB shows an active job with a live lock (busy remote worker)', async () => {
await inEmptyHome(async () => {
expect(await hasLiveJobsWorker(stubEngine('2'))).toBe(true);
});
});
test('true when a local worker is registered (this-host ground truth)', async () => {
await inEmptyHome(async () => {
const cleanup = registerWorker({
pid: process.pid,
queue: 'default',
nice_requested: null,
nice_effective: null,
started_at: Date.now(),
});
try {
expect(await hasLiveJobsWorker(stubEngine('0'))).toBe(true);
} finally {
cleanup();
}
});
});
test('GBRAIN_REMEDIATION_ASSUME_WORKER=1 skips the probe (remote idle worker)', async () => {
await inEmptyHome(
async () => {
expect(await hasLiveJobsWorker(stubEngine(new Error('should not be called')))).toBe(true);
},
{ GBRAIN_REMEDIATION_ASSUME_WORKER: '1' },
);
});
test('fails OPEN when the DB probe throws', async () => {
await inEmptyHome(async () => {
expect(await hasLiveJobsWorker(stubEngine(new Error('relation minion_jobs missing')))).toBe(true);
});
});
});