fix(runtime): cancelled workflows no longer continue external commands (#119)

* fix(runtime): stop process-backed work on cancellation

* fix(runtime): invalidate cancelled resume state

* Revert "fix(runtime): invalidate cancelled resume state"

This reverts commit 0f7d291f98.

* fix(runtime): narrow cancellation to safe child processes

* fix(runtime): stop after completed search cancellation

* fix(runtime): halt direct pipelines after cancellation

Preserve completed in-flight stage results while preventing later direct pipeline stages from starting after parent cancellation.

* fix(runtime): consume aborted approval resumes

* fix(runtime): cancelled workflows no longer continue external commands

* fix(workflow): propagate custom parent cancellation

* fix(runtime): preserve pre-aborted resume state

* fix(runtime): stop lazy handoff after cancellation

* fix(workflow): close remaining cancellation boundaries

* fix(runtime): preserve workflow resumes during setup cancellation

* fix(runtime): close final cancellation persistence gaps

* fix(runtime): stop lazy handoff after cancellation

* fix(runtime): interrupt blocked lazy handoff reads

* fix(runtime): terminate cancellation process trees

* fix(runtime): await process tree termination

* fix(runtime): terminate workflow process trees

* fix(runtime): bridge CLI cancellation

* fix(cli): preserve cancellation lifecycle

* fix(cli): abort stalled signal-aware commands

* fix(cli): release aborted interactive prompts

* fix(cli): preserve sequential prompt input

* fix(cli): preserve buffered prompt input

* fix(cli): handle prompt EOF after buffered input

* fix(runtime): preserve UTF-8 subprocess output

* fix(workflow): preserve retryable resume before execution

* fix(workflow): roll back cancelled resume replacement

* fix(state): roll back cancelled monitor snapshot

* fix(resume): preserve cancelled gate capabilities

* fix(resume): close cancellation rollback windows

* fix(resume): harden cancellation state cleanup

* fix(runtime): close resumed cancellation gaps

* fix(runtime): preserve cancellation cleanup

* fix(runtime): close cancellation lifecycle gaps

* fix(runtime): harden resumed cancellation boundaries

* fix(workflow): consume timed-out resume capabilities

* fix(workflow): preserve resume policy boundaries

* fix(runtime): preserve cancellation cleanup liveness

* fix(runtime): harden cancellation cleanup

* fix(runtime): stop lazy output after cancellation

* fix(runtime): settle cancellation cleanup

* fix(runtime): preserve safe input resumes

* fix(runtime): prevent consumed resume replays

* fix(runtime): serialize approval resume consumption

* fix(runtime): prevent concurrent safe gate forks

* fix(runtime): close cancellation review gaps

* fix(llm): restore cache after cancelled refresh

* fix(runtime): close remaining cancellation windows

* fix(runtime): preserve cancellation recovery invariants

* fix(runtime): prevent stale resume recovery

* fix(runtime): preserve resume claim recovery

* fix(runtime): retry pre-dispatch claims safely

* fix(state): synchronize rollback-safe reads

* fix(resume): discard cancelled pipeline successors

* fix(runtime): harden cancellation and state locking

* fix(runtime): recover durable cancellation failures

* fix(runtime): preserve legacy workflow cancellation

---------

Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
xingzhou
2026-08-13 10:54:20 -07:00
committed by GitHub
co-authored by Peter Steinberger
parent 0ac962e90b
commit c440ca57d1
36 changed files with 10287 additions and 956 deletions
+236
View File
@@ -0,0 +1,236 @@
import { spawn, type ChildProcess } from "node:child_process";
const ABORT_FORCE_KILL_AFTER_MS = 250;
const forceTerminationCallbacks = new WeakMap<AbortSignal, Set<() => void>>();
type ProcessResult = {
stdout: string;
stderr: string;
code: number | null;
};
type RunAbortableProcessOptions = {
command: string;
argv: string[];
env: NodeJS.ProcessEnv;
cwd?: string;
stdin?: string | null;
signal?: AbortSignal;
forceTerminationSignal?: AbortSignal;
killSignal?: NodeJS.Signals | (() => NodeJS.Signals | undefined);
maxOutputBytes?: number;
outputLimitMessage?: string;
notFoundMessage: string;
};
export function forceTerminateAbortableProcesses(signal: AbortSignal) {
for (const terminate of forceTerminationCallbacks.get(signal) ?? []) terminate();
}
function abortError(signal: AbortSignal) {
if (signal.reason instanceof Error) return signal.reason;
const error = new Error("This operation was aborted");
error.name = "AbortError";
return error;
}
function terminateProcessTree(child: ChildProcess, signal: NodeJS.Signals): Promise<void> {
if (!child.pid) return Promise.resolve();
if (process.platform === "win32") {
return new Promise((resolve) => {
const taskkill = spawn("taskkill", ["/pid", String(child.pid), "/T", "/F"], {
stdio: "ignore",
windowsHide: true,
});
taskkill.once("error", () => {
child.kill(signal);
resolve();
});
taskkill.once("close", resolve);
});
}
try {
process.kill(-child.pid, signal);
} catch {
child.kill(signal);
}
return Promise.resolve();
}
export function runAbortableProcess({
command,
argv,
env,
cwd,
stdin,
signal,
forceTerminationSignal,
killSignal,
maxOutputBytes,
outputLimitMessage,
notFoundMessage,
}: RunAbortableProcessOptions): Promise<ProcessResult> {
return new Promise((resolve, reject) => {
if (
maxOutputBytes !== undefined &&
(!Number.isSafeInteger(maxOutputBytes) || maxOutputBytes < 0)
) {
reject(new Error("maxOutputBytes must be a non-negative safe integer"));
return;
}
try {
signal?.throwIfAborted();
} catch (err) {
reject(err);
return;
}
const child = spawn(command, argv, {
env,
cwd,
stdio: ["pipe", "pipe", "pipe"],
// Create a dedicated POSIX process group only when this runner owns a
// cancellation signal for it. Direct APIs without one must retain the
// caller's terminal process group so Ctrl-C still reaches their child.
detached: process.platform !== "win32" && signal !== undefined,
});
let stdout = "";
let stderr = "";
let stdoutBytes = 0;
let stderrBytes = 0;
let terminationError: Error | undefined;
let forceKillTimer: ReturnType<typeof setTimeout> | undefined;
let processClosed = false;
let forceKillRequested = false;
let forceKillIssued = false;
let settled = false;
const forceTerminationRegistrations: Set<() => void>[] = [];
let forceTerminate: (() => void) | undefined;
const cleanup = () => {
signal?.removeEventListener("abort", onAbort);
if (forceKillTimer) clearTimeout(forceKillTimer);
if (forceTerminate) {
for (const listeners of forceTerminationRegistrations) listeners.delete(forceTerminate);
}
};
const failTerminationWhenTreeIsStopped = () => {
if (!terminationError || !processClosed || !forceKillIssued || settled) return;
settled = true;
cleanup();
reject(terminationError);
};
const fail = (error: Error) => {
if (settled) return;
settled = true;
cleanup();
reject(error);
};
const forceKill = () => {
if (forceKillRequested) return;
forceKillRequested = true;
void terminateProcessTree(child, "SIGKILL").finally(() => {
forceKillIssued = true;
failTerminationWhenTreeIsStopped();
});
};
const startTermination = (error: Error) => {
if (settled || terminationError) return;
terminationError = error;
const initialKillSignal =
(typeof killSignal === "function" ? killSignal() : killSignal) ?? "SIGTERM";
if (initialKillSignal === "SIGKILL") {
forceKill();
return;
}
void terminateProcessTree(child, initialKillSignal);
forceKillTimer = setTimeout(() => {
forceKillTimer = undefined;
forceKill();
}, ABORT_FORCE_KILL_AFTER_MS);
};
forceTerminate = () => {
if (settled) return;
if (!terminationError) {
terminationError = signal ? abortError(signal) : new Error("Process termination requested");
}
if (forceKillTimer) {
clearTimeout(forceKillTimer);
forceKillTimer = undefined;
}
forceKill();
};
const onAbort = () => {
if (!signal) return;
startTermination(abortError(signal));
};
const appendOutput = (stream: "stdout" | "stderr", data: string) => {
const bytes = Buffer.byteLength(data);
const total = stream === "stdout" ? stdoutBytes + bytes : stderrBytes + bytes;
if (maxOutputBytes !== undefined && total > maxOutputBytes) {
startTermination(
new Error(outputLimitMessage ?? `Process output exceeded ${maxOutputBytes} bytes`),
);
return;
}
if (stream === "stdout") {
stdoutBytes = total;
stdout += data;
} else {
stderrBytes = total;
stderr += data;
}
};
child.stdout?.setEncoding("utf8");
child.stderr?.setEncoding("utf8");
child.stdout?.on("data", (data: string) => appendOutput("stdout", data));
child.stderr?.on("data", (data: string) => appendOutput("stderr", data));
if (typeof stdin === "string") child.stdin?.write(stdin);
child.stdin?.end();
child.on("error", (error: NodeJS.ErrnoException) => {
if (terminationError) {
processClosed = true;
failTerminationWhenTreeIsStopped();
return;
}
if (error.code === "ENOENT") {
fail(new Error(notFoundMessage));
return;
}
fail(error);
});
child.on("close", (code) => {
if (settled) return;
processClosed = true;
if (terminationError) {
failTerminationWhenTreeIsStopped();
return;
}
settled = true;
cleanup();
resolve({ stdout, stderr, code });
});
for (const registrationSignal of new Set(
[signal, forceTerminationSignal].filter(
(candidate): candidate is AbortSignal => candidate !== undefined,
),
)) {
let listeners = forceTerminationCallbacks.get(registrationSignal);
if (!listeners) {
listeners = new Set();
forceTerminationCallbacks.set(registrationSignal, listeners);
}
listeners.add(forceTerminate);
forceTerminationRegistrations.push(listeners);
}
if (signal) {
signal.addEventListener("abort", onAbort, { once: true });
if (signal.aborted) onAbort();
}
});
}
+134 -261
View File
@@ -1,24 +1,31 @@
import { parsePipeline } from "./parser.js";
import { createDefaultRegistry } from "./commands/registry.js";
import { runPipeline } from "./runtime.js";
import { decodeResumeToken, parseResumeArgs, resolveApprovalId } from "./resume.js";
import { cleanupApprovalIndexByStateKey, deleteApprovalId } from "./state/store.js";
import {
WorkflowResumeArgumentError,
loadWorkflowFile,
resolveWorkflowArgs,
runWorkflowFile,
} from "./workflows/file.js";
import { parseResumeArgs } from "./resume.js";
import { resumeToolRequest } from "./core/tool_runtime.js";
import { loadWorkflowFile, resolveWorkflowArgs, runWorkflowFile } from "./workflows/file.js";
import { renderWorkflowGraph } from "./workflows/graph.js";
import type { WorkflowGraphFormat } from "./workflows/graph.js";
import { deleteStateJson } from "./state/store.js";
import {
finalizePipelineToolRun,
loadPipelineResumeState,
validatePipelineInputResponse,
} from "./pipeline_resume_state.js";
import { finalizePipelineToolRun } from "./pipeline_resume_state.js";
import { forceTerminateAbortableProcesses } from "./abortable_process.js";
export async function runCli(argv) {
const cancellation = createCliCancellation();
try {
await runCliWithSignal(argv, cancellation.signal, cancellation.signal);
} finally {
if (cancellation.exitCode !== undefined) {
process.exitCode = cancellation.exitCode;
}
cancellation.dispose();
}
}
async function runCliWithSignal(
argv,
signal: AbortSignal,
forceTerminationSignal: AbortSignal = signal,
) {
const registry = createDefaultRegistry();
if (argv.length === 0 || argv.includes("-h") || argv.includes("--help")) {
@@ -48,7 +55,7 @@ export async function runCli(argv) {
}
if (argv[0] === "doctor") {
await handleDoctor({ argv: argv.slice(1), registry });
await handleDoctor({ argv: argv.slice(1), registry, signal, forceTerminationSignal });
return;
}
@@ -58,17 +65,57 @@ export async function runCli(argv) {
}
if (argv[0] === "run") {
await handleRun({ argv: argv.slice(1), registry });
await handleRun({ argv: argv.slice(1), registry, signal, forceTerminationSignal });
return;
}
if (argv[0] === "resume") {
await handleResume({ argv: argv.slice(1), registry });
await handleResume({ argv: argv.slice(1), registry, signal, forceTerminationSignal });
return;
}
// Default: treat argv as a pipeline string.
await handleRun({ argv, registry });
await handleRun({ argv, registry, signal, forceTerminationSignal });
}
function createCliCancellation() {
const controller = new AbortController();
let received: NodeJS.Signals | undefined;
const terminatingSignals: NodeJS.Signals[] = ["SIGINT", "SIGTERM", "SIGHUP", "SIGQUIT"];
const abort = (receivedSignal: NodeJS.Signals) => {
if (received) {
forceTerminateAbortableProcesses(controller.signal);
return;
}
received = receivedSignal;
controller.abort(new Error(`Received ${receivedSignal}`));
};
const listeners = new Map<NodeJS.Signals, () => void>();
for (const terminatingSignal of terminatingSignals) {
const listener = () => abort(terminatingSignal);
listeners.set(terminatingSignal, listener);
process.on(terminatingSignal, listener);
}
return {
signal: controller.signal,
get exitCode() {
if (!received) return undefined;
return (
{
SIGINT: 130,
SIGHUP: 129,
SIGQUIT: 131,
SIGTERM: 143,
} as const
)[received];
},
dispose() {
for (const [terminatingSignal, listener] of listeners) {
process.removeListener(terminatingSignal, listener);
}
},
};
}
async function handleGraph({ argv }) {
@@ -132,7 +179,17 @@ function isWorkflowGraphFormat(value: string): value is WorkflowGraphFormat {
return value === "mermaid" || value === "dot" || value === "ascii";
}
async function handleRun({ argv, registry }) {
async function handleRun({
argv,
registry,
signal,
forceTerminationSignal,
}: {
argv;
registry;
signal: AbortSignal;
forceTerminationSignal: AbortSignal;
}) {
const parsed = parseRunArgs(argv);
const { mode, argsJson } = parsed;
const normalizedMode = normalizeMode(mode);
@@ -173,6 +230,8 @@ async function handleRun({ argv, registry }) {
mode: normalizedMode,
registry,
dryRun,
signal,
forceTerminationSignal,
},
});
@@ -276,6 +335,9 @@ async function handleRun({ argv, registry }) {
env: process.env,
mode: normalizedMode,
dryRun,
signal,
forceTerminationSignal,
haltAfterStageOnAbort: true,
});
if (normalizedMode === "tool") {
@@ -283,6 +345,7 @@ async function handleRun({ argv, registry }) {
env: process.env,
pipeline,
output,
signal,
});
writeToolEnvelope({
ok: true,
@@ -504,28 +567,20 @@ async function resolveWorkflowFile(candidate) {
return resolved;
}
async function handleResume({ argv, registry }) {
const mode = "tool";
let approved: boolean | undefined;
let response: unknown = undefined;
let cancel = false;
let payload: any;
let resolvedApprovalId: string | null = null;
async function handleResume({
argv,
registry,
signal,
forceTerminationSignal,
}: {
argv;
registry;
signal: AbortSignal;
forceTerminationSignal: AbortSignal;
}) {
let parsed;
try {
const parsed = parseResumeArgs(argv);
approved = parsed.approved;
response = parsed.response;
cancel = parsed.cancel === true;
resolvedApprovalId = parsed.approvalId;
// Resolve short approval ID to token if provided
let token: string;
if (parsed.approvalId) {
token = await resolveApprovalId(parsed.approvalId, process.env);
} else {
token = parsed.token!;
}
payload = decodeResumeToken(token);
parsed = parseResumeArgs(argv);
} catch (err) {
writeToolEnvelope({
ok: false,
@@ -535,230 +590,35 @@ async function handleResume({ argv, registry }) {
return;
}
// Helper: clean up approval ID index after successful use
const cleanupIndex = async () => {
if (resolvedApprovalId) {
await deleteApprovalId({ env: process.env, approvalId: resolvedApprovalId });
} else if (payload.stateKey) {
await cleanupApprovalIndexByStateKey({ env: process.env, stateKey: payload.stateKey });
}
};
if (cancel === true) {
await cleanupIndex();
if (payload.kind === "workflow-file" && payload.stateKey) {
await deleteStateJson({ env: process.env, key: payload.stateKey });
}
if (payload.kind === "pipeline-resume" && payload.stateKey) {
await deleteStateJson({ env: process.env, key: payload.stateKey });
}
writeToolEnvelope({
ok: true,
status: "cancelled",
output: [],
requiresApproval: null,
requiresInput: null,
});
return;
}
if (payload.kind === "workflow-file") {
try {
const output = await runWorkflowFile({
filePath: payload.filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env: process.env,
mode: "tool",
registry,
},
resume: payload,
approved,
response,
cancel,
});
if (output.status === "needs_approval") {
writeToolEnvelope({
ok: true,
status: "needs_approval",
output: [],
requiresApproval: output.requiresApproval ?? null,
requiresInput: null,
});
return;
}
if (output.status === "needs_input") {
writeToolEnvelope({
ok: true,
status: "needs_input",
output: [],
requiresApproval: null,
requiresInput: output.requiresInput ?? null,
});
return;
}
await cleanupIndex();
if (output.status === "cancelled") {
writeToolEnvelope({
ok: true,
status: "cancelled",
output: [],
requiresApproval: null,
requiresInput: null,
});
return;
}
writeToolEnvelope({
ok: true,
status: "ok",
output: output.output,
requiresApproval: null,
requiresInput: null,
});
return;
} catch (err) {
if (err instanceof WorkflowResumeArgumentError) {
writeToolEnvelope({ ok: false, error: { type: "parse_error", message: err.message } });
process.exitCode = 2;
return;
}
// Don't clean up index on error — allow retry by --id
writeToolEnvelope({
ok: false,
error: { type: "runtime_error", message: err?.message ?? String(err) },
});
process.exitCode = 1;
return;
}
}
const previousStateKey = payload.stateKey;
let resumeState;
let envelope;
try {
resumeState = await loadPipelineResumeState(process.env, previousStateKey);
} catch (err) {
writeToolEnvelope({
envelope = await resumeToolRequest({
token: parsed.token ?? undefined,
approvalId: parsed.approvalId ?? undefined,
approved: parsed.approved,
response: parsed.response,
cancel: parsed.cancel,
ctx: {
cwd: process.cwd(),
env: process.env,
mode: "tool",
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
registry,
signal,
forceTerminationSignal,
},
});
} catch (err: any) {
envelope = {
ok: false,
error: { type: "runtime_error", message: err?.message ?? String(err) },
});
process.exitCode = 1;
return;
};
}
if (resumeState.haltType === "input_request") {
if (approved !== undefined) {
writeToolEnvelope({
ok: false,
error: {
type: "parse_error",
message: "pipeline input resumes require --response-json <json>",
},
});
process.exitCode = 2;
return;
}
if (response === undefined) {
writeToolEnvelope({
ok: false,
error: {
type: "parse_error",
message: "pipeline input resumes require --response-json <json>",
},
});
process.exitCode = 2;
return;
}
try {
validatePipelineInputResponse(resumeState.inputSchema, response);
} catch (err) {
writeToolEnvelope({
ok: false,
error: { type: "parse_error", message: err?.message ?? String(err) },
});
process.exitCode = 2;
return;
}
} else {
if (response !== undefined) {
writeToolEnvelope({
ok: false,
error: {
type: "parse_error",
message: "approval resumes require --approve yes|no, not --response-json",
},
});
process.exitCode = 2;
return;
}
if (approved !== true) {
await cleanupIndex();
await deleteStateJson({ env: process.env, key: previousStateKey });
writeToolEnvelope({
ok: true,
status: "cancelled",
output: [],
requiresApproval: null,
requiresInput: null,
});
return;
}
}
const isSameStageInput =
resumeState.haltType === "input_request" && resumeState.resumeMode === "same_stage";
const remaining = resumeState.pipeline.slice(resumeState.resumeAtIndex);
const input = isSameStageInput
? resumeState.items
: resumeState.haltType === "input_request"
? [response]
: resumeState.items;
const requestInputResume = isSameStageInput
? {
state: resumeState.commandInput!,
response,
onConsumed: async () => {
await cleanupIndex();
await deleteStateJson({ env: process.env, key: previousStateKey });
},
}
: undefined;
try {
const output = await runPipeline({
pipeline: remaining,
registry,
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env: process.env,
mode,
input,
requestInputResume,
});
await cleanupIndex();
const finalized = await finalizePipelineToolRun({
env: process.env,
pipeline: remaining,
output,
previousStateKey,
});
writeToolEnvelope({
ok: true,
status: finalized.status,
output: finalized.output,
requiresApproval: finalized.requiresApproval,
requiresInput: finalized.requiresInput,
});
} catch (err) {
// Don't clean up index on error — allow retry by --id
writeToolEnvelope({
ok: false,
error: { type: "runtime_error", message: err?.message ?? String(err) },
});
process.exitCode = 1;
writeToolEnvelope(envelope);
if (!envelope.ok) {
process.exitCode = envelope.error?.type === "parse_error" ? 2 : 1;
}
}
@@ -773,7 +633,17 @@ async function readVersion() {
return pkg.version ?? "0.0.0";
}
async function handleDoctor({ argv, registry }) {
async function handleDoctor({
argv,
registry,
signal,
forceTerminationSignal,
}: {
argv;
registry;
signal: AbortSignal;
forceTerminationSignal: AbortSignal;
}) {
const mode = "tool";
const pipeline = "exec --json --shell 'echo [1]'";
const output: any = await (async () => {
@@ -788,6 +658,9 @@ async function handleDoctor({ argv, registry }) {
stderr: process.stderr,
env: process.env,
mode,
signal,
forceTerminationSignal,
haltAfterStageOnAbort: true,
});
} catch (err: any) {
return { error: err };
+2
View File
@@ -18,6 +18,7 @@ export const approveCommand = {
required: [],
},
sideEffects: [],
resumeSafeBeforeInput: true,
},
help() {
return `approve — require confirmation to continue\n\nUsage:\n ... | approve --prompt "Send these emails?"\n ... | approve --emit --prompt "Send these emails?"\n ... | approve --emit --preview-from-stdin --limit 5 --prompt "Proceed?"\n\nModes:\n - Interactive (default): prompts on TTY and passes items through if approved.\n - Emit (--emit): returns an approval request object and stops the pipeline.\n\nNotes:\n - In tool mode (or non-interactive), this emits an approval request and halts.\n`;
@@ -53,6 +54,7 @@ export const approveCommand = {
ctx.stdout.write(`${prompt} [y/N] `);
const answer = await readLineFromStream(ctx.stdin, {
timeoutMs: parseApprovalTimeoutMs(ctx.env),
signal: ctx.signal,
});
if (!/^y(es)?$/i.test(String(answer).trim())) {
+3 -1
View File
@@ -57,6 +57,8 @@ export const askCommand = {
required: [],
},
sideEffects: [],
resumeSafeBeforeInput: true,
resumeSafeAfterInput: true,
},
help() {
return [
@@ -154,7 +156,7 @@ export const askCommand = {
ctx.stdout.write(`${prompt}\n> `);
const { readLineFromStream } = await import("../../read_line.js");
const raw = await readLineFromStream(ctx.stdin, { timeoutMs: 0 });
const raw = await readLineFromStream(ctx.stdin, { timeoutMs: 0, signal: ctx.signal });
const text = String(raw ?? "").trim();
let lastError;
+6 -1
View File
@@ -25,7 +25,12 @@ export const diffLastCommand = {
for await (const item of input) afterItems.push(item);
const after = afterItems.length === 1 ? afterItems[0] : afterItems;
const { before, changed } = await diffAndStore({ env: ctx.env, key, value: after });
const { before, changed } = await diffAndStore({
env: ctx.env,
key,
value: after,
signal: ctx.signal,
});
return {
output: (async function* () {
+17 -36
View File
@@ -1,4 +1,4 @@
import { spawn } from "node:child_process";
import { runAbortableProcess } from "../../abortable_process.js";
import { resolveInlineShellCommand } from "../../shell.js";
export const execCommand = {
@@ -58,12 +58,14 @@ export const execCommand = {
cwd,
stdin: stdinPayload,
signal: ctx.signal,
forceTerminationSignal: ctx.forceTerminationSignal,
})
: await runProcess(cmd[0], cmd.slice(1), {
env: ctx.env,
cwd,
stdin: stdinPayload,
signal: ctx.signal,
forceTerminationSignal: ctx.forceTerminationSignal,
});
if (args.json) {
@@ -86,45 +88,24 @@ export const execCommand = {
},
};
function runProcess(command, argv, { env, cwd, stdin, signal }) {
return new Promise<any>((resolve, reject) => {
const child = spawn(command, argv, {
env,
cwd,
signal,
stdio: ["pipe", "pipe", "pipe"],
});
let stdout = "";
let stderr = "";
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (d) => {
stdout += d;
});
child.stderr.on("data", (d) => {
stderr += d;
});
if (typeof stdin === "string") {
child.stdin.setDefaultEncoding("utf8");
child.stdin.write(stdin);
}
child.stdin.end();
child.on("error", reject);
child.on("close", (code) => {
if (code === 0) return resolve({ stdout, stderr });
reject(new Error(`exec failed (${code}): ${stderr.trim() || stdout.trim() || command}`));
});
async function runProcess(command, argv, { env, cwd, stdin, signal, forceTerminationSignal }) {
const { stdout, stderr, code } = await runAbortableProcess({
command,
argv,
env,
cwd,
stdin,
signal,
forceTerminationSignal,
notFoundMessage: `exec command not found: ${command}`,
});
if (code === 0) return { stdout, stderr };
throw new Error(`exec failed (${code}): ${stderr.trim() || stdout.trim() || command}`);
}
function runShellLine(commandLine, { env, cwd, stdin, signal }) {
function runShellLine(commandLine, { env, cwd, stdin, signal, forceTerminationSignal }) {
const shell = resolveInlineShellCommand({ command: commandLine, env });
return runProcess(shell.command, shell.argv, { env, cwd, stdin, signal });
return runProcess(shell.command, shell.argv, { env, cwd, stdin, signal, forceTerminationSignal });
}
function encodeStdin(items, mode) {
+12 -29
View File
@@ -1,31 +1,4 @@
import { spawn } from "node:child_process";
function run(cmd: string, argv: string[], env: Record<string, string | undefined>, cwd?: string) {
return new Promise<{ stdout: string; stderr: string; code: number | null }>((resolve, reject) => {
const child = spawn(cmd, argv, {
env: { ...process.env, ...env },
cwd,
stdio: ["ignore", "pipe", "pipe"],
});
let stdout = "";
let stderr = "";
child.stdout?.on("data", (d) => (stdout += String(d)));
child.stderr?.on("data", (d) => (stderr += String(d)));
child.on("error", (err: any) => {
if (err?.code === "ENOENT") {
reject(new Error("gog not found on PATH (install: https://github.com/steipete/gogcli)"));
return;
}
reject(err);
});
child.on("close", (code) => {
resolve({ stdout, stderr, code });
});
});
}
import { runAbortableProcess } from "../../abortable_process.js";
export const gogGmailSearchCommand = {
name: "gog.gmail.search",
@@ -55,10 +28,12 @@ export const gogGmailSearchCommand = {
);
},
async run({ input, args, ctx }) {
ctx.signal?.throwIfAborted();
// Drain input
for await (const _item of input) {
// no-op
}
ctx.signal?.throwIfAborted();
const query = String(args.query ?? "newer_than:1d");
const max = Number(args.max ?? args.limit ?? 20);
@@ -75,7 +50,15 @@ export const gogGmailSearchCommand = {
const gogBin = isScript ? process.execPath : gogBinRaw;
const argv = isScript ? [gogBinRaw, ...argvBase] : argvBase;
const res = await run(gogBin, argv, ctx.env, process.cwd());
const res = await runAbortableProcess({
command: gogBin,
argv,
env: { ...process.env, ...ctx.env },
cwd: process.cwd(),
signal: ctx.signal,
forceTerminationSignal: ctx.forceTerminationSignal,
notFoundMessage: "gog not found on PATH (install: https://github.com/steipete/gogcli)",
});
if (res.code !== 0) {
throw new Error(`gog.gmail.search failed (${res.code ?? "?"}): ${res.stderr.slice(0, 400)}`);
}
+6
View File
@@ -72,6 +72,7 @@ export const gogGmailSendCommand = {
);
},
async run({ input, args, ctx }) {
ctx.signal?.throwIfAborted();
const dryRun = Boolean(args.dryRun ?? args["dry-run"] ?? false);
const gogBinRaw = String(ctx.env.GOG_BIN ?? "gog");
const isScript = /\.(mjs|cjs|js|ts)$/i.test(gogBinRaw);
@@ -80,6 +81,10 @@ export const gogGmailSendCommand = {
const results: any[] = [];
for await (const item of input) {
if (ctx.signal?.aborted) {
if (results.length > 0) break;
ctx.signal.throwIfAborted();
}
const draft = parseDraft(item);
if (dryRun) {
@@ -111,6 +116,7 @@ export const gogGmailSendCommand = {
}
results.push(parsed);
if (ctx.signal?.aborted) break;
}
return {
+120 -28
View File
@@ -5,12 +5,14 @@ import { Ajv } from "ajv";
import type { ErrorObject } from "ajv";
import {
diffAndStore,
ensureDirectory,
atomicWriteWasPublished,
isJsonSyntaxError,
readStateJson,
readStateJsonWithLock,
stableStringify,
withFileLock,
writeFileAtomic,
writeStateJson,
} from "../../state/store.js";
import { createCompileCached } from "../../validation.js";
import type { LobsterCommand } from "../types.js";
@@ -178,6 +180,7 @@ type Adapter = {
env: any;
args: any;
payload: Record<string, any>;
signal?: AbortSignal;
}) => Promise<LlmResponseEnvelope>;
};
@@ -187,6 +190,7 @@ type DirectAdapter =
args: any;
payload: Record<string, any>;
ctx: any;
signal?: AbortSignal;
}) => Promise<LlmResponseEnvelope>)
| {
source?: string;
@@ -195,6 +199,7 @@ type DirectAdapter =
args: any;
payload: Record<string, any>;
ctx: any;
signal?: AbortSignal;
}) => Promise<LlmResponseEnvelope>;
};
@@ -385,7 +390,7 @@ async function runLlmInvoke({
});
if (stateKey && !forceRefresh) {
const stored = await readReusableLlmState(env, stateKey);
const stored = await readReusableLlmState(env, stateKey, ctx.signal);
const reused = pickReusableState(stored, cacheKey, config.stateType);
if (reused) {
return {
@@ -397,7 +402,7 @@ async function runLlmInvoke({
}
if (!disableCache && !forceRefresh) {
const cache = await readCacheEntry(env, cacheKey, config.cacheNamespace);
const cache = await readCacheEntry(env, cacheKey, config.cacheNamespace, ctx.signal);
if (cache) {
return {
output: streamOf(cache.items.map((item) => ({ ...item, source: "cache", cached: true }))),
@@ -438,7 +443,9 @@ async function runLlmInvoke({
let responseEnvelope: LlmResponseEnvelope;
try {
responseEnvelope = await adapter.invoke({ env, args, payload });
ctx.signal?.throwIfAborted();
responseEnvelope = await adapter.invoke({ env, args, payload, signal: ctx.signal });
ctx.signal?.throwIfAborted();
} catch (err: any) {
throw new Error(`${config.name} request failed: ${err?.message ?? String(err)}`);
}
@@ -463,27 +470,35 @@ async function runLlmInvoke({
});
if (!validator) {
ctx.signal?.throwIfAborted();
await persistOutputs({
env,
stateKey,
cacheKey,
items: normalized,
stateType: config.stateType,
signal: ctx.signal,
afterStore: disableCache
? undefined
: () => writeCacheEntry(env, cacheKey, normalized, config.cacheNamespace, ctx.signal),
});
if (!disableCache) await writeCacheEntry(env, cacheKey, normalized, config.cacheNamespace);
return { output: streamOf(normalized) };
}
const structured = normalized[0]?.output?.data ?? null;
if (validator(structured)) {
ctx.signal?.throwIfAborted();
await persistOutputs({
env,
stateKey,
cacheKey,
items: normalized,
stateType: config.stateType,
signal: ctx.signal,
afterStore: disableCache
? undefined
: () => writeCacheEntry(env, cacheKey, normalized, config.cacheNamespace, ctx.signal),
});
if (!disableCache) await writeCacheEntry(env, cacheKey, normalized, config.cacheNamespace);
return { output: streamOf(normalized) };
}
@@ -547,8 +562,8 @@ function resolveAdapter({
return {
provider,
source: typeof direct === "function" ? provider : (direct.source ?? provider),
async invoke({ payload }) {
return invoke({ env, args, payload, ctx });
async invoke({ payload, signal }) {
return invoke({ env, args, payload, ctx, signal });
},
};
}
@@ -563,8 +578,8 @@ function resolveAdapter({
return {
provider,
source: config.sourceForProvider?.(provider) ?? "openclaw",
async invoke({ payload }) {
return invokeOpenClawAdapter({ endpoint, token, payload });
async invoke({ payload, signal }) {
return invokeOpenClawAdapter({ endpoint, token, payload, signal });
},
};
}
@@ -578,8 +593,13 @@ function resolveAdapter({
return {
provider,
source: config.sourceForProvider?.(provider) ?? "pi",
async invoke({ payload }) {
return invokeHttpAdapter({ endpoint: buildAdapterEndpoint(adapterUrl), token, payload });
async invoke({ payload, signal }) {
return invokeHttpAdapter({
endpoint: buildAdapterEndpoint(adapterUrl),
token,
payload,
signal,
});
},
};
}
@@ -592,8 +612,13 @@ function resolveAdapter({
return {
provider,
source: config.sourceForProvider?.(provider) ?? "http",
async invoke({ payload }) {
return invokeHttpAdapter({ endpoint: buildAdapterEndpoint(adapterUrl), token, payload });
async invoke({ payload, signal }) {
return invokeHttpAdapter({
endpoint: buildAdapterEndpoint(adapterUrl),
token,
payload,
signal,
});
},
};
}
@@ -621,13 +646,16 @@ async function invokeOpenClawAdapter({
endpoint,
token,
payload,
signal,
}: {
endpoint: URL;
token: string;
payload: any;
signal?: AbortSignal;
}) {
const res = await fetch(endpoint, {
method: "POST",
signal,
headers: {
"content-type": "application/json",
...(token ? { authorization: `Bearer ${token}` } : null),
@@ -670,13 +698,16 @@ async function invokeHttpAdapter({
endpoint,
token,
payload,
signal,
}: {
endpoint: URL;
token: string;
payload: any;
signal?: AbortSignal;
}) {
const res = await fetch(endpoint, {
method: "POST",
signal,
headers: {
"content-type": "application/json",
...(token ? { authorization: `Bearer ${token}` } : null),
@@ -876,14 +907,21 @@ async function persistOutputs({
cacheKey,
items,
stateType,
signal,
afterStore,
}: {
env: any;
stateKey: string | null;
cacheKey: string;
items: NormalizedInvocationItem[];
stateType: string;
signal?: AbortSignal;
afterStore?: () => Promise<void>;
}) {
if (!stateKey) return;
if (!stateKey) {
await afterStore?.();
return;
}
const record = {
type: stateType,
version: STATE_VERSION,
@@ -891,12 +929,18 @@ async function persistOutputs({
items,
storedAt: new Date().toISOString(),
};
await writeStateJson({ env, key: stateKey, value: record });
await diffAndStore({
env,
key: stateKey,
value: record,
signal,
afterStore: afterStore ? () => afterStore() : undefined,
});
}
async function readReusableLlmState(env: any, stateKey: string) {
async function readReusableLlmState(env: any, stateKey: string, signal?: AbortSignal) {
try {
return await readStateJson({ env, key: stateKey });
return await readStateJsonWithLock({ env, key: stateKey, signal });
} catch (err: any) {
if (isJsonSyntaxError(err)) return null;
throw err;
@@ -920,16 +964,28 @@ async function readCacheEntry(
env: any,
key: string,
cacheNamespace: string,
signal?: AbortSignal,
): Promise<CacheEntry | null> {
const filePath = path.join(getCacheDir(env), cacheNamespace, `${key}.json`);
const read = async () => {
try {
const text = await fsp.readFile(filePath, "utf8");
const parsed = JSON.parse(text) as Partial<CacheEntry>;
if (parsed?.cacheKey !== key || !Array.isArray(parsed.items)) return null;
return parsed as CacheEntry;
} catch (err: any) {
if (err?.code === "ENOENT") return null;
if (isJsonSyntaxError(err)) return null;
throw err;
}
};
try {
const text = await fsp.readFile(filePath, "utf8");
const parsed = JSON.parse(text) as Partial<CacheEntry>;
if (parsed?.cacheKey !== key || !Array.isArray(parsed.items)) return null;
return parsed as CacheEntry;
return await withFileLock({ filePath, signal, task: read });
} catch (err: any) {
if (err?.code === "ENOENT") return null;
if (isJsonSyntaxError(err)) return null;
// A cache mounted read-only cannot have an active local writer because a
// writer first creates the same coordination lock. Preserve its reusable
// entries instead of requiring a lock-directory write for a read.
if (["EACCES", "EPERM", "EROFS"].includes(err?.code)) return read();
throw err;
}
}
@@ -939,14 +995,50 @@ async function writeCacheEntry(
key: string,
items: NormalizedInvocationItem[],
cacheNamespace: string,
signal?: AbortSignal,
) {
const dir = path.join(getCacheDir(env), cacheNamespace);
signal?.throwIfAborted();
await ensureDirectory(dir);
const filePath = path.join(dir, `${key}.json`);
await writeFileAtomic(
const content =
JSON.stringify({ items, cacheKey: key, storedAt: new Date().toISOString() }, null, 2) + "\n";
await withFileLock({
filePath,
JSON.stringify({ items, cacheKey: key, storedAt: new Date().toISOString() }, null, 2) + "\n",
);
signal,
task: async () => {
let previousContent: Buffer | null = null;
try {
previousContent = await fsp.readFile(filePath);
} catch (err: any) {
if (err?.code !== "ENOENT") throw err;
}
const restorePreviousContent = async () => {
// No cache reader or competing cache writer can observe this entry
// until this lock is released. Restore an entry replaced by a refresh;
// only remove the just-published file when none existed beforehand.
if (previousContent === null) await fsp.rm(filePath, { force: true });
else await writeFileAtomic(filePath, previousContent);
};
let cacheWasPublished = false;
try {
signal?.throwIfAborted();
const result = await writeFileAtomic(filePath, content, { signal });
cacheWasPublished = true;
if (result?.signalAbortedAfterCommit || signal?.aborted) {
await restorePreviousContent();
cacheWasPublished = false;
signal?.throwIfAborted();
throw new Error("LLM cache publication cancelled");
}
} catch (err) {
if (cacheWasPublished || atomicWriteWasPublished(err)) {
await restorePreviousContent();
}
throw err;
}
},
});
}
function getCacheDir(env: any) {
+30 -33
View File
@@ -1,12 +1,15 @@
import { execFile } from "node:child_process";
import { runAbortableProcess } from "../../abortable_process.js";
import type { LobsterCommand } from "../types.js";
const OPENCLAW_AGENT_MAX_OUTPUT_BYTES = 10 * 1024 * 1024;
type AgentCliRunner = (params: {
executable: string;
argv: string[];
cwd: string;
env: NodeJS.ProcessEnv;
signal?: AbortSignal;
forceTerminationSignal?: AbortSignal;
}) => Promise<unknown>;
export const openclawAgentCommand = createOpenClawAgentCommand();
@@ -92,6 +95,7 @@ export function createOpenClawAgentCommand(
cwd: ctx?.cwd ?? process.cwd(),
env,
signal: ctx?.signal,
forceTerminationSignal: ctx?.forceTerminationSignal,
});
return { output: streamOf([response]) };
},
@@ -104,39 +108,32 @@ export function runOpenClawAgentCli(params: {
cwd: string;
env: NodeJS.ProcessEnv;
signal?: AbortSignal;
forceTerminationSignal?: AbortSignal;
}): Promise<unknown> {
return new Promise((resolve, reject) => {
execFile(
params.executable,
params.argv,
{
cwd: params.cwd,
env: params.env,
signal: params.signal,
encoding: "utf8",
maxBuffer: 10 * 1024 * 1024,
},
(error, stdout, stderr) => {
if (error) {
if (error.name === "AbortError") {
reject(error);
return;
}
const detail = String(stderr || stdout || error.message).trim();
reject(new Error(`openclaw.agent failed: ${detail}`));
return;
}
try {
const parsed = JSON.parse(String(stdout).trim() || "null");
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
throw new Error("response must be an object");
}
resolve(parsed);
} catch {
reject(new Error("openclaw.agent expected JSON output from `openclaw agent --json`"));
}
},
);
return runAbortableProcess({
command: params.executable,
argv: params.argv,
cwd: params.cwd,
env: params.env,
signal: params.signal,
forceTerminationSignal: params.forceTerminationSignal,
maxOutputBytes: OPENCLAW_AGENT_MAX_OUTPUT_BYTES,
outputLimitMessage: `openclaw.agent output exceeded ${OPENCLAW_AGENT_MAX_OUTPUT_BYTES} bytes`,
notFoundMessage: "openclaw.agent could not find the OpenClaw CLI",
}).then(({ code, stdout, stderr }) => {
if (code !== 0) {
const detail = String(stderr || stdout || `exited with code ${code}`).trim();
throw new Error(`openclaw.agent failed: ${detail}`);
}
try {
const parsed = JSON.parse(stdout.trim() || "null");
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
throw new Error("response must be an object");
}
return parsed;
} catch {
throw new Error("openclaw.agent expected JSON output from `openclaw agent --json`");
}
});
}
+1
View File
@@ -91,6 +91,7 @@ function createInvokeCommand(commandName: string) {
const invokeOnce = async (argsValue: unknown) => {
const res = await fetch(endpoint, {
method: "POST",
signal: ctx.signal,
headers: {
"content-type": "application/json",
...(token ? { authorization: `Bearer ${token}` } : null),
+3 -22
View File
@@ -1,6 +1,4 @@
import { promises as fsp } from "node:fs";
import { defaultStateDir, ensureDirectory, keyToPath, writeFileAtomic } from "../../state/store.js";
import { readStateJsonWithLock, writeStateJson } from "../../state/store.js";
export const stateGetCommand = {
name: "state.get",
@@ -22,20 +20,7 @@ export const stateGetCommand = {
const key = args._[0];
if (!key) throw new Error("state.get requires a key");
const stateDir = defaultStateDir(ctx.env);
const filePath = keyToPath(stateDir, key);
let value = null;
try {
const text = await fsp.readFile(filePath, "utf8");
value = JSON.parse(text);
} catch (err) {
if (err?.code === "ENOENT") {
value = null;
} else {
throw err;
}
}
const value = await readStateJsonWithLock({ env: ctx.env, key, signal: ctx.signal });
return { output: asStream([value]) };
},
@@ -66,11 +51,7 @@ export const stateSetCommand = {
const value = items.length === 1 ? items[0] : items;
const stateDir = defaultStateDir(ctx.env);
const filePath = keyToPath(stateDir, key);
await ensureDirectory(stateDir);
await writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n");
await writeStateJson({ env: ctx.env, key, value, signal: ctx.signal });
return { output: asStream([value]) };
},
+12
View File
@@ -3,6 +3,18 @@ export type CommandMeta = {
argsSchema?: unknown;
examples?: Array<{ args: Record<string, unknown>; description?: string }>;
sideEffects?: string[];
/**
* The command may create an input/approval gate before it begins execution.
* Commands that omit this are treated conservatively when a resumed pipeline
* is cancelled: its original capability cannot be replayed after dispatch.
*/
resumeSafeBeforeInput?: boolean;
/**
* The command remains side-effect-free after returning a resumed input until
* the next pipeline stage dispatches. This is intentionally opt-in so a
* command that acts on a resumed response consumes its capability first.
*/
resumeSafeAfterInput?: boolean;
};
export type LobsterCommand = {
+161 -21
View File
@@ -7,12 +7,20 @@ import { decodeResumeToken, kindFromStateKey } from "../resume.js";
import { runPipeline } from "../runtime.js";
import { encodeToken } from "../token.js";
import {
deleteStateJson,
deleteStateJsonWithBoundedResumeCleanup,
deleteUnconsumedResumeState,
deleteApprovalId,
findStateKeyByApprovalId,
cleanupApprovalIndexByStateKey,
consumeResumeState,
restoreConsumedResumeState,
stateJsonExists,
} from "../state/store.js";
import { WorkflowResumeArgumentError, runWorkflowFile } from "../workflows/file.js";
import {
WorkflowResumeArgumentError,
alternateWorkflowResumeStateKey,
runWorkflowFile,
} from "../workflows/file.js";
import {
finalizePipelineToolRun,
loadPipelineResumeState,
@@ -27,6 +35,7 @@ type ToolRunContext = {
stdout?: NodeJS.WritableStream;
stderr?: NodeJS.WritableStream;
signal?: AbortSignal;
forceTerminationSignal?: AbortSignal;
registry?: any;
llmAdapters?: Record<string, any>;
};
@@ -130,12 +139,15 @@ export async function runToolRequest({
cwd: runtime.cwd,
llmAdapters: runtime.llmAdapters,
signal: runtime.signal,
forceTerminationSignal: runtime.forceTerminationSignal,
haltAfterStageOnAbort: true,
});
const finalized = await finalizePipelineToolRun({
env: runtime.env,
pipeline: parsed,
output,
signal: runtime.signal,
});
return okEnvelope(
finalized.status,
@@ -193,30 +205,71 @@ export async function resumeToolRequest({
}
// Helper: clean up approval ID index after successful use
const cleanupIndex = async () => {
const cleanupIndex = async (stateKey = payload?.stateKey) => {
if (resolvedApprovalId) {
await deleteApprovalId({ env: runtime.env, approvalId: resolvedApprovalId });
} else if (payload?.stateKey) {
await cleanupApprovalIndexByStateKey({ env: runtime.env, stateKey: payload.stateKey });
} else if (stateKey) {
await cleanupApprovalIndexByStateKey({ env: runtime.env, stateKey });
}
};
if (cancel === true) {
await cleanupIndex();
let stateKeys = payload.stateKey ? [payload.stateKey] : [];
if (payload.kind === "workflow-file" && payload.stateKey) {
await deleteStateJson({ env: runtime.env, key: payload.stateKey });
const alternateStateKey = alternateWorkflowResumeStateKey(payload.stateKey);
if (alternateStateKey) {
// Delete a non-authoritative spelling first. If cancellation interrupts
// its lock wait, the state that makes this capability resumable remains.
const [primaryExists, alternateExists] = await Promise.all([
stateJsonExists({ env: runtime.env, key: payload.stateKey }),
stateJsonExists({ env: runtime.env, key: alternateStateKey }),
]);
if (primaryExists || !alternateExists) {
stateKeys = [alternateStateKey, payload.stateKey];
} else {
stateKeys = [payload.stateKey, alternateStateKey];
}
}
}
if (payload.kind === "pipeline-resume" && payload.stateKey) {
await deleteStateJson({ env: runtime.env, key: payload.stateKey });
// Keep the capability indexed until every state deletion succeeds. A
// cancelled request must not orphan a resume state by dropping its
// approval ID while waiting on another writer's state lock.
const deletionResults = [];
for (const stateKey of new Set(stateKeys)) {
deletionResults.push(
await deleteUnconsumedResumeState({
env: runtime.env,
key: stateKey,
signal: runtime.signal,
}),
);
}
if (
stateKeys.length > 0 &&
(deletionResults.includes("claimed") ||
deletionResults.every((result) => result === "missing"))
) {
return errorEnvelope("runtime_error", "Resume state not found");
}
if (resolvedApprovalId) {
await cleanupIndex();
} else {
for (const stateKey of stateKeys) await cleanupIndex(stateKey);
}
return okEnvelope("cancelled", [], null, null);
}
if (payload.kind === "workflow-file") {
let workflowResumeStateKey = payload.stateKey;
try {
const output = await runWorkflowFile({
filePath: payload.filePath,
ctx: runtime,
ctx: {
...runtime,
_onResumeStateResolved: (stateKey) => {
workflowResumeStateKey = stateKey;
},
},
resume: payload,
approved,
response,
@@ -224,13 +277,12 @@ export async function resumeToolRequest({
});
if (output.status === "needs_approval") {
// Don't clean up index — next gate will issue a new approvalId
return okEnvelope("needs_approval", [], output.requiresApproval ?? null, null);
}
if (output.status === "needs_input") {
return okEnvelope("needs_input", [], null, output.requiresInput ?? null);
}
await cleanupIndex();
await cleanupIndex(workflowResumeStateKey);
if (output.status === "cancelled") {
return okEnvelope("cancelled", [], null, null);
}
@@ -239,15 +291,30 @@ export async function resumeToolRequest({
if (err instanceof WorkflowResumeArgumentError) {
return errorEnvelope("parse_error", err.message);
}
// Don't clean up index on error — allow retry by --id
// Non-abort failures and cancellations before step execution remain retryable.
return errorEnvelope("runtime_error", err?.message ?? String(err));
}
}
if (runtime.signal?.aborted) {
return errorEnvelope(
"runtime_error",
runtime.signal.reason instanceof Error
? runtime.signal.reason.message
: "This operation was aborted",
);
}
let resumeState;
try {
resumeState = await loadPipelineResumeState(runtime.env, payload.stateKey);
// No state has been claimed yet, so a cancelled resume can return before
// touching the lock and leave its capability safely retryable.
resumeState = await loadPipelineResumeState(runtime.env, payload.stateKey, runtime.signal);
} catch (err: any) {
// Approval rejection historically reaches the signal-aware deletion path
// below. Keep its direct cancellation propagation while other resume modes
// retain their structured tool envelope.
if (runtime.signal?.aborted && approved === false) throw err;
return errorEnvelope("runtime_error", err?.message ?? String(err));
}
@@ -271,8 +338,18 @@ export async function resumeToolRequest({
);
}
if (approved !== true) {
// Keep the approval ID usable while this may still be waiting on a
// concurrent state writer. Dropping the index first would orphan the
// capability if cancellation interrupts the deletion.
const deletion = await deleteUnconsumedResumeState({
env: runtime.env,
key: payload.stateKey,
signal: runtime.signal,
});
if (deletion !== "deleted") {
return errorEnvelope("runtime_error", "Pipeline resume state not found");
}
await cleanupIndex();
await deleteStateJson({ env: runtime.env, key: payload.stateKey });
return okEnvelope("cancelled", [], null, null);
}
}
@@ -285,14 +362,14 @@ export async function resumeToolRequest({
: resumeState.haltType === "input_request"
? [response]
: resumeState.items;
const abortedBeforeResume = runtime.signal?.aborted === true;
let pipelineResumeStateRestored = false;
let pipelineExecutionStarted = false;
let pipelineResumeStateClaimId: string | undefined;
const requestInputResume = isSameStageInput
? {
state: resumeState.commandInput!,
response,
onConsumed: async () => {
await cleanupIndex();
await deleteStateJson({ env: runtime.env, key: payload.stateKey });
},
}
: undefined;
@@ -308,17 +385,53 @@ export async function resumeToolRequest({
cwd: runtime.cwd,
llmAdapters: runtime.llmAdapters,
signal: runtime.signal,
forceTerminationSignal: runtime.forceTerminationSignal,
haltAfterStageOnAbort: true,
input,
requestInputResume,
onExecutionStart: async () => {
const consumption = await consumeResumeState({
env: runtime.env,
key: payload.stateKey,
expectedState: resumeState,
signal: runtime.signal,
});
if (!consumption.consumed) {
throw new Error("Pipeline resume state not found");
}
pipelineResumeStateClaimId = consumption.claimId;
if (consumption.signalAbortedAfterCommit) {
const restored = await restoreConsumedResumeState({
env: runtime.env,
key: payload.stateKey,
expectedState: resumeState,
claimId: consumption.claimId,
});
if (restored) {
pipelineResumeStateRestored = true;
pipelineResumeStateClaimId = undefined;
}
runtime.signal?.throwIfAborted();
}
runtime.signal?.throwIfAborted();
pipelineExecutionStarted = true;
},
});
await cleanupIndex();
const finalized = await finalizePipelineToolRun({
env: runtime.env,
pipeline: remaining,
output,
previousStateKey: payload.stateKey,
previousState: resumeState,
previousStateConsumed: pipelineExecutionStarted,
restorePreviousStateOnAbort: !pipelineExecutionStarted,
onPreviousStateRestored: () => {
pipelineResumeStateRestored = true;
},
signal: runtime.signal,
});
if (finalized.status === "ok" && pipelineExecutionStarted) await cleanupIndex();
return okEnvelope(
finalized.status,
finalized.output,
@@ -326,7 +439,33 @@ export async function resumeToolRequest({
finalized.requiresInput,
);
} catch (err: any) {
// Don't clean up index on error — allow retry by --id
const abortedResume = runtime.signal?.aborted === true;
if (
abortedResume &&
!pipelineExecutionStarted &&
!pipelineResumeStateRestored &&
pipelineResumeStateClaimId
) {
pipelineResumeStateRestored = await restoreConsumedResumeState({
env: runtime.env,
key: payload.stateKey,
expectedState: resumeState,
claimId: pipelineResumeStateClaimId,
}).catch(() => false);
}
if (pipelineExecutionStarted && !pipelineResumeStateRestored) {
if (abortedResume && !abortedBeforeResume) {
await deleteStateJsonWithBoundedResumeCleanup({
env: runtime.env,
key: payload.stateKey,
}).catch(() => {});
}
// Keep the short approval ID through the pre-dispatch claim window. Once
// the unsafe stage has actually been entered, the tombstone makes retry
// unsafe and the old index may be retired just as it was before this fix.
await cleanupIndex().catch(() => {});
}
// Non-abort failures and pre-aborted resumes remain retryable by token or approval ID.
return errorEnvelope("runtime_error", err?.message ?? String(err));
}
}
@@ -340,6 +479,7 @@ export function createToolContext(ctx: ToolRunContext = {}) {
stdout: ctx.stdout ?? createCaptureStream(),
stderr: ctx.stderr ?? createCaptureStream(),
signal: ctx.signal,
forceTerminationSignal: ctx.forceTerminationSignal,
registry: ctx.registry ?? createDefaultRegistry(),
llmAdapters: ctx.llmAdapters,
};
+5 -1
View File
@@ -177,6 +177,7 @@ export function createStageRequestInput({
getInactiveReason,
isOutputStarted,
resume,
onResumedInput,
}: {
ctx: any;
stageIndex: number;
@@ -186,6 +187,7 @@ export function createStageRequestInput({
getInactiveReason?: () => string | undefined;
isOutputStarted: () => boolean;
resume?: CommandInputResume;
onResumedInput?: () => void | Promise<void>;
}) {
let requestIndex = 0;
const history: CommandInputHistoryEntry[] = [...(resume?.state.history ?? [])];
@@ -209,6 +211,7 @@ export function createStageRequestInput({
const response = snapshotJson(historical.response, "requestInput response");
validateRequestInputResponse(metadata.responseSchema, response, "requestInput");
requestIndex += 1;
await onResumedInput?.();
return response;
}
@@ -234,6 +237,7 @@ export function createStageRequestInput({
response: historyResponse,
});
requestIndex += 1;
await onResumedInput?.();
return response;
}
@@ -479,7 +483,7 @@ function assertJsonSerializable(value: unknown, label: string, seen: WeakSet<obj
async function requestInputInteractively(ctx: any, metadata: RequestInputMetadata) {
ctx.stdout.write(`${metadata.prompt}\n> `);
const { readLineFromStream } = await import("./read_line.js");
const raw = await readLineFromStream(ctx.stdin, { timeoutMs: 0 });
const raw = await readLineFromStream(ctx.stdin, { timeoutMs: 0, signal: ctx.signal });
let response;
try {
response = JSON.parse(String(raw ?? "").trim());
+300 -73
View File
@@ -3,9 +3,13 @@ import { randomUUID } from "node:crypto";
import { encodeToken } from "./token.js";
import {
cleanupApprovalIndexByStateKey,
consumeResumeState,
createApprovalIndex,
deleteResumeStateWithRollback,
deleteStateJson,
readStateJson,
isConsumedResumeState,
readStateJsonWithLock,
restoreConsumedResumeState,
writeStateJson,
} from "./state/store.js";
import { compileCached } from "./validation.js";
@@ -20,6 +24,7 @@ export type PipelineResumeState = {
inputSchema?: unknown;
prompt?: string;
commandInput?: CommandInputState;
supersededResumeStateKeys?: string[];
createdAt: string;
};
@@ -44,6 +49,7 @@ export type PipelineRunOutput = {
items: unknown[];
halted?: boolean;
haltedAt?: { index: number } | null;
executionStarted?: boolean;
};
export type PipelineToolRunResolution =
@@ -97,90 +103,163 @@ export async function finalizePipelineToolRun(params: {
pipeline: PipelineResumeState["pipeline"];
output: PipelineRunOutput;
previousStateKey?: string;
previousState?: PipelineResumeState;
previousStateConsumed?: boolean;
restorePreviousStateOnAbort?: boolean;
onPreviousStateRestored?: () => void;
signal?: AbortSignal;
}): Promise<PipelineToolRunResolution> {
params.signal?.throwIfAborted();
const { approval, inputRequest } = extractPipelineHalt(params.output);
if (approval) {
const nextStateKey = await savePipelineResumeState(params.env, {
pipeline: params.pipeline,
resumeAtIndex: (params.output.haltedAt?.index ?? -1) + 1,
items: approval.items,
haltType: "approval_request",
prompt: approval.prompt,
createdAt: new Date().toISOString(),
});
if (params.previousStateKey) {
await cleanupApprovalIndexByStateKey({ env: params.env, stateKey: params.previousStateKey });
await deleteStateJson({ env: params.env, key: params.previousStateKey });
}
let approvalId: string | null;
let nextStateKey: string | undefined;
try {
nextStateKey = await savePipelineResumeState(
params.env,
{
pipeline: params.pipeline,
resumeAtIndex: (params.output.haltedAt?.index ?? -1) + 1,
items: approval.items,
haltType: "approval_request",
prompt: approval.prompt,
supersededResumeStateKeys: collectSupersededPipelineResumeStateKeys(
params.previousStateKey,
params.previousState,
),
createdAt: new Date().toISOString(),
},
params.signal,
);
let approvalId: string | null;
approvalId = await createApprovalIndex({ env: params.env, stateKey: nextStateKey });
const replaced = await replacePipelineResumeState({
env: params.env,
previousStateKey: params.previousStateKey,
expectedPreviousState: params.previousState,
previousStateConsumed: params.previousStateConsumed,
replacementStateKey: nextStateKey,
signal: params.signal,
});
if (!replaced) throw new Error("Pipeline resume state not found");
const resumeToken = encodeToken({
protocolVersion: 1,
v: 1,
kind: "pipeline-resume",
stateKey: nextStateKey,
});
await retirePreviousPipelineApprovalIndex(params.env, params.previousStateKey, nextStateKey);
return {
status: "needs_approval",
output: [],
requiresApproval: {
...approval,
resumeToken,
...(approvalId ? { approvalId } : null),
},
requiresInput: null,
};
} catch (err) {
await deleteStateJson({ env: params.env, key: nextStateKey }).catch(() => {});
try {
if (!params.previousStateConsumed) await restorePreviousPipelineResumeState(params);
} finally {
if (nextStateKey) await discardPipelineResumeState(params.env, nextStateKey);
}
throw err;
}
const resumeToken = encodeToken({
protocolVersion: 1,
v: 1,
kind: "pipeline-resume",
stateKey: nextStateKey,
});
return {
status: "needs_approval",
output: [],
requiresApproval: {
...approval,
resumeToken,
...(approvalId ? { approvalId } : null),
},
requiresInput: null,
};
}
if (inputRequest) {
const resumeMode = inputRequest.commandInput ? "same_stage" : "next_stage";
const nextStateKey = await savePipelineResumeState(params.env, {
pipeline: params.pipeline,
resumeAtIndex:
resumeMode === "same_stage"
? (params.output.haltedAt?.index ?? -1)
: (params.output.haltedAt?.index ?? -1) + 1,
items: resumeMode === "same_stage" ? (inputRequest.items ?? []) : [],
haltType: "input_request",
resumeMode,
inputSchema: inputRequest.responseSchema,
prompt: inputRequest.prompt,
...(inputRequest.commandInput ? { commandInput: inputRequest.commandInput } : null),
createdAt: new Date().toISOString(),
});
if (params.previousStateKey) {
await cleanupApprovalIndexByStateKey({ env: params.env, stateKey: params.previousStateKey });
await deleteStateJson({ env: params.env, key: params.previousStateKey });
let nextStateKey: string | undefined;
try {
nextStateKey = await savePipelineResumeState(
params.env,
{
pipeline: params.pipeline,
resumeAtIndex:
resumeMode === "same_stage"
? (params.output.haltedAt?.index ?? -1)
: (params.output.haltedAt?.index ?? -1) + 1,
items: resumeMode === "same_stage" ? (inputRequest.items ?? []) : [],
haltType: "input_request",
resumeMode,
inputSchema: inputRequest.responseSchema,
prompt: inputRequest.prompt,
...(inputRequest.commandInput ? { commandInput: inputRequest.commandInput } : null),
supersededResumeStateKeys: collectSupersededPipelineResumeStateKeys(
params.previousStateKey,
params.previousState,
),
createdAt: new Date().toISOString(),
},
params.signal,
);
const replaced = await replacePipelineResumeState({
env: params.env,
previousStateKey: params.previousStateKey,
expectedPreviousState: params.previousState,
previousStateConsumed: params.previousStateConsumed,
replacementStateKey: nextStateKey,
signal: params.signal,
});
if (!replaced) throw new Error("Pipeline resume state not found");
const resumeToken = encodeToken({
protocolVersion: 1,
v: 1,
kind: "pipeline-resume",
stateKey: nextStateKey,
});
await retirePreviousPipelineApprovalIndex(params.env, params.previousStateKey, nextStateKey);
return {
status: "needs_input",
output: [],
requiresApproval: null,
requiresInput: {
type: "input_request",
prompt: inputRequest.prompt,
responseSchema: inputRequest.responseSchema,
...(inputRequest.defaults !== undefined ? { defaults: inputRequest.defaults } : null),
...(inputRequest.subject !== undefined ? { subject: inputRequest.subject } : null),
resumeToken,
},
};
} catch (err) {
try {
if (!params.previousStateConsumed) await restorePreviousPipelineResumeState(params);
} finally {
if (nextStateKey) await discardPipelineResumeState(params.env, nextStateKey);
}
throw err;
}
const resumeToken = encodeToken({
protocolVersion: 1,
v: 1,
kind: "pipeline-resume",
stateKey: nextStateKey,
});
return {
status: "needs_input",
output: [],
requiresApproval: null,
requiresInput: {
type: "input_request",
prompt: inputRequest.prompt,
responseSchema: inputRequest.responseSchema,
...(inputRequest.defaults !== undefined ? { defaults: inputRequest.defaults } : null),
...(inputRequest.subject !== undefined ? { subject: inputRequest.subject } : null),
resumeToken,
},
};
}
params.signal?.throwIfAborted();
if (params.previousStateKey) {
await cleanupApprovalIndexByStateKey({ env: params.env, stateKey: params.previousStateKey });
await deleteStateJson({ env: params.env, key: params.previousStateKey });
try {
if (params.previousStateConsumed) {
await deleteStateJson({
env: params.env,
key: params.previousStateKey,
signal: params.signal,
});
params.signal?.throwIfAborted();
} else {
const deleted = await deleteResumeStateWithRollback({
env: params.env,
key: params.previousStateKey,
expectedState: params.previousState,
signal: params.signal,
});
if (!deleted) throw new Error("Pipeline resume state not found");
}
await cleanupSupersededPipelineResumeStates(
params.env,
params.previousState?.supersededResumeStateKeys,
);
} catch (err) {
if (!params.previousStateConsumed) await restorePreviousPipelineResumeState(params);
throw err;
}
}
return {
status: "ok",
@@ -193,18 +272,159 @@ export async function finalizePipelineToolRun(params: {
export async function savePipelineResumeState(
env: Record<string, string | undefined>,
state: PipelineResumeState,
signal?: AbortSignal,
) {
const stateKey = `pipeline_resume_${randomUUID()}`;
await writeStateJson({ env, key: stateKey, value: state });
return stateKey;
try {
signal?.throwIfAborted();
await writeStateJson({ env, key: stateKey, value: state, signal });
signal?.throwIfAborted();
return stateKey;
} catch (err) {
if (signal?.aborted) await discardPipelineResumeState(env, stateKey);
throw err;
}
}
async function replacePipelineResumeState({
env,
previousStateKey,
expectedPreviousState,
previousStateConsumed,
replacementStateKey,
signal,
}: {
env: Record<string, string | undefined>;
previousStateKey?: string;
expectedPreviousState?: PipelineResumeState;
previousStateConsumed?: boolean;
replacementStateKey: string;
signal?: AbortSignal;
}) {
if (!previousStateKey || previousStateKey === replacementStateKey) {
signal?.throwIfAborted();
return true;
}
// The current resume has already crossed an unsafe boundary and owns the
// predecessor's consumed marker. It may safely publish the next gate, but
// must retain that marker rather than attempting a stale snapshot CAS.
if (previousStateConsumed) {
signal?.throwIfAborted();
return true;
}
if (!expectedPreviousState) return false;
let claimId: string | undefined;
try {
const consumption = await consumeResumeState({
env,
key: previousStateKey,
expectedState: expectedPreviousState,
signal,
});
if (!consumption.consumed) return false;
claimId = consumption.claimId;
// The predecessor remains as a durable tombstone until terminal cleanup.
// A concurrent caller can therefore never turn the same approval into a
// second successor capability.
signal?.throwIfAborted();
return true;
} catch (err) {
// A cancellation after the atomic marker publication has not exposed the
// successor token yet. Restore only the marker created by this caller so a
// competing transition can never be overwritten.
if (claimId && signal?.aborted) {
await restoreConsumedResumeState({
env,
key: previousStateKey,
expectedState: expectedPreviousState,
claimId,
}).catch(() => {});
}
throw err;
}
}
async function restorePreviousPipelineResumeState({
env,
previousStateKey,
previousState,
restorePreviousStateOnAbort,
onPreviousStateRestored,
signal,
}: {
env: Record<string, string | undefined>;
previousStateKey?: string;
previousState?: PipelineResumeState;
restorePreviousStateOnAbort?: boolean;
onPreviousStateRestored?: () => void;
signal?: AbortSignal;
}) {
if (!signal?.aborted || !restorePreviousStateOnAbort || !previousStateKey || !previousState) {
return;
}
// Safe terminal cleanup restores its own claimed marker while still holding
// the state lock. Never recreate a missing snapshot here: this caller may
// have only observed it before another resume completed.
if ((await readStateJsonWithLock({ env, key: previousStateKey, signal })) !== null) {
onPreviousStateRestored?.();
}
}
async function retirePreviousPipelineApprovalIndex(
env: Record<string, string | undefined>,
previousStateKey: string | undefined,
replacementStateKey: string,
) {
if (!previousStateKey || previousStateKey === replacementStateKey) return;
// This runs only after replacement deletion has passed its cancellation
// checkpoint. Do not add a later cancellation check: the transition is
// committed once the old approval capability is retired.
await cleanupApprovalIndexByStateKey({ env, stateKey: previousStateKey }).catch(() => {});
}
function collectSupersededPipelineResumeStateKeys(
previousStateKey: string | undefined,
previousState: PipelineResumeState | undefined,
) {
return [
...(previousState?.supersededResumeStateKeys ?? []),
...(previousStateKey ? [previousStateKey] : []),
].filter((stateKey, index, all) => stateKey && all.indexOf(stateKey) === index);
}
async function cleanupSupersededPipelineResumeStates(
env: Record<string, string | undefined>,
stateKeys: string[] | undefined,
) {
for (const stateKey of stateKeys ?? []) {
try {
// Retire only a non-executable marker after the successor itself has
// committed. A restored state must remain available for retry.
if (isConsumedResumeState(await readStateJsonWithLock({ env, key: stateKey }))) {
await deleteStateJson({ env, key: stateKey });
}
} catch {
// Leaving a tombstone is safe if best-effort cleanup cannot complete.
}
}
}
async function discardPipelineResumeState(
env: Record<string, string | undefined>,
stateKey: string,
) {
await cleanupApprovalIndexByStateKey({ env, stateKey }).catch(() => {});
await deleteStateJson({ env, key: stateKey }).catch(() => {});
}
export async function loadPipelineResumeState(
env: Record<string, string | undefined>,
stateKey: string,
signal?: AbortSignal,
) {
const stored = await readStateJson({ env, key: stateKey });
if (!stored || typeof stored !== "object") {
const stored = await readStateJsonWithLock({ env, key: stateKey, signal });
if (!stored || typeof stored !== "object" || isConsumedResumeState(stored)) {
throw new Error("Pipeline resume state not found");
}
const data = stored as Partial<PipelineResumeState>;
@@ -219,6 +439,13 @@ export async function loadPipelineResumeState(
throw new Error("Invalid pipeline resume state");
}
if (!Array.isArray(data.items)) throw new Error("Invalid pipeline resume state");
if (
data.supersededResumeStateKeys !== undefined &&
(!Array.isArray(data.supersededResumeStateKeys) ||
data.supersededResumeStateKeys.some((stateKey) => typeof stateKey !== "string"))
) {
throw new Error("Invalid pipeline resume state");
}
if (
data.haltType !== undefined &&
!["approval_request", "input_request"].includes(data.haltType)
+60 -9
View File
@@ -1,9 +1,22 @@
export function readLineFromStream(stream: NodeJS.ReadableStream, opts?: { timeoutMs?: number }) {
const unreadInput = new WeakMap<NodeJS.ReadableStream, string>();
type ObservableReadableStream = NodeJS.ReadableStream & {
readableEnded?: boolean;
destroyed?: boolean;
closed?: boolean;
};
export function readLineFromStream(
stream: NodeJS.ReadableStream,
opts?: { timeoutMs?: number; signal?: AbortSignal },
) {
const timeoutMs = Number(opts?.timeoutMs ?? 0);
const signal = opts?.signal;
const observableStream = stream as ObservableReadableStream;
return new Promise<string>((resolve, reject) => {
let settled = false;
let buf = "";
let buf = unreadInput.get(stream) ?? "";
unreadInput.delete(stream);
let timer: NodeJS.Timeout | null = null;
const cleanup = () => {
@@ -11,6 +24,8 @@ export function readLineFromStream(stream: NodeJS.ReadableStream, opts?: { timeo
stream.off("end", onEnd);
stream.off("close", onClose);
stream.off("error", onError);
stream.pause();
signal?.removeEventListener("abort", onAbort);
if (timer) clearTimeout(timer);
};
@@ -24,21 +39,50 @@ export function readLineFromStream(stream: NodeJS.ReadableStream, opts?: { timeo
const fail = (err: Error) => {
if (settled) return;
settled = true;
unreadInput.delete(stream);
cleanup();
reject(err);
};
const onData = (chunk: Buffer | string) => {
buf += Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk);
const consumeLine = () => {
const idx = buf.indexOf("\n");
if (idx !== -1) {
finish(buf.slice(0, idx));
}
if (idx === -1) return false;
unreadInput.set(stream, buf.slice(idx + 1));
finish(buf.slice(0, idx));
return true;
};
const onEnd = () => finish(buf);
const onClose = () => finish(buf);
const onData = (chunk: Buffer | string) => {
buf += Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk);
consumeLine();
};
const drainBuffered = () => {
let chunk: Buffer | string | null;
while (!settled && (chunk = stream.read()) !== null) onData(chunk);
};
const onEnd = () => {
drainBuffered();
if (settled) return;
unreadInput.delete(stream);
finish(buf);
};
const onClose = () => {
drainBuffered();
if (settled) return;
unreadInput.delete(stream);
finish(buf);
};
const onError = (err: Error) => fail(err);
const onAbort = () => {
const reason = signal?.reason;
fail(reason instanceof Error ? reason : new Error("Input read aborted"));
};
if (signal?.aborted) {
onAbort();
return;
}
if (timeoutMs > 0) {
timer = setTimeout(() => {
@@ -50,5 +94,12 @@ export function readLineFromStream(stream: NodeJS.ReadableStream, opts?: { timeo
stream.on("end", onEnd);
stream.on("close", onClose);
stream.on("error", onError);
signal?.addEventListener("abort", onAbort, { once: true });
drainBuffered();
if (!settled && !consumeLine()) {
if (observableStream.readableEnded || observableStream.destroyed || observableStream.closed)
onEnd();
else stream.resume();
}
});
}
+167 -22
View File
@@ -20,9 +20,12 @@ export async function runPipeline({
cwd = undefined,
llmAdapters = undefined,
signal = undefined,
forceTerminationSignal = undefined,
haltAfterStageOnAbort = false,
dryRun = false,
requestInputResume = undefined,
requestInputEnabled = true,
onExecutionStart = undefined,
}: {
pipeline: any[];
registry: any;
@@ -35,9 +38,12 @@ export async function runPipeline({
cwd?: string | undefined;
llmAdapters?: Record<string, any> | undefined;
signal?: AbortSignal | undefined;
forceTerminationSignal?: AbortSignal | undefined;
haltAfterStageOnAbort?: boolean;
dryRun?: boolean;
requestInputResume?: CommandInputResume | undefined;
requestInputEnabled?: boolean;
onExecutionStart?: (() => void | Promise<void>) | undefined;
}) {
if (dryRun) {
return dryRunPipeline({ pipeline, registry, stderr });
@@ -48,6 +54,16 @@ export async function runPipeline({
let halted = false;
let haltedAt = null;
let pipelineOutputStarted = false;
let executionStarted = false;
let executionStart: Promise<void> | undefined;
const markExecutionStarted = async () => {
if (executionStart) return executionStart;
executionStart = (async () => {
await onExecutionStart?.();
executionStarted = true;
})();
return executionStart;
};
const baseCtx = {
stdin,
@@ -59,14 +75,19 @@ export async function runPipeline({
cwd,
llmAdapters,
signal,
forceTerminationSignal,
};
for (let idx = 0; idx < pipeline.length; idx++) {
if (haltAfterStageOnAbort) signal?.throwIfAborted();
const stage = pipeline[idx];
const command = registry.get(stage.name);
if (!command) {
throw new Error(`Unknown command: ${stage.name}`);
}
if (command.meta?.resumeSafeBeforeInput !== true) {
await markExecutionStarted();
}
const inputTracker = createInputTracker(stream);
const stageResume = idx === 0 ? requestInputResume : undefined;
@@ -102,6 +123,8 @@ export async function runPipeline({
getInactiveReason: () => inactiveReason,
isOutputStarted: () => pipelineOutputStarted || commandOutputStarted,
resume: stageResume,
onResumedInput:
command.meta?.resumeSafeAfterInput === true ? undefined : markExecutionStarted,
})
: createUnsupportedRequestInput(),
};
@@ -120,15 +143,17 @@ export async function runPipeline({
rendered = true;
}
const terminalOutput = Boolean(result?.halt);
let stageHalted = Boolean(terminalOutput || (haltAfterStageOnAbort && signal?.aborted));
const output = result?.output;
if (Array.isArray(output)) {
stream = output;
await finishStage();
} else if (output && !result?.halt && idx < pipeline.length - 1) {
} else if (output && idx < pipeline.length - 1 && !terminalOutput) {
commandActive = false;
inactiveReason = "requestInput cannot suspend from lazy output before downstream stages";
assertRequestInputResumeConsumed(stageResume);
stream = trackCommandOutput(
const trackedOutput = trackCommandOutput(
output,
() => {
commandOutputStarted = true;
@@ -137,22 +162,34 @@ export async function runPipeline({
(err) => assertNoUnconsumedResumeAfterError(stageResume, err),
finishStage,
);
stream = haltAfterStageOnAbort
? throwIfAbortedAfterDrain(trackedOutput, signal)
: trackedOutput;
} else {
stream = output
? trackCommandOutput(
output,
() => {
commandOutputStarted = true;
},
() => assertRequestInputResumeConsumed(stageResume),
(err) => assertNoUnconsumedResumeAfterError(stageResume, err),
finishStage,
)
: [];
if (!output) await finishStage();
if (output) {
const trackedOutput = trackCommandOutput(
output,
() => {
commandOutputStarted = true;
},
() => assertRequestInputResumeConsumed(stageResume),
(err) => assertNoUnconsumedResumeAfterError(stageResume, err),
finishStage,
);
// Terminal output is drained after the stage loop as well. It needs the
// same abort-aware read as a handoff, otherwise a stalled final iterator
// can prevent the tool cancellation from settling forever.
stream = haltAfterStageOnAbort
? throwIfAbortedAfterDrain(trackedOutput, signal)
: trackedOutput;
} else {
stream = [];
await finishStage();
}
}
if (result?.halt) {
stageHalted ||= Boolean(haltAfterStageOnAbort && signal?.aborted);
if (stageHalted) {
halted = true;
haltedAt = { index: idx, stage };
break;
@@ -170,9 +207,10 @@ export async function runPipeline({
throw err;
}
}
if (haltAfterStageOnAbort) signal?.throwIfAborted();
assertRequestInputResumeConsumed(requestInputResume);
return { items, rendered, halted, haltedAt };
return { items, rendered, halted, haltedAt, executionStarted };
function haltForInputRequest(err: unknown) {
if (!(err instanceof InputRequestSuspension)) return false;
@@ -214,7 +252,7 @@ function dryRunPipeline({
lines.push("");
stderr.write(lines.join("\n"));
// Return rendered:true so the CLI does not print an empty JSON array to stdout.
return { items: [], rendered: true, halted: false, haltedAt: null };
return { items: [], rendered: true, halted: false, haltedAt: null, executionStarted: false };
}
function formatStageArgs(args: Record<string, unknown>) {
@@ -238,6 +276,82 @@ function streamFromItems(items: unknown[]) {
})();
}
function throwIfAbortedAfterDrain(input: AsyncIterable<unknown>, signal?: AbortSignal) {
return (async function* () {
const iterator = input[Symbol.asyncIterator]();
let completed = false;
try {
while (true) {
const next = await nextWithAbort(iterator, signal);
if (next.done) {
completed = true;
break;
}
signal?.throwIfAborted();
yield next.value;
}
signal?.throwIfAborted();
} finally {
if (!completed) await closeAfterAbortedRead(input, iterator, signal);
}
})();
}
type CancellableLazyOutput = {
abort?: (reason?: unknown) => void | Promise<void>;
};
async function closeAfterAbortedRead(
input: AsyncIterable<unknown>,
iterator: AsyncIterator<unknown>,
signal?: AbortSignal,
) {
const cancellable = iterator as AsyncIterator<unknown> & CancellableLazyOutput;
const inputCancellable = input as AsyncIterable<unknown> & CancellableLazyOutput;
const abort = cancellable.abort ?? inputCancellable.abort;
try {
// A source that owns a pending timer, socket, or process can expose this
// small cancellation hook. It must release the pending next() operation.
if (signal?.aborted && abort) await abort(signal.reason);
if (typeof iterator.return !== "function") return;
const close = iterator.return();
if (signal?.aborted && !abort) {
// Legacy iterators cannot interrupt an in-flight next(). Keep the
// existing prompt cancellation behavior, while resource-owning sources
// opt into the abort hook above so their cleanup is awaited.
void Promise.resolve(close).catch(() => {});
return;
}
await close;
} catch (err) {
if (!signal?.aborted) throw err;
}
}
async function nextWithAbort(iterator: AsyncIterator<unknown>, signal?: AbortSignal) {
if (!signal) return iterator.next();
signal.throwIfAborted();
let onAbort!: () => void;
const aborted = new Promise<never>((_resolve, reject) => {
onAbort = () => {
try {
signal.throwIfAborted();
} catch (err) {
reject(err);
}
};
signal.addEventListener("abort", onAbort, { once: true });
});
if (signal.aborted) onAbort();
try {
return await Promise.race([iterator.next(), aborted]);
} finally {
signal.removeEventListener("abort", onAbort);
}
}
function trackCommandOutput(
output: AsyncIterable<unknown> | Iterable<unknown>,
markOutput: () => void,
@@ -248,13 +362,27 @@ function trackCommandOutput(
suppressCloseErrors?: boolean;
}) => Promise<void>,
) {
return (async function* () {
const source = output as AsyncIterable<unknown> & CancellableLazyOutput & AsyncIterator<unknown>;
let sourceIterator: AsyncIterator<unknown> | undefined;
const tracked = (async function* () {
let completed = false;
try {
for await (const item of output) {
assertResumeConsumed();
markOutput();
yield item;
if (typeof source[Symbol.asyncIterator] === "function") {
sourceIterator = source[Symbol.asyncIterator]();
const iteratorInput: AsyncIterable<unknown> = {
[Symbol.asyncIterator]: () => sourceIterator!,
};
for await (const item of iteratorInput) {
assertResumeConsumed();
markOutput();
yield item;
}
} else {
for (const item of output as Iterable<unknown>) {
assertResumeConsumed();
markOutput();
yield item;
}
}
completed = true;
} catch (err) {
@@ -265,6 +393,23 @@ function trackCommandOutput(
await finishStage({ assertResume: completed });
}
})();
// Iterator-owned abort hooks are only discoverable after iterator acquisition.
// A getter lets the abort-aware wrapper find that hook once a read is pending,
// without performing acquisition outside the generator's guarded cleanup path.
Object.defineProperty(tracked, "abort", {
configurable: true,
get() {
const cancellationOwner =
sourceIterator && typeof (sourceIterator as CancellableLazyOutput).abort === "function"
? (sourceIterator as CancellableLazyOutput)
: source;
const abort = cancellationOwner.abort;
return typeof abort === "function"
? (reason?: unknown) => abort.call(cancellationOwner, reason)
: undefined;
},
});
return tracked;
}
function assertNoUnconsumedResumeAfterError(resume: CommandInputResume | undefined, err: unknown) {
+6 -2
View File
@@ -3,7 +3,7 @@ import { runPipelineInternal } from "./runtime.js";
import { encodeToken, decodeToken } from "./token.js";
import { compileCached } from "../validation.js";
import { validateCommandInputState, type CommandInputState } from "../input_request.js";
import { deleteStateJson, readStateJson, writeStateJson } from "../state/store.js";
import { deleteStateJson, readStateJsonWithLock, writeStateJson } from "../state/store.js";
type SdkResumePayload = {
protocolVersion: 1;
@@ -449,7 +449,11 @@ async function loadSdkCommandInputResumeState(
options: any,
stateKey: string,
): Promise<SdkCommandInputResumeState> {
const stored = await readStateJson({ env: sdkStateEnv(options), key: stateKey });
const stored = await readStateJsonWithLock({
env: sdkStateEnv(options),
key: stateKey,
signal: options?.signal,
});
if (!stored || typeof stored !== "object") throw new Error("SDK resume state not found");
const data = stored as Partial<SdkCommandInputResumeState>;
if (
+14 -93
View File
@@ -14,56 +14,12 @@
* });
*/
import { promises as fsp } from "node:fs";
import os from "node:os";
import path from "node:path";
import { ensureDirectory, isJsonSyntaxError, writeFileAtomic } from "../../state/store.js";
import { diffAndStore } from "../../state/store.js";
/**
* Get the state directory
* @param {Object} ctx
* @returns {string}
*/
function getStateDir(ctx) {
return (
ctx?.stateDir ||
(ctx?.env?.LOBSTER_STATE_DIR && String(ctx.env.LOBSTER_STATE_DIR).trim()) ||
path.join(os.homedir(), ".lobster", "state")
);
}
/**
* Convert a key to a safe file path
* @param {string} stateDir
* @param {string} key
* @returns {string}
*/
function keyToPath(stateDir, key) {
const safe = String(key)
.toLowerCase()
.replace(/[^a-z0-9._-]+/g, "_")
.replace(/_+/g, "_")
.replace(/^_+|_+$/g, "");
if (!safe) throw new Error("state key is empty/invalid");
return path.join(stateDir, `${safe}.json`);
}
/**
* Stable JSON stringify for comparison
* @param {any} value
* @returns {string}
*/
function stableStringify(value) {
return JSON.stringify(value, (_k, v) => {
if (v && typeof v === "object" && !Array.isArray(v)) {
return Object.fromEntries(
Object.keys(v)
.sort()
.map((k) => [k, v[k]]),
);
}
return v;
});
function stateEnv(ctx) {
return ctx?.stateDir
? { ...(ctx?.env ?? process.env), LOBSTER_STATE_DIR: ctx.stateDir }
: (ctx?.env ?? process.env);
}
/**
@@ -95,26 +51,12 @@ export function diffLast(key, options: any = {}) {
const value = items.length === 1 ? items[0] : items;
const stateDir = getStateDir(ctx);
const filePath = keyToPath(stateDir, key);
// Read previous value
let before = null;
try {
const text = await fsp.readFile(filePath, "utf8");
before = JSON.parse(text);
} catch (err) {
if (err?.code !== "ENOENT" && !isJsonSyntaxError(err)) {
throw err;
}
}
// Compare
const changed = stableStringify(before) !== stableStringify(value);
// Store new value
await ensureDirectory(stateDir);
await writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n");
const { before, after, changed } = await diffAndStore({
env: stateEnv(ctx),
key,
value,
signal: ctx?.signal,
});
// Build result
const result = {
@@ -122,7 +64,7 @@ export function diffLast(key, options: any = {}) {
key,
changed,
before,
after: value,
after,
};
// If changesOnly and no change, output suppressed marker
@@ -150,27 +92,6 @@ export function diffLast(key, options: any = {}) {
* @param {Object} [ctx]
* @returns {Promise<{before: any, after: any, changed: boolean}>}
*/
export async function diffAndStoreValue(key, value, ctx = {}) {
const stateDir = getStateDir(ctx);
const filePath = keyToPath(stateDir, key);
// Read previous value
let before = null;
try {
const text = await fsp.readFile(filePath, "utf8");
before = JSON.parse(text);
} catch (err) {
if (err?.code !== "ENOENT" && !isJsonSyntaxError(err)) {
throw err;
}
}
// Compare
const changed = stableStringify(before) !== stableStringify(value);
// Store new value
await ensureDirectory(stateDir);
await writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n");
return { before, after: value, changed };
export async function diffAndStoreValue(key, value, ctx: any = {}) {
return diffAndStore({ env: stateEnv(ctx), key, value, signal: ctx?.signal });
}
+11 -105
View File
@@ -15,77 +15,12 @@
* .pipe(stateSet('my-key'));
*/
import { randomBytes } from "node:crypto";
import { promises as fsp } from "node:fs";
import os from "node:os";
import path from "node:path";
import { readStateJsonWithLock, writeStateJson } from "../../state/store.js";
/**
* Write a file atomically (stage to a sibling temp file, fsync, then rename).
* `rename(2)` is atomic on a single filesystem, so a concurrent reader or a
* crash never observes a truncated/partial file. Plain `fsp.writeFile`
* truncates the target up front, leaving a corruption window on SIGKILL/OOM/
* power loss. New state files are private by default; existing file modes are
* preserved across replacement. Kept local to keep the SDK self-contained.
* @param {string} filePath
* @param {string} data
*/
async function writeFileAtomic(filePath, data) {
const dir = path.dirname(filePath);
const tmpPath = path.join(
dir,
`.${path.basename(filePath)}.${randomBytes(6).toString("hex")}.tmp`,
);
let mode = 0o600;
let handle;
let cleanup = true;
try {
try {
mode = (await fsp.stat(filePath)).mode & 0o777;
} catch (err) {
if (err?.code !== "ENOENT") throw err;
}
handle = await fsp.open(tmpPath, "wx", mode);
await handle.writeFile(data, "utf8");
await handle.sync();
await handle.close();
handle = undefined;
await fsp.chmod(tmpPath, mode);
await fsp.rename(tmpPath, filePath);
cleanup = false;
} finally {
if (handle) await handle.close().catch(() => {});
if (cleanup) await fsp.rm(tmpPath, { force: true }).catch(() => {});
}
}
/**
* Get the state directory
* @param {Object} ctx
* @returns {string}
*/
function getStateDir(ctx) {
return (
ctx?.stateDir ||
(ctx?.env?.LOBSTER_STATE_DIR && String(ctx.env.LOBSTER_STATE_DIR).trim()) ||
path.join(os.homedir(), ".lobster", "state")
);
}
/**
* Convert a key to a safe file path
* @param {string} stateDir
* @param {string} key
* @returns {string}
*/
function keyToPath(stateDir, key) {
const safe = String(key)
.toLowerCase()
.replace(/[^a-z0-9._-]+/g, "_")
.replace(/_+/g, "_")
.replace(/^_+|_+$/g, "");
if (!safe) throw new Error("state key is empty/invalid");
return path.join(stateDir, `${safe}.json`);
function stateEnv(ctx) {
return ctx?.stateDir
? { ...(ctx?.env ?? process.env), LOBSTER_STATE_DIR: ctx.stateDir }
: (ctx?.env ?? process.env);
}
/**
@@ -107,19 +42,7 @@ export function stateGet(key) {
// no-op
}
const stateDir = getStateDir(ctx);
const filePath = keyToPath(stateDir, key);
let value = null;
try {
const text = await fsp.readFile(filePath, "utf8");
value = JSON.parse(text);
} catch (err) {
if (err?.code !== "ENOENT") {
throw err;
}
// File doesn't exist, return null
}
const value = await readStateJsonWithLock({ env: stateEnv(ctx), key, signal: ctx?.signal });
return {
output: (async function* () {
@@ -152,11 +75,7 @@ export function stateSet(key) {
const value = items.length === 1 ? items[0] : items;
const stateDir = getStateDir(ctx);
const filePath = keyToPath(stateDir, key);
await fsp.mkdir(stateDir, { recursive: true });
await writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n");
await writeStateJson({ env: stateEnv(ctx), key, value, signal: ctx?.signal });
// Pass through the value
return {
@@ -189,17 +108,8 @@ export const state = {
* @param {Object} [ctx]
* @returns {Promise<any>}
*/
export async function readState(key, ctx = {}) {
const stateDir = getStateDir(ctx);
const filePath = keyToPath(stateDir, key);
try {
const text = await fsp.readFile(filePath, "utf8");
return JSON.parse(text);
} catch (err) {
if (err?.code === "ENOENT") return null;
throw err;
}
export async function readState(key, ctx: any = {}) {
return readStateJsonWithLock({ env: stateEnv(ctx), key, signal: ctx?.signal });
}
/**
@@ -209,10 +119,6 @@ export async function readState(key, ctx = {}) {
* @param {Object} [ctx]
* @returns {Promise<void>}
*/
export async function writeState(key, value, ctx = {}) {
const stateDir = getStateDir(ctx);
const filePath = keyToPath(stateDir, key);
await fsp.mkdir(stateDir, { recursive: true });
await writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n");
export async function writeState(key, value, ctx: any = {}) {
await writeStateJson({ env: stateEnv(ctx), key, value, signal: ctx?.signal });
}
+703 -13
View File
@@ -3,6 +3,22 @@ import path from "node:path";
import { promises as fsp } from "node:fs";
import { randomBytes } from "node:crypto";
const CONSUMED_RESUME_STATE_TYPE = "lobster.consumed-resume-state.v1";
export type ConsumedResumeState = {
type: typeof CONSUMED_RESUME_STATE_TYPE;
consumedAt: string;
claimId: string;
};
export function isConsumedResumeState(value: unknown): value is ConsumedResumeState {
return (
value !== null &&
typeof value === "object" &&
(value as { type?: unknown }).type === CONSUMED_RESUME_STATE_TYPE
);
}
export function defaultStateDir(env) {
return (
(env?.LOBSTER_STATE_DIR && String(env.LOBSTER_STATE_DIR).trim()) ||
@@ -36,6 +52,7 @@ export function stableStringify(value) {
type AtomicWriteOptions = {
renameFile?: typeof fsp.rename;
syncParentDir?: (filePath: string) => Promise<void>;
signal?: AbortSignal;
};
type AtomicExclusiveWriteOptions = {
@@ -43,6 +60,29 @@ type AtomicExclusiveWriteOptions = {
syncParentDir?: (filePath: string) => Promise<void>;
};
type PublishedAtomicWriteError = NodeJS.ErrnoException & {
atomicWritePublished?: true;
};
function markAtomicWritePublished(err: unknown) {
if (err && (typeof err === "object" || typeof err === "function")) {
Object.defineProperty(err, "atomicWritePublished", {
value: true,
configurable: true,
});
}
return err;
}
export function atomicWriteWasPublished(err: unknown): err is PublishedAtomicWriteError {
return Boolean((err as PublishedAtomicWriteError | undefined)?.atomicWritePublished);
}
const STATE_LOCK_RETRY_MS = 10;
const STATE_LOCK_ORPHAN_MS = 30_000;
const STATE_LOCK_HEARTBEAT_MS = 250;
const TERMINAL_RESUME_CLEANUP_TIMEOUT_MS = STATE_LOCK_RETRY_MS * 10;
function isDirectorySyncUnsupportedError(err: any): boolean {
return [
"EACCES",
@@ -101,6 +141,267 @@ export function isJsonSyntaxError(err) {
return err instanceof SyntaxError;
}
async function waitForStateLock(signal?: AbortSignal) {
await new Promise<void>((resolve, reject) => {
const onAbort = () => {
clearTimeout(timer);
try {
signal?.throwIfAborted();
} catch (err) {
reject(err);
}
};
const timer = setTimeout(() => {
signal?.removeEventListener("abort", onAbort);
resolve();
}, STATE_LOCK_RETRY_MS);
if (signal?.aborted) {
onAbort();
return;
}
signal?.addEventListener("abort", onAbort, { once: true });
});
}
function isProcessAlive(pid: number) {
try {
process.kill(pid, 0);
return true;
} catch (err: any) {
return err?.code !== "ESRCH";
}
}
type StateLockOwner = {
pid: number;
processStartIdentity: string | null;
nonce: string;
};
function parseStateLockOwner(ownerText: string): StateLockOwner | null {
const parts = ownerText.trim().split(":");
if (parts.length !== 3) return null;
const pid = Number(parts[0]);
const processStartIdentity = parts[1] || null;
const nonce = parts[2];
if (!Number.isInteger(pid) || pid <= 0 || !nonce) return null;
return { pid, processStartIdentity, nonce };
}
async function readProcessStartIdentity(pid: number): Promise<string | null> {
if (process.platform !== "linux") return null;
try {
const stat = await fsp.readFile(`/proc/${pid}/stat`, "utf8");
const closeParen = stat.lastIndexOf(")");
if (closeParen < 0) return null;
// The remainder starts at procfs field 3; starttime is field 22.
const fields = stat
.slice(closeParen + 1)
.trim()
.split(/\s+/);
return fields[19] || null;
} catch {
return null;
}
}
async function isStateLockOld(lockPath: string) {
try {
const stat = await fsp.stat(lockPath);
return Date.now() - stat.mtimeMs >= STATE_LOCK_ORPHAN_MS;
} catch (err: any) {
if (err?.code === "ENOENT") return true;
throw err;
}
}
async function hasExpiredStateLockLease(lockPath: string) {
let first;
try {
first = await fsp.stat(lockPath);
} catch (err: any) {
if (err?.code === "ENOENT") return true;
throw err;
}
if (Date.now() - first.mtimeMs < STATE_LOCK_ORPHAN_MS) return false;
await new Promise<void>((resolve) => setTimeout(resolve, STATE_LOCK_HEARTBEAT_MS));
try {
const second = await fsp.stat(lockPath);
return second.mtimeMs === first.mtimeMs && Date.now() - second.mtimeMs >= STATE_LOCK_ORPHAN_MS;
} catch (err: any) {
if (err?.code === "ENOENT") return true;
throw err;
}
}
async function reclaimOrphanedStateLock(lockPath: string) {
let observedLock: { dev: number; ino: number };
try {
const stat = await fsp.lstat(lockPath);
observedLock = { dev: stat.dev, ino: stat.ino };
} catch (err: any) {
if (err?.code === "ENOENT") return true;
throw err;
}
let stale = false;
try {
const ownerPath = path.join(lockPath, "owner");
const ownerText = (await fsp.readFile(ownerPath, "utf8")).trim();
const owner = parseStateLockOwner(ownerText);
if (owner && isProcessAlive(owner.pid)) {
const processStartIdentity = await readProcessStartIdentity(owner.pid);
if (owner.processStartIdentity && processStartIdentity) {
stale = owner.processStartIdentity !== processStartIdentity;
} else {
// A live PID without a matching process-instance identity may have
// been reused. Require a conservative expired lease and a second
// unchanged observation before reclaiming it.
stale = await hasExpiredStateLockLease(ownerPath);
}
} else if (owner) {
stale = true;
} else {
stale = await isStateLockOld(lockPath);
}
} catch (err: any) {
if (err?.code === "ENOENT") {
stale = await isStateLockOld(lockPath);
} else {
throw err;
}
}
if (!stale) return false;
// Claim reclamation inside the observed directory before removing it. Renaming
// `lockPath` directly is unsafe: another reclaimer can replace that pathname
// with a live lock after the stale observation, and a later recursive cleanup
// would then delete the new owner's lock while it is in use.
const reclaimPath = path.join(lockPath, ".reclaiming");
try {
await fsp.mkdir(reclaimPath, { mode: 0o700 });
} catch (err: any) {
if (err?.code === "ENOENT") return true;
if (err?.code === "EEXIST") return false;
throw err;
}
try {
const currentLock = await fsp.lstat(lockPath);
if (currentLock.dev !== observedLock.dev || currentLock.ino !== observedLock.ino) {
return false;
}
await fsp.rm(lockPath, { recursive: true, force: true });
} finally {
// If the path changed before the reclamation marker was claimed, leave the
// replacement lock intact and only remove our harmless marker.
await fsp.rmdir(reclaimPath).catch(() => {});
}
return true;
}
async function stillOwnsStateLock(ownerPath: string, owner: string) {
for (let attempt = 0; attempt < 2; attempt++) {
try {
return (await fsp.readFile(ownerPath, "utf8")) === owner;
} catch (err: any) {
if (err?.code === "ENOENT" || attempt === 1) return false;
await new Promise<void>((resolve) => setTimeout(resolve, STATE_LOCK_RETRY_MS));
}
}
return false;
}
async function withStateKeyLock<T>({
env,
key,
signal,
task,
}: {
env: Record<string, string | undefined>;
key: string;
signal?: AbortSignal;
task: () => Promise<T>;
}): Promise<T> {
const stateDir = defaultStateDir(env);
return withFileLock({
filePath: keyToPath(stateDir, key),
signal,
task,
});
}
/**
* Serialize a transition for an arbitrary durable file. Readers that need a
* publish-or-rollback decision use the same lock as writers, rather than
* observing an atomic rename that cancellation may still need to undo.
*/
export async function withFileLock<T>({
filePath,
signal,
task,
}: {
filePath: string;
signal?: AbortSignal;
task: () => Promise<T>;
}): Promise<T> {
await ensureDirectory(path.dirname(filePath));
const lockPath = `${filePath}.lock`;
let acquired = false;
let owner: string | undefined;
let ownerWritten = false;
let heartbeat: ReturnType<typeof setInterval> | undefined;
try {
while (!acquired) {
signal?.throwIfAborted();
try {
await fsp.mkdir(lockPath, { mode: 0o700 });
acquired = true;
const processStartIdentity = await readProcessStartIdentity(process.pid);
owner = `${process.pid}:${processStartIdentity ?? ""}:${randomBytes(6).toString("hex")}\n`;
await fsp.writeFile(path.join(lockPath, "owner"), owner, {
encoding: "utf8",
mode: 0o600,
});
ownerWritten = true;
const ownerPath = path.join(lockPath, "owner");
heartbeat = setInterval(() => {
void fsp.utimes(ownerPath, new Date(), new Date()).catch(() => {});
}, STATE_LOCK_HEARTBEAT_MS);
heartbeat.unref?.();
} catch (err: any) {
if (acquired) throw err;
if (err?.code !== "EEXIST") throw err;
if (!(await reclaimOrphanedStateLock(lockPath))) {
await waitForStateLock(signal);
}
}
}
return await task();
} finally {
if (heartbeat) clearInterval(heartbeat);
if (acquired) {
const ownerPath = path.join(lockPath, "owner");
if (!ownerWritten || !owner || (await stillOwnsStateLock(ownerPath, owner))) {
// Detach an owned lock before best-effort cleanup. If a filesystem error
// prevents removing the detached directory, the canonical lock path is
// already free for a subsequent operation; leaving it in place would
// instead look like a live lock owned by this still-running process.
const cleanupPath = `${lockPath}.release-${process.pid}-${randomBytes(6).toString("hex")}`;
try {
await fsp.rename(lockPath, cleanupPath);
await fsp.rm(cleanupPath, { recursive: true, force: true }).catch(() => {});
} catch (err: any) {
if (err?.code !== "ENOENT") {
await fsp.rm(lockPath, { recursive: true, force: true }).catch(() => {});
}
}
}
}
}
}
function isLinkUnsupportedError(err: any): boolean {
return ["ENOSYS", "ENOTSUP", "EOPNOTSUPP", "EPERM", "EXDEV"].includes(err?.code);
}
@@ -148,9 +449,18 @@ export async function writeFileAtomic(filePath, data, options: AtomicWriteOption
await handle.sync();
await handle.close();
handle = undefined;
options.signal?.throwIfAborted();
await renameFile(tmpPath, filePath);
await syncDir(filePath);
cleanup = false;
// The rename is the irreversible publication point. Keep propagating a
// directory-sync failure, but mark it so a state transition can reconcile
// the visible replacement before deciding whether dispatch is safe.
try {
await syncDir(filePath);
} catch (err) {
throw markAtomicWritePublished(err);
}
return { signalAbortedAfterCommit: options.signal?.aborted === true };
} finally {
if (handle) await handle.close().catch(() => {});
if (cleanup) await fsp.rm(tmpPath, { force: true }).catch(() => {});
@@ -202,28 +512,282 @@ export async function writeFileAtomicExclusive(
}
}
export async function readStateJson({ env, key }) {
export async function readStateJson({
env,
key,
signal = undefined,
}: {
env: Record<string, string | undefined>;
key: string;
signal?: AbortSignal;
}) {
signal?.throwIfAborted();
const stateDir = defaultStateDir(env);
const filePath = keyToPath(stateDir, key);
try {
const text = await fsp.readFile(filePath, "utf8");
return JSON.parse(text);
const value = JSON.parse(text);
signal?.throwIfAborted();
return value;
} catch (err) {
if (err?.code === "ENOENT") return null;
throw err;
}
}
export async function writeStateJson({ env, key, value }) {
/**
* Read a state record only after any in-progress publish-or-rollback transition
* for the same key has settled. Callers that compose state with another durable
* resource use this to avoid observing a value that cancellation may undo.
*/
export async function readStateJsonWithLock({
env,
key,
signal = undefined,
}: {
env: Record<string, string | undefined>;
key: string;
signal?: AbortSignal;
}) {
try {
return await withStateKeyLock({ env, key, signal, task: () => readStateJson({ env, key }) });
} catch (err: any) {
// A readable state directory can deliberately be mounted read-only. In that
// case no writer can begin a paired publish-or-rollback transition, so use
// the non-mutating JSON read instead of requiring creation of a lock path.
if (["EACCES", "EPERM", "EROFS"].includes(err?.code)) {
return readStateJson({ env, key, signal });
}
throw err;
}
}
async function writeStateJsonUnlocked({
env,
key,
value,
signal = undefined,
atomicWriteOptions = undefined,
}: {
env: Record<string, string | undefined>;
key: string;
value: unknown;
signal?: AbortSignal;
atomicWriteOptions?: Omit<AtomicWriteOptions, "signal">;
}) {
const stateDir = defaultStateDir(env);
const filePath = keyToPath(stateDir, key);
await ensureDirectory(stateDir);
await writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n");
return writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n", {
...atomicWriteOptions,
signal,
});
}
export async function deleteStateJson({ env, key }) {
export async function writeStateJson({
env,
key,
value,
signal = undefined,
atomicWriteOptions = undefined,
}: {
env: Record<string, string | undefined>;
key: string;
value: unknown;
signal?: AbortSignal;
atomicWriteOptions?: Omit<AtomicWriteOptions, "signal">;
}) {
return withStateKeyLock({
env,
key,
signal,
task: () => writeStateJsonUnlocked({ env, key, value, signal, atomicWriteOptions }),
});
}
/**
* Retire a resume capability before its first unsafe execution boundary. The
* replacement is deliberately persistent: a failed later unlink must not make
* the original token replayable.
*/
export async function consumeResumeState({
env,
key,
expectedState,
signal = undefined,
}: {
env: Record<string, string | undefined>;
key: string;
expectedState: unknown;
signal?: AbortSignal;
}) {
return withStateKeyLock({
env,
key,
signal,
task: async () => {
const currentState = await readStateJson({ env, key });
// A resume snapshot is loaded before command/workflow setup. Re-check it
// while holding the state lock so two callers cannot both turn the same
// approval into an executable invocation.
if (stableStringify(currentState) !== stableStringify(expectedState)) {
return { consumed: false as const };
}
const claimId = randomBytes(16).toString("hex");
try {
const result = await writeStateJsonUnlocked({
env,
key,
value: {
type: CONSUMED_RESUME_STATE_TYPE,
consumedAt: new Date().toISOString(),
claimId,
},
signal,
});
return {
consumed: true as const,
claimId,
signalAbortedAfterCommit: result?.signalAbortedAfterCommit === true,
};
} catch (err) {
if (atomicWriteWasPublished(err)) {
const latest = await readStateJson({ env, key });
if (isConsumedResumeState(latest) && latest.claimId === claimId) {
await writeStateJsonUnlocked({ env, key, value: expectedState });
}
}
throw err;
}
},
});
}
/**
* Undo a just-published consumed marker only when it is still owned by the
* caller's pre-dispatch claim. This never overwrites a replacement state or a
* concurrent claimant's marker.
*/
export async function restoreConsumedResumeState({
env,
key,
expectedState,
claimId,
}: {
env: Record<string, string | undefined>;
key: string;
expectedState: unknown;
claimId: string;
}) {
return withStateKeyLock({
env,
key,
task: async () => {
const currentState = await readStateJson({ env, key });
if (!isConsumedResumeState(currentState) || currentState.claimId !== claimId) {
return false;
}
await writeStateJsonUnlocked({ env, key, value: expectedState });
return true;
},
});
}
/**
* Retire an unconsumed capability at a safe terminal boundary. The snapshot is
* first replaced by a caller-owned marker under the state lock, so a
* cancellation can restore only the state this caller claimed. A stale resume
* that merely observed an earlier snapshot can never recreate a state another
* resume has already settled.
*/
export async function deleteResumeStateWithRollback({
env,
key,
expectedState,
signal = undefined,
}: {
env: Record<string, string | undefined>;
key: string;
expectedState: unknown;
signal?: AbortSignal;
}): Promise<boolean> {
return withStateKeyLock({
env,
key,
signal,
task: async () => {
signal?.throwIfAborted();
const currentState = await readStateJson({ env, key });
if (stableStringify(currentState) !== stableStringify(expectedState)) return false;
const claimId = randomBytes(16).toString("hex");
let claimPublished = false;
let claimMayBePublished = false;
let deleted = false;
try {
let result;
try {
result = await writeStateJsonUnlocked({
env,
key,
value: {
type: CONSUMED_RESUME_STATE_TYPE,
consumedAt: new Date().toISOString(),
claimId,
},
signal,
});
} catch (err) {
claimMayBePublished = atomicWriteWasPublished(err);
throw err;
}
claimPublished = true;
if (result?.signalAbortedAfterCommit) signal?.throwIfAborted();
signal?.throwIfAborted();
await deleteStateJsonUnlocked({ env, key });
deleted = true;
signal?.throwIfAborted();
return true;
} catch (err) {
if ((signal?.aborted && claimPublished) || claimMayBePublished) {
const latest = await readStateJson({ env, key });
if (
(deleted && latest === null) ||
(isConsumedResumeState(latest) && latest.claimId === claimId)
) {
await writeStateJsonUnlocked({ env, key, value: expectedState });
}
}
throw err;
}
},
});
}
/**
* Check physical state presence without parsing it. Cancellation uses this to
* retain the authoritative workflow spelling even if the state file is corrupt.
*/
export async function stateJsonExists({
env,
key,
}: {
env: Record<string, string | undefined>;
key: string;
}) {
const filePath = keyToPath(defaultStateDir(env), key);
try {
await fsp.access(filePath);
return true;
} catch (err: any) {
if (err?.code === "ENOENT") return false;
throw err;
}
}
async function deleteStateJsonUnlocked({ env, key }) {
const stateDir = defaultStateDir(env);
const filePath = keyToPath(stateDir, key);
try {
@@ -234,6 +798,82 @@ export async function deleteStateJson({ env, key }) {
}
}
export async function deleteStateJson({
env,
key,
signal = undefined,
}: {
env: Record<string, string | undefined>;
key: string;
signal?: AbortSignal;
}) {
return withStateKeyLock({
env,
key,
signal,
task: () => deleteStateJsonUnlocked({ env, key }),
});
}
/**
* After an effect has started, its consumed marker already prevents replay.
* Give terminal cleanup a small, bounded opportunity to remove that marker,
* but never let a live state writer turn cancellation into an unbounded wait.
*/
export async function deleteStateJsonWithBoundedResumeCleanup({
env,
key,
}: {
env: Record<string, string | undefined>;
key: string;
}) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), TERMINAL_RESUME_CLEANUP_TIMEOUT_MS);
try {
await deleteStateJson({ env, key, signal: controller.signal });
} finally {
clearTimeout(timeout);
}
}
/**
* Retire a resume capability only while it is still a live, unclaimed state.
* A consumed marker belongs to a resume that has already started its atomic
* predecessor-to-successor handoff, so cancellation must not delete it and
* report success while that successor remains executable.
*/
export async function deleteUnconsumedResumeState({
env,
key,
signal = undefined,
}: {
env: Record<string, string | undefined>;
key: string;
signal?: AbortSignal;
}): Promise<"deleted" | "missing" | "claimed"> {
return withStateKeyLock({
env,
key,
signal,
task: async () => {
let currentState: unknown;
try {
currentState = await readStateJson({ env, key });
} catch (err) {
// A corrupt state is not resumable, so explicit cancellation may still
// remove it. This keeps the legacy workflow-alias recovery behavior.
if (!isJsonSyntaxError(err)) throw err;
await deleteStateJsonUnlocked({ env, key });
return "deleted";
}
if (currentState === null) return "missing";
if (isConsumedResumeState(currentState)) return "claimed";
await deleteStateJsonUnlocked({ env, key });
return "deleted";
},
});
}
function sanitizeApprovalId(approvalId: string): string {
return approvalId.replace(/[^a-f0-9]/g, "");
}
@@ -383,12 +1023,62 @@ export async function cleanupApprovalIndexByStateKey({
}
}
export async function diffAndStore({ env, key, value }) {
const before = await readStateJson({ env, key }).catch((err) => {
if (isJsonSyntaxError(err)) return null;
throw err;
export async function diffAndStore({
env,
key,
value,
signal = undefined,
atomicWriteOptions = undefined,
afterStore = undefined,
}: {
env: Record<string, string | undefined>;
key: string;
value: unknown;
signal?: AbortSignal;
atomicWriteOptions?: Omit<AtomicWriteOptions, "signal">;
afterStore?: (snapshot: { before: unknown; after: unknown; changed: boolean }) => Promise<void>;
}) {
return withStateKeyLock({
env,
key,
signal,
task: async () => {
const filePath = keyToPath(defaultStateDir(env), key);
let beforeExists = true;
try {
await fsp.access(filePath);
} catch (err: any) {
if (err?.code !== "ENOENT") throw err;
beforeExists = false;
}
const before = await readStateJson({ env, key }).catch((err) => {
if (isJsonSyntaxError(err)) return null;
throw err;
});
const changed = stableStringify(before) !== stableStringify(value);
const snapshot = { before, after: value, changed };
let stored = false;
try {
signal?.throwIfAborted();
await writeStateJsonUnlocked({ env, key, value, signal, atomicWriteOptions });
stored = true;
signal?.throwIfAborted();
await afterStore?.(snapshot);
} catch (err) {
// A caller can publish another resource while this state lock is held.
// If that coordinated publication fails, restore the state snapshot before
// releasing the lock so readers never reuse a cancelled result.
const stateWasPublished = stored || atomicWriteWasPublished(err);
if (stateWasPublished && (signal?.aborted || afterStore || atomicWriteWasPublished(err))) {
if (!beforeExists) {
await deleteStateJsonUnlocked({ env, key });
} else {
await writeStateJsonUnlocked({ env, key, value: before });
}
}
throw err;
}
return snapshot;
},
});
const changed = stableStringify(before) !== stableStringify(value);
await writeStateJson({ env, key, value });
return { before, after: value, changed };
}
+517 -171
View File
File diff suppressed because it is too large Load Diff
+26 -32
View File
@@ -1,35 +1,17 @@
import { spawn } from "node:child_process";
import { runAbortableProcess } from "../abortable_process.js";
function runProcess(command, argv, { env, cwd }) {
return new Promise((resolve, reject) => {
const child = spawn(command, argv, { env, cwd, stdio: ["ignore", "pipe", "pipe"] });
let stdout = "";
let stderr = "";
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (d) => {
stdout += d;
});
child.stderr.on("data", (d) => {
stderr += d;
});
child.on("error", (err: any) => {
if (err?.code === "ENOENT") {
reject(new Error("gh not found on PATH (install GitHub CLI)"));
return;
}
reject(err);
});
child.on("close", (code) => {
if (code === 0) return resolve({ stdout, stderr });
reject(new Error(`gh failed (${code}): ${stderr.trim() || stdout.trim()}`));
});
async function runProcess(command, argv, { env, cwd, signal, forceTerminationSignal }) {
const { stdout, stderr, code } = await runAbortableProcess({
command,
argv,
env,
cwd,
signal,
forceTerminationSignal,
notFoundMessage: "gh not found on PATH (install GitHub CLI)",
});
if (code === 0) return { stdout, stderr };
throw new Error(`gh failed (${code}): ${stderr.trim() || stdout.trim()}`);
}
import { diffAndStore } from "../state/store.js";
@@ -83,6 +65,7 @@ function formatPrChangeMessage({ repo, pr, changedFields, prInfo }) {
}
export async function runGithubPrMonitorWorkflow({ args, ctx }) {
ctx.signal?.throwIfAborted();
const repo = args.repo;
const pr = args.pr;
if (!repo || !pr) throw new Error("github.pr.monitor requires args.repo and args.pr");
@@ -101,7 +84,13 @@ export async function runGithubPrMonitorWorkflow({ args, ctx }) {
"number,title,url,state,isDraft,mergeable,reviewDecision,author,baseRefName,headRefName,updatedAt",
];
const { stdout } = (await runProcess("gh", argv, { env: ctx.env, cwd: process.cwd() })) as any;
const { stdout } = (await runProcess("gh", argv, {
env: ctx.env,
cwd: process.cwd(),
signal: ctx.signal,
forceTerminationSignal: ctx.forceTerminationSignal,
})) as any;
ctx.signal?.throwIfAborted();
let current;
try {
@@ -110,7 +99,12 @@ export async function runGithubPrMonitorWorkflow({ args, ctx }) {
throw new Error("gh returned non-JSON output");
}
const { changed, before } = await diffAndStore({ env: ctx.env, key, value: current });
const { changed, before } = await diffAndStore({
env: ctx.env,
key,
value: current,
signal: ctx.signal,
});
if (changesOnly && !changed) {
return {
File diff suppressed because it is too large Load Diff
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env node
import { pathToFileURL } from "node:url";
import { access } from "node:fs/promises";
const [runnerPath, mockGog] = process.argv.slice(2);
const { runAbortableProcess } = await import(pathToFileURL(runnerPath).href);
async function waitFor(path) {
for (let attempt = 0; attempt < 300; attempt += 1) {
try {
await access(path);
return;
} catch {
await new Promise((resolve) => setTimeout(resolve, 10));
}
}
throw new Error(`Timed out waiting for ${path}`);
}
const controller = new AbortController();
const run = runAbortableProcess({
command: process.execPath,
argv: [mockGog, "gmail", "search"],
env: process.env,
signal: controller.signal,
notFoundMessage: "mock gog not found",
});
await waitFor(process.env.MOCK_GOG_SEARCH_STARTED_FILE);
await waitFor(process.env.MOCK_GOG_DESCENDANT_STARTED_FILE);
controller.abort(new Error("cancelled by short-lived caller"));
await run.catch(() => undefined);
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env node
import { readFileSync, writeFileSync } from "node:fs";
import { pathToFileURL } from "node:url";
const [runnerPath] = process.argv.slice(2);
const { runAbortableProcess } = await import(pathToFileURL(runnerPath).href);
function processGroupId() {
const stat = readFileSync("/proc/self/stat", "utf8");
const fields = stat
.slice(stat.lastIndexOf(")") + 1)
.trim()
.split(/\s+/);
return Number(fields[2]);
}
writeFileSync(process.env.LOBSTER_TERMINAL_DIRECT_GROUP_FILE, String(processGroupId()), "utf8");
await runAbortableProcess({
command: process.execPath,
argv: [
"-e",
`const { writeFileSync } = require("node:fs");
writeFileSync(process.env.LOBSTER_TERMINAL_DIRECT_STARTED_FILE, String(process.pid));
setTimeout(() => writeFileSync(process.env.LOBSTER_TERMINAL_DIRECT_COMPLETED_FILE, "completed"), 700);`,
],
env: process.env,
notFoundMessage: "node missing",
});
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env node
import { writeFileSync } from "node:fs";
function mark(path, value) {
if (path) writeFileSync(path, value, "utf8");
}
process.once("SIGTERM", () => {
mark(process.env.MOCK_GH_TERMINATED_FILE, "SIGTERM");
const terminationDelayMs = Number(process.env.MOCK_GH_TERMINATION_DELAY_MS ?? 0);
setTimeout(() => process.exit(143), terminationDelayMs);
});
mark(process.env.MOCK_GH_STARTED_FILE, String(process.pid));
setTimeout(
() => {
mark(process.env.MOCK_GH_COMPLETED_FILE, "completed");
process.stdout.write(
JSON.stringify({
number: 1,
title: "Fixture PR",
url: "https://example.invalid/pr/1",
state: "OPEN",
isDraft: false,
mergeable: "MERGEABLE",
reviewDecision: "",
updatedAt: "2026-07-11T00:00:00Z",
baseRefName: "main",
headRefName: "fixture",
}),
);
},
Number(process.env.MOCK_GH_COMPLETION_DELAY_MS ?? 1200),
);
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env node
import { appendFileSync, writeFileSync } from "node:fs";
import { spawn } from "node:child_process";
const argv = process.argv.slice(2);
function mark(path, value) {
if (path) writeFileSync(path, value, "utf8");
}
function startDescendant() {
if (!process.env.MOCK_GOG_DESCENDANT_STARTED_FILE) return;
const helper = spawn(
process.execPath,
[
"-e",
`const { writeFileSync } = require("node:fs");
process.once("SIGTERM", () => {});
writeFileSync(process.env.MOCK_GOG_DESCENDANT_STARTED_FILE, String(process.pid));
setTimeout(() => writeFileSync(process.env.MOCK_GOG_DESCENDANT_COMPLETED_FILE, "completed"), 650);`,
],
{ env: process.env, stdio: "ignore" },
);
helper.unref();
}
function waitForCompletion({ startedFile, terminatedFile, completedFile, output }) {
const terminationDelayMs = Number(process.env.MOCK_GOG_TERMINATION_DELAY_MS ?? 0);
const completionDelayMs = Number(process.env.MOCK_GOG_COMPLETION_DELAY_MS ?? 1200);
process.once("SIGTERM", () => {
mark(terminatedFile, "SIGTERM");
setTimeout(() => process.exit(143), terminationDelayMs);
});
mark(startedFile, String(process.pid));
startDescendant();
setTimeout(() => {
mark(completedFile, "completed");
process.stdout.write(JSON.stringify(output));
}, completionDelayMs);
}
if (argv[0] === "gmail" && argv[1] === "search") {
waitForCompletion({
startedFile: process.env.MOCK_GOG_SEARCH_STARTED_FILE,
terminatedFile: process.env.MOCK_GOG_SEARCH_TERMINATED_FILE,
completedFile: process.env.MOCK_GOG_SEARCH_COMPLETED_FILE,
output: [{ to: "user@example.com", subject: "Reply", body: "Hello" }],
});
} else if (argv[0] === "gmail" && argv[1] === "send") {
if (process.env.MOCK_GOG_SEND_INVOCATIONS_FILE) {
appendFileSync(process.env.MOCK_GOG_SEND_INVOCATIONS_FILE, `${process.pid}\n`, "utf8");
}
waitForCompletion({
startedFile: process.env.MOCK_GOG_SEND_STARTED_FILE,
terminatedFile: process.env.MOCK_GOG_SEND_TERMINATED_FILE,
completedFile: process.env.MOCK_GOG_SEND_COMPLETED_FILE,
output: { ok: true },
});
} else {
process.stderr.write(`mock-gog-cancellation: unsupported args: ${argv.join(" ")}\n`);
process.exit(2);
}
+17
View File
@@ -1,3 +1,5 @@
import { spawn } from "node:child_process";
const writeResponse = () => {
process.stdout.write(
JSON.stringify({
@@ -8,6 +10,21 @@ const writeResponse = () => {
);
};
if (process.argv.includes("--spawn-descendant")) {
const helper = spawn(
process.execPath,
[
"-e",
`const { writeFileSync } = require("node:fs");
process.once("SIGTERM", () => {});
writeFileSync(process.env.MOCK_OPENCLAW_AGENT_DESCENDANT_STARTED_FILE, String(process.pid));
setTimeout(() => writeFileSync(process.env.MOCK_OPENCLAW_AGENT_DESCENDANT_COMPLETED_FILE, "completed"), 650);`,
],
{ env: process.env, stdio: "ignore" },
);
helper.unref();
}
if (process.argv.includes("--sleep")) {
setTimeout(writeResponse, 10_000);
} else {
+446
View File
@@ -1,11 +1,13 @@
import test from "node:test";
import assert from "node:assert/strict";
import http from "node:http";
import { promises as fsp } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { createDefaultRegistry } from "../src/commands/registry.js";
import { keyToPath } from "../src/state/store.js";
function streamOf(items: any[]) {
return (async function* () {
@@ -167,6 +169,450 @@ test("llm.invoke uses Pi adapter over local HTTP bridge", async () => {
}
});
test("llm.invoke does not retry schema validation after adapter cancellation", async () => {
const registry = createDefaultRegistry();
const cmd = registry.get("llm.invoke");
assert.ok(cmd);
const controller = new AbortController();
let calls = 0;
await assert.rejects(
cmd.run({
input: streamOf([]),
args: {
_: [],
provider: "cancel-test",
prompt: "Decide",
"output-schema": '{"type":"object","required":["decision"]}',
"max-validation-retries": 2,
},
ctx: {
...baseCtx({}, registry),
signal: controller.signal,
llmAdapters: {
"cancel-test": {
source: "cancel-test",
async invoke() {
calls += 1;
controller.abort(new Error("adapter cancelled during validation"));
return {
ok: true,
result: {
runId: "cancelled-attempt",
output: { data: { unexpected: true } },
},
};
},
},
},
},
} as any),
/adapter cancelled during validation/,
);
assert.equal(calls, 1, "cancellation after an invalid response must suppress retries");
});
test("llm.invoke aborts while waiting for its reusable run-state lock", async () => {
const registry = createDefaultRegistry();
const cmd = registry.get("llm.invoke");
assert.ok(cmd);
const cacheDir = await mkdtemp(path.join(tmpdir(), "lobster-llm-state-lock-abort-"));
const stateDir = path.join(cacheDir, "state");
const stateKey = "blocked-run-state";
const lockPath = `${keyToPath(stateDir, stateKey)}.lock`;
await fsp.mkdir(lockPath, { recursive: true });
await fsp.writeFile(path.join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
try {
const controller = new AbortController();
const pending = cmd.run({
input: streamOf([]),
args: {
_: [],
provider: "state-lock-abort-test",
prompt: "Decide",
"state-key": stateKey,
},
ctx: {
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
signal: controller.signal,
llmAdapters: {
"state-lock-abort-test": {
source: "state-lock-abort-test",
async invoke() {
throw new Error("adapter must not run while the state read is locked");
},
},
},
},
} as any);
const completion = pending.then(
() => ({ kind: "success" as const }),
(error) => ({ kind: "error" as const, error }),
);
await new Promise((resolve) => setImmediate(resolve));
controller.abort(new Error("LLM state read cancelled"));
const early = await Promise.race([
completion,
new Promise<{ kind: "timeout" }>((resolve) =>
setTimeout(() => resolve({ kind: "timeout" }), 75),
),
]);
if (early.kind === "timeout") await fsp.rm(lockPath, { recursive: true, force: true });
const settled = early.kind === "timeout" ? await completion : early;
assert.notEqual(
early.kind,
"timeout",
"state-key reads must observe cancellation while locked",
);
assert.equal(settled.kind, "error");
if (settled.kind === "error") {
assert.match(settled.error?.message ?? "", /LLM state read cancelled/);
}
} finally {
await fsp.rm(cacheDir, { recursive: true, force: true });
}
});
test("llm.invoke does not publish a reusable cache entry when cancellation races cache commit", async () => {
const registry = createDefaultRegistry();
const cmd = registry.get("llm.invoke");
assert.ok(cmd);
const cacheDir = await mkdtemp(path.join(tmpdir(), "lobster-cache-cancel-publication-"));
const stateDir = path.join(cacheDir, "state");
const controller = new AbortController();
const originalRename = fsp.rename;
let cacheCommitAborted = false;
let calls = 0;
const adapter = {
source: "cache-cancel-test",
async invoke() {
calls += 1;
return {
ok: true,
result: {
runId: `call-${calls}`,
output: { data: { call: calls } },
},
};
},
};
const args = {
_: [],
provider: "cache-cancel-test",
prompt: "Decide",
"state-key": "cancelled-cache-publication",
};
try {
Object.defineProperty(fsp, "rename", {
configurable: true,
writable: true,
async value(from: Parameters<typeof fsp.rename>[0], to: Parameters<typeof fsp.rename>[1]) {
const result = await originalRename(from, to);
if (
!cacheCommitAborted &&
String(to).startsWith(`${path.join(cacheDir, "llm.invoke")}${path.sep}`) &&
String(to).endsWith(".json")
) {
cacheCommitAborted = true;
controller.abort(new Error("cancelled during cache publication"));
}
return result;
},
});
await assert.rejects(
cmd.run({
input: streamOf([]),
args,
ctx: {
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
signal: controller.signal,
llmAdapters: { "cache-cancel-test": adapter },
},
} as any),
/cancelled during cache publication/,
);
} finally {
Object.defineProperty(fsp, "rename", {
configurable: true,
writable: true,
value: originalRename,
});
}
const retried = await cmd.run({
input: streamOf([]),
args,
ctx: {
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
llmAdapters: { "cache-cancel-test": adapter },
},
} as any);
const retriedItems = await collect(retried.output!);
assert.equal(
calls,
2,
"a cancelled invocation must not satisfy a later request from cache or run state",
);
assert.equal(retriedItems[0]?.source, "cache-cancel-test");
await rm(cacheDir, { recursive: true, force: true });
});
test("llm.invoke restores the previous cache entry when a refresh is cancelled after commit", async () => {
const registry = createDefaultRegistry();
const cmd = registry.get("llm.invoke");
assert.ok(cmd);
const cacheDir = await mkdtemp(path.join(tmpdir(), "lobster-cache-refresh-cancel-"));
const stateDir = path.join(cacheDir, "state");
let calls = 0;
const adapter = {
source: "cache-refresh-cancel-test",
async invoke() {
calls += 1;
return {
ok: true,
result: {
runId: `call-${calls}`,
output: { data: { call: calls } },
},
};
},
};
const args = {
_: [],
provider: "cache-refresh-cancel-test",
prompt: "Decide",
"state-key": "refresh-cache-publication",
};
try {
const first = await cmd.run({
input: streamOf([]),
args,
ctx: {
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
llmAdapters: { "cache-refresh-cancel-test": adapter },
},
} as any);
assert.deepEqual((await collect(first.output!))[0]?.output.data, { call: 1 });
const controller = new AbortController();
const originalRename = fsp.rename;
let cacheCommitAborted = false;
try {
Object.defineProperty(fsp, "rename", {
configurable: true,
writable: true,
async value(from: Parameters<typeof fsp.rename>[0], to: Parameters<typeof fsp.rename>[1]) {
const result = await originalRename(from, to);
if (
!cacheCommitAborted &&
String(to).startsWith(`${path.join(cacheDir, "llm.invoke")}${path.sep}`)
) {
cacheCommitAborted = true;
controller.abort(new Error("cancelled during cache refresh publication"));
}
return result;
},
});
await assert.rejects(
cmd.run({
input: streamOf([]),
args: { ...args, refresh: true },
ctx: {
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
signal: controller.signal,
llmAdapters: { "cache-refresh-cancel-test": adapter },
},
} as any),
/cancelled during cache refresh publication/,
);
} finally {
Object.defineProperty(fsp, "rename", {
configurable: true,
writable: true,
value: originalRename,
});
}
const recovered = await cmd.run({
input: streamOf([]),
args,
ctx: {
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
llmAdapters: { "cache-refresh-cancel-test": adapter },
},
} as any);
const recoveredItems = await collect(recovered.output!);
assert.equal(calls, 2, "the cancelled refresh must restore the existing run state");
assert.equal(recoveredItems[0]?.source, "run_state");
assert.deepEqual(recoveredItems[0]?.output.data, { call: 1 });
const recoveredCache = await cmd.run({
input: streamOf([]),
args: { _: [], provider: "cache-refresh-cancel-test", prompt: "Decide" },
ctx: {
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
llmAdapters: { "cache-refresh-cancel-test": adapter },
},
} as any);
const recoveredCacheItems = await collect(recoveredCache.output!);
assert.equal(recoveredCacheItems[0]?.source, "cache");
assert.deepEqual(recoveredCacheItems[0]?.output.data, { call: 1 });
} finally {
await rm(cacheDir, { recursive: true, force: true });
}
});
test("llm.invoke rolls back cache and run-state publications after a cache directory sync failure", async () => {
const registry = createDefaultRegistry();
const cmd = registry.get("llm.invoke");
assert.ok(cmd);
const cacheDir = await mkdtemp(path.join(tmpdir(), "lobster-cache-dir-sync-failure-"));
const stateDir = path.join(cacheDir, "state");
const cacheNamespaceDir = path.join(cacheDir, "llm.invoke");
const originalOpen = fsp.open;
const fault = Object.assign(new Error("cache directory sync failed"), { code: "EIO" });
let failNextCacheDirectorySync = true;
let calls = 0;
const adapter = {
source: "cache-dir-sync-test",
async invoke() {
calls += 1;
return { ok: true, result: { runId: `call-${calls}`, output: { data: { call: calls } } } };
},
};
const args = {
_: [],
provider: "cache-dir-sync-test",
prompt: "Decide",
"state-key": "cache-directory-sync-failure",
};
try {
await fsp.mkdir(cacheNamespaceDir, { recursive: true });
Object.defineProperty(fsp, "open", {
configurable: true,
writable: true,
async value(...openArgs: any[]) {
const handle = await (originalOpen as any)(...openArgs);
if (
failNextCacheDirectorySync &&
String(openArgs[0]) === cacheNamespaceDir &&
openArgs[1] === "r"
) {
failNextCacheDirectorySync = false;
return new Proxy(handle, {
get(target, property, receiver) {
if (property === "sync") return async () => Promise.reject(fault);
const value = Reflect.get(target, property, receiver);
return typeof value === "function" ? value.bind(target) : value;
},
});
}
return handle;
},
});
await assert.rejects(
cmd.run({
input: streamOf([]),
args,
ctx: {
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
llmAdapters: { "cache-dir-sync-test": adapter },
},
} as any),
/cache directory sync failed/,
);
} finally {
Object.defineProperty(fsp, "open", {
configurable: true,
writable: true,
value: originalOpen,
});
}
const retried = await cmd.run({
input: streamOf([]),
args,
ctx: {
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
llmAdapters: { "cache-dir-sync-test": adapter },
},
} as any);
const items = await collect(retried.output!);
assert.equal(calls, 2, "a failed publication must not be reused from cache or run state");
assert.equal(items[0]?.source, "cache-dir-sync-test");
await rm(cacheDir, { recursive: true, force: true });
});
test("llm.invoke reads a populated cache when lock creation is forbidden", async () => {
const registry = createDefaultRegistry();
const cmd = registry.get("llm.invoke");
assert.ok(cmd);
const cacheDir = await mkdtemp(path.join(tmpdir(), "lobster-readonly-cache-"));
const originalMkdir = fsp.mkdir;
let calls = 0;
const adapter = {
source: "readonly-cache-test",
async invoke() {
calls += 1;
return { ok: true, result: { runId: `call-${calls}`, output: { data: { call: calls } } } };
},
};
const args = { _: [], provider: "readonly-cache-test", prompt: "Decide" };
try {
const first = await cmd.run({
input: streamOf([]),
args,
ctx: {
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir }, registry),
llmAdapters: { "readonly-cache-test": adapter },
},
} as any);
await collect(first.output!);
Object.defineProperty(fsp, "mkdir", {
configurable: true,
writable: true,
async value(
filePath: Parameters<typeof fsp.mkdir>[0],
options?: Parameters<typeof fsp.mkdir>[1],
) {
if (String(filePath).endsWith(".lock")) {
throw Object.assign(new Error("read-only cache directory"), { code: "EACCES" });
}
return originalMkdir(filePath, options);
},
});
const cached = await cmd.run({
input: streamOf([]),
args,
ctx: {
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir }, registry),
llmAdapters: { "readonly-cache-test": adapter },
},
} as any);
const items = await collect(cached.output!);
assert.equal(calls, 1);
assert.equal(items[0]?.source, "cache");
} finally {
Object.defineProperty(fsp, "mkdir", {
configurable: true,
writable: true,
value: originalMkdir,
});
await rm(cacheDir, { recursive: true, force: true });
}
});
function baseCtx(envOverrides: Record<string, string>, registry?: any) {
return {
stdin: process.stdin,
+82
View File
@@ -1,5 +1,8 @@
import test from "node:test";
import assert from "node:assert/strict";
import { access, mkdtemp, readFile, rm } from "node:fs/promises";
import { readFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import {
createOpenClawAgentCommand,
@@ -12,6 +15,34 @@ function streamOf(items: unknown[]) {
})();
}
async function fileExists(filePath: string) {
try {
await access(filePath);
return true;
} catch {
return false;
}
}
async function waitForFile(filePath: string, timeoutMs = 2000) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (await fileExists(filePath)) return;
await new Promise((resolve) => setTimeout(resolve, 10));
}
throw new Error(`Timed out waiting for ${filePath}`);
}
function processIsRunning(pid: number) {
try {
const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
const state = stat.slice(stat.lastIndexOf(")") + 2, stat.lastIndexOf(")") + 3);
return state !== "Z";
} catch {
return false;
}
}
test("openclaw.agent delegates agent, session, and model selection to OpenClaw", async () => {
const calls: Array<Record<string, unknown>> = [];
const cmd = createOpenClawAgentCommand(async (params) => {
@@ -103,6 +134,18 @@ test("OpenClaw CLI runner parses structured JSON output", async () => {
});
});
test("OpenClaw CLI runner preserves the 10 MiB output limit", async () => {
await assert.rejects(
runOpenClawAgentCli({
executable: process.execPath,
argv: ["-e", "process.stdout.write('x'.repeat(10 * 1024 * 1024 + 1))"],
cwd: process.cwd(),
env: process.env,
}),
/openclaw\.agent output exceeded 10485760 bytes/,
);
});
test("OpenClaw CLI runner preserves workflow cancellation", async () => {
const fixturePath = path.join(process.cwd(), "test", "fixtures", "mock-openclaw-agent.mjs");
const controller = new AbortController();
@@ -117,3 +160,42 @@ test("OpenClaw CLI runner preserves workflow cancellation", async () => {
await assert.rejects(pending, (error: Error) => error.name === "AbortError");
});
test(
"OpenClaw CLI runner terminates descendant processes on cancellation",
{ skip: process.platform === "win32" },
async () => {
const dir = await mkdtemp(path.join(tmpdir(), "lobster-openclaw-agent-cancel-"));
try {
const fixturePath = path.join(process.cwd(), "test", "fixtures", "mock-openclaw-agent.mjs");
const descendantStarted = path.join(dir, "descendant-started");
const descendantCompleted = path.join(dir, "descendant-completed");
const controller = new AbortController();
const pending = runOpenClawAgentCli({
executable: process.execPath,
argv: [fixturePath, "--spawn-descendant", "--sleep"],
cwd: process.cwd(),
env: {
...process.env,
MOCK_OPENCLAW_AGENT_DESCENDANT_STARTED_FILE: descendantStarted,
MOCK_OPENCLAW_AGENT_DESCENDANT_COMPLETED_FILE: descendantCompleted,
},
signal: controller.signal,
});
await waitForFile(descendantStarted);
const descendantPid = Number(await readFile(descendantStarted, "utf8"));
controller.abort();
await assert.rejects(pending, (error: Error) => error.name === "AbortError");
await new Promise((resolve) => setTimeout(resolve, 700));
assert.equal(processIsRunning(descendantPid), false);
assert.equal(
await fileExists(descendantCompleted),
false,
"the OpenClaw child process must not outlive cancellation",
);
} finally {
await rm(dir, { recursive: true, force: true });
}
},
);
+63
View File
@@ -1,5 +1,8 @@
import test from "node:test";
import assert from "node:assert/strict";
import { spawn } from "node:child_process";
import { once } from "node:events";
import { EventEmitter } from "node:events";
import { PassThrough } from "node:stream";
import { readLineFromStream } from "../src/read_line.js";
@@ -14,6 +17,55 @@ test("readLineFromStream resolves on newline", async () => {
assert.equal(value, "yes");
});
test("readLineFromStream accepts sequential reads from one stream", async () => {
const input = new PassThrough();
const first = readLineFromStream(input);
input.write("yes\n");
assert.equal(await first, "yes");
const second = readLineFromStream(input, { timeoutMs: 50 });
input.write("no\n");
assert.equal(await second, "no");
input.end();
});
test("readLineFromStream preserves the next line from a combined input chunk", async () => {
const input = new PassThrough();
const first = readLineFromStream(input);
input.end("yes\nno\n");
assert.equal(await first, "yes");
assert.equal(await readLineFromStream(input), "no");
});
test("readLineFromStream returns buffered input after a child pipe reaches EOF", async () => {
const child = spawn(process.execPath, ["-e", "process.stdout.write('yes\\npartial')"], {
stdio: ["ignore", "pipe", "inherit"],
});
assert.ok(child.stdout);
const first = readLineFromStream(child.stdout);
assert.equal(await first, "yes");
await once(child, "close");
assert.equal(await readLineFromStream(child.stdout, { timeoutMs: 50 }), "partial");
});
test("readLineFromStream drains a buffer that remains readable after EOF", async () => {
let buffered: Buffer | null = Buffer.from("partial");
const input = Object.assign(new EventEmitter(), {
readableEnded: true,
closed: true,
read() {
const value = buffered;
buffered = null;
return value;
},
pause() {},
resume() {},
}) as unknown as NodeJS.ReadableStream;
assert.equal(await readLineFromStream(input), "partial");
});
test("readLineFromStream resolves on end without newline", async () => {
const input = new PassThrough();
const promise = readLineFromStream(input);
@@ -31,3 +83,14 @@ test("readLineFromStream times out when no input arrives", async () => {
/Timed out waiting for input/,
);
});
test("readLineFromStream rejects when its signal is aborted", async () => {
const input = new PassThrough();
const controller = new AbortController();
const promise = readLineFromStream(input, { signal: controller.signal });
controller.abort(new Error("input cancelled"));
await assert.rejects(() => promise, /input cancelled/);
assert.equal(input.readableFlowing, false);
input.end();
});
+1 -1
View File
@@ -8,7 +8,7 @@ import path from "node:path";
import { resumeToolRequest, runToolRequest } from "../src/core/tool_runtime.js";
import { runPipeline } from "../src/runtime.js";
import { decodeResumeToken } from "../src/resume.js";
import { readStateJson, writeStateJson } from "../src/state/store.js";
import { readStateJsonWithLock as readStateJson, writeStateJson } from "../src/state/store.js";
const responseSchema = {
type: "object",
+603 -1
View File
@@ -9,9 +9,13 @@ import { diffLast, diffAndStoreValue } from "../src/sdk/primitives/diff.js";
import { stateSet, readState, writeState } from "../src/sdk/primitives/state.js";
import {
createApprovalIndex,
consumeResumeState,
deleteResumeStateWithRollback,
diffAndStore,
keyToPath,
withFileLock,
writeStateJson,
readStateJson,
readStateJsonWithLock as readStateJson,
writeFileAtomic,
writeFileAtomicExclusive,
} from "../src/state/store.js";
@@ -84,6 +88,52 @@ test("state.get returns null for missing key", async () => {
assert.deepEqual(output.items, [null]);
});
test("ordinary state reads work when creating a coordination lock is forbidden", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-readonly-state-"));
const env = { ...process.env, LOBSTER_STATE_DIR: tmp };
const key = "demo";
const value = { readable: true };
const statePath = keyToPath(tmp, key);
const lockPath = `${statePath}.lock`;
const originalMkdir = fsp.mkdir;
await fsp.writeFile(statePath, JSON.stringify(value), "utf8");
Object.defineProperty(fsp, "mkdir", {
configurable: true,
writable: true,
async value(
filePath: Parameters<typeof fsp.mkdir>[0],
options?: Parameters<typeof fsp.mkdir>[1],
) {
if (String(filePath) === lockPath) {
throw Object.assign(new Error("read-only state directory"), { code: "EACCES" });
}
return originalMkdir(filePath, options);
},
});
try {
assert.deepEqual(await readState(key, { env }), value);
const registry = createDefaultRegistry();
const output = await runPipeline({
pipeline: [{ name: "state.get", args: { _: [key] }, raw: `state.get ${key}` }],
registry,
input: [],
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
});
assert.deepEqual(output.items, [value]);
} finally {
Object.defineProperty(fsp, "mkdir", {
configurable: true,
writable: true,
value: originalMkdir,
});
await fsp.rm(tmp, { recursive: true, force: true });
}
});
// --- Atomic-write behavior proofs (issues #108, #109) ---
//
// Plain fsp.writeFile truncates the target before writing, so a concurrent
@@ -191,6 +241,94 @@ test("writeFileAtomic propagates parent directory sync failures", async () => {
assert.deepEqual(leftovers, []);
});
test("consumeResumeState restores a published marker when parent sync fails", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-consume-dir-sync-"));
const env = { LOBSTER_STATE_DIR: tmp };
const expectedState = { haltType: "approval_request", pipeline: [] };
const fault = Object.assign(new Error("dir sync failed after resume claim"), { code: "EIO" });
const originalOpen = fsp.open;
let failNextDirectorySync = true;
await writeStateJson({ env, key: "resume", value: expectedState });
Object.defineProperty(fsp, "open", {
configurable: true,
writable: true,
async value(...args: any[]) {
const handle = await (originalOpen as any)(...args);
if (failNextDirectorySync && String(args[0]) === tmp && args[1] === "r") {
failNextDirectorySync = false;
return new Proxy(handle, {
get(target, property) {
if (property === "sync") return async () => Promise.reject(fault);
const value = Reflect.get(target, property);
return typeof value === "function" ? value.bind(target) : value;
},
});
}
return handle;
},
});
try {
await assert.rejects(
() => consumeResumeState({ env, key: "resume", expectedState }),
(err: NodeJS.ErrnoException) => err?.code === "EIO",
);
assert.deepEqual(await readStateJson({ env, key: "resume" }), expectedState);
} finally {
Object.defineProperty(fsp, "open", {
configurable: true,
writable: true,
value: originalOpen,
});
}
});
test("deleteResumeStateWithRollback restores a published marker when parent sync fails", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-delete-resume-dir-sync-"));
const env = { LOBSTER_STATE_DIR: tmp };
const expectedState = { haltType: "input_request", pipeline: [] };
const fault = Object.assign(new Error("dir sync failed after terminal resume claim"), {
code: "EIO",
});
const originalOpen = fsp.open;
let failNextDirectorySync = true;
await writeStateJson({ env, key: "resume", value: expectedState });
Object.defineProperty(fsp, "open", {
configurable: true,
writable: true,
async value(...args: any[]) {
const handle = await (originalOpen as any)(...args);
if (failNextDirectorySync && String(args[0]) === tmp && args[1] === "r") {
failNextDirectorySync = false;
return new Proxy(handle, {
get(target, property) {
if (property === "sync") return async () => Promise.reject(fault);
const value = Reflect.get(target, property);
return typeof value === "function" ? value.bind(target) : value;
},
});
}
return handle;
},
});
try {
await assert.rejects(
() => deleteResumeStateWithRollback({ env, key: "resume", expectedState }),
(err: NodeJS.ErrnoException) => err?.code === "EIO",
);
assert.deepEqual(await readStateJson({ env, key: "resume" }), expectedState);
} finally {
Object.defineProperty(fsp, "open", {
configurable: true,
writable: true,
value: originalOpen,
});
}
});
test("readStateJson surfaces malformed authoritative state", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-state-corrupt-"));
const env = { LOBSTER_STATE_DIR: tmp };
@@ -333,6 +471,470 @@ test("diffAndStore treats corrupt previous state as a miss and rewrites atomical
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { ok: true });
});
test("diffAndStore rolls back a state publication when its parent-directory sync fails", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-dir-sync-"));
const env = { LOBSTER_STATE_DIR: tmp };
await writeStateJson({ env, key: "snapshot", value: { version: "before" } });
const fault = Object.assign(new Error("dir sync failed after state publication"), {
code: "EIO",
});
await assert.rejects(
() =>
diffAndStore({
env,
key: "snapshot",
value: { version: "after" },
atomicWriteOptions: {
async syncParentDir() {
throw fault;
},
},
}),
(err: NodeJS.ErrnoException) => err?.code === "EIO",
);
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { version: "before" });
await fsp.rm(tmp, { recursive: true, force: true });
});
test("diffAndStore does not publish a snapshot after cancellation before atomic replace", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-cancel-publish-"));
const env = { LOBSTER_STATE_DIR: tmp };
await writeStateJson({ env, key: "snapshot", value: { version: "before" } });
const controller = new AbortController();
const signal = controller.signal;
const throwIfAborted = signal.throwIfAborted.bind(signal);
let signalChecks = 0;
Object.defineProperty(signal, "throwIfAborted", {
value() {
signalChecks += 1;
if (signalChecks === 2) controller.abort(new Error("abort before state publish"));
throwIfAborted();
},
});
await assert.rejects(
() => diffAndStore({ env, key: "snapshot", value: { version: "after" }, signal }),
/abort before state publish/,
);
assert.equal(signalChecks, 2);
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { version: "before" });
const leftovers = (await fsp.readdir(tmp)).filter((file) => file.includes(".tmp"));
assert.deepEqual(leftovers, []);
});
test("diffAndStore restores the previous snapshot when cancellation arrives during atomic rename", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-cancel-rename-"));
const env = { LOBSTER_STATE_DIR: tmp };
await writeStateJson({ env, key: "snapshot", value: { version: "before" } });
const controller = new AbortController();
await assert.rejects(
() =>
diffAndStore({
env,
key: "snapshot",
value: { version: "after" },
signal: controller.signal,
atomicWriteOptions: {
async renameFile(from, to) {
await fsp.rename(from, to);
controller.abort(new Error("abort during atomic rename"));
},
},
}),
/abort during atomic rename/,
);
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { version: "before" });
const leftovers = (await fsp.readdir(tmp)).filter((file) => file.includes(".tmp"));
assert.deepEqual(leftovers, []);
});
test("diffAndStore removes a newly published snapshot when cancellation arrives during atomic rename", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-cancel-new-rename-"));
const env = { LOBSTER_STATE_DIR: tmp };
const controller = new AbortController();
await assert.rejects(
() =>
diffAndStore({
env,
key: "snapshot",
value: { version: "after" },
signal: controller.signal,
atomicWriteOptions: {
async renameFile(from, to) {
await fsp.rename(from, to);
controller.abort(new Error("abort during initial atomic rename"));
},
},
}),
/abort during initial atomic rename/,
);
assert.equal(await readStateJson({ env, key: "snapshot" }), null);
const leftovers = (await fsp.readdir(tmp)).filter((file) => file.includes(".tmp"));
assert.deepEqual(leftovers, []);
});
test("diffAndStore restores an existing null snapshot when cancellation arrives during atomic rename", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-cancel-null-rename-"));
const env = { LOBSTER_STATE_DIR: tmp };
await writeStateJson({ env, key: "snapshot", value: null });
const snapshotPath = keyToPath(tmp, "snapshot");
const controller = new AbortController();
await assert.rejects(
() =>
diffAndStore({
env,
key: "snapshot",
value: { version: "after" },
signal: controller.signal,
atomicWriteOptions: {
async renameFile(from, to) {
await fsp.rename(from, to);
controller.abort(new Error("abort during null snapshot rename"));
},
},
}),
/abort during null snapshot rename/,
);
assert.equal(await readStateJson({ env, key: "snapshot" }), null);
assert.equal(await fsp.readFile(snapshotPath, "utf8"), "null\n");
});
test("diffAndStore serializes cancellation rollback before a concurrent snapshot update", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-cancel-concurrent-"));
const env = { LOBSTER_STATE_DIR: tmp };
await writeStateJson({ env, key: "snapshot", value: { version: "before" } });
const controller = new AbortController();
let publishCancelledSnapshot!: () => void;
const cancelledSnapshotPublished = new Promise<void>((resolve) => {
publishCancelledSnapshot = resolve;
});
let allowCancellation!: () => void;
const waitForCancellation = new Promise<void>((resolve) => {
allowCancellation = resolve;
});
const cancelled = diffAndStore({
env,
key: "snapshot",
value: { version: "cancelled-A" },
signal: controller.signal,
atomicWriteOptions: {
async renameFile(from, to) {
await fsp.rename(from, to);
publishCancelledSnapshot();
await waitForCancellation;
},
},
});
await cancelledSnapshotPublished;
let successfulSnapshotPublished = false;
const successful = writeState("snapshot", { version: "successful-B" }, { env }).then(() => {
successfulSnapshotPublished = true;
});
await new Promise((resolve) => setTimeout(resolve, 20));
assert.equal(successfulSnapshotPublished, false, "the next writer must wait for rollback");
controller.abort(new Error("abort during concurrent atomic rename"));
allowCancellation();
await assert.rejects(cancelled, /abort during concurrent atomic rename/);
await successful;
assert.equal(successfulSnapshotPublished, true);
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { version: "successful-B" });
const leftovers = (await fsp.readdir(tmp)).filter(
(file) => file.includes(".tmp") || file.endsWith(".lock"),
);
assert.deepEqual(leftovers, []);
});
test("state.get waits for a diff publication to commit or roll back", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-state-read-transaction-"));
const env = { ...process.env, LOBSTER_STATE_DIR: tmp };
await writeStateJson({ env, key: "snapshot", value: { version: "before" } });
let markPublished!: () => void;
const published = new Promise<void>((resolve) => {
markPublished = resolve;
});
let release!: () => void;
const releasePublication = new Promise<void>((resolve) => {
release = resolve;
});
const transaction = diffAndStore({
env,
key: "snapshot",
value: { version: "new" },
afterStore: async () => {
markPublished();
await releasePublication;
throw new Error("paired publication failed");
},
});
await published;
const getCmd = createDefaultRegistry().get("state.get");
const pendingRead = getCmd.run({
input: streamOf([]),
args: { _: ["snapshot"] },
ctx: { stdin: process.stdin, stdout: process.stdout, stderr: process.stderr, env },
});
const early = await Promise.race([
pendingRead.then(() => "settled" as const),
new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 25)),
]);
assert.equal(early, "pending", "state.get must not expose a snapshot pending rollback");
release();
await assert.rejects(transaction, /paired publication failed/);
const result = await pendingRead;
const items = [];
for await (const item of result.output) items.push(item);
assert.deepEqual(items, [{ version: "before" }]);
await fsp.rm(tmp, { recursive: true, force: true });
});
test("state.set stops waiting for a live state lock when its signal is aborted", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-state-lock-abort-"));
const env = { LOBSTER_STATE_DIR: tmp };
const key = "blocked";
const lockPath = `${keyToPath(tmp, key)}.lock`;
await fsp.mkdir(lockPath);
await fsp.writeFile(path.join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
const controller = new AbortController();
const stateSet = createDefaultRegistry().get("state.set");
const pending = stateSet.run({
input: streamOf([{ value: true }]),
args: { _: [key] },
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
signal: controller.signal,
},
});
const completion = pending.then(
() => ({ kind: "success" as const }),
(error) => ({ kind: "error" as const, error }),
);
await new Promise((resolve) => setImmediate(resolve));
controller.abort(new Error("state lock cancelled"));
const early = await Promise.race([
completion,
new Promise<{ kind: "timeout" }>((resolve) =>
setTimeout(() => resolve({ kind: "timeout" }), 75),
),
]);
if (early.kind === "timeout") await fsp.rm(lockPath, { recursive: true, force: true });
const settled = early.kind === "timeout" ? await completion : early;
assert.notEqual(early.kind, "timeout", "state.set must not remain blocked after cancellation");
assert.equal(settled.kind, "error");
if (settled.kind === "error") assert.match(settled.error?.message ?? "", /state lock cancelled/);
await fsp.rm(lockPath, { recursive: true, force: true });
});
test("diffAndStore does not reclaim a live fallback lock after a short heartbeat gap", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-state-lock-lease-"));
const env = { LOBSTER_STATE_DIR: tmp };
const key = "snapshot";
const lockPath = `${keyToPath(tmp, key)}.lock`;
const ownerPath = path.join(lockPath, "owner");
await fsp.mkdir(lockPath);
await fsp.writeFile(ownerPath, `${process.pid}::live-writer\n`, "utf8");
const briefGap = new Date(Date.now() - 2_000);
await fsp.utimes(lockPath, briefGap, briefGap);
await fsp.utimes(ownerPath, briefGap, briefGap);
const controller = new AbortController();
const abort = setTimeout(
() => controller.abort(new Error("live fallback lock remained held")),
100,
);
try {
await assert.rejects(
() => diffAndStore({ env, key, value: { version: "new" }, signal: controller.signal }),
/live fallback lock remained held/,
);
} finally {
clearTimeout(abort);
await fsp.rm(lockPath, { recursive: true, force: true });
}
assert.equal(await readStateJson({ env, key }), null);
});
test("diffAndStore reclaims an old lock with a malformed owner", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-malformed-lock-"));
const env = { LOBSTER_STATE_DIR: tmp };
const lockPath = `${keyToPath(tmp, "snapshot")}.lock`;
await fsp.mkdir(lockPath);
await fsp.writeFile(path.join(lockPath, "owner"), "\n", "utf8");
const staleAt = new Date(Date.now() - 10_000);
await fsp.utimes(lockPath, staleAt, staleAt);
await diffAndStore({ env, key: "snapshot", value: { version: "recovered" } });
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { version: "recovered" });
await assert.rejects(fsp.access(lockPath));
});
test(
"diffAndStore reclaims an old lock after its owner PID is reused",
{ skip: process.platform !== "linux" },
async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-reused-pid-lock-"));
const env = { LOBSTER_STATE_DIR: tmp };
const lockPath = `${keyToPath(tmp, "snapshot")}.lock`;
const ownerPath = path.join(lockPath, "owner");
await fsp.mkdir(lockPath);
await fsp.writeFile(ownerPath, `${process.pid}:0:stale-owner\n`, "utf8");
const staleAt = new Date(Date.now() - 10_000);
await fsp.utimes(lockPath, staleAt, staleAt);
await fsp.utimes(ownerPath, staleAt, staleAt);
const controller = new AbortController();
const timeout = setTimeout(
() => controller.abort(new Error("reused lock was not reclaimed")),
250,
);
try {
await diffAndStore({
env,
key: "snapshot",
value: { version: "recovered" },
signal: controller.signal,
});
} finally {
clearTimeout(timeout);
}
assert.equal(controller.signal.aborted, false);
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { version: "recovered" });
await assert.rejects(fsp.access(lockPath));
},
);
test("withFileLock does not reclaim a replacement lock after observing a stale one", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-state-lock-replacement-"));
const filePath = path.join(tmp, "snapshot.json");
const lockPath = `${filePath}.lock`;
await fsp.mkdir(lockPath);
await fsp.writeFile(path.join(lockPath, "owner"), `${process.pid}:0:stale-owner\n`, "utf8");
const staleAt = new Date(Date.now() - 10_000);
await fsp.utimes(lockPath, staleAt, staleAt);
await fsp.utimes(path.join(lockPath, "owner"), staleAt, staleAt);
const originalReadFile = fsp.readFile;
let replaced = false;
let replacementActive = false;
let overlap = false;
let replacement: Promise<void> | undefined;
let releaseReplacement!: () => void;
const replacementReleased = new Promise<void>((resolve) => {
releaseReplacement = resolve;
});
let replacementStarted!: () => void;
const replacementEntered = new Promise<void>((resolve) => {
replacementStarted = resolve;
});
Object.defineProperty(fsp, "readFile", {
configurable: true,
writable: true,
async value(
filePathArg: Parameters<typeof fsp.readFile>[0],
options?: Parameters<typeof fsp.readFile>[1],
) {
const result = await originalReadFile(filePathArg, options);
if (!replaced && String(filePathArg) === path.join(lockPath, "owner")) {
replaced = true;
await fsp.rm(lockPath, { recursive: true, force: true });
replacement = withFileLock({
filePath,
task: async () => {
replacementActive = true;
replacementStarted();
await replacementReleased;
replacementActive = false;
},
});
await replacementEntered;
}
return result;
},
});
try {
const original = withFileLock({
filePath,
task: async () => {
if (replacementActive) overlap = true;
},
});
await replacementEntered;
await new Promise((resolve) => setTimeout(resolve, 25));
assert.equal(overlap, false, "the stale reclaimer must not enter beside the replacement");
releaseReplacement();
if (!replacement) throw new Error("replacement lock did not start");
await Promise.all([original, replacement]);
} finally {
Object.defineProperty(fsp, "readFile", {
configurable: true,
writable: true,
value: originalReadFile,
});
await fsp.rm(tmp, { recursive: true, force: true });
}
assert.equal(replaced, true);
assert.equal(overlap, false);
});
test("withFileLock releases its key when best-effort cleanup cannot remove the detached lock", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-state-lock-release-failure-"));
const filePath = path.join(tmp, "snapshot.json");
const lockPath = `${filePath}.lock`;
const originalRm = fsp.rm;
let failReleaseOnce = true;
Object.defineProperty(fsp, "rm", {
configurable: true,
writable: true,
async value(filePathArg: Parameters<typeof fsp.rm>[0], options?: Parameters<typeof fsp.rm>[1]) {
if (failReleaseOnce && String(filePathArg).startsWith(lockPath)) {
failReleaseOnce = false;
throw Object.assign(new Error("simulated lock cleanup failure"), { code: "EIO" });
}
return originalRm(filePathArg, options);
},
});
let second: Promise<string> | undefined;
try {
await withFileLock({ filePath, task: async () => {} });
second = withFileLock({ filePath, task: async () => "reacquired" });
const settled = await Promise.race([
second,
new Promise<"timed out">((resolve) => setTimeout(() => resolve("timed out"), 500)),
]);
if (settled === "timed out") await originalRm(lockPath, { recursive: true, force: true });
assert.equal(settled, "reacquired", "a failed cleanup must not poison the live state lock");
await second;
} finally {
Object.defineProperty(fsp, "rm", {
configurable: true,
writable: true,
value: originalRm,
});
await originalRm(tmp, { recursive: true, force: true });
}
});
test("SDK diff primitives treat corrupt previous state as a miss (#112)", async () => {
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-sdk-diff-corrupt-"));
const ctx = { env: { LOBSTER_STATE_DIR: tmp } };
+683 -1
View File
@@ -8,7 +8,7 @@ import os from "node:os";
import { createDefaultRegistry } from "../src/commands/registry.js";
import { runWorkflowFile } from "../src/workflows/file.js";
import { decodeResumeToken } from "../src/resume.js";
import { readStateJson } from "../src/state/store.js";
import { keyToPath, readStateJsonWithLock as readStateJson } from "../src/state/store.js";
function streamOf(items: unknown[]) {
return (async function* () {
@@ -92,6 +92,61 @@ test("workflow file runs with approval and resume", async () => {
assert.deepEqual(resumeStateFiles, []);
});
test("sequential workflow approvals reclaim superseded resume markers", async () => {
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-sequential-approval-"));
try {
const stateDir = path.join(tmpDir, "state");
const filePath = path.join(tmpDir, "workflow.lobster");
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
await fsp.writeFile(
filePath,
JSON.stringify({
steps: [
{ id: "first", approval: "First?" },
{ id: "second", approval: "Second?" },
{ id: "finish", command: "node -e \"process.stdout.write('{}')\"" },
],
}),
"utf8",
);
const ctx = {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool" as const,
};
const first = await runWorkflowFile({ filePath, ctx });
assert.equal(first.status, "needs_approval");
const firstPayload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
assert.equal(firstPayload.kind, "workflow-file");
const second = await runWorkflowFile({
filePath,
ctx,
resume: firstPayload,
approved: true,
});
assert.equal(second.status, "needs_approval");
const secondPayload = decodeResumeToken(second.requiresApproval?.resumeToken ?? "");
assert.equal(secondPayload.kind, "workflow-file");
const completed = await runWorkflowFile({
filePath,
ctx,
resume: secondPayload,
approved: true,
});
assert.equal(completed.status, "ok");
const stateFiles = await fsp.readdir(stateDir);
assert.deepEqual(
stateFiles.filter((name) => name.startsWith("workflow_resume_")),
[],
);
} finally {
await fsp.rm(tmpDir, { recursive: true, force: true });
}
});
test("workflow resume cancellation cleans up resume state", async () => {
const workflow = {
steps: [
@@ -154,6 +209,562 @@ test("workflow resume cancellation cleans up resume state", async () => {
assert.deepEqual(resumeStateFiles, []);
});
test("direct workflow resume consumes its capability after cancellation starts an effect", async () => {
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-direct-cancel-"));
const stateDir = path.join(tmpDir, "state");
const effectPath = path.join(tmpDir, "effects.log");
const filePath = path.join(tmpDir, "workflow.lobster");
await fsp.writeFile(
filePath,
JSON.stringify({
steps: [
{
id: "approve",
command:
"node -e \"process.stdout.write(JSON.stringify({requiresApproval:{prompt:'Proceed?',items:[{id:1}]}}))\"",
approval: "required",
},
{
id: "effect",
run: `node -e "require('fs').appendFileSync(process.argv[1], 'run\\n'); setInterval(() => {}, 1000)" ${JSON.stringify(effectPath)}`,
condition: "$approve.approved",
},
],
}),
"utf8",
);
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
const first = await runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
});
assert.equal(first.status, "needs_approval");
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
assert.equal(payload.kind, "workflow-file");
assert.ok(payload.stateKey);
const controller = new AbortController();
const resumed = runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
signal: controller.signal,
},
resume: payload,
approved: true,
});
for (let attempt = 0; attempt < 100; attempt++) {
try {
await fsp.access(effectPath);
break;
} catch {
if (attempt === 99) throw new Error("workflow effect did not start");
await new Promise((resolve) => setTimeout(resolve, 10));
}
}
controller.abort(new Error("cancel after dispatch"));
await assert.rejects(() => resumed, /cancel after dispatch/);
assert.equal(await readStateJson({ env, key: payload.stateKey! }), null);
const stateFiles = await fsp.readdir(stateDir);
assert.equal(
stateFiles.some((file) => file.startsWith("approval_")),
false,
);
await assert.rejects(
() =>
runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
resume: payload,
approved: true,
}),
/Workflow resume state not found/,
);
assert.equal((await fsp.readFile(effectPath, "utf8")).trim().split(/\r?\n/).length, 1);
});
test("workflow template setup failure preserves an unstarted approval capability", async () => {
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-template-resume-"));
try {
const stateDir = path.join(tmpDir, "state");
const filePath = path.join(tmpDir, "workflow.lobster");
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
const broken = {
steps: [
{ id: "approve", approval: "Continue?" },
{
id: "effect",
run: "node -e \"process.stdout.write('should not run')\"",
stdin: "$missing.stdout",
},
],
};
await fsp.writeFile(filePath, JSON.stringify(broken), "utf8");
const first = await runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
});
assert.equal(first.status, "needs_approval");
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
assert.equal(payload.kind, "workflow-file");
assert.ok(payload.stateKey);
await assert.rejects(
() =>
runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
resume: payload,
approved: true,
}),
/Unknown step reference: missing\.stdout/,
);
assert.notEqual(await readStateJson({ env, key: payload.stateKey! }), null);
await fsp.writeFile(
filePath,
JSON.stringify({
steps: [
{ id: "approve", approval: "Continue?" },
{ id: "effect", run: "node -e \"process.stdout.write('ok')\"" },
],
}),
"utf8",
);
const retried = await runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
resume: payload,
approved: true,
});
assert.equal(retried.status, "ok");
assert.deepEqual(retried.output, ["ok"]);
} finally {
await fsp.rm(tmpDir, { recursive: true, force: true });
}
});
test("workflow resume consumes its capability after a step timeout starts an effect", async () => {
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-step-timeout-"));
const stateDir = path.join(tmpDir, "state");
const effectPath = path.join(tmpDir, "effects.log");
const filePath = path.join(tmpDir, "workflow.lobster");
await fsp.writeFile(
filePath,
JSON.stringify({
steps: [
{
id: "approve",
command:
"node -e \"process.stdout.write(JSON.stringify({requiresApproval:{prompt:'Proceed?',items:[{id:1}]}}))\"",
approval: "required",
},
{
id: "effect",
run: `node -e "require('fs').appendFileSync(process.argv[1], 'run\\n'); setInterval(() => {}, 1000)" ${JSON.stringify(effectPath)}`,
condition: "$approve.approved",
timeout_ms: 1500,
},
],
}),
"utf8",
);
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
const first = await runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
});
assert.equal(first.status, "needs_approval");
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
assert.equal(payload.kind, "workflow-file");
assert.ok(payload.stateKey);
await assert.rejects(
() =>
runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
resume: payload,
approved: true,
}),
/timed out|timeout|abort|cancel/i,
);
assert.equal(await readStateJson({ env, key: payload.stateKey! }), null);
assert.equal((await fsp.readFile(effectPath, "utf8")).trim().split(/\r?\n/).length, 1);
await assert.rejects(
() =>
runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
resume: payload,
approved: true,
}),
/Workflow resume state not found/,
);
assert.equal((await fsp.readFile(effectPath, "utf8")).trim().split(/\r?\n/).length, 1);
});
test("workflow resume applies on_error after a timed-out effect", async () => {
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-timeout-on-error-"));
const stateDir = path.join(tmpDir, "state");
const effectPath = path.join(tmpDir, "effects.log");
const filePath = path.join(tmpDir, "workflow.lobster");
await fsp.writeFile(
filePath,
JSON.stringify({
steps: [
{
id: "approve",
command:
"node -e \"process.stdout.write(JSON.stringify({requiresApproval:{prompt:'Proceed?',items:[{id:1}]}}))\"",
approval: "required",
},
{
id: "effect",
run: `node -e "require('fs').appendFileSync(process.argv[1], 'run\\n'); setInterval(() => {}, 1000)" ${JSON.stringify(effectPath)}`,
condition: "$approve.approved",
timeout_ms: 1500,
on_error: "continue",
},
{ id: "after", run: "echo continued" },
],
}),
"utf8",
);
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
const first = await runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
});
assert.equal(first.status, "needs_approval");
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
assert.equal(payload.kind, "workflow-file");
assert.ok(payload.stateKey);
const resumed = await runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
resume: payload,
approved: true,
});
assert.equal(resumed.status, "ok");
assert.deepEqual(resumed.output, ["continued\n"]);
assert.equal(await readStateJson({ env, key: payload.stateKey! }), null);
assert.equal((await fsp.readFile(effectPath, "utf8")).trim().split(/\r?\n/).length, 1);
});
test("workflow resume retries a timed-out effect before consuming its capability", async () => {
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-timeout-retry-"));
const stateDir = path.join(tmpDir, "state");
const attemptsPath = path.join(tmpDir, "attempts");
const filePath = path.join(tmpDir, "workflow.lobster");
await fsp.writeFile(
filePath,
JSON.stringify({
steps: [
{
id: "approve",
command:
"node -e \"process.stdout.write(JSON.stringify({requiresApproval:{prompt:'Proceed?',items:[{id:1}]}}))\"",
approval: "required",
},
{
id: "effect",
run: `node -e "const fs=require('fs'); const file=process.argv[1]; const attempt=fs.existsSync(file) ? Number(fs.readFileSync(file, 'utf8')) : 0; fs.writeFileSync(file, String(attempt + 1)); if (attempt === 0) setInterval(() => {}, 1000); else process.stdout.write('retried');" ${JSON.stringify(attemptsPath)}`,
condition: "$approve.approved",
timeout_ms: 1500,
retry: { max: 2, delay_ms: 10 },
},
],
}),
"utf8",
);
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
const first = await runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
});
assert.equal(first.status, "needs_approval");
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
assert.equal(payload.kind, "workflow-file");
assert.ok(payload.stateKey);
const resumed = await runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
resume: payload,
approved: true,
});
assert.equal(resumed.status, "ok");
assert.deepEqual(resumed.output, ["retried"]);
assert.equal(await fsp.readFile(attemptsPath, "utf8"), "2");
assert.equal(await readStateJson({ env, key: payload.stateKey! }), null);
});
test("workflow resume retries a claim blocked by a state lock before dispatch", async () => {
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-claim-retry-"));
try {
const stateDir = path.join(tmpDir, "state");
const filePath = path.join(tmpDir, "workflow.lobster");
await fsp.writeFile(
filePath,
JSON.stringify({
steps: [
{ id: "approve", approval: "Proceed?" },
{
id: "effect",
run: "printf ran",
condition: "$approve.approved",
timeout_ms: 1000,
retry: { max: 2, delay_ms: 250 },
},
],
}),
"utf8",
);
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
const first = await runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
});
assert.equal(first.status, "needs_approval");
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
assert.equal(payload.kind, "workflow-file");
assert.ok(payload.stateKey);
const lockPath = `${keyToPath(stateDir, payload.stateKey!)}.lock`;
const originalRename = fsp.rename;
const originalRm = fsp.rm;
let releasedInitialReadLocks = 0;
let lockReplaced!: () => void;
const replacedInitialReadLocks = new Promise<void>((resolve) => {
lockReplaced = resolve;
});
Object.defineProperty(fsp, "rename", {
configurable: true,
writable: true,
async value(
oldPath: Parameters<typeof fsp.rename>[0],
newPath: Parameters<typeof fsp.rename>[1],
) {
const result = await originalRename(oldPath, newPath);
if (String(oldPath) === lockPath && ++releasedInitialReadLocks === 2) {
await fsp.mkdir(lockPath);
await fsp.writeFile(
path.join(lockPath, "owner"),
`${process.pid}::live-writer\n`,
"utf8",
);
lockReplaced();
}
return result;
},
});
try {
const resumedRun = runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
resume: payload,
approved: true,
});
await replacedInitialReadLocks;
const release = setTimeout(
() => void originalRm(lockPath, { recursive: true, force: true }),
1200,
);
const resumed = await resumedRun;
clearTimeout(release);
assert.equal(resumed.status, "ok");
assert.deepEqual(resumed.output, ["ran"]);
assert.equal(await readStateJson({ env, key: payload.stateKey! }), null);
} finally {
Object.defineProperty(fsp, "rename", {
configurable: true,
writable: true,
value: originalRename,
});
await fsp.rm(lockPath, { recursive: true, force: true });
}
} finally {
await fsp.rm(tmpDir, { recursive: true, force: true });
}
});
test("workflow resume consumes its capability after a parallel timeout starts an effect", async () => {
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-parallel-timeout-"));
const stateDir = path.join(tmpDir, "state");
const effectPath = path.join(tmpDir, "effects.log");
const filePath = path.join(tmpDir, "workflow.lobster");
await fsp.writeFile(
filePath,
JSON.stringify({
steps: [
{
id: "approve",
command:
"node -e \"process.stdout.write(JSON.stringify({requiresApproval:{prompt:'Proceed?',items:[{id:1}]}}))\"",
approval: "required",
},
{
id: "effect",
condition: "$approve.approved",
parallel: {
timeout_ms: 1500,
branches: [
{
id: "side-effect",
run: `node -e "require('fs').appendFileSync(process.argv[1], 'run\\n'); setInterval(() => {}, 1000)" ${JSON.stringify(effectPath)}`,
},
],
},
},
],
}),
"utf8",
);
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
const first = await runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
});
assert.equal(first.status, "needs_approval");
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
assert.equal(payload.kind, "workflow-file");
assert.ok(payload.stateKey);
await assert.rejects(
() =>
runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
resume: payload,
approved: true,
}),
/timed out|timeout|abort|cancel/i,
);
assert.equal(await readStateJson({ env, key: payload.stateKey! }), null);
assert.equal((await fsp.readFile(effectPath, "utf8")).trim().split(/\r?\n/).length, 1);
await assert.rejects(
() =>
runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env,
mode: "tool",
},
resume: payload,
approved: true,
}),
/Workflow resume state not found/,
);
assert.equal((await fsp.readFile(effectPath, "utf8")).trim().split(/\r?\n/).length, 1);
});
test("workflow resume accepts workflow-resume_ state key aliases and cleans up state", async () => {
const workflow = {
steps: [
@@ -383,6 +994,7 @@ test("workflow pipeline requestInput resume invariant bypasses on_error", async
let sideEffects = 0;
const choose = {
name: "choose",
meta: { resumeSafeBeforeInput: true },
async run({ ctx }: any) {
calls += 1;
if (calls > 1) return { output: streamOf([{ skipped: true }]) };
@@ -1246,6 +1858,76 @@ test("workflow conditions reject standalone bare identifiers", async () => {
);
});
test("parallel wait:any fails promptly when a registered loser ignores cancellation", async () => {
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-parallel-unsettled-"));
const filePath = path.join(tmpDir, "workflow.lobster");
const registry = {
get(name: string) {
if (name === "winner") {
return {
name,
help: () => "winner",
async run() {
return { output: [{ winner: true }] };
},
};
}
if (name === "never") {
return {
name,
help: () => "never",
async run() {
await new Promise<void>(() => {});
},
};
}
return undefined;
},
};
try {
await fsp.writeFile(
filePath,
JSON.stringify({
steps: [
{
id: "parallel",
parallel: {
wait: "any",
branches: [
{ id: "winner", pipeline: "winner" },
{ id: "never", pipeline: "never" },
],
},
},
],
}),
"utf8",
);
await assert.rejects(
Promise.race([
runWorkflowFile({
filePath,
ctx: {
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
env: { ...process.env },
mode: "tool",
registry,
},
}),
new Promise<never>((_resolve, reject) =>
setTimeout(() => reject(new Error("workflow did not finish")), 1_000),
),
]),
/Parallel branches did not settle after cancellation/,
);
} finally {
await fsp.rm(tmpDir, { recursive: true, force: true });
}
});
test("workflow conditions reject unknown step refs even under negation", async () => {
const workflow = {
steps: [