mirror of
https://github.com/openclaw/lobster.git
synced 2026-08-14 08:52:48 +00:00
fix(runtime): cancelled workflows no longer continue external commands (#119)
* fix(runtime): stop process-backed work on cancellation
* fix(runtime): invalidate cancelled resume state
* Revert "fix(runtime): invalidate cancelled resume state"
This reverts commit 0f7d291f98.
* fix(runtime): narrow cancellation to safe child processes
* fix(runtime): stop after completed search cancellation
* fix(runtime): halt direct pipelines after cancellation
Preserve completed in-flight stage results while preventing later direct pipeline stages from starting after parent cancellation.
* fix(runtime): consume aborted approval resumes
* fix(runtime): cancelled workflows no longer continue external commands
* fix(workflow): propagate custom parent cancellation
* fix(runtime): preserve pre-aborted resume state
* fix(runtime): stop lazy handoff after cancellation
* fix(workflow): close remaining cancellation boundaries
* fix(runtime): preserve workflow resumes during setup cancellation
* fix(runtime): close final cancellation persistence gaps
* fix(runtime): stop lazy handoff after cancellation
* fix(runtime): interrupt blocked lazy handoff reads
* fix(runtime): terminate cancellation process trees
* fix(runtime): await process tree termination
* fix(runtime): terminate workflow process trees
* fix(runtime): bridge CLI cancellation
* fix(cli): preserve cancellation lifecycle
* fix(cli): abort stalled signal-aware commands
* fix(cli): release aborted interactive prompts
* fix(cli): preserve sequential prompt input
* fix(cli): preserve buffered prompt input
* fix(cli): handle prompt EOF after buffered input
* fix(runtime): preserve UTF-8 subprocess output
* fix(workflow): preserve retryable resume before execution
* fix(workflow): roll back cancelled resume replacement
* fix(state): roll back cancelled monitor snapshot
* fix(resume): preserve cancelled gate capabilities
* fix(resume): close cancellation rollback windows
* fix(resume): harden cancellation state cleanup
* fix(runtime): close resumed cancellation gaps
* fix(runtime): preserve cancellation cleanup
* fix(runtime): close cancellation lifecycle gaps
* fix(runtime): harden resumed cancellation boundaries
* fix(workflow): consume timed-out resume capabilities
* fix(workflow): preserve resume policy boundaries
* fix(runtime): preserve cancellation cleanup liveness
* fix(runtime): harden cancellation cleanup
* fix(runtime): stop lazy output after cancellation
* fix(runtime): settle cancellation cleanup
* fix(runtime): preserve safe input resumes
* fix(runtime): prevent consumed resume replays
* fix(runtime): serialize approval resume consumption
* fix(runtime): prevent concurrent safe gate forks
* fix(runtime): close cancellation review gaps
* fix(llm): restore cache after cancelled refresh
* fix(runtime): close remaining cancellation windows
* fix(runtime): preserve cancellation recovery invariants
* fix(runtime): prevent stale resume recovery
* fix(runtime): preserve resume claim recovery
* fix(runtime): retry pre-dispatch claims safely
* fix(state): synchronize rollback-safe reads
* fix(resume): discard cancelled pipeline successors
* fix(runtime): harden cancellation and state locking
* fix(runtime): recover durable cancellation failures
* fix(runtime): preserve legacy workflow cancellation
---------
Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
co-authored by
Peter Steinberger
parent
0ac962e90b
commit
c440ca57d1
@@ -0,0 +1,236 @@
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
|
||||
const ABORT_FORCE_KILL_AFTER_MS = 250;
|
||||
const forceTerminationCallbacks = new WeakMap<AbortSignal, Set<() => void>>();
|
||||
|
||||
type ProcessResult = {
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
code: number | null;
|
||||
};
|
||||
|
||||
type RunAbortableProcessOptions = {
|
||||
command: string;
|
||||
argv: string[];
|
||||
env: NodeJS.ProcessEnv;
|
||||
cwd?: string;
|
||||
stdin?: string | null;
|
||||
signal?: AbortSignal;
|
||||
forceTerminationSignal?: AbortSignal;
|
||||
killSignal?: NodeJS.Signals | (() => NodeJS.Signals | undefined);
|
||||
maxOutputBytes?: number;
|
||||
outputLimitMessage?: string;
|
||||
notFoundMessage: string;
|
||||
};
|
||||
|
||||
export function forceTerminateAbortableProcesses(signal: AbortSignal) {
|
||||
for (const terminate of forceTerminationCallbacks.get(signal) ?? []) terminate();
|
||||
}
|
||||
|
||||
function abortError(signal: AbortSignal) {
|
||||
if (signal.reason instanceof Error) return signal.reason;
|
||||
const error = new Error("This operation was aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function terminateProcessTree(child: ChildProcess, signal: NodeJS.Signals): Promise<void> {
|
||||
if (!child.pid) return Promise.resolve();
|
||||
|
||||
if (process.platform === "win32") {
|
||||
return new Promise((resolve) => {
|
||||
const taskkill = spawn("taskkill", ["/pid", String(child.pid), "/T", "/F"], {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
taskkill.once("error", () => {
|
||||
child.kill(signal);
|
||||
resolve();
|
||||
});
|
||||
taskkill.once("close", resolve);
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
process.kill(-child.pid, signal);
|
||||
} catch {
|
||||
child.kill(signal);
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
export function runAbortableProcess({
|
||||
command,
|
||||
argv,
|
||||
env,
|
||||
cwd,
|
||||
stdin,
|
||||
signal,
|
||||
forceTerminationSignal,
|
||||
killSignal,
|
||||
maxOutputBytes,
|
||||
outputLimitMessage,
|
||||
notFoundMessage,
|
||||
}: RunAbortableProcessOptions): Promise<ProcessResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (
|
||||
maxOutputBytes !== undefined &&
|
||||
(!Number.isSafeInteger(maxOutputBytes) || maxOutputBytes < 0)
|
||||
) {
|
||||
reject(new Error("maxOutputBytes must be a non-negative safe integer"));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
const child = spawn(command, argv, {
|
||||
env,
|
||||
cwd,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
// Create a dedicated POSIX process group only when this runner owns a
|
||||
// cancellation signal for it. Direct APIs without one must retain the
|
||||
// caller's terminal process group so Ctrl-C still reaches their child.
|
||||
detached: process.platform !== "win32" && signal !== undefined,
|
||||
});
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let stdoutBytes = 0;
|
||||
let stderrBytes = 0;
|
||||
let terminationError: Error | undefined;
|
||||
let forceKillTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let processClosed = false;
|
||||
let forceKillRequested = false;
|
||||
let forceKillIssued = false;
|
||||
let settled = false;
|
||||
const forceTerminationRegistrations: Set<() => void>[] = [];
|
||||
let forceTerminate: (() => void) | undefined;
|
||||
const cleanup = () => {
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
if (forceKillTimer) clearTimeout(forceKillTimer);
|
||||
if (forceTerminate) {
|
||||
for (const listeners of forceTerminationRegistrations) listeners.delete(forceTerminate);
|
||||
}
|
||||
};
|
||||
const failTerminationWhenTreeIsStopped = () => {
|
||||
if (!terminationError || !processClosed || !forceKillIssued || settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(terminationError);
|
||||
};
|
||||
const fail = (error: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const forceKill = () => {
|
||||
if (forceKillRequested) return;
|
||||
forceKillRequested = true;
|
||||
void terminateProcessTree(child, "SIGKILL").finally(() => {
|
||||
forceKillIssued = true;
|
||||
failTerminationWhenTreeIsStopped();
|
||||
});
|
||||
};
|
||||
const startTermination = (error: Error) => {
|
||||
if (settled || terminationError) return;
|
||||
terminationError = error;
|
||||
const initialKillSignal =
|
||||
(typeof killSignal === "function" ? killSignal() : killSignal) ?? "SIGTERM";
|
||||
if (initialKillSignal === "SIGKILL") {
|
||||
forceKill();
|
||||
return;
|
||||
}
|
||||
void terminateProcessTree(child, initialKillSignal);
|
||||
forceKillTimer = setTimeout(() => {
|
||||
forceKillTimer = undefined;
|
||||
forceKill();
|
||||
}, ABORT_FORCE_KILL_AFTER_MS);
|
||||
};
|
||||
forceTerminate = () => {
|
||||
if (settled) return;
|
||||
if (!terminationError) {
|
||||
terminationError = signal ? abortError(signal) : new Error("Process termination requested");
|
||||
}
|
||||
if (forceKillTimer) {
|
||||
clearTimeout(forceKillTimer);
|
||||
forceKillTimer = undefined;
|
||||
}
|
||||
forceKill();
|
||||
};
|
||||
const onAbort = () => {
|
||||
if (!signal) return;
|
||||
startTermination(abortError(signal));
|
||||
};
|
||||
const appendOutput = (stream: "stdout" | "stderr", data: string) => {
|
||||
const bytes = Buffer.byteLength(data);
|
||||
const total = stream === "stdout" ? stdoutBytes + bytes : stderrBytes + bytes;
|
||||
if (maxOutputBytes !== undefined && total > maxOutputBytes) {
|
||||
startTermination(
|
||||
new Error(outputLimitMessage ?? `Process output exceeded ${maxOutputBytes} bytes`),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (stream === "stdout") {
|
||||
stdoutBytes = total;
|
||||
stdout += data;
|
||||
} else {
|
||||
stderrBytes = total;
|
||||
stderr += data;
|
||||
}
|
||||
};
|
||||
|
||||
child.stdout?.setEncoding("utf8");
|
||||
child.stderr?.setEncoding("utf8");
|
||||
child.stdout?.on("data", (data: string) => appendOutput("stdout", data));
|
||||
child.stderr?.on("data", (data: string) => appendOutput("stderr", data));
|
||||
if (typeof stdin === "string") child.stdin?.write(stdin);
|
||||
child.stdin?.end();
|
||||
|
||||
child.on("error", (error: NodeJS.ErrnoException) => {
|
||||
if (terminationError) {
|
||||
processClosed = true;
|
||||
failTerminationWhenTreeIsStopped();
|
||||
return;
|
||||
}
|
||||
if (error.code === "ENOENT") {
|
||||
fail(new Error(notFoundMessage));
|
||||
return;
|
||||
}
|
||||
fail(error);
|
||||
});
|
||||
child.on("close", (code) => {
|
||||
if (settled) return;
|
||||
processClosed = true;
|
||||
if (terminationError) {
|
||||
failTerminationWhenTreeIsStopped();
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
cleanup();
|
||||
resolve({ stdout, stderr, code });
|
||||
});
|
||||
|
||||
for (const registrationSignal of new Set(
|
||||
[signal, forceTerminationSignal].filter(
|
||||
(candidate): candidate is AbortSignal => candidate !== undefined,
|
||||
),
|
||||
)) {
|
||||
let listeners = forceTerminationCallbacks.get(registrationSignal);
|
||||
if (!listeners) {
|
||||
listeners = new Set();
|
||||
forceTerminationCallbacks.set(registrationSignal, listeners);
|
||||
}
|
||||
listeners.add(forceTerminate);
|
||||
forceTerminationRegistrations.push(listeners);
|
||||
}
|
||||
|
||||
if (signal) {
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
if (signal.aborted) onAbort();
|
||||
}
|
||||
});
|
||||
}
|
||||
+134
-261
@@ -1,24 +1,31 @@
|
||||
import { parsePipeline } from "./parser.js";
|
||||
import { createDefaultRegistry } from "./commands/registry.js";
|
||||
import { runPipeline } from "./runtime.js";
|
||||
import { decodeResumeToken, parseResumeArgs, resolveApprovalId } from "./resume.js";
|
||||
import { cleanupApprovalIndexByStateKey, deleteApprovalId } from "./state/store.js";
|
||||
import {
|
||||
WorkflowResumeArgumentError,
|
||||
loadWorkflowFile,
|
||||
resolveWorkflowArgs,
|
||||
runWorkflowFile,
|
||||
} from "./workflows/file.js";
|
||||
import { parseResumeArgs } from "./resume.js";
|
||||
import { resumeToolRequest } from "./core/tool_runtime.js";
|
||||
import { loadWorkflowFile, resolveWorkflowArgs, runWorkflowFile } from "./workflows/file.js";
|
||||
import { renderWorkflowGraph } from "./workflows/graph.js";
|
||||
import type { WorkflowGraphFormat } from "./workflows/graph.js";
|
||||
import { deleteStateJson } from "./state/store.js";
|
||||
import {
|
||||
finalizePipelineToolRun,
|
||||
loadPipelineResumeState,
|
||||
validatePipelineInputResponse,
|
||||
} from "./pipeline_resume_state.js";
|
||||
import { finalizePipelineToolRun } from "./pipeline_resume_state.js";
|
||||
import { forceTerminateAbortableProcesses } from "./abortable_process.js";
|
||||
|
||||
export async function runCli(argv) {
|
||||
const cancellation = createCliCancellation();
|
||||
try {
|
||||
await runCliWithSignal(argv, cancellation.signal, cancellation.signal);
|
||||
} finally {
|
||||
if (cancellation.exitCode !== undefined) {
|
||||
process.exitCode = cancellation.exitCode;
|
||||
}
|
||||
cancellation.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
async function runCliWithSignal(
|
||||
argv,
|
||||
signal: AbortSignal,
|
||||
forceTerminationSignal: AbortSignal = signal,
|
||||
) {
|
||||
const registry = createDefaultRegistry();
|
||||
|
||||
if (argv.length === 0 || argv.includes("-h") || argv.includes("--help")) {
|
||||
@@ -48,7 +55,7 @@ export async function runCli(argv) {
|
||||
}
|
||||
|
||||
if (argv[0] === "doctor") {
|
||||
await handleDoctor({ argv: argv.slice(1), registry });
|
||||
await handleDoctor({ argv: argv.slice(1), registry, signal, forceTerminationSignal });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -58,17 +65,57 @@ export async function runCli(argv) {
|
||||
}
|
||||
|
||||
if (argv[0] === "run") {
|
||||
await handleRun({ argv: argv.slice(1), registry });
|
||||
await handleRun({ argv: argv.slice(1), registry, signal, forceTerminationSignal });
|
||||
return;
|
||||
}
|
||||
|
||||
if (argv[0] === "resume") {
|
||||
await handleResume({ argv: argv.slice(1), registry });
|
||||
await handleResume({ argv: argv.slice(1), registry, signal, forceTerminationSignal });
|
||||
return;
|
||||
}
|
||||
|
||||
// Default: treat argv as a pipeline string.
|
||||
await handleRun({ argv, registry });
|
||||
await handleRun({ argv, registry, signal, forceTerminationSignal });
|
||||
}
|
||||
|
||||
function createCliCancellation() {
|
||||
const controller = new AbortController();
|
||||
let received: NodeJS.Signals | undefined;
|
||||
const terminatingSignals: NodeJS.Signals[] = ["SIGINT", "SIGTERM", "SIGHUP", "SIGQUIT"];
|
||||
const abort = (receivedSignal: NodeJS.Signals) => {
|
||||
if (received) {
|
||||
forceTerminateAbortableProcesses(controller.signal);
|
||||
return;
|
||||
}
|
||||
received = receivedSignal;
|
||||
controller.abort(new Error(`Received ${receivedSignal}`));
|
||||
};
|
||||
const listeners = new Map<NodeJS.Signals, () => void>();
|
||||
for (const terminatingSignal of terminatingSignals) {
|
||||
const listener = () => abort(terminatingSignal);
|
||||
listeners.set(terminatingSignal, listener);
|
||||
process.on(terminatingSignal, listener);
|
||||
}
|
||||
|
||||
return {
|
||||
signal: controller.signal,
|
||||
get exitCode() {
|
||||
if (!received) return undefined;
|
||||
return (
|
||||
{
|
||||
SIGINT: 130,
|
||||
SIGHUP: 129,
|
||||
SIGQUIT: 131,
|
||||
SIGTERM: 143,
|
||||
} as const
|
||||
)[received];
|
||||
},
|
||||
dispose() {
|
||||
for (const [terminatingSignal, listener] of listeners) {
|
||||
process.removeListener(terminatingSignal, listener);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function handleGraph({ argv }) {
|
||||
@@ -132,7 +179,17 @@ function isWorkflowGraphFormat(value: string): value is WorkflowGraphFormat {
|
||||
return value === "mermaid" || value === "dot" || value === "ascii";
|
||||
}
|
||||
|
||||
async function handleRun({ argv, registry }) {
|
||||
async function handleRun({
|
||||
argv,
|
||||
registry,
|
||||
signal,
|
||||
forceTerminationSignal,
|
||||
}: {
|
||||
argv;
|
||||
registry;
|
||||
signal: AbortSignal;
|
||||
forceTerminationSignal: AbortSignal;
|
||||
}) {
|
||||
const parsed = parseRunArgs(argv);
|
||||
const { mode, argsJson } = parsed;
|
||||
const normalizedMode = normalizeMode(mode);
|
||||
@@ -173,6 +230,8 @@ async function handleRun({ argv, registry }) {
|
||||
mode: normalizedMode,
|
||||
registry,
|
||||
dryRun,
|
||||
signal,
|
||||
forceTerminationSignal,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -276,6 +335,9 @@ async function handleRun({ argv, registry }) {
|
||||
env: process.env,
|
||||
mode: normalizedMode,
|
||||
dryRun,
|
||||
signal,
|
||||
forceTerminationSignal,
|
||||
haltAfterStageOnAbort: true,
|
||||
});
|
||||
|
||||
if (normalizedMode === "tool") {
|
||||
@@ -283,6 +345,7 @@ async function handleRun({ argv, registry }) {
|
||||
env: process.env,
|
||||
pipeline,
|
||||
output,
|
||||
signal,
|
||||
});
|
||||
writeToolEnvelope({
|
||||
ok: true,
|
||||
@@ -504,28 +567,20 @@ async function resolveWorkflowFile(candidate) {
|
||||
return resolved;
|
||||
}
|
||||
|
||||
async function handleResume({ argv, registry }) {
|
||||
const mode = "tool";
|
||||
let approved: boolean | undefined;
|
||||
let response: unknown = undefined;
|
||||
let cancel = false;
|
||||
let payload: any;
|
||||
let resolvedApprovalId: string | null = null;
|
||||
async function handleResume({
|
||||
argv,
|
||||
registry,
|
||||
signal,
|
||||
forceTerminationSignal,
|
||||
}: {
|
||||
argv;
|
||||
registry;
|
||||
signal: AbortSignal;
|
||||
forceTerminationSignal: AbortSignal;
|
||||
}) {
|
||||
let parsed;
|
||||
try {
|
||||
const parsed = parseResumeArgs(argv);
|
||||
approved = parsed.approved;
|
||||
response = parsed.response;
|
||||
cancel = parsed.cancel === true;
|
||||
resolvedApprovalId = parsed.approvalId;
|
||||
|
||||
// Resolve short approval ID to token if provided
|
||||
let token: string;
|
||||
if (parsed.approvalId) {
|
||||
token = await resolveApprovalId(parsed.approvalId, process.env);
|
||||
} else {
|
||||
token = parsed.token!;
|
||||
}
|
||||
payload = decodeResumeToken(token);
|
||||
parsed = parseResumeArgs(argv);
|
||||
} catch (err) {
|
||||
writeToolEnvelope({
|
||||
ok: false,
|
||||
@@ -535,230 +590,35 @@ async function handleResume({ argv, registry }) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Helper: clean up approval ID index after successful use
|
||||
const cleanupIndex = async () => {
|
||||
if (resolvedApprovalId) {
|
||||
await deleteApprovalId({ env: process.env, approvalId: resolvedApprovalId });
|
||||
} else if (payload.stateKey) {
|
||||
await cleanupApprovalIndexByStateKey({ env: process.env, stateKey: payload.stateKey });
|
||||
}
|
||||
};
|
||||
|
||||
if (cancel === true) {
|
||||
await cleanupIndex();
|
||||
if (payload.kind === "workflow-file" && payload.stateKey) {
|
||||
await deleteStateJson({ env: process.env, key: payload.stateKey });
|
||||
}
|
||||
if (payload.kind === "pipeline-resume" && payload.stateKey) {
|
||||
await deleteStateJson({ env: process.env, key: payload.stateKey });
|
||||
}
|
||||
writeToolEnvelope({
|
||||
ok: true,
|
||||
status: "cancelled",
|
||||
output: [],
|
||||
requiresApproval: null,
|
||||
requiresInput: null,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (payload.kind === "workflow-file") {
|
||||
try {
|
||||
const output = await runWorkflowFile({
|
||||
filePath: payload.filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env: process.env,
|
||||
mode: "tool",
|
||||
registry,
|
||||
},
|
||||
resume: payload,
|
||||
approved,
|
||||
response,
|
||||
cancel,
|
||||
});
|
||||
|
||||
if (output.status === "needs_approval") {
|
||||
writeToolEnvelope({
|
||||
ok: true,
|
||||
status: "needs_approval",
|
||||
output: [],
|
||||
requiresApproval: output.requiresApproval ?? null,
|
||||
requiresInput: null,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (output.status === "needs_input") {
|
||||
writeToolEnvelope({
|
||||
ok: true,
|
||||
status: "needs_input",
|
||||
output: [],
|
||||
requiresApproval: null,
|
||||
requiresInput: output.requiresInput ?? null,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await cleanupIndex();
|
||||
if (output.status === "cancelled") {
|
||||
writeToolEnvelope({
|
||||
ok: true,
|
||||
status: "cancelled",
|
||||
output: [],
|
||||
requiresApproval: null,
|
||||
requiresInput: null,
|
||||
});
|
||||
return;
|
||||
}
|
||||
writeToolEnvelope({
|
||||
ok: true,
|
||||
status: "ok",
|
||||
output: output.output,
|
||||
requiresApproval: null,
|
||||
requiresInput: null,
|
||||
});
|
||||
return;
|
||||
} catch (err) {
|
||||
if (err instanceof WorkflowResumeArgumentError) {
|
||||
writeToolEnvelope({ ok: false, error: { type: "parse_error", message: err.message } });
|
||||
process.exitCode = 2;
|
||||
return;
|
||||
}
|
||||
// Don't clean up index on error — allow retry by --id
|
||||
writeToolEnvelope({
|
||||
ok: false,
|
||||
error: { type: "runtime_error", message: err?.message ?? String(err) },
|
||||
});
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
const previousStateKey = payload.stateKey;
|
||||
let resumeState;
|
||||
let envelope;
|
||||
try {
|
||||
resumeState = await loadPipelineResumeState(process.env, previousStateKey);
|
||||
} catch (err) {
|
||||
writeToolEnvelope({
|
||||
envelope = await resumeToolRequest({
|
||||
token: parsed.token ?? undefined,
|
||||
approvalId: parsed.approvalId ?? undefined,
|
||||
approved: parsed.approved,
|
||||
response: parsed.response,
|
||||
cancel: parsed.cancel,
|
||||
ctx: {
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
mode: "tool",
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
registry,
|
||||
signal,
|
||||
forceTerminationSignal,
|
||||
},
|
||||
});
|
||||
} catch (err: any) {
|
||||
envelope = {
|
||||
ok: false,
|
||||
error: { type: "runtime_error", message: err?.message ?? String(err) },
|
||||
});
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
};
|
||||
}
|
||||
if (resumeState.haltType === "input_request") {
|
||||
if (approved !== undefined) {
|
||||
writeToolEnvelope({
|
||||
ok: false,
|
||||
error: {
|
||||
type: "parse_error",
|
||||
message: "pipeline input resumes require --response-json <json>",
|
||||
},
|
||||
});
|
||||
process.exitCode = 2;
|
||||
return;
|
||||
}
|
||||
if (response === undefined) {
|
||||
writeToolEnvelope({
|
||||
ok: false,
|
||||
error: {
|
||||
type: "parse_error",
|
||||
message: "pipeline input resumes require --response-json <json>",
|
||||
},
|
||||
});
|
||||
process.exitCode = 2;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
validatePipelineInputResponse(resumeState.inputSchema, response);
|
||||
} catch (err) {
|
||||
writeToolEnvelope({
|
||||
ok: false,
|
||||
error: { type: "parse_error", message: err?.message ?? String(err) },
|
||||
});
|
||||
process.exitCode = 2;
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
if (response !== undefined) {
|
||||
writeToolEnvelope({
|
||||
ok: false,
|
||||
error: {
|
||||
type: "parse_error",
|
||||
message: "approval resumes require --approve yes|no, not --response-json",
|
||||
},
|
||||
});
|
||||
process.exitCode = 2;
|
||||
return;
|
||||
}
|
||||
if (approved !== true) {
|
||||
await cleanupIndex();
|
||||
await deleteStateJson({ env: process.env, key: previousStateKey });
|
||||
writeToolEnvelope({
|
||||
ok: true,
|
||||
status: "cancelled",
|
||||
output: [],
|
||||
requiresApproval: null,
|
||||
requiresInput: null,
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const isSameStageInput =
|
||||
resumeState.haltType === "input_request" && resumeState.resumeMode === "same_stage";
|
||||
const remaining = resumeState.pipeline.slice(resumeState.resumeAtIndex);
|
||||
const input = isSameStageInput
|
||||
? resumeState.items
|
||||
: resumeState.haltType === "input_request"
|
||||
? [response]
|
||||
: resumeState.items;
|
||||
const requestInputResume = isSameStageInput
|
||||
? {
|
||||
state: resumeState.commandInput!,
|
||||
response,
|
||||
onConsumed: async () => {
|
||||
await cleanupIndex();
|
||||
await deleteStateJson({ env: process.env, key: previousStateKey });
|
||||
},
|
||||
}
|
||||
: undefined;
|
||||
|
||||
try {
|
||||
const output = await runPipeline({
|
||||
pipeline: remaining,
|
||||
registry,
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env: process.env,
|
||||
mode,
|
||||
input,
|
||||
requestInputResume,
|
||||
});
|
||||
await cleanupIndex();
|
||||
const finalized = await finalizePipelineToolRun({
|
||||
env: process.env,
|
||||
pipeline: remaining,
|
||||
output,
|
||||
previousStateKey,
|
||||
});
|
||||
writeToolEnvelope({
|
||||
ok: true,
|
||||
status: finalized.status,
|
||||
output: finalized.output,
|
||||
requiresApproval: finalized.requiresApproval,
|
||||
requiresInput: finalized.requiresInput,
|
||||
});
|
||||
} catch (err) {
|
||||
// Don't clean up index on error — allow retry by --id
|
||||
writeToolEnvelope({
|
||||
ok: false,
|
||||
error: { type: "runtime_error", message: err?.message ?? String(err) },
|
||||
});
|
||||
process.exitCode = 1;
|
||||
writeToolEnvelope(envelope);
|
||||
if (!envelope.ok) {
|
||||
process.exitCode = envelope.error?.type === "parse_error" ? 2 : 1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -773,7 +633,17 @@ async function readVersion() {
|
||||
return pkg.version ?? "0.0.0";
|
||||
}
|
||||
|
||||
async function handleDoctor({ argv, registry }) {
|
||||
async function handleDoctor({
|
||||
argv,
|
||||
registry,
|
||||
signal,
|
||||
forceTerminationSignal,
|
||||
}: {
|
||||
argv;
|
||||
registry;
|
||||
signal: AbortSignal;
|
||||
forceTerminationSignal: AbortSignal;
|
||||
}) {
|
||||
const mode = "tool";
|
||||
const pipeline = "exec --json --shell 'echo [1]'";
|
||||
const output: any = await (async () => {
|
||||
@@ -788,6 +658,9 @@ async function handleDoctor({ argv, registry }) {
|
||||
stderr: process.stderr,
|
||||
env: process.env,
|
||||
mode,
|
||||
signal,
|
||||
forceTerminationSignal,
|
||||
haltAfterStageOnAbort: true,
|
||||
});
|
||||
} catch (err: any) {
|
||||
return { error: err };
|
||||
|
||||
@@ -18,6 +18,7 @@ export const approveCommand = {
|
||||
required: [],
|
||||
},
|
||||
sideEffects: [],
|
||||
resumeSafeBeforeInput: true,
|
||||
},
|
||||
help() {
|
||||
return `approve — require confirmation to continue\n\nUsage:\n ... | approve --prompt "Send these emails?"\n ... | approve --emit --prompt "Send these emails?"\n ... | approve --emit --preview-from-stdin --limit 5 --prompt "Proceed?"\n\nModes:\n - Interactive (default): prompts on TTY and passes items through if approved.\n - Emit (--emit): returns an approval request object and stops the pipeline.\n\nNotes:\n - In tool mode (or non-interactive), this emits an approval request and halts.\n`;
|
||||
@@ -53,6 +54,7 @@ export const approveCommand = {
|
||||
ctx.stdout.write(`${prompt} [y/N] `);
|
||||
const answer = await readLineFromStream(ctx.stdin, {
|
||||
timeoutMs: parseApprovalTimeoutMs(ctx.env),
|
||||
signal: ctx.signal,
|
||||
});
|
||||
|
||||
if (!/^y(es)?$/i.test(String(answer).trim())) {
|
||||
|
||||
@@ -57,6 +57,8 @@ export const askCommand = {
|
||||
required: [],
|
||||
},
|
||||
sideEffects: [],
|
||||
resumeSafeBeforeInput: true,
|
||||
resumeSafeAfterInput: true,
|
||||
},
|
||||
help() {
|
||||
return [
|
||||
@@ -154,7 +156,7 @@ export const askCommand = {
|
||||
|
||||
ctx.stdout.write(`${prompt}\n> `);
|
||||
const { readLineFromStream } = await import("../../read_line.js");
|
||||
const raw = await readLineFromStream(ctx.stdin, { timeoutMs: 0 });
|
||||
const raw = await readLineFromStream(ctx.stdin, { timeoutMs: 0, signal: ctx.signal });
|
||||
const text = String(raw ?? "").trim();
|
||||
|
||||
let lastError;
|
||||
|
||||
@@ -25,7 +25,12 @@ export const diffLastCommand = {
|
||||
for await (const item of input) afterItems.push(item);
|
||||
|
||||
const after = afterItems.length === 1 ? afterItems[0] : afterItems;
|
||||
const { before, changed } = await diffAndStore({ env: ctx.env, key, value: after });
|
||||
const { before, changed } = await diffAndStore({
|
||||
env: ctx.env,
|
||||
key,
|
||||
value: after,
|
||||
signal: ctx.signal,
|
||||
});
|
||||
|
||||
return {
|
||||
output: (async function* () {
|
||||
|
||||
+17
-36
@@ -1,4 +1,4 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { runAbortableProcess } from "../../abortable_process.js";
|
||||
import { resolveInlineShellCommand } from "../../shell.js";
|
||||
|
||||
export const execCommand = {
|
||||
@@ -58,12 +58,14 @@ export const execCommand = {
|
||||
cwd,
|
||||
stdin: stdinPayload,
|
||||
signal: ctx.signal,
|
||||
forceTerminationSignal: ctx.forceTerminationSignal,
|
||||
})
|
||||
: await runProcess(cmd[0], cmd.slice(1), {
|
||||
env: ctx.env,
|
||||
cwd,
|
||||
stdin: stdinPayload,
|
||||
signal: ctx.signal,
|
||||
forceTerminationSignal: ctx.forceTerminationSignal,
|
||||
});
|
||||
|
||||
if (args.json) {
|
||||
@@ -86,45 +88,24 @@ export const execCommand = {
|
||||
},
|
||||
};
|
||||
|
||||
function runProcess(command, argv, { env, cwd, stdin, signal }) {
|
||||
return new Promise<any>((resolve, reject) => {
|
||||
const child = spawn(command, argv, {
|
||||
env,
|
||||
cwd,
|
||||
signal,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
|
||||
child.stdout.on("data", (d) => {
|
||||
stdout += d;
|
||||
});
|
||||
child.stderr.on("data", (d) => {
|
||||
stderr += d;
|
||||
});
|
||||
|
||||
if (typeof stdin === "string") {
|
||||
child.stdin.setDefaultEncoding("utf8");
|
||||
child.stdin.write(stdin);
|
||||
}
|
||||
child.stdin.end();
|
||||
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (code === 0) return resolve({ stdout, stderr });
|
||||
reject(new Error(`exec failed (${code}): ${stderr.trim() || stdout.trim() || command}`));
|
||||
});
|
||||
async function runProcess(command, argv, { env, cwd, stdin, signal, forceTerminationSignal }) {
|
||||
const { stdout, stderr, code } = await runAbortableProcess({
|
||||
command,
|
||||
argv,
|
||||
env,
|
||||
cwd,
|
||||
stdin,
|
||||
signal,
|
||||
forceTerminationSignal,
|
||||
notFoundMessage: `exec command not found: ${command}`,
|
||||
});
|
||||
if (code === 0) return { stdout, stderr };
|
||||
throw new Error(`exec failed (${code}): ${stderr.trim() || stdout.trim() || command}`);
|
||||
}
|
||||
|
||||
function runShellLine(commandLine, { env, cwd, stdin, signal }) {
|
||||
function runShellLine(commandLine, { env, cwd, stdin, signal, forceTerminationSignal }) {
|
||||
const shell = resolveInlineShellCommand({ command: commandLine, env });
|
||||
return runProcess(shell.command, shell.argv, { env, cwd, stdin, signal });
|
||||
return runProcess(shell.command, shell.argv, { env, cwd, stdin, signal, forceTerminationSignal });
|
||||
}
|
||||
|
||||
function encodeStdin(items, mode) {
|
||||
|
||||
@@ -1,31 +1,4 @@
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
function run(cmd: string, argv: string[], env: Record<string, string | undefined>, cwd?: string) {
|
||||
return new Promise<{ stdout: string; stderr: string; code: number | null }>((resolve, reject) => {
|
||||
const child = spawn(cmd, argv, {
|
||||
env: { ...process.env, ...env },
|
||||
cwd,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout?.on("data", (d) => (stdout += String(d)));
|
||||
child.stderr?.on("data", (d) => (stderr += String(d)));
|
||||
|
||||
child.on("error", (err: any) => {
|
||||
if (err?.code === "ENOENT") {
|
||||
reject(new Error("gog not found on PATH (install: https://github.com/steipete/gogcli)"));
|
||||
return;
|
||||
}
|
||||
reject(err);
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
resolve({ stdout, stderr, code });
|
||||
});
|
||||
});
|
||||
}
|
||||
import { runAbortableProcess } from "../../abortable_process.js";
|
||||
|
||||
export const gogGmailSearchCommand = {
|
||||
name: "gog.gmail.search",
|
||||
@@ -55,10 +28,12 @@ export const gogGmailSearchCommand = {
|
||||
);
|
||||
},
|
||||
async run({ input, args, ctx }) {
|
||||
ctx.signal?.throwIfAborted();
|
||||
// Drain input
|
||||
for await (const _item of input) {
|
||||
// no-op
|
||||
}
|
||||
ctx.signal?.throwIfAborted();
|
||||
|
||||
const query = String(args.query ?? "newer_than:1d");
|
||||
const max = Number(args.max ?? args.limit ?? 20);
|
||||
@@ -75,7 +50,15 @@ export const gogGmailSearchCommand = {
|
||||
const gogBin = isScript ? process.execPath : gogBinRaw;
|
||||
const argv = isScript ? [gogBinRaw, ...argvBase] : argvBase;
|
||||
|
||||
const res = await run(gogBin, argv, ctx.env, process.cwd());
|
||||
const res = await runAbortableProcess({
|
||||
command: gogBin,
|
||||
argv,
|
||||
env: { ...process.env, ...ctx.env },
|
||||
cwd: process.cwd(),
|
||||
signal: ctx.signal,
|
||||
forceTerminationSignal: ctx.forceTerminationSignal,
|
||||
notFoundMessage: "gog not found on PATH (install: https://github.com/steipete/gogcli)",
|
||||
});
|
||||
if (res.code !== 0) {
|
||||
throw new Error(`gog.gmail.search failed (${res.code ?? "?"}): ${res.stderr.slice(0, 400)}`);
|
||||
}
|
||||
|
||||
@@ -72,6 +72,7 @@ export const gogGmailSendCommand = {
|
||||
);
|
||||
},
|
||||
async run({ input, args, ctx }) {
|
||||
ctx.signal?.throwIfAborted();
|
||||
const dryRun = Boolean(args.dryRun ?? args["dry-run"] ?? false);
|
||||
const gogBinRaw = String(ctx.env.GOG_BIN ?? "gog");
|
||||
const isScript = /\.(mjs|cjs|js|ts)$/i.test(gogBinRaw);
|
||||
@@ -80,6 +81,10 @@ export const gogGmailSendCommand = {
|
||||
const results: any[] = [];
|
||||
|
||||
for await (const item of input) {
|
||||
if (ctx.signal?.aborted) {
|
||||
if (results.length > 0) break;
|
||||
ctx.signal.throwIfAborted();
|
||||
}
|
||||
const draft = parseDraft(item);
|
||||
|
||||
if (dryRun) {
|
||||
@@ -111,6 +116,7 @@ export const gogGmailSendCommand = {
|
||||
}
|
||||
|
||||
results.push(parsed);
|
||||
if (ctx.signal?.aborted) break;
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -5,12 +5,14 @@ import { Ajv } from "ajv";
|
||||
import type { ErrorObject } from "ajv";
|
||||
|
||||
import {
|
||||
diffAndStore,
|
||||
ensureDirectory,
|
||||
atomicWriteWasPublished,
|
||||
isJsonSyntaxError,
|
||||
readStateJson,
|
||||
readStateJsonWithLock,
|
||||
stableStringify,
|
||||
withFileLock,
|
||||
writeFileAtomic,
|
||||
writeStateJson,
|
||||
} from "../../state/store.js";
|
||||
import { createCompileCached } from "../../validation.js";
|
||||
import type { LobsterCommand } from "../types.js";
|
||||
@@ -178,6 +180,7 @@ type Adapter = {
|
||||
env: any;
|
||||
args: any;
|
||||
payload: Record<string, any>;
|
||||
signal?: AbortSignal;
|
||||
}) => Promise<LlmResponseEnvelope>;
|
||||
};
|
||||
|
||||
@@ -187,6 +190,7 @@ type DirectAdapter =
|
||||
args: any;
|
||||
payload: Record<string, any>;
|
||||
ctx: any;
|
||||
signal?: AbortSignal;
|
||||
}) => Promise<LlmResponseEnvelope>)
|
||||
| {
|
||||
source?: string;
|
||||
@@ -195,6 +199,7 @@ type DirectAdapter =
|
||||
args: any;
|
||||
payload: Record<string, any>;
|
||||
ctx: any;
|
||||
signal?: AbortSignal;
|
||||
}) => Promise<LlmResponseEnvelope>;
|
||||
};
|
||||
|
||||
@@ -385,7 +390,7 @@ async function runLlmInvoke({
|
||||
});
|
||||
|
||||
if (stateKey && !forceRefresh) {
|
||||
const stored = await readReusableLlmState(env, stateKey);
|
||||
const stored = await readReusableLlmState(env, stateKey, ctx.signal);
|
||||
const reused = pickReusableState(stored, cacheKey, config.stateType);
|
||||
if (reused) {
|
||||
return {
|
||||
@@ -397,7 +402,7 @@ async function runLlmInvoke({
|
||||
}
|
||||
|
||||
if (!disableCache && !forceRefresh) {
|
||||
const cache = await readCacheEntry(env, cacheKey, config.cacheNamespace);
|
||||
const cache = await readCacheEntry(env, cacheKey, config.cacheNamespace, ctx.signal);
|
||||
if (cache) {
|
||||
return {
|
||||
output: streamOf(cache.items.map((item) => ({ ...item, source: "cache", cached: true }))),
|
||||
@@ -438,7 +443,9 @@ async function runLlmInvoke({
|
||||
|
||||
let responseEnvelope: LlmResponseEnvelope;
|
||||
try {
|
||||
responseEnvelope = await adapter.invoke({ env, args, payload });
|
||||
ctx.signal?.throwIfAborted();
|
||||
responseEnvelope = await adapter.invoke({ env, args, payload, signal: ctx.signal });
|
||||
ctx.signal?.throwIfAborted();
|
||||
} catch (err: any) {
|
||||
throw new Error(`${config.name} request failed: ${err?.message ?? String(err)}`);
|
||||
}
|
||||
@@ -463,27 +470,35 @@ async function runLlmInvoke({
|
||||
});
|
||||
|
||||
if (!validator) {
|
||||
ctx.signal?.throwIfAborted();
|
||||
await persistOutputs({
|
||||
env,
|
||||
stateKey,
|
||||
cacheKey,
|
||||
items: normalized,
|
||||
stateType: config.stateType,
|
||||
signal: ctx.signal,
|
||||
afterStore: disableCache
|
||||
? undefined
|
||||
: () => writeCacheEntry(env, cacheKey, normalized, config.cacheNamespace, ctx.signal),
|
||||
});
|
||||
if (!disableCache) await writeCacheEntry(env, cacheKey, normalized, config.cacheNamespace);
|
||||
return { output: streamOf(normalized) };
|
||||
}
|
||||
|
||||
const structured = normalized[0]?.output?.data ?? null;
|
||||
if (validator(structured)) {
|
||||
ctx.signal?.throwIfAborted();
|
||||
await persistOutputs({
|
||||
env,
|
||||
stateKey,
|
||||
cacheKey,
|
||||
items: normalized,
|
||||
stateType: config.stateType,
|
||||
signal: ctx.signal,
|
||||
afterStore: disableCache
|
||||
? undefined
|
||||
: () => writeCacheEntry(env, cacheKey, normalized, config.cacheNamespace, ctx.signal),
|
||||
});
|
||||
if (!disableCache) await writeCacheEntry(env, cacheKey, normalized, config.cacheNamespace);
|
||||
return { output: streamOf(normalized) };
|
||||
}
|
||||
|
||||
@@ -547,8 +562,8 @@ function resolveAdapter({
|
||||
return {
|
||||
provider,
|
||||
source: typeof direct === "function" ? provider : (direct.source ?? provider),
|
||||
async invoke({ payload }) {
|
||||
return invoke({ env, args, payload, ctx });
|
||||
async invoke({ payload, signal }) {
|
||||
return invoke({ env, args, payload, ctx, signal });
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -563,8 +578,8 @@ function resolveAdapter({
|
||||
return {
|
||||
provider,
|
||||
source: config.sourceForProvider?.(provider) ?? "openclaw",
|
||||
async invoke({ payload }) {
|
||||
return invokeOpenClawAdapter({ endpoint, token, payload });
|
||||
async invoke({ payload, signal }) {
|
||||
return invokeOpenClawAdapter({ endpoint, token, payload, signal });
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -578,8 +593,13 @@ function resolveAdapter({
|
||||
return {
|
||||
provider,
|
||||
source: config.sourceForProvider?.(provider) ?? "pi",
|
||||
async invoke({ payload }) {
|
||||
return invokeHttpAdapter({ endpoint: buildAdapterEndpoint(adapterUrl), token, payload });
|
||||
async invoke({ payload, signal }) {
|
||||
return invokeHttpAdapter({
|
||||
endpoint: buildAdapterEndpoint(adapterUrl),
|
||||
token,
|
||||
payload,
|
||||
signal,
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -592,8 +612,13 @@ function resolveAdapter({
|
||||
return {
|
||||
provider,
|
||||
source: config.sourceForProvider?.(provider) ?? "http",
|
||||
async invoke({ payload }) {
|
||||
return invokeHttpAdapter({ endpoint: buildAdapterEndpoint(adapterUrl), token, payload });
|
||||
async invoke({ payload, signal }) {
|
||||
return invokeHttpAdapter({
|
||||
endpoint: buildAdapterEndpoint(adapterUrl),
|
||||
token,
|
||||
payload,
|
||||
signal,
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -621,13 +646,16 @@ async function invokeOpenClawAdapter({
|
||||
endpoint,
|
||||
token,
|
||||
payload,
|
||||
signal,
|
||||
}: {
|
||||
endpoint: URL;
|
||||
token: string;
|
||||
payload: any;
|
||||
signal?: AbortSignal;
|
||||
}) {
|
||||
const res = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
signal,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...(token ? { authorization: `Bearer ${token}` } : null),
|
||||
@@ -670,13 +698,16 @@ async function invokeHttpAdapter({
|
||||
endpoint,
|
||||
token,
|
||||
payload,
|
||||
signal,
|
||||
}: {
|
||||
endpoint: URL;
|
||||
token: string;
|
||||
payload: any;
|
||||
signal?: AbortSignal;
|
||||
}) {
|
||||
const res = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
signal,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...(token ? { authorization: `Bearer ${token}` } : null),
|
||||
@@ -876,14 +907,21 @@ async function persistOutputs({
|
||||
cacheKey,
|
||||
items,
|
||||
stateType,
|
||||
signal,
|
||||
afterStore,
|
||||
}: {
|
||||
env: any;
|
||||
stateKey: string | null;
|
||||
cacheKey: string;
|
||||
items: NormalizedInvocationItem[];
|
||||
stateType: string;
|
||||
signal?: AbortSignal;
|
||||
afterStore?: () => Promise<void>;
|
||||
}) {
|
||||
if (!stateKey) return;
|
||||
if (!stateKey) {
|
||||
await afterStore?.();
|
||||
return;
|
||||
}
|
||||
const record = {
|
||||
type: stateType,
|
||||
version: STATE_VERSION,
|
||||
@@ -891,12 +929,18 @@ async function persistOutputs({
|
||||
items,
|
||||
storedAt: new Date().toISOString(),
|
||||
};
|
||||
await writeStateJson({ env, key: stateKey, value: record });
|
||||
await diffAndStore({
|
||||
env,
|
||||
key: stateKey,
|
||||
value: record,
|
||||
signal,
|
||||
afterStore: afterStore ? () => afterStore() : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
async function readReusableLlmState(env: any, stateKey: string) {
|
||||
async function readReusableLlmState(env: any, stateKey: string, signal?: AbortSignal) {
|
||||
try {
|
||||
return await readStateJson({ env, key: stateKey });
|
||||
return await readStateJsonWithLock({ env, key: stateKey, signal });
|
||||
} catch (err: any) {
|
||||
if (isJsonSyntaxError(err)) return null;
|
||||
throw err;
|
||||
@@ -920,16 +964,28 @@ async function readCacheEntry(
|
||||
env: any,
|
||||
key: string,
|
||||
cacheNamespace: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CacheEntry | null> {
|
||||
const filePath = path.join(getCacheDir(env), cacheNamespace, `${key}.json`);
|
||||
const read = async () => {
|
||||
try {
|
||||
const text = await fsp.readFile(filePath, "utf8");
|
||||
const parsed = JSON.parse(text) as Partial<CacheEntry>;
|
||||
if (parsed?.cacheKey !== key || !Array.isArray(parsed.items)) return null;
|
||||
return parsed as CacheEntry;
|
||||
} catch (err: any) {
|
||||
if (err?.code === "ENOENT") return null;
|
||||
if (isJsonSyntaxError(err)) return null;
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
try {
|
||||
const text = await fsp.readFile(filePath, "utf8");
|
||||
const parsed = JSON.parse(text) as Partial<CacheEntry>;
|
||||
if (parsed?.cacheKey !== key || !Array.isArray(parsed.items)) return null;
|
||||
return parsed as CacheEntry;
|
||||
return await withFileLock({ filePath, signal, task: read });
|
||||
} catch (err: any) {
|
||||
if (err?.code === "ENOENT") return null;
|
||||
if (isJsonSyntaxError(err)) return null;
|
||||
// A cache mounted read-only cannot have an active local writer because a
|
||||
// writer first creates the same coordination lock. Preserve its reusable
|
||||
// entries instead of requiring a lock-directory write for a read.
|
||||
if (["EACCES", "EPERM", "EROFS"].includes(err?.code)) return read();
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -939,14 +995,50 @@ async function writeCacheEntry(
|
||||
key: string,
|
||||
items: NormalizedInvocationItem[],
|
||||
cacheNamespace: string,
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
const dir = path.join(getCacheDir(env), cacheNamespace);
|
||||
signal?.throwIfAborted();
|
||||
await ensureDirectory(dir);
|
||||
const filePath = path.join(dir, `${key}.json`);
|
||||
await writeFileAtomic(
|
||||
const content =
|
||||
JSON.stringify({ items, cacheKey: key, storedAt: new Date().toISOString() }, null, 2) + "\n";
|
||||
await withFileLock({
|
||||
filePath,
|
||||
JSON.stringify({ items, cacheKey: key, storedAt: new Date().toISOString() }, null, 2) + "\n",
|
||||
);
|
||||
signal,
|
||||
task: async () => {
|
||||
let previousContent: Buffer | null = null;
|
||||
try {
|
||||
previousContent = await fsp.readFile(filePath);
|
||||
} catch (err: any) {
|
||||
if (err?.code !== "ENOENT") throw err;
|
||||
}
|
||||
const restorePreviousContent = async () => {
|
||||
// No cache reader or competing cache writer can observe this entry
|
||||
// until this lock is released. Restore an entry replaced by a refresh;
|
||||
// only remove the just-published file when none existed beforehand.
|
||||
if (previousContent === null) await fsp.rm(filePath, { force: true });
|
||||
else await writeFileAtomic(filePath, previousContent);
|
||||
};
|
||||
let cacheWasPublished = false;
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
const result = await writeFileAtomic(filePath, content, { signal });
|
||||
cacheWasPublished = true;
|
||||
if (result?.signalAbortedAfterCommit || signal?.aborted) {
|
||||
await restorePreviousContent();
|
||||
cacheWasPublished = false;
|
||||
signal?.throwIfAborted();
|
||||
throw new Error("LLM cache publication cancelled");
|
||||
}
|
||||
} catch (err) {
|
||||
if (cacheWasPublished || atomicWriteWasPublished(err)) {
|
||||
await restorePreviousContent();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function getCacheDir(env: any) {
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { runAbortableProcess } from "../../abortable_process.js";
|
||||
import type { LobsterCommand } from "../types.js";
|
||||
|
||||
const OPENCLAW_AGENT_MAX_OUTPUT_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
type AgentCliRunner = (params: {
|
||||
executable: string;
|
||||
argv: string[];
|
||||
cwd: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
signal?: AbortSignal;
|
||||
forceTerminationSignal?: AbortSignal;
|
||||
}) => Promise<unknown>;
|
||||
|
||||
export const openclawAgentCommand = createOpenClawAgentCommand();
|
||||
@@ -92,6 +95,7 @@ export function createOpenClawAgentCommand(
|
||||
cwd: ctx?.cwd ?? process.cwd(),
|
||||
env,
|
||||
signal: ctx?.signal,
|
||||
forceTerminationSignal: ctx?.forceTerminationSignal,
|
||||
});
|
||||
return { output: streamOf([response]) };
|
||||
},
|
||||
@@ -104,39 +108,32 @@ export function runOpenClawAgentCli(params: {
|
||||
cwd: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
signal?: AbortSignal;
|
||||
forceTerminationSignal?: AbortSignal;
|
||||
}): Promise<unknown> {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile(
|
||||
params.executable,
|
||||
params.argv,
|
||||
{
|
||||
cwd: params.cwd,
|
||||
env: params.env,
|
||||
signal: params.signal,
|
||||
encoding: "utf8",
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
},
|
||||
(error, stdout, stderr) => {
|
||||
if (error) {
|
||||
if (error.name === "AbortError") {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
const detail = String(stderr || stdout || error.message).trim();
|
||||
reject(new Error(`openclaw.agent failed: ${detail}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(String(stdout).trim() || "null");
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
||||
throw new Error("response must be an object");
|
||||
}
|
||||
resolve(parsed);
|
||||
} catch {
|
||||
reject(new Error("openclaw.agent expected JSON output from `openclaw agent --json`"));
|
||||
}
|
||||
},
|
||||
);
|
||||
return runAbortableProcess({
|
||||
command: params.executable,
|
||||
argv: params.argv,
|
||||
cwd: params.cwd,
|
||||
env: params.env,
|
||||
signal: params.signal,
|
||||
forceTerminationSignal: params.forceTerminationSignal,
|
||||
maxOutputBytes: OPENCLAW_AGENT_MAX_OUTPUT_BYTES,
|
||||
outputLimitMessage: `openclaw.agent output exceeded ${OPENCLAW_AGENT_MAX_OUTPUT_BYTES} bytes`,
|
||||
notFoundMessage: "openclaw.agent could not find the OpenClaw CLI",
|
||||
}).then(({ code, stdout, stderr }) => {
|
||||
if (code !== 0) {
|
||||
const detail = String(stderr || stdout || `exited with code ${code}`).trim();
|
||||
throw new Error(`openclaw.agent failed: ${detail}`);
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(stdout.trim() || "null");
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
||||
throw new Error("response must be an object");
|
||||
}
|
||||
return parsed;
|
||||
} catch {
|
||||
throw new Error("openclaw.agent expected JSON output from `openclaw agent --json`");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -91,6 +91,7 @@ function createInvokeCommand(commandName: string) {
|
||||
const invokeOnce = async (argsValue: unknown) => {
|
||||
const res = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
signal: ctx.signal,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...(token ? { authorization: `Bearer ${token}` } : null),
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import { promises as fsp } from "node:fs";
|
||||
|
||||
import { defaultStateDir, ensureDirectory, keyToPath, writeFileAtomic } from "../../state/store.js";
|
||||
import { readStateJsonWithLock, writeStateJson } from "../../state/store.js";
|
||||
|
||||
export const stateGetCommand = {
|
||||
name: "state.get",
|
||||
@@ -22,20 +20,7 @@ export const stateGetCommand = {
|
||||
const key = args._[0];
|
||||
if (!key) throw new Error("state.get requires a key");
|
||||
|
||||
const stateDir = defaultStateDir(ctx.env);
|
||||
const filePath = keyToPath(stateDir, key);
|
||||
|
||||
let value = null;
|
||||
try {
|
||||
const text = await fsp.readFile(filePath, "utf8");
|
||||
value = JSON.parse(text);
|
||||
} catch (err) {
|
||||
if (err?.code === "ENOENT") {
|
||||
value = null;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
const value = await readStateJsonWithLock({ env: ctx.env, key, signal: ctx.signal });
|
||||
|
||||
return { output: asStream([value]) };
|
||||
},
|
||||
@@ -66,11 +51,7 @@ export const stateSetCommand = {
|
||||
|
||||
const value = items.length === 1 ? items[0] : items;
|
||||
|
||||
const stateDir = defaultStateDir(ctx.env);
|
||||
const filePath = keyToPath(stateDir, key);
|
||||
|
||||
await ensureDirectory(stateDir);
|
||||
await writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n");
|
||||
await writeStateJson({ env: ctx.env, key, value, signal: ctx.signal });
|
||||
|
||||
return { output: asStream([value]) };
|
||||
},
|
||||
|
||||
@@ -3,6 +3,18 @@ export type CommandMeta = {
|
||||
argsSchema?: unknown;
|
||||
examples?: Array<{ args: Record<string, unknown>; description?: string }>;
|
||||
sideEffects?: string[];
|
||||
/**
|
||||
* The command may create an input/approval gate before it begins execution.
|
||||
* Commands that omit this are treated conservatively when a resumed pipeline
|
||||
* is cancelled: its original capability cannot be replayed after dispatch.
|
||||
*/
|
||||
resumeSafeBeforeInput?: boolean;
|
||||
/**
|
||||
* The command remains side-effect-free after returning a resumed input until
|
||||
* the next pipeline stage dispatches. This is intentionally opt-in so a
|
||||
* command that acts on a resumed response consumes its capability first.
|
||||
*/
|
||||
resumeSafeAfterInput?: boolean;
|
||||
};
|
||||
|
||||
export type LobsterCommand = {
|
||||
|
||||
+161
-21
@@ -7,12 +7,20 @@ import { decodeResumeToken, kindFromStateKey } from "../resume.js";
|
||||
import { runPipeline } from "../runtime.js";
|
||||
import { encodeToken } from "../token.js";
|
||||
import {
|
||||
deleteStateJson,
|
||||
deleteStateJsonWithBoundedResumeCleanup,
|
||||
deleteUnconsumedResumeState,
|
||||
deleteApprovalId,
|
||||
findStateKeyByApprovalId,
|
||||
cleanupApprovalIndexByStateKey,
|
||||
consumeResumeState,
|
||||
restoreConsumedResumeState,
|
||||
stateJsonExists,
|
||||
} from "../state/store.js";
|
||||
import { WorkflowResumeArgumentError, runWorkflowFile } from "../workflows/file.js";
|
||||
import {
|
||||
WorkflowResumeArgumentError,
|
||||
alternateWorkflowResumeStateKey,
|
||||
runWorkflowFile,
|
||||
} from "../workflows/file.js";
|
||||
import {
|
||||
finalizePipelineToolRun,
|
||||
loadPipelineResumeState,
|
||||
@@ -27,6 +35,7 @@ type ToolRunContext = {
|
||||
stdout?: NodeJS.WritableStream;
|
||||
stderr?: NodeJS.WritableStream;
|
||||
signal?: AbortSignal;
|
||||
forceTerminationSignal?: AbortSignal;
|
||||
registry?: any;
|
||||
llmAdapters?: Record<string, any>;
|
||||
};
|
||||
@@ -130,12 +139,15 @@ export async function runToolRequest({
|
||||
cwd: runtime.cwd,
|
||||
llmAdapters: runtime.llmAdapters,
|
||||
signal: runtime.signal,
|
||||
forceTerminationSignal: runtime.forceTerminationSignal,
|
||||
haltAfterStageOnAbort: true,
|
||||
});
|
||||
|
||||
const finalized = await finalizePipelineToolRun({
|
||||
env: runtime.env,
|
||||
pipeline: parsed,
|
||||
output,
|
||||
signal: runtime.signal,
|
||||
});
|
||||
return okEnvelope(
|
||||
finalized.status,
|
||||
@@ -193,30 +205,71 @@ export async function resumeToolRequest({
|
||||
}
|
||||
|
||||
// Helper: clean up approval ID index after successful use
|
||||
const cleanupIndex = async () => {
|
||||
const cleanupIndex = async (stateKey = payload?.stateKey) => {
|
||||
if (resolvedApprovalId) {
|
||||
await deleteApprovalId({ env: runtime.env, approvalId: resolvedApprovalId });
|
||||
} else if (payload?.stateKey) {
|
||||
await cleanupApprovalIndexByStateKey({ env: runtime.env, stateKey: payload.stateKey });
|
||||
} else if (stateKey) {
|
||||
await cleanupApprovalIndexByStateKey({ env: runtime.env, stateKey });
|
||||
}
|
||||
};
|
||||
|
||||
if (cancel === true) {
|
||||
await cleanupIndex();
|
||||
let stateKeys = payload.stateKey ? [payload.stateKey] : [];
|
||||
if (payload.kind === "workflow-file" && payload.stateKey) {
|
||||
await deleteStateJson({ env: runtime.env, key: payload.stateKey });
|
||||
const alternateStateKey = alternateWorkflowResumeStateKey(payload.stateKey);
|
||||
if (alternateStateKey) {
|
||||
// Delete a non-authoritative spelling first. If cancellation interrupts
|
||||
// its lock wait, the state that makes this capability resumable remains.
|
||||
const [primaryExists, alternateExists] = await Promise.all([
|
||||
stateJsonExists({ env: runtime.env, key: payload.stateKey }),
|
||||
stateJsonExists({ env: runtime.env, key: alternateStateKey }),
|
||||
]);
|
||||
if (primaryExists || !alternateExists) {
|
||||
stateKeys = [alternateStateKey, payload.stateKey];
|
||||
} else {
|
||||
stateKeys = [payload.stateKey, alternateStateKey];
|
||||
}
|
||||
}
|
||||
}
|
||||
if (payload.kind === "pipeline-resume" && payload.stateKey) {
|
||||
await deleteStateJson({ env: runtime.env, key: payload.stateKey });
|
||||
// Keep the capability indexed until every state deletion succeeds. A
|
||||
// cancelled request must not orphan a resume state by dropping its
|
||||
// approval ID while waiting on another writer's state lock.
|
||||
const deletionResults = [];
|
||||
for (const stateKey of new Set(stateKeys)) {
|
||||
deletionResults.push(
|
||||
await deleteUnconsumedResumeState({
|
||||
env: runtime.env,
|
||||
key: stateKey,
|
||||
signal: runtime.signal,
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (
|
||||
stateKeys.length > 0 &&
|
||||
(deletionResults.includes("claimed") ||
|
||||
deletionResults.every((result) => result === "missing"))
|
||||
) {
|
||||
return errorEnvelope("runtime_error", "Resume state not found");
|
||||
}
|
||||
if (resolvedApprovalId) {
|
||||
await cleanupIndex();
|
||||
} else {
|
||||
for (const stateKey of stateKeys) await cleanupIndex(stateKey);
|
||||
}
|
||||
return okEnvelope("cancelled", [], null, null);
|
||||
}
|
||||
|
||||
if (payload.kind === "workflow-file") {
|
||||
let workflowResumeStateKey = payload.stateKey;
|
||||
try {
|
||||
const output = await runWorkflowFile({
|
||||
filePath: payload.filePath,
|
||||
ctx: runtime,
|
||||
ctx: {
|
||||
...runtime,
|
||||
_onResumeStateResolved: (stateKey) => {
|
||||
workflowResumeStateKey = stateKey;
|
||||
},
|
||||
},
|
||||
resume: payload,
|
||||
approved,
|
||||
response,
|
||||
@@ -224,13 +277,12 @@ export async function resumeToolRequest({
|
||||
});
|
||||
|
||||
if (output.status === "needs_approval") {
|
||||
// Don't clean up index — next gate will issue a new approvalId
|
||||
return okEnvelope("needs_approval", [], output.requiresApproval ?? null, null);
|
||||
}
|
||||
if (output.status === "needs_input") {
|
||||
return okEnvelope("needs_input", [], null, output.requiresInput ?? null);
|
||||
}
|
||||
await cleanupIndex();
|
||||
await cleanupIndex(workflowResumeStateKey);
|
||||
if (output.status === "cancelled") {
|
||||
return okEnvelope("cancelled", [], null, null);
|
||||
}
|
||||
@@ -239,15 +291,30 @@ export async function resumeToolRequest({
|
||||
if (err instanceof WorkflowResumeArgumentError) {
|
||||
return errorEnvelope("parse_error", err.message);
|
||||
}
|
||||
// Don't clean up index on error — allow retry by --id
|
||||
// Non-abort failures and cancellations before step execution remain retryable.
|
||||
return errorEnvelope("runtime_error", err?.message ?? String(err));
|
||||
}
|
||||
}
|
||||
|
||||
if (runtime.signal?.aborted) {
|
||||
return errorEnvelope(
|
||||
"runtime_error",
|
||||
runtime.signal.reason instanceof Error
|
||||
? runtime.signal.reason.message
|
||||
: "This operation was aborted",
|
||||
);
|
||||
}
|
||||
|
||||
let resumeState;
|
||||
try {
|
||||
resumeState = await loadPipelineResumeState(runtime.env, payload.stateKey);
|
||||
// No state has been claimed yet, so a cancelled resume can return before
|
||||
// touching the lock and leave its capability safely retryable.
|
||||
resumeState = await loadPipelineResumeState(runtime.env, payload.stateKey, runtime.signal);
|
||||
} catch (err: any) {
|
||||
// Approval rejection historically reaches the signal-aware deletion path
|
||||
// below. Keep its direct cancellation propagation while other resume modes
|
||||
// retain their structured tool envelope.
|
||||
if (runtime.signal?.aborted && approved === false) throw err;
|
||||
return errorEnvelope("runtime_error", err?.message ?? String(err));
|
||||
}
|
||||
|
||||
@@ -271,8 +338,18 @@ export async function resumeToolRequest({
|
||||
);
|
||||
}
|
||||
if (approved !== true) {
|
||||
// Keep the approval ID usable while this may still be waiting on a
|
||||
// concurrent state writer. Dropping the index first would orphan the
|
||||
// capability if cancellation interrupts the deletion.
|
||||
const deletion = await deleteUnconsumedResumeState({
|
||||
env: runtime.env,
|
||||
key: payload.stateKey,
|
||||
signal: runtime.signal,
|
||||
});
|
||||
if (deletion !== "deleted") {
|
||||
return errorEnvelope("runtime_error", "Pipeline resume state not found");
|
||||
}
|
||||
await cleanupIndex();
|
||||
await deleteStateJson({ env: runtime.env, key: payload.stateKey });
|
||||
return okEnvelope("cancelled", [], null, null);
|
||||
}
|
||||
}
|
||||
@@ -285,14 +362,14 @@ export async function resumeToolRequest({
|
||||
: resumeState.haltType === "input_request"
|
||||
? [response]
|
||||
: resumeState.items;
|
||||
const abortedBeforeResume = runtime.signal?.aborted === true;
|
||||
let pipelineResumeStateRestored = false;
|
||||
let pipelineExecutionStarted = false;
|
||||
let pipelineResumeStateClaimId: string | undefined;
|
||||
const requestInputResume = isSameStageInput
|
||||
? {
|
||||
state: resumeState.commandInput!,
|
||||
response,
|
||||
onConsumed: async () => {
|
||||
await cleanupIndex();
|
||||
await deleteStateJson({ env: runtime.env, key: payload.stateKey });
|
||||
},
|
||||
}
|
||||
: undefined;
|
||||
|
||||
@@ -308,17 +385,53 @@ export async function resumeToolRequest({
|
||||
cwd: runtime.cwd,
|
||||
llmAdapters: runtime.llmAdapters,
|
||||
signal: runtime.signal,
|
||||
forceTerminationSignal: runtime.forceTerminationSignal,
|
||||
haltAfterStageOnAbort: true,
|
||||
input,
|
||||
requestInputResume,
|
||||
onExecutionStart: async () => {
|
||||
const consumption = await consumeResumeState({
|
||||
env: runtime.env,
|
||||
key: payload.stateKey,
|
||||
expectedState: resumeState,
|
||||
signal: runtime.signal,
|
||||
});
|
||||
if (!consumption.consumed) {
|
||||
throw new Error("Pipeline resume state not found");
|
||||
}
|
||||
pipelineResumeStateClaimId = consumption.claimId;
|
||||
if (consumption.signalAbortedAfterCommit) {
|
||||
const restored = await restoreConsumedResumeState({
|
||||
env: runtime.env,
|
||||
key: payload.stateKey,
|
||||
expectedState: resumeState,
|
||||
claimId: consumption.claimId,
|
||||
});
|
||||
if (restored) {
|
||||
pipelineResumeStateRestored = true;
|
||||
pipelineResumeStateClaimId = undefined;
|
||||
}
|
||||
runtime.signal?.throwIfAborted();
|
||||
}
|
||||
runtime.signal?.throwIfAborted();
|
||||
pipelineExecutionStarted = true;
|
||||
},
|
||||
});
|
||||
|
||||
await cleanupIndex();
|
||||
const finalized = await finalizePipelineToolRun({
|
||||
env: runtime.env,
|
||||
pipeline: remaining,
|
||||
output,
|
||||
previousStateKey: payload.stateKey,
|
||||
previousState: resumeState,
|
||||
previousStateConsumed: pipelineExecutionStarted,
|
||||
restorePreviousStateOnAbort: !pipelineExecutionStarted,
|
||||
onPreviousStateRestored: () => {
|
||||
pipelineResumeStateRestored = true;
|
||||
},
|
||||
signal: runtime.signal,
|
||||
});
|
||||
if (finalized.status === "ok" && pipelineExecutionStarted) await cleanupIndex();
|
||||
return okEnvelope(
|
||||
finalized.status,
|
||||
finalized.output,
|
||||
@@ -326,7 +439,33 @@ export async function resumeToolRequest({
|
||||
finalized.requiresInput,
|
||||
);
|
||||
} catch (err: any) {
|
||||
// Don't clean up index on error — allow retry by --id
|
||||
const abortedResume = runtime.signal?.aborted === true;
|
||||
if (
|
||||
abortedResume &&
|
||||
!pipelineExecutionStarted &&
|
||||
!pipelineResumeStateRestored &&
|
||||
pipelineResumeStateClaimId
|
||||
) {
|
||||
pipelineResumeStateRestored = await restoreConsumedResumeState({
|
||||
env: runtime.env,
|
||||
key: payload.stateKey,
|
||||
expectedState: resumeState,
|
||||
claimId: pipelineResumeStateClaimId,
|
||||
}).catch(() => false);
|
||||
}
|
||||
if (pipelineExecutionStarted && !pipelineResumeStateRestored) {
|
||||
if (abortedResume && !abortedBeforeResume) {
|
||||
await deleteStateJsonWithBoundedResumeCleanup({
|
||||
env: runtime.env,
|
||||
key: payload.stateKey,
|
||||
}).catch(() => {});
|
||||
}
|
||||
// Keep the short approval ID through the pre-dispatch claim window. Once
|
||||
// the unsafe stage has actually been entered, the tombstone makes retry
|
||||
// unsafe and the old index may be retired just as it was before this fix.
|
||||
await cleanupIndex().catch(() => {});
|
||||
}
|
||||
// Non-abort failures and pre-aborted resumes remain retryable by token or approval ID.
|
||||
return errorEnvelope("runtime_error", err?.message ?? String(err));
|
||||
}
|
||||
}
|
||||
@@ -340,6 +479,7 @@ export function createToolContext(ctx: ToolRunContext = {}) {
|
||||
stdout: ctx.stdout ?? createCaptureStream(),
|
||||
stderr: ctx.stderr ?? createCaptureStream(),
|
||||
signal: ctx.signal,
|
||||
forceTerminationSignal: ctx.forceTerminationSignal,
|
||||
registry: ctx.registry ?? createDefaultRegistry(),
|
||||
llmAdapters: ctx.llmAdapters,
|
||||
};
|
||||
|
||||
@@ -177,6 +177,7 @@ export function createStageRequestInput({
|
||||
getInactiveReason,
|
||||
isOutputStarted,
|
||||
resume,
|
||||
onResumedInput,
|
||||
}: {
|
||||
ctx: any;
|
||||
stageIndex: number;
|
||||
@@ -186,6 +187,7 @@ export function createStageRequestInput({
|
||||
getInactiveReason?: () => string | undefined;
|
||||
isOutputStarted: () => boolean;
|
||||
resume?: CommandInputResume;
|
||||
onResumedInput?: () => void | Promise<void>;
|
||||
}) {
|
||||
let requestIndex = 0;
|
||||
const history: CommandInputHistoryEntry[] = [...(resume?.state.history ?? [])];
|
||||
@@ -209,6 +211,7 @@ export function createStageRequestInput({
|
||||
const response = snapshotJson(historical.response, "requestInput response");
|
||||
validateRequestInputResponse(metadata.responseSchema, response, "requestInput");
|
||||
requestIndex += 1;
|
||||
await onResumedInput?.();
|
||||
return response;
|
||||
}
|
||||
|
||||
@@ -234,6 +237,7 @@ export function createStageRequestInput({
|
||||
response: historyResponse,
|
||||
});
|
||||
requestIndex += 1;
|
||||
await onResumedInput?.();
|
||||
return response;
|
||||
}
|
||||
|
||||
@@ -479,7 +483,7 @@ function assertJsonSerializable(value: unknown, label: string, seen: WeakSet<obj
|
||||
async function requestInputInteractively(ctx: any, metadata: RequestInputMetadata) {
|
||||
ctx.stdout.write(`${metadata.prompt}\n> `);
|
||||
const { readLineFromStream } = await import("./read_line.js");
|
||||
const raw = await readLineFromStream(ctx.stdin, { timeoutMs: 0 });
|
||||
const raw = await readLineFromStream(ctx.stdin, { timeoutMs: 0, signal: ctx.signal });
|
||||
let response;
|
||||
try {
|
||||
response = JSON.parse(String(raw ?? "").trim());
|
||||
|
||||
+300
-73
@@ -3,9 +3,13 @@ import { randomUUID } from "node:crypto";
|
||||
import { encodeToken } from "./token.js";
|
||||
import {
|
||||
cleanupApprovalIndexByStateKey,
|
||||
consumeResumeState,
|
||||
createApprovalIndex,
|
||||
deleteResumeStateWithRollback,
|
||||
deleteStateJson,
|
||||
readStateJson,
|
||||
isConsumedResumeState,
|
||||
readStateJsonWithLock,
|
||||
restoreConsumedResumeState,
|
||||
writeStateJson,
|
||||
} from "./state/store.js";
|
||||
import { compileCached } from "./validation.js";
|
||||
@@ -20,6 +24,7 @@ export type PipelineResumeState = {
|
||||
inputSchema?: unknown;
|
||||
prompt?: string;
|
||||
commandInput?: CommandInputState;
|
||||
supersededResumeStateKeys?: string[];
|
||||
createdAt: string;
|
||||
};
|
||||
|
||||
@@ -44,6 +49,7 @@ export type PipelineRunOutput = {
|
||||
items: unknown[];
|
||||
halted?: boolean;
|
||||
haltedAt?: { index: number } | null;
|
||||
executionStarted?: boolean;
|
||||
};
|
||||
|
||||
export type PipelineToolRunResolution =
|
||||
@@ -97,90 +103,163 @@ export async function finalizePipelineToolRun(params: {
|
||||
pipeline: PipelineResumeState["pipeline"];
|
||||
output: PipelineRunOutput;
|
||||
previousStateKey?: string;
|
||||
previousState?: PipelineResumeState;
|
||||
previousStateConsumed?: boolean;
|
||||
restorePreviousStateOnAbort?: boolean;
|
||||
onPreviousStateRestored?: () => void;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<PipelineToolRunResolution> {
|
||||
params.signal?.throwIfAborted();
|
||||
const { approval, inputRequest } = extractPipelineHalt(params.output);
|
||||
if (approval) {
|
||||
const nextStateKey = await savePipelineResumeState(params.env, {
|
||||
pipeline: params.pipeline,
|
||||
resumeAtIndex: (params.output.haltedAt?.index ?? -1) + 1,
|
||||
items: approval.items,
|
||||
haltType: "approval_request",
|
||||
prompt: approval.prompt,
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
if (params.previousStateKey) {
|
||||
await cleanupApprovalIndexByStateKey({ env: params.env, stateKey: params.previousStateKey });
|
||||
await deleteStateJson({ env: params.env, key: params.previousStateKey });
|
||||
}
|
||||
let approvalId: string | null;
|
||||
let nextStateKey: string | undefined;
|
||||
try {
|
||||
nextStateKey = await savePipelineResumeState(
|
||||
params.env,
|
||||
{
|
||||
pipeline: params.pipeline,
|
||||
resumeAtIndex: (params.output.haltedAt?.index ?? -1) + 1,
|
||||
items: approval.items,
|
||||
haltType: "approval_request",
|
||||
prompt: approval.prompt,
|
||||
supersededResumeStateKeys: collectSupersededPipelineResumeStateKeys(
|
||||
params.previousStateKey,
|
||||
params.previousState,
|
||||
),
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
params.signal,
|
||||
);
|
||||
let approvalId: string | null;
|
||||
approvalId = await createApprovalIndex({ env: params.env, stateKey: nextStateKey });
|
||||
const replaced = await replacePipelineResumeState({
|
||||
env: params.env,
|
||||
previousStateKey: params.previousStateKey,
|
||||
expectedPreviousState: params.previousState,
|
||||
previousStateConsumed: params.previousStateConsumed,
|
||||
replacementStateKey: nextStateKey,
|
||||
signal: params.signal,
|
||||
});
|
||||
if (!replaced) throw new Error("Pipeline resume state not found");
|
||||
const resumeToken = encodeToken({
|
||||
protocolVersion: 1,
|
||||
v: 1,
|
||||
kind: "pipeline-resume",
|
||||
stateKey: nextStateKey,
|
||||
});
|
||||
await retirePreviousPipelineApprovalIndex(params.env, params.previousStateKey, nextStateKey);
|
||||
return {
|
||||
status: "needs_approval",
|
||||
output: [],
|
||||
requiresApproval: {
|
||||
...approval,
|
||||
resumeToken,
|
||||
...(approvalId ? { approvalId } : null),
|
||||
},
|
||||
requiresInput: null,
|
||||
};
|
||||
} catch (err) {
|
||||
await deleteStateJson({ env: params.env, key: nextStateKey }).catch(() => {});
|
||||
try {
|
||||
if (!params.previousStateConsumed) await restorePreviousPipelineResumeState(params);
|
||||
} finally {
|
||||
if (nextStateKey) await discardPipelineResumeState(params.env, nextStateKey);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const resumeToken = encodeToken({
|
||||
protocolVersion: 1,
|
||||
v: 1,
|
||||
kind: "pipeline-resume",
|
||||
stateKey: nextStateKey,
|
||||
});
|
||||
return {
|
||||
status: "needs_approval",
|
||||
output: [],
|
||||
requiresApproval: {
|
||||
...approval,
|
||||
resumeToken,
|
||||
...(approvalId ? { approvalId } : null),
|
||||
},
|
||||
requiresInput: null,
|
||||
};
|
||||
}
|
||||
|
||||
if (inputRequest) {
|
||||
const resumeMode = inputRequest.commandInput ? "same_stage" : "next_stage";
|
||||
const nextStateKey = await savePipelineResumeState(params.env, {
|
||||
pipeline: params.pipeline,
|
||||
resumeAtIndex:
|
||||
resumeMode === "same_stage"
|
||||
? (params.output.haltedAt?.index ?? -1)
|
||||
: (params.output.haltedAt?.index ?? -1) + 1,
|
||||
items: resumeMode === "same_stage" ? (inputRequest.items ?? []) : [],
|
||||
haltType: "input_request",
|
||||
resumeMode,
|
||||
inputSchema: inputRequest.responseSchema,
|
||||
prompt: inputRequest.prompt,
|
||||
...(inputRequest.commandInput ? { commandInput: inputRequest.commandInput } : null),
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
if (params.previousStateKey) {
|
||||
await cleanupApprovalIndexByStateKey({ env: params.env, stateKey: params.previousStateKey });
|
||||
await deleteStateJson({ env: params.env, key: params.previousStateKey });
|
||||
let nextStateKey: string | undefined;
|
||||
try {
|
||||
nextStateKey = await savePipelineResumeState(
|
||||
params.env,
|
||||
{
|
||||
pipeline: params.pipeline,
|
||||
resumeAtIndex:
|
||||
resumeMode === "same_stage"
|
||||
? (params.output.haltedAt?.index ?? -1)
|
||||
: (params.output.haltedAt?.index ?? -1) + 1,
|
||||
items: resumeMode === "same_stage" ? (inputRequest.items ?? []) : [],
|
||||
haltType: "input_request",
|
||||
resumeMode,
|
||||
inputSchema: inputRequest.responseSchema,
|
||||
prompt: inputRequest.prompt,
|
||||
...(inputRequest.commandInput ? { commandInput: inputRequest.commandInput } : null),
|
||||
supersededResumeStateKeys: collectSupersededPipelineResumeStateKeys(
|
||||
params.previousStateKey,
|
||||
params.previousState,
|
||||
),
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
params.signal,
|
||||
);
|
||||
const replaced = await replacePipelineResumeState({
|
||||
env: params.env,
|
||||
previousStateKey: params.previousStateKey,
|
||||
expectedPreviousState: params.previousState,
|
||||
previousStateConsumed: params.previousStateConsumed,
|
||||
replacementStateKey: nextStateKey,
|
||||
signal: params.signal,
|
||||
});
|
||||
if (!replaced) throw new Error("Pipeline resume state not found");
|
||||
const resumeToken = encodeToken({
|
||||
protocolVersion: 1,
|
||||
v: 1,
|
||||
kind: "pipeline-resume",
|
||||
stateKey: nextStateKey,
|
||||
});
|
||||
await retirePreviousPipelineApprovalIndex(params.env, params.previousStateKey, nextStateKey);
|
||||
return {
|
||||
status: "needs_input",
|
||||
output: [],
|
||||
requiresApproval: null,
|
||||
requiresInput: {
|
||||
type: "input_request",
|
||||
prompt: inputRequest.prompt,
|
||||
responseSchema: inputRequest.responseSchema,
|
||||
...(inputRequest.defaults !== undefined ? { defaults: inputRequest.defaults } : null),
|
||||
...(inputRequest.subject !== undefined ? { subject: inputRequest.subject } : null),
|
||||
resumeToken,
|
||||
},
|
||||
};
|
||||
} catch (err) {
|
||||
try {
|
||||
if (!params.previousStateConsumed) await restorePreviousPipelineResumeState(params);
|
||||
} finally {
|
||||
if (nextStateKey) await discardPipelineResumeState(params.env, nextStateKey);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const resumeToken = encodeToken({
|
||||
protocolVersion: 1,
|
||||
v: 1,
|
||||
kind: "pipeline-resume",
|
||||
stateKey: nextStateKey,
|
||||
});
|
||||
return {
|
||||
status: "needs_input",
|
||||
output: [],
|
||||
requiresApproval: null,
|
||||
requiresInput: {
|
||||
type: "input_request",
|
||||
prompt: inputRequest.prompt,
|
||||
responseSchema: inputRequest.responseSchema,
|
||||
...(inputRequest.defaults !== undefined ? { defaults: inputRequest.defaults } : null),
|
||||
...(inputRequest.subject !== undefined ? { subject: inputRequest.subject } : null),
|
||||
resumeToken,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
params.signal?.throwIfAborted();
|
||||
if (params.previousStateKey) {
|
||||
await cleanupApprovalIndexByStateKey({ env: params.env, stateKey: params.previousStateKey });
|
||||
await deleteStateJson({ env: params.env, key: params.previousStateKey });
|
||||
try {
|
||||
if (params.previousStateConsumed) {
|
||||
await deleteStateJson({
|
||||
env: params.env,
|
||||
key: params.previousStateKey,
|
||||
signal: params.signal,
|
||||
});
|
||||
params.signal?.throwIfAborted();
|
||||
} else {
|
||||
const deleted = await deleteResumeStateWithRollback({
|
||||
env: params.env,
|
||||
key: params.previousStateKey,
|
||||
expectedState: params.previousState,
|
||||
signal: params.signal,
|
||||
});
|
||||
if (!deleted) throw new Error("Pipeline resume state not found");
|
||||
}
|
||||
await cleanupSupersededPipelineResumeStates(
|
||||
params.env,
|
||||
params.previousState?.supersededResumeStateKeys,
|
||||
);
|
||||
} catch (err) {
|
||||
if (!params.previousStateConsumed) await restorePreviousPipelineResumeState(params);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
return {
|
||||
status: "ok",
|
||||
@@ -193,18 +272,159 @@ export async function finalizePipelineToolRun(params: {
|
||||
export async function savePipelineResumeState(
|
||||
env: Record<string, string | undefined>,
|
||||
state: PipelineResumeState,
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
const stateKey = `pipeline_resume_${randomUUID()}`;
|
||||
await writeStateJson({ env, key: stateKey, value: state });
|
||||
return stateKey;
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
await writeStateJson({ env, key: stateKey, value: state, signal });
|
||||
signal?.throwIfAborted();
|
||||
return stateKey;
|
||||
} catch (err) {
|
||||
if (signal?.aborted) await discardPipelineResumeState(env, stateKey);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function replacePipelineResumeState({
|
||||
env,
|
||||
previousStateKey,
|
||||
expectedPreviousState,
|
||||
previousStateConsumed,
|
||||
replacementStateKey,
|
||||
signal,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
previousStateKey?: string;
|
||||
expectedPreviousState?: PipelineResumeState;
|
||||
previousStateConsumed?: boolean;
|
||||
replacementStateKey: string;
|
||||
signal?: AbortSignal;
|
||||
}) {
|
||||
if (!previousStateKey || previousStateKey === replacementStateKey) {
|
||||
signal?.throwIfAborted();
|
||||
return true;
|
||||
}
|
||||
// The current resume has already crossed an unsafe boundary and owns the
|
||||
// predecessor's consumed marker. It may safely publish the next gate, but
|
||||
// must retain that marker rather than attempting a stale snapshot CAS.
|
||||
if (previousStateConsumed) {
|
||||
signal?.throwIfAborted();
|
||||
return true;
|
||||
}
|
||||
if (!expectedPreviousState) return false;
|
||||
|
||||
let claimId: string | undefined;
|
||||
try {
|
||||
const consumption = await consumeResumeState({
|
||||
env,
|
||||
key: previousStateKey,
|
||||
expectedState: expectedPreviousState,
|
||||
signal,
|
||||
});
|
||||
if (!consumption.consumed) return false;
|
||||
claimId = consumption.claimId;
|
||||
// The predecessor remains as a durable tombstone until terminal cleanup.
|
||||
// A concurrent caller can therefore never turn the same approval into a
|
||||
// second successor capability.
|
||||
signal?.throwIfAborted();
|
||||
return true;
|
||||
} catch (err) {
|
||||
// A cancellation after the atomic marker publication has not exposed the
|
||||
// successor token yet. Restore only the marker created by this caller so a
|
||||
// competing transition can never be overwritten.
|
||||
if (claimId && signal?.aborted) {
|
||||
await restoreConsumedResumeState({
|
||||
env,
|
||||
key: previousStateKey,
|
||||
expectedState: expectedPreviousState,
|
||||
claimId,
|
||||
}).catch(() => {});
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function restorePreviousPipelineResumeState({
|
||||
env,
|
||||
previousStateKey,
|
||||
previousState,
|
||||
restorePreviousStateOnAbort,
|
||||
onPreviousStateRestored,
|
||||
signal,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
previousStateKey?: string;
|
||||
previousState?: PipelineResumeState;
|
||||
restorePreviousStateOnAbort?: boolean;
|
||||
onPreviousStateRestored?: () => void;
|
||||
signal?: AbortSignal;
|
||||
}) {
|
||||
if (!signal?.aborted || !restorePreviousStateOnAbort || !previousStateKey || !previousState) {
|
||||
return;
|
||||
}
|
||||
// Safe terminal cleanup restores its own claimed marker while still holding
|
||||
// the state lock. Never recreate a missing snapshot here: this caller may
|
||||
// have only observed it before another resume completed.
|
||||
if ((await readStateJsonWithLock({ env, key: previousStateKey, signal })) !== null) {
|
||||
onPreviousStateRestored?.();
|
||||
}
|
||||
}
|
||||
|
||||
async function retirePreviousPipelineApprovalIndex(
|
||||
env: Record<string, string | undefined>,
|
||||
previousStateKey: string | undefined,
|
||||
replacementStateKey: string,
|
||||
) {
|
||||
if (!previousStateKey || previousStateKey === replacementStateKey) return;
|
||||
// This runs only after replacement deletion has passed its cancellation
|
||||
// checkpoint. Do not add a later cancellation check: the transition is
|
||||
// committed once the old approval capability is retired.
|
||||
await cleanupApprovalIndexByStateKey({ env, stateKey: previousStateKey }).catch(() => {});
|
||||
}
|
||||
|
||||
function collectSupersededPipelineResumeStateKeys(
|
||||
previousStateKey: string | undefined,
|
||||
previousState: PipelineResumeState | undefined,
|
||||
) {
|
||||
return [
|
||||
...(previousState?.supersededResumeStateKeys ?? []),
|
||||
...(previousStateKey ? [previousStateKey] : []),
|
||||
].filter((stateKey, index, all) => stateKey && all.indexOf(stateKey) === index);
|
||||
}
|
||||
|
||||
async function cleanupSupersededPipelineResumeStates(
|
||||
env: Record<string, string | undefined>,
|
||||
stateKeys: string[] | undefined,
|
||||
) {
|
||||
for (const stateKey of stateKeys ?? []) {
|
||||
try {
|
||||
// Retire only a non-executable marker after the successor itself has
|
||||
// committed. A restored state must remain available for retry.
|
||||
if (isConsumedResumeState(await readStateJsonWithLock({ env, key: stateKey }))) {
|
||||
await deleteStateJson({ env, key: stateKey });
|
||||
}
|
||||
} catch {
|
||||
// Leaving a tombstone is safe if best-effort cleanup cannot complete.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function discardPipelineResumeState(
|
||||
env: Record<string, string | undefined>,
|
||||
stateKey: string,
|
||||
) {
|
||||
await cleanupApprovalIndexByStateKey({ env, stateKey }).catch(() => {});
|
||||
await deleteStateJson({ env, key: stateKey }).catch(() => {});
|
||||
}
|
||||
|
||||
export async function loadPipelineResumeState(
|
||||
env: Record<string, string | undefined>,
|
||||
stateKey: string,
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
const stored = await readStateJson({ env, key: stateKey });
|
||||
if (!stored || typeof stored !== "object") {
|
||||
const stored = await readStateJsonWithLock({ env, key: stateKey, signal });
|
||||
if (!stored || typeof stored !== "object" || isConsumedResumeState(stored)) {
|
||||
throw new Error("Pipeline resume state not found");
|
||||
}
|
||||
const data = stored as Partial<PipelineResumeState>;
|
||||
@@ -219,6 +439,13 @@ export async function loadPipelineResumeState(
|
||||
throw new Error("Invalid pipeline resume state");
|
||||
}
|
||||
if (!Array.isArray(data.items)) throw new Error("Invalid pipeline resume state");
|
||||
if (
|
||||
data.supersededResumeStateKeys !== undefined &&
|
||||
(!Array.isArray(data.supersededResumeStateKeys) ||
|
||||
data.supersededResumeStateKeys.some((stateKey) => typeof stateKey !== "string"))
|
||||
) {
|
||||
throw new Error("Invalid pipeline resume state");
|
||||
}
|
||||
if (
|
||||
data.haltType !== undefined &&
|
||||
!["approval_request", "input_request"].includes(data.haltType)
|
||||
|
||||
+60
-9
@@ -1,9 +1,22 @@
|
||||
export function readLineFromStream(stream: NodeJS.ReadableStream, opts?: { timeoutMs?: number }) {
|
||||
const unreadInput = new WeakMap<NodeJS.ReadableStream, string>();
|
||||
type ObservableReadableStream = NodeJS.ReadableStream & {
|
||||
readableEnded?: boolean;
|
||||
destroyed?: boolean;
|
||||
closed?: boolean;
|
||||
};
|
||||
|
||||
export function readLineFromStream(
|
||||
stream: NodeJS.ReadableStream,
|
||||
opts?: { timeoutMs?: number; signal?: AbortSignal },
|
||||
) {
|
||||
const timeoutMs = Number(opts?.timeoutMs ?? 0);
|
||||
const signal = opts?.signal;
|
||||
const observableStream = stream as ObservableReadableStream;
|
||||
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
let settled = false;
|
||||
let buf = "";
|
||||
let buf = unreadInput.get(stream) ?? "";
|
||||
unreadInput.delete(stream);
|
||||
let timer: NodeJS.Timeout | null = null;
|
||||
|
||||
const cleanup = () => {
|
||||
@@ -11,6 +24,8 @@ export function readLineFromStream(stream: NodeJS.ReadableStream, opts?: { timeo
|
||||
stream.off("end", onEnd);
|
||||
stream.off("close", onClose);
|
||||
stream.off("error", onError);
|
||||
stream.pause();
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
|
||||
@@ -24,21 +39,50 @@ export function readLineFromStream(stream: NodeJS.ReadableStream, opts?: { timeo
|
||||
const fail = (err: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
unreadInput.delete(stream);
|
||||
cleanup();
|
||||
reject(err);
|
||||
};
|
||||
|
||||
const onData = (chunk: Buffer | string) => {
|
||||
buf += Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk);
|
||||
const consumeLine = () => {
|
||||
const idx = buf.indexOf("\n");
|
||||
if (idx !== -1) {
|
||||
finish(buf.slice(0, idx));
|
||||
}
|
||||
if (idx === -1) return false;
|
||||
unreadInput.set(stream, buf.slice(idx + 1));
|
||||
finish(buf.slice(0, idx));
|
||||
return true;
|
||||
};
|
||||
|
||||
const onEnd = () => finish(buf);
|
||||
const onClose = () => finish(buf);
|
||||
const onData = (chunk: Buffer | string) => {
|
||||
buf += Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk);
|
||||
consumeLine();
|
||||
};
|
||||
const drainBuffered = () => {
|
||||
let chunk: Buffer | string | null;
|
||||
while (!settled && (chunk = stream.read()) !== null) onData(chunk);
|
||||
};
|
||||
|
||||
const onEnd = () => {
|
||||
drainBuffered();
|
||||
if (settled) return;
|
||||
unreadInput.delete(stream);
|
||||
finish(buf);
|
||||
};
|
||||
const onClose = () => {
|
||||
drainBuffered();
|
||||
if (settled) return;
|
||||
unreadInput.delete(stream);
|
||||
finish(buf);
|
||||
};
|
||||
const onError = (err: Error) => fail(err);
|
||||
const onAbort = () => {
|
||||
const reason = signal?.reason;
|
||||
fail(reason instanceof Error ? reason : new Error("Input read aborted"));
|
||||
};
|
||||
|
||||
if (signal?.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
|
||||
if (timeoutMs > 0) {
|
||||
timer = setTimeout(() => {
|
||||
@@ -50,5 +94,12 @@ export function readLineFromStream(stream: NodeJS.ReadableStream, opts?: { timeo
|
||||
stream.on("end", onEnd);
|
||||
stream.on("close", onClose);
|
||||
stream.on("error", onError);
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
drainBuffered();
|
||||
if (!settled && !consumeLine()) {
|
||||
if (observableStream.readableEnded || observableStream.destroyed || observableStream.closed)
|
||||
onEnd();
|
||||
else stream.resume();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
+167
-22
@@ -20,9 +20,12 @@ export async function runPipeline({
|
||||
cwd = undefined,
|
||||
llmAdapters = undefined,
|
||||
signal = undefined,
|
||||
forceTerminationSignal = undefined,
|
||||
haltAfterStageOnAbort = false,
|
||||
dryRun = false,
|
||||
requestInputResume = undefined,
|
||||
requestInputEnabled = true,
|
||||
onExecutionStart = undefined,
|
||||
}: {
|
||||
pipeline: any[];
|
||||
registry: any;
|
||||
@@ -35,9 +38,12 @@ export async function runPipeline({
|
||||
cwd?: string | undefined;
|
||||
llmAdapters?: Record<string, any> | undefined;
|
||||
signal?: AbortSignal | undefined;
|
||||
forceTerminationSignal?: AbortSignal | undefined;
|
||||
haltAfterStageOnAbort?: boolean;
|
||||
dryRun?: boolean;
|
||||
requestInputResume?: CommandInputResume | undefined;
|
||||
requestInputEnabled?: boolean;
|
||||
onExecutionStart?: (() => void | Promise<void>) | undefined;
|
||||
}) {
|
||||
if (dryRun) {
|
||||
return dryRunPipeline({ pipeline, registry, stderr });
|
||||
@@ -48,6 +54,16 @@ export async function runPipeline({
|
||||
let halted = false;
|
||||
let haltedAt = null;
|
||||
let pipelineOutputStarted = false;
|
||||
let executionStarted = false;
|
||||
let executionStart: Promise<void> | undefined;
|
||||
const markExecutionStarted = async () => {
|
||||
if (executionStart) return executionStart;
|
||||
executionStart = (async () => {
|
||||
await onExecutionStart?.();
|
||||
executionStarted = true;
|
||||
})();
|
||||
return executionStart;
|
||||
};
|
||||
|
||||
const baseCtx = {
|
||||
stdin,
|
||||
@@ -59,14 +75,19 @@ export async function runPipeline({
|
||||
cwd,
|
||||
llmAdapters,
|
||||
signal,
|
||||
forceTerminationSignal,
|
||||
};
|
||||
|
||||
for (let idx = 0; idx < pipeline.length; idx++) {
|
||||
if (haltAfterStageOnAbort) signal?.throwIfAborted();
|
||||
const stage = pipeline[idx];
|
||||
const command = registry.get(stage.name);
|
||||
if (!command) {
|
||||
throw new Error(`Unknown command: ${stage.name}`);
|
||||
}
|
||||
if (command.meta?.resumeSafeBeforeInput !== true) {
|
||||
await markExecutionStarted();
|
||||
}
|
||||
|
||||
const inputTracker = createInputTracker(stream);
|
||||
const stageResume = idx === 0 ? requestInputResume : undefined;
|
||||
@@ -102,6 +123,8 @@ export async function runPipeline({
|
||||
getInactiveReason: () => inactiveReason,
|
||||
isOutputStarted: () => pipelineOutputStarted || commandOutputStarted,
|
||||
resume: stageResume,
|
||||
onResumedInput:
|
||||
command.meta?.resumeSafeAfterInput === true ? undefined : markExecutionStarted,
|
||||
})
|
||||
: createUnsupportedRequestInput(),
|
||||
};
|
||||
@@ -120,15 +143,17 @@ export async function runPipeline({
|
||||
rendered = true;
|
||||
}
|
||||
|
||||
const terminalOutput = Boolean(result?.halt);
|
||||
let stageHalted = Boolean(terminalOutput || (haltAfterStageOnAbort && signal?.aborted));
|
||||
const output = result?.output;
|
||||
if (Array.isArray(output)) {
|
||||
stream = output;
|
||||
await finishStage();
|
||||
} else if (output && !result?.halt && idx < pipeline.length - 1) {
|
||||
} else if (output && idx < pipeline.length - 1 && !terminalOutput) {
|
||||
commandActive = false;
|
||||
inactiveReason = "requestInput cannot suspend from lazy output before downstream stages";
|
||||
assertRequestInputResumeConsumed(stageResume);
|
||||
stream = trackCommandOutput(
|
||||
const trackedOutput = trackCommandOutput(
|
||||
output,
|
||||
() => {
|
||||
commandOutputStarted = true;
|
||||
@@ -137,22 +162,34 @@ export async function runPipeline({
|
||||
(err) => assertNoUnconsumedResumeAfterError(stageResume, err),
|
||||
finishStage,
|
||||
);
|
||||
stream = haltAfterStageOnAbort
|
||||
? throwIfAbortedAfterDrain(trackedOutput, signal)
|
||||
: trackedOutput;
|
||||
} else {
|
||||
stream = output
|
||||
? trackCommandOutput(
|
||||
output,
|
||||
() => {
|
||||
commandOutputStarted = true;
|
||||
},
|
||||
() => assertRequestInputResumeConsumed(stageResume),
|
||||
(err) => assertNoUnconsumedResumeAfterError(stageResume, err),
|
||||
finishStage,
|
||||
)
|
||||
: [];
|
||||
if (!output) await finishStage();
|
||||
if (output) {
|
||||
const trackedOutput = trackCommandOutput(
|
||||
output,
|
||||
() => {
|
||||
commandOutputStarted = true;
|
||||
},
|
||||
() => assertRequestInputResumeConsumed(stageResume),
|
||||
(err) => assertNoUnconsumedResumeAfterError(stageResume, err),
|
||||
finishStage,
|
||||
);
|
||||
// Terminal output is drained after the stage loop as well. It needs the
|
||||
// same abort-aware read as a handoff, otherwise a stalled final iterator
|
||||
// can prevent the tool cancellation from settling forever.
|
||||
stream = haltAfterStageOnAbort
|
||||
? throwIfAbortedAfterDrain(trackedOutput, signal)
|
||||
: trackedOutput;
|
||||
} else {
|
||||
stream = [];
|
||||
await finishStage();
|
||||
}
|
||||
}
|
||||
|
||||
if (result?.halt) {
|
||||
stageHalted ||= Boolean(haltAfterStageOnAbort && signal?.aborted);
|
||||
if (stageHalted) {
|
||||
halted = true;
|
||||
haltedAt = { index: idx, stage };
|
||||
break;
|
||||
@@ -170,9 +207,10 @@ export async function runPipeline({
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
if (haltAfterStageOnAbort) signal?.throwIfAborted();
|
||||
assertRequestInputResumeConsumed(requestInputResume);
|
||||
|
||||
return { items, rendered, halted, haltedAt };
|
||||
return { items, rendered, halted, haltedAt, executionStarted };
|
||||
|
||||
function haltForInputRequest(err: unknown) {
|
||||
if (!(err instanceof InputRequestSuspension)) return false;
|
||||
@@ -214,7 +252,7 @@ function dryRunPipeline({
|
||||
lines.push("");
|
||||
stderr.write(lines.join("\n"));
|
||||
// Return rendered:true so the CLI does not print an empty JSON array to stdout.
|
||||
return { items: [], rendered: true, halted: false, haltedAt: null };
|
||||
return { items: [], rendered: true, halted: false, haltedAt: null, executionStarted: false };
|
||||
}
|
||||
|
||||
function formatStageArgs(args: Record<string, unknown>) {
|
||||
@@ -238,6 +276,82 @@ function streamFromItems(items: unknown[]) {
|
||||
})();
|
||||
}
|
||||
|
||||
function throwIfAbortedAfterDrain(input: AsyncIterable<unknown>, signal?: AbortSignal) {
|
||||
return (async function* () {
|
||||
const iterator = input[Symbol.asyncIterator]();
|
||||
let completed = false;
|
||||
try {
|
||||
while (true) {
|
||||
const next = await nextWithAbort(iterator, signal);
|
||||
if (next.done) {
|
||||
completed = true;
|
||||
break;
|
||||
}
|
||||
signal?.throwIfAborted();
|
||||
yield next.value;
|
||||
}
|
||||
signal?.throwIfAborted();
|
||||
} finally {
|
||||
if (!completed) await closeAfterAbortedRead(input, iterator, signal);
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
type CancellableLazyOutput = {
|
||||
abort?: (reason?: unknown) => void | Promise<void>;
|
||||
};
|
||||
|
||||
async function closeAfterAbortedRead(
|
||||
input: AsyncIterable<unknown>,
|
||||
iterator: AsyncIterator<unknown>,
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
const cancellable = iterator as AsyncIterator<unknown> & CancellableLazyOutput;
|
||||
const inputCancellable = input as AsyncIterable<unknown> & CancellableLazyOutput;
|
||||
const abort = cancellable.abort ?? inputCancellable.abort;
|
||||
try {
|
||||
// A source that owns a pending timer, socket, or process can expose this
|
||||
// small cancellation hook. It must release the pending next() operation.
|
||||
if (signal?.aborted && abort) await abort(signal.reason);
|
||||
if (typeof iterator.return !== "function") return;
|
||||
const close = iterator.return();
|
||||
if (signal?.aborted && !abort) {
|
||||
// Legacy iterators cannot interrupt an in-flight next(). Keep the
|
||||
// existing prompt cancellation behavior, while resource-owning sources
|
||||
// opt into the abort hook above so their cleanup is awaited.
|
||||
void Promise.resolve(close).catch(() => {});
|
||||
return;
|
||||
}
|
||||
await close;
|
||||
} catch (err) {
|
||||
if (!signal?.aborted) throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function nextWithAbort(iterator: AsyncIterator<unknown>, signal?: AbortSignal) {
|
||||
if (!signal) return iterator.next();
|
||||
signal.throwIfAborted();
|
||||
|
||||
let onAbort!: () => void;
|
||||
const aborted = new Promise<never>((_resolve, reject) => {
|
||||
onAbort = () => {
|
||||
try {
|
||||
signal.throwIfAborted();
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
};
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
if (signal.aborted) onAbort();
|
||||
|
||||
try {
|
||||
return await Promise.race([iterator.next(), aborted]);
|
||||
} finally {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
}
|
||||
}
|
||||
|
||||
function trackCommandOutput(
|
||||
output: AsyncIterable<unknown> | Iterable<unknown>,
|
||||
markOutput: () => void,
|
||||
@@ -248,13 +362,27 @@ function trackCommandOutput(
|
||||
suppressCloseErrors?: boolean;
|
||||
}) => Promise<void>,
|
||||
) {
|
||||
return (async function* () {
|
||||
const source = output as AsyncIterable<unknown> & CancellableLazyOutput & AsyncIterator<unknown>;
|
||||
let sourceIterator: AsyncIterator<unknown> | undefined;
|
||||
const tracked = (async function* () {
|
||||
let completed = false;
|
||||
try {
|
||||
for await (const item of output) {
|
||||
assertResumeConsumed();
|
||||
markOutput();
|
||||
yield item;
|
||||
if (typeof source[Symbol.asyncIterator] === "function") {
|
||||
sourceIterator = source[Symbol.asyncIterator]();
|
||||
const iteratorInput: AsyncIterable<unknown> = {
|
||||
[Symbol.asyncIterator]: () => sourceIterator!,
|
||||
};
|
||||
for await (const item of iteratorInput) {
|
||||
assertResumeConsumed();
|
||||
markOutput();
|
||||
yield item;
|
||||
}
|
||||
} else {
|
||||
for (const item of output as Iterable<unknown>) {
|
||||
assertResumeConsumed();
|
||||
markOutput();
|
||||
yield item;
|
||||
}
|
||||
}
|
||||
completed = true;
|
||||
} catch (err) {
|
||||
@@ -265,6 +393,23 @@ function trackCommandOutput(
|
||||
await finishStage({ assertResume: completed });
|
||||
}
|
||||
})();
|
||||
// Iterator-owned abort hooks are only discoverable after iterator acquisition.
|
||||
// A getter lets the abort-aware wrapper find that hook once a read is pending,
|
||||
// without performing acquisition outside the generator's guarded cleanup path.
|
||||
Object.defineProperty(tracked, "abort", {
|
||||
configurable: true,
|
||||
get() {
|
||||
const cancellationOwner =
|
||||
sourceIterator && typeof (sourceIterator as CancellableLazyOutput).abort === "function"
|
||||
? (sourceIterator as CancellableLazyOutput)
|
||||
: source;
|
||||
const abort = cancellationOwner.abort;
|
||||
return typeof abort === "function"
|
||||
? (reason?: unknown) => abort.call(cancellationOwner, reason)
|
||||
: undefined;
|
||||
},
|
||||
});
|
||||
return tracked;
|
||||
}
|
||||
|
||||
function assertNoUnconsumedResumeAfterError(resume: CommandInputResume | undefined, err: unknown) {
|
||||
|
||||
+6
-2
@@ -3,7 +3,7 @@ import { runPipelineInternal } from "./runtime.js";
|
||||
import { encodeToken, decodeToken } from "./token.js";
|
||||
import { compileCached } from "../validation.js";
|
||||
import { validateCommandInputState, type CommandInputState } from "../input_request.js";
|
||||
import { deleteStateJson, readStateJson, writeStateJson } from "../state/store.js";
|
||||
import { deleteStateJson, readStateJsonWithLock, writeStateJson } from "../state/store.js";
|
||||
|
||||
type SdkResumePayload = {
|
||||
protocolVersion: 1;
|
||||
@@ -449,7 +449,11 @@ async function loadSdkCommandInputResumeState(
|
||||
options: any,
|
||||
stateKey: string,
|
||||
): Promise<SdkCommandInputResumeState> {
|
||||
const stored = await readStateJson({ env: sdkStateEnv(options), key: stateKey });
|
||||
const stored = await readStateJsonWithLock({
|
||||
env: sdkStateEnv(options),
|
||||
key: stateKey,
|
||||
signal: options?.signal,
|
||||
});
|
||||
if (!stored || typeof stored !== "object") throw new Error("SDK resume state not found");
|
||||
const data = stored as Partial<SdkCommandInputResumeState>;
|
||||
if (
|
||||
|
||||
+14
-93
@@ -14,56 +14,12 @@
|
||||
* });
|
||||
*/
|
||||
|
||||
import { promises as fsp } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { ensureDirectory, isJsonSyntaxError, writeFileAtomic } from "../../state/store.js";
|
||||
import { diffAndStore } from "../../state/store.js";
|
||||
|
||||
/**
|
||||
* Get the state directory
|
||||
* @param {Object} ctx
|
||||
* @returns {string}
|
||||
*/
|
||||
function getStateDir(ctx) {
|
||||
return (
|
||||
ctx?.stateDir ||
|
||||
(ctx?.env?.LOBSTER_STATE_DIR && String(ctx.env.LOBSTER_STATE_DIR).trim()) ||
|
||||
path.join(os.homedir(), ".lobster", "state")
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a key to a safe file path
|
||||
* @param {string} stateDir
|
||||
* @param {string} key
|
||||
* @returns {string}
|
||||
*/
|
||||
function keyToPath(stateDir, key) {
|
||||
const safe = String(key)
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9._-]+/g, "_")
|
||||
.replace(/_+/g, "_")
|
||||
.replace(/^_+|_+$/g, "");
|
||||
if (!safe) throw new Error("state key is empty/invalid");
|
||||
return path.join(stateDir, `${safe}.json`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable JSON stringify for comparison
|
||||
* @param {any} value
|
||||
* @returns {string}
|
||||
*/
|
||||
function stableStringify(value) {
|
||||
return JSON.stringify(value, (_k, v) => {
|
||||
if (v && typeof v === "object" && !Array.isArray(v)) {
|
||||
return Object.fromEntries(
|
||||
Object.keys(v)
|
||||
.sort()
|
||||
.map((k) => [k, v[k]]),
|
||||
);
|
||||
}
|
||||
return v;
|
||||
});
|
||||
function stateEnv(ctx) {
|
||||
return ctx?.stateDir
|
||||
? { ...(ctx?.env ?? process.env), LOBSTER_STATE_DIR: ctx.stateDir }
|
||||
: (ctx?.env ?? process.env);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -95,26 +51,12 @@ export function diffLast(key, options: any = {}) {
|
||||
|
||||
const value = items.length === 1 ? items[0] : items;
|
||||
|
||||
const stateDir = getStateDir(ctx);
|
||||
const filePath = keyToPath(stateDir, key);
|
||||
|
||||
// Read previous value
|
||||
let before = null;
|
||||
try {
|
||||
const text = await fsp.readFile(filePath, "utf8");
|
||||
before = JSON.parse(text);
|
||||
} catch (err) {
|
||||
if (err?.code !== "ENOENT" && !isJsonSyntaxError(err)) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Compare
|
||||
const changed = stableStringify(before) !== stableStringify(value);
|
||||
|
||||
// Store new value
|
||||
await ensureDirectory(stateDir);
|
||||
await writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n");
|
||||
const { before, after, changed } = await diffAndStore({
|
||||
env: stateEnv(ctx),
|
||||
key,
|
||||
value,
|
||||
signal: ctx?.signal,
|
||||
});
|
||||
|
||||
// Build result
|
||||
const result = {
|
||||
@@ -122,7 +64,7 @@ export function diffLast(key, options: any = {}) {
|
||||
key,
|
||||
changed,
|
||||
before,
|
||||
after: value,
|
||||
after,
|
||||
};
|
||||
|
||||
// If changesOnly and no change, output suppressed marker
|
||||
@@ -150,27 +92,6 @@ export function diffLast(key, options: any = {}) {
|
||||
* @param {Object} [ctx]
|
||||
* @returns {Promise<{before: any, after: any, changed: boolean}>}
|
||||
*/
|
||||
export async function diffAndStoreValue(key, value, ctx = {}) {
|
||||
const stateDir = getStateDir(ctx);
|
||||
const filePath = keyToPath(stateDir, key);
|
||||
|
||||
// Read previous value
|
||||
let before = null;
|
||||
try {
|
||||
const text = await fsp.readFile(filePath, "utf8");
|
||||
before = JSON.parse(text);
|
||||
} catch (err) {
|
||||
if (err?.code !== "ENOENT" && !isJsonSyntaxError(err)) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Compare
|
||||
const changed = stableStringify(before) !== stableStringify(value);
|
||||
|
||||
// Store new value
|
||||
await ensureDirectory(stateDir);
|
||||
await writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n");
|
||||
|
||||
return { before, after: value, changed };
|
||||
export async function diffAndStoreValue(key, value, ctx: any = {}) {
|
||||
return diffAndStore({ env: stateEnv(ctx), key, value, signal: ctx?.signal });
|
||||
}
|
||||
|
||||
+11
-105
@@ -15,77 +15,12 @@
|
||||
* .pipe(stateSet('my-key'));
|
||||
*/
|
||||
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { promises as fsp } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { readStateJsonWithLock, writeStateJson } from "../../state/store.js";
|
||||
|
||||
/**
|
||||
* Write a file atomically (stage to a sibling temp file, fsync, then rename).
|
||||
* `rename(2)` is atomic on a single filesystem, so a concurrent reader or a
|
||||
* crash never observes a truncated/partial file. Plain `fsp.writeFile`
|
||||
* truncates the target up front, leaving a corruption window on SIGKILL/OOM/
|
||||
* power loss. New state files are private by default; existing file modes are
|
||||
* preserved across replacement. Kept local to keep the SDK self-contained.
|
||||
* @param {string} filePath
|
||||
* @param {string} data
|
||||
*/
|
||||
async function writeFileAtomic(filePath, data) {
|
||||
const dir = path.dirname(filePath);
|
||||
const tmpPath = path.join(
|
||||
dir,
|
||||
`.${path.basename(filePath)}.${randomBytes(6).toString("hex")}.tmp`,
|
||||
);
|
||||
let mode = 0o600;
|
||||
let handle;
|
||||
let cleanup = true;
|
||||
try {
|
||||
try {
|
||||
mode = (await fsp.stat(filePath)).mode & 0o777;
|
||||
} catch (err) {
|
||||
if (err?.code !== "ENOENT") throw err;
|
||||
}
|
||||
handle = await fsp.open(tmpPath, "wx", mode);
|
||||
await handle.writeFile(data, "utf8");
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
handle = undefined;
|
||||
await fsp.chmod(tmpPath, mode);
|
||||
await fsp.rename(tmpPath, filePath);
|
||||
cleanup = false;
|
||||
} finally {
|
||||
if (handle) await handle.close().catch(() => {});
|
||||
if (cleanup) await fsp.rm(tmpPath, { force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the state directory
|
||||
* @param {Object} ctx
|
||||
* @returns {string}
|
||||
*/
|
||||
function getStateDir(ctx) {
|
||||
return (
|
||||
ctx?.stateDir ||
|
||||
(ctx?.env?.LOBSTER_STATE_DIR && String(ctx.env.LOBSTER_STATE_DIR).trim()) ||
|
||||
path.join(os.homedir(), ".lobster", "state")
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a key to a safe file path
|
||||
* @param {string} stateDir
|
||||
* @param {string} key
|
||||
* @returns {string}
|
||||
*/
|
||||
function keyToPath(stateDir, key) {
|
||||
const safe = String(key)
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9._-]+/g, "_")
|
||||
.replace(/_+/g, "_")
|
||||
.replace(/^_+|_+$/g, "");
|
||||
if (!safe) throw new Error("state key is empty/invalid");
|
||||
return path.join(stateDir, `${safe}.json`);
|
||||
function stateEnv(ctx) {
|
||||
return ctx?.stateDir
|
||||
? { ...(ctx?.env ?? process.env), LOBSTER_STATE_DIR: ctx.stateDir }
|
||||
: (ctx?.env ?? process.env);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -107,19 +42,7 @@ export function stateGet(key) {
|
||||
// no-op
|
||||
}
|
||||
|
||||
const stateDir = getStateDir(ctx);
|
||||
const filePath = keyToPath(stateDir, key);
|
||||
|
||||
let value = null;
|
||||
try {
|
||||
const text = await fsp.readFile(filePath, "utf8");
|
||||
value = JSON.parse(text);
|
||||
} catch (err) {
|
||||
if (err?.code !== "ENOENT") {
|
||||
throw err;
|
||||
}
|
||||
// File doesn't exist, return null
|
||||
}
|
||||
const value = await readStateJsonWithLock({ env: stateEnv(ctx), key, signal: ctx?.signal });
|
||||
|
||||
return {
|
||||
output: (async function* () {
|
||||
@@ -152,11 +75,7 @@ export function stateSet(key) {
|
||||
|
||||
const value = items.length === 1 ? items[0] : items;
|
||||
|
||||
const stateDir = getStateDir(ctx);
|
||||
const filePath = keyToPath(stateDir, key);
|
||||
|
||||
await fsp.mkdir(stateDir, { recursive: true });
|
||||
await writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n");
|
||||
await writeStateJson({ env: stateEnv(ctx), key, value, signal: ctx?.signal });
|
||||
|
||||
// Pass through the value
|
||||
return {
|
||||
@@ -189,17 +108,8 @@ export const state = {
|
||||
* @param {Object} [ctx]
|
||||
* @returns {Promise<any>}
|
||||
*/
|
||||
export async function readState(key, ctx = {}) {
|
||||
const stateDir = getStateDir(ctx);
|
||||
const filePath = keyToPath(stateDir, key);
|
||||
|
||||
try {
|
||||
const text = await fsp.readFile(filePath, "utf8");
|
||||
return JSON.parse(text);
|
||||
} catch (err) {
|
||||
if (err?.code === "ENOENT") return null;
|
||||
throw err;
|
||||
}
|
||||
export async function readState(key, ctx: any = {}) {
|
||||
return readStateJsonWithLock({ env: stateEnv(ctx), key, signal: ctx?.signal });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -209,10 +119,6 @@ export async function readState(key, ctx = {}) {
|
||||
* @param {Object} [ctx]
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export async function writeState(key, value, ctx = {}) {
|
||||
const stateDir = getStateDir(ctx);
|
||||
const filePath = keyToPath(stateDir, key);
|
||||
|
||||
await fsp.mkdir(stateDir, { recursive: true });
|
||||
await writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n");
|
||||
export async function writeState(key, value, ctx: any = {}) {
|
||||
await writeStateJson({ env: stateEnv(ctx), key, value, signal: ctx?.signal });
|
||||
}
|
||||
|
||||
+703
-13
@@ -3,6 +3,22 @@ import path from "node:path";
|
||||
import { promises as fsp } from "node:fs";
|
||||
import { randomBytes } from "node:crypto";
|
||||
|
||||
const CONSUMED_RESUME_STATE_TYPE = "lobster.consumed-resume-state.v1";
|
||||
|
||||
export type ConsumedResumeState = {
|
||||
type: typeof CONSUMED_RESUME_STATE_TYPE;
|
||||
consumedAt: string;
|
||||
claimId: string;
|
||||
};
|
||||
|
||||
export function isConsumedResumeState(value: unknown): value is ConsumedResumeState {
|
||||
return (
|
||||
value !== null &&
|
||||
typeof value === "object" &&
|
||||
(value as { type?: unknown }).type === CONSUMED_RESUME_STATE_TYPE
|
||||
);
|
||||
}
|
||||
|
||||
export function defaultStateDir(env) {
|
||||
return (
|
||||
(env?.LOBSTER_STATE_DIR && String(env.LOBSTER_STATE_DIR).trim()) ||
|
||||
@@ -36,6 +52,7 @@ export function stableStringify(value) {
|
||||
type AtomicWriteOptions = {
|
||||
renameFile?: typeof fsp.rename;
|
||||
syncParentDir?: (filePath: string) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
type AtomicExclusiveWriteOptions = {
|
||||
@@ -43,6 +60,29 @@ type AtomicExclusiveWriteOptions = {
|
||||
syncParentDir?: (filePath: string) => Promise<void>;
|
||||
};
|
||||
|
||||
type PublishedAtomicWriteError = NodeJS.ErrnoException & {
|
||||
atomicWritePublished?: true;
|
||||
};
|
||||
|
||||
function markAtomicWritePublished(err: unknown) {
|
||||
if (err && (typeof err === "object" || typeof err === "function")) {
|
||||
Object.defineProperty(err, "atomicWritePublished", {
|
||||
value: true,
|
||||
configurable: true,
|
||||
});
|
||||
}
|
||||
return err;
|
||||
}
|
||||
|
||||
export function atomicWriteWasPublished(err: unknown): err is PublishedAtomicWriteError {
|
||||
return Boolean((err as PublishedAtomicWriteError | undefined)?.atomicWritePublished);
|
||||
}
|
||||
|
||||
const STATE_LOCK_RETRY_MS = 10;
|
||||
const STATE_LOCK_ORPHAN_MS = 30_000;
|
||||
const STATE_LOCK_HEARTBEAT_MS = 250;
|
||||
const TERMINAL_RESUME_CLEANUP_TIMEOUT_MS = STATE_LOCK_RETRY_MS * 10;
|
||||
|
||||
function isDirectorySyncUnsupportedError(err: any): boolean {
|
||||
return [
|
||||
"EACCES",
|
||||
@@ -101,6 +141,267 @@ export function isJsonSyntaxError(err) {
|
||||
return err instanceof SyntaxError;
|
||||
}
|
||||
|
||||
async function waitForStateLock(signal?: AbortSignal) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
};
|
||||
const timer = setTimeout(() => {
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve();
|
||||
}, STATE_LOCK_RETRY_MS);
|
||||
if (signal?.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
function isProcessAlive(pid: number) {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch (err: any) {
|
||||
return err?.code !== "ESRCH";
|
||||
}
|
||||
}
|
||||
|
||||
type StateLockOwner = {
|
||||
pid: number;
|
||||
processStartIdentity: string | null;
|
||||
nonce: string;
|
||||
};
|
||||
|
||||
function parseStateLockOwner(ownerText: string): StateLockOwner | null {
|
||||
const parts = ownerText.trim().split(":");
|
||||
if (parts.length !== 3) return null;
|
||||
const pid = Number(parts[0]);
|
||||
const processStartIdentity = parts[1] || null;
|
||||
const nonce = parts[2];
|
||||
if (!Number.isInteger(pid) || pid <= 0 || !nonce) return null;
|
||||
return { pid, processStartIdentity, nonce };
|
||||
}
|
||||
|
||||
async function readProcessStartIdentity(pid: number): Promise<string | null> {
|
||||
if (process.platform !== "linux") return null;
|
||||
try {
|
||||
const stat = await fsp.readFile(`/proc/${pid}/stat`, "utf8");
|
||||
const closeParen = stat.lastIndexOf(")");
|
||||
if (closeParen < 0) return null;
|
||||
// The remainder starts at procfs field 3; starttime is field 22.
|
||||
const fields = stat
|
||||
.slice(closeParen + 1)
|
||||
.trim()
|
||||
.split(/\s+/);
|
||||
return fields[19] || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function isStateLockOld(lockPath: string) {
|
||||
try {
|
||||
const stat = await fsp.stat(lockPath);
|
||||
return Date.now() - stat.mtimeMs >= STATE_LOCK_ORPHAN_MS;
|
||||
} catch (err: any) {
|
||||
if (err?.code === "ENOENT") return true;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function hasExpiredStateLockLease(lockPath: string) {
|
||||
let first;
|
||||
try {
|
||||
first = await fsp.stat(lockPath);
|
||||
} catch (err: any) {
|
||||
if (err?.code === "ENOENT") return true;
|
||||
throw err;
|
||||
}
|
||||
if (Date.now() - first.mtimeMs < STATE_LOCK_ORPHAN_MS) return false;
|
||||
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, STATE_LOCK_HEARTBEAT_MS));
|
||||
try {
|
||||
const second = await fsp.stat(lockPath);
|
||||
return second.mtimeMs === first.mtimeMs && Date.now() - second.mtimeMs >= STATE_LOCK_ORPHAN_MS;
|
||||
} catch (err: any) {
|
||||
if (err?.code === "ENOENT") return true;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function reclaimOrphanedStateLock(lockPath: string) {
|
||||
let observedLock: { dev: number; ino: number };
|
||||
try {
|
||||
const stat = await fsp.lstat(lockPath);
|
||||
observedLock = { dev: stat.dev, ino: stat.ino };
|
||||
} catch (err: any) {
|
||||
if (err?.code === "ENOENT") return true;
|
||||
throw err;
|
||||
}
|
||||
|
||||
let stale = false;
|
||||
try {
|
||||
const ownerPath = path.join(lockPath, "owner");
|
||||
const ownerText = (await fsp.readFile(ownerPath, "utf8")).trim();
|
||||
const owner = parseStateLockOwner(ownerText);
|
||||
if (owner && isProcessAlive(owner.pid)) {
|
||||
const processStartIdentity = await readProcessStartIdentity(owner.pid);
|
||||
if (owner.processStartIdentity && processStartIdentity) {
|
||||
stale = owner.processStartIdentity !== processStartIdentity;
|
||||
} else {
|
||||
// A live PID without a matching process-instance identity may have
|
||||
// been reused. Require a conservative expired lease and a second
|
||||
// unchanged observation before reclaiming it.
|
||||
stale = await hasExpiredStateLockLease(ownerPath);
|
||||
}
|
||||
} else if (owner) {
|
||||
stale = true;
|
||||
} else {
|
||||
stale = await isStateLockOld(lockPath);
|
||||
}
|
||||
} catch (err: any) {
|
||||
if (err?.code === "ENOENT") {
|
||||
stale = await isStateLockOld(lockPath);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
if (!stale) return false;
|
||||
|
||||
// Claim reclamation inside the observed directory before removing it. Renaming
|
||||
// `lockPath` directly is unsafe: another reclaimer can replace that pathname
|
||||
// with a live lock after the stale observation, and a later recursive cleanup
|
||||
// would then delete the new owner's lock while it is in use.
|
||||
const reclaimPath = path.join(lockPath, ".reclaiming");
|
||||
try {
|
||||
await fsp.mkdir(reclaimPath, { mode: 0o700 });
|
||||
} catch (err: any) {
|
||||
if (err?.code === "ENOENT") return true;
|
||||
if (err?.code === "EEXIST") return false;
|
||||
throw err;
|
||||
}
|
||||
|
||||
try {
|
||||
const currentLock = await fsp.lstat(lockPath);
|
||||
if (currentLock.dev !== observedLock.dev || currentLock.ino !== observedLock.ino) {
|
||||
return false;
|
||||
}
|
||||
await fsp.rm(lockPath, { recursive: true, force: true });
|
||||
} finally {
|
||||
// If the path changed before the reclamation marker was claimed, leave the
|
||||
// replacement lock intact and only remove our harmless marker.
|
||||
await fsp.rmdir(reclaimPath).catch(() => {});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async function stillOwnsStateLock(ownerPath: string, owner: string) {
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
return (await fsp.readFile(ownerPath, "utf8")) === owner;
|
||||
} catch (err: any) {
|
||||
if (err?.code === "ENOENT" || attempt === 1) return false;
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, STATE_LOCK_RETRY_MS));
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function withStateKeyLock<T>({
|
||||
env,
|
||||
key,
|
||||
signal,
|
||||
task,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
key: string;
|
||||
signal?: AbortSignal;
|
||||
task: () => Promise<T>;
|
||||
}): Promise<T> {
|
||||
const stateDir = defaultStateDir(env);
|
||||
return withFileLock({
|
||||
filePath: keyToPath(stateDir, key),
|
||||
signal,
|
||||
task,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize a transition for an arbitrary durable file. Readers that need a
|
||||
* publish-or-rollback decision use the same lock as writers, rather than
|
||||
* observing an atomic rename that cancellation may still need to undo.
|
||||
*/
|
||||
export async function withFileLock<T>({
|
||||
filePath,
|
||||
signal,
|
||||
task,
|
||||
}: {
|
||||
filePath: string;
|
||||
signal?: AbortSignal;
|
||||
task: () => Promise<T>;
|
||||
}): Promise<T> {
|
||||
await ensureDirectory(path.dirname(filePath));
|
||||
const lockPath = `${filePath}.lock`;
|
||||
let acquired = false;
|
||||
let owner: string | undefined;
|
||||
let ownerWritten = false;
|
||||
let heartbeat: ReturnType<typeof setInterval> | undefined;
|
||||
try {
|
||||
while (!acquired) {
|
||||
signal?.throwIfAborted();
|
||||
try {
|
||||
await fsp.mkdir(lockPath, { mode: 0o700 });
|
||||
acquired = true;
|
||||
const processStartIdentity = await readProcessStartIdentity(process.pid);
|
||||
owner = `${process.pid}:${processStartIdentity ?? ""}:${randomBytes(6).toString("hex")}\n`;
|
||||
await fsp.writeFile(path.join(lockPath, "owner"), owner, {
|
||||
encoding: "utf8",
|
||||
mode: 0o600,
|
||||
});
|
||||
ownerWritten = true;
|
||||
const ownerPath = path.join(lockPath, "owner");
|
||||
heartbeat = setInterval(() => {
|
||||
void fsp.utimes(ownerPath, new Date(), new Date()).catch(() => {});
|
||||
}, STATE_LOCK_HEARTBEAT_MS);
|
||||
heartbeat.unref?.();
|
||||
} catch (err: any) {
|
||||
if (acquired) throw err;
|
||||
if (err?.code !== "EEXIST") throw err;
|
||||
if (!(await reclaimOrphanedStateLock(lockPath))) {
|
||||
await waitForStateLock(signal);
|
||||
}
|
||||
}
|
||||
}
|
||||
return await task();
|
||||
} finally {
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
if (acquired) {
|
||||
const ownerPath = path.join(lockPath, "owner");
|
||||
if (!ownerWritten || !owner || (await stillOwnsStateLock(ownerPath, owner))) {
|
||||
// Detach an owned lock before best-effort cleanup. If a filesystem error
|
||||
// prevents removing the detached directory, the canonical lock path is
|
||||
// already free for a subsequent operation; leaving it in place would
|
||||
// instead look like a live lock owned by this still-running process.
|
||||
const cleanupPath = `${lockPath}.release-${process.pid}-${randomBytes(6).toString("hex")}`;
|
||||
try {
|
||||
await fsp.rename(lockPath, cleanupPath);
|
||||
await fsp.rm(cleanupPath, { recursive: true, force: true }).catch(() => {});
|
||||
} catch (err: any) {
|
||||
if (err?.code !== "ENOENT") {
|
||||
await fsp.rm(lockPath, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isLinkUnsupportedError(err: any): boolean {
|
||||
return ["ENOSYS", "ENOTSUP", "EOPNOTSUPP", "EPERM", "EXDEV"].includes(err?.code);
|
||||
}
|
||||
@@ -148,9 +449,18 @@ export async function writeFileAtomic(filePath, data, options: AtomicWriteOption
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
handle = undefined;
|
||||
options.signal?.throwIfAborted();
|
||||
await renameFile(tmpPath, filePath);
|
||||
await syncDir(filePath);
|
||||
cleanup = false;
|
||||
// The rename is the irreversible publication point. Keep propagating a
|
||||
// directory-sync failure, but mark it so a state transition can reconcile
|
||||
// the visible replacement before deciding whether dispatch is safe.
|
||||
try {
|
||||
await syncDir(filePath);
|
||||
} catch (err) {
|
||||
throw markAtomicWritePublished(err);
|
||||
}
|
||||
return { signalAbortedAfterCommit: options.signal?.aborted === true };
|
||||
} finally {
|
||||
if (handle) await handle.close().catch(() => {});
|
||||
if (cleanup) await fsp.rm(tmpPath, { force: true }).catch(() => {});
|
||||
@@ -202,28 +512,282 @@ export async function writeFileAtomicExclusive(
|
||||
}
|
||||
}
|
||||
|
||||
export async function readStateJson({ env, key }) {
|
||||
export async function readStateJson({
|
||||
env,
|
||||
key,
|
||||
signal = undefined,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
key: string;
|
||||
signal?: AbortSignal;
|
||||
}) {
|
||||
signal?.throwIfAborted();
|
||||
const stateDir = defaultStateDir(env);
|
||||
const filePath = keyToPath(stateDir, key);
|
||||
|
||||
try {
|
||||
const text = await fsp.readFile(filePath, "utf8");
|
||||
return JSON.parse(text);
|
||||
const value = JSON.parse(text);
|
||||
signal?.throwIfAborted();
|
||||
return value;
|
||||
} catch (err) {
|
||||
if (err?.code === "ENOENT") return null;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeStateJson({ env, key, value }) {
|
||||
/**
|
||||
* Read a state record only after any in-progress publish-or-rollback transition
|
||||
* for the same key has settled. Callers that compose state with another durable
|
||||
* resource use this to avoid observing a value that cancellation may undo.
|
||||
*/
|
||||
export async function readStateJsonWithLock({
|
||||
env,
|
||||
key,
|
||||
signal = undefined,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
key: string;
|
||||
signal?: AbortSignal;
|
||||
}) {
|
||||
try {
|
||||
return await withStateKeyLock({ env, key, signal, task: () => readStateJson({ env, key }) });
|
||||
} catch (err: any) {
|
||||
// A readable state directory can deliberately be mounted read-only. In that
|
||||
// case no writer can begin a paired publish-or-rollback transition, so use
|
||||
// the non-mutating JSON read instead of requiring creation of a lock path.
|
||||
if (["EACCES", "EPERM", "EROFS"].includes(err?.code)) {
|
||||
return readStateJson({ env, key, signal });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function writeStateJsonUnlocked({
|
||||
env,
|
||||
key,
|
||||
value,
|
||||
signal = undefined,
|
||||
atomicWriteOptions = undefined,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
key: string;
|
||||
value: unknown;
|
||||
signal?: AbortSignal;
|
||||
atomicWriteOptions?: Omit<AtomicWriteOptions, "signal">;
|
||||
}) {
|
||||
const stateDir = defaultStateDir(env);
|
||||
const filePath = keyToPath(stateDir, key);
|
||||
|
||||
await ensureDirectory(stateDir);
|
||||
await writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n");
|
||||
return writeFileAtomic(filePath, JSON.stringify(value, null, 2) + "\n", {
|
||||
...atomicWriteOptions,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
|
||||
export async function deleteStateJson({ env, key }) {
|
||||
export async function writeStateJson({
|
||||
env,
|
||||
key,
|
||||
value,
|
||||
signal = undefined,
|
||||
atomicWriteOptions = undefined,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
key: string;
|
||||
value: unknown;
|
||||
signal?: AbortSignal;
|
||||
atomicWriteOptions?: Omit<AtomicWriteOptions, "signal">;
|
||||
}) {
|
||||
return withStateKeyLock({
|
||||
env,
|
||||
key,
|
||||
signal,
|
||||
task: () => writeStateJsonUnlocked({ env, key, value, signal, atomicWriteOptions }),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Retire a resume capability before its first unsafe execution boundary. The
|
||||
* replacement is deliberately persistent: a failed later unlink must not make
|
||||
* the original token replayable.
|
||||
*/
|
||||
export async function consumeResumeState({
|
||||
env,
|
||||
key,
|
||||
expectedState,
|
||||
signal = undefined,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
key: string;
|
||||
expectedState: unknown;
|
||||
signal?: AbortSignal;
|
||||
}) {
|
||||
return withStateKeyLock({
|
||||
env,
|
||||
key,
|
||||
signal,
|
||||
task: async () => {
|
||||
const currentState = await readStateJson({ env, key });
|
||||
// A resume snapshot is loaded before command/workflow setup. Re-check it
|
||||
// while holding the state lock so two callers cannot both turn the same
|
||||
// approval into an executable invocation.
|
||||
if (stableStringify(currentState) !== stableStringify(expectedState)) {
|
||||
return { consumed: false as const };
|
||||
}
|
||||
const claimId = randomBytes(16).toString("hex");
|
||||
try {
|
||||
const result = await writeStateJsonUnlocked({
|
||||
env,
|
||||
key,
|
||||
value: {
|
||||
type: CONSUMED_RESUME_STATE_TYPE,
|
||||
consumedAt: new Date().toISOString(),
|
||||
claimId,
|
||||
},
|
||||
signal,
|
||||
});
|
||||
return {
|
||||
consumed: true as const,
|
||||
claimId,
|
||||
signalAbortedAfterCommit: result?.signalAbortedAfterCommit === true,
|
||||
};
|
||||
} catch (err) {
|
||||
if (atomicWriteWasPublished(err)) {
|
||||
const latest = await readStateJson({ env, key });
|
||||
if (isConsumedResumeState(latest) && latest.claimId === claimId) {
|
||||
await writeStateJsonUnlocked({ env, key, value: expectedState });
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Undo a just-published consumed marker only when it is still owned by the
|
||||
* caller's pre-dispatch claim. This never overwrites a replacement state or a
|
||||
* concurrent claimant's marker.
|
||||
*/
|
||||
export async function restoreConsumedResumeState({
|
||||
env,
|
||||
key,
|
||||
expectedState,
|
||||
claimId,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
key: string;
|
||||
expectedState: unknown;
|
||||
claimId: string;
|
||||
}) {
|
||||
return withStateKeyLock({
|
||||
env,
|
||||
key,
|
||||
task: async () => {
|
||||
const currentState = await readStateJson({ env, key });
|
||||
if (!isConsumedResumeState(currentState) || currentState.claimId !== claimId) {
|
||||
return false;
|
||||
}
|
||||
await writeStateJsonUnlocked({ env, key, value: expectedState });
|
||||
return true;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Retire an unconsumed capability at a safe terminal boundary. The snapshot is
|
||||
* first replaced by a caller-owned marker under the state lock, so a
|
||||
* cancellation can restore only the state this caller claimed. A stale resume
|
||||
* that merely observed an earlier snapshot can never recreate a state another
|
||||
* resume has already settled.
|
||||
*/
|
||||
export async function deleteResumeStateWithRollback({
|
||||
env,
|
||||
key,
|
||||
expectedState,
|
||||
signal = undefined,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
key: string;
|
||||
expectedState: unknown;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<boolean> {
|
||||
return withStateKeyLock({
|
||||
env,
|
||||
key,
|
||||
signal,
|
||||
task: async () => {
|
||||
signal?.throwIfAborted();
|
||||
const currentState = await readStateJson({ env, key });
|
||||
if (stableStringify(currentState) !== stableStringify(expectedState)) return false;
|
||||
|
||||
const claimId = randomBytes(16).toString("hex");
|
||||
let claimPublished = false;
|
||||
let claimMayBePublished = false;
|
||||
let deleted = false;
|
||||
try {
|
||||
let result;
|
||||
try {
|
||||
result = await writeStateJsonUnlocked({
|
||||
env,
|
||||
key,
|
||||
value: {
|
||||
type: CONSUMED_RESUME_STATE_TYPE,
|
||||
consumedAt: new Date().toISOString(),
|
||||
claimId,
|
||||
},
|
||||
signal,
|
||||
});
|
||||
} catch (err) {
|
||||
claimMayBePublished = atomicWriteWasPublished(err);
|
||||
throw err;
|
||||
}
|
||||
claimPublished = true;
|
||||
if (result?.signalAbortedAfterCommit) signal?.throwIfAborted();
|
||||
signal?.throwIfAborted();
|
||||
await deleteStateJsonUnlocked({ env, key });
|
||||
deleted = true;
|
||||
signal?.throwIfAborted();
|
||||
return true;
|
||||
} catch (err) {
|
||||
if ((signal?.aborted && claimPublished) || claimMayBePublished) {
|
||||
const latest = await readStateJson({ env, key });
|
||||
if (
|
||||
(deleted && latest === null) ||
|
||||
(isConsumedResumeState(latest) && latest.claimId === claimId)
|
||||
) {
|
||||
await writeStateJsonUnlocked({ env, key, value: expectedState });
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check physical state presence without parsing it. Cancellation uses this to
|
||||
* retain the authoritative workflow spelling even if the state file is corrupt.
|
||||
*/
|
||||
export async function stateJsonExists({
|
||||
env,
|
||||
key,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
key: string;
|
||||
}) {
|
||||
const filePath = keyToPath(defaultStateDir(env), key);
|
||||
try {
|
||||
await fsp.access(filePath);
|
||||
return true;
|
||||
} catch (err: any) {
|
||||
if (err?.code === "ENOENT") return false;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteStateJsonUnlocked({ env, key }) {
|
||||
const stateDir = defaultStateDir(env);
|
||||
const filePath = keyToPath(stateDir, key);
|
||||
try {
|
||||
@@ -234,6 +798,82 @@ export async function deleteStateJson({ env, key }) {
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteStateJson({
|
||||
env,
|
||||
key,
|
||||
signal = undefined,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
key: string;
|
||||
signal?: AbortSignal;
|
||||
}) {
|
||||
return withStateKeyLock({
|
||||
env,
|
||||
key,
|
||||
signal,
|
||||
task: () => deleteStateJsonUnlocked({ env, key }),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* After an effect has started, its consumed marker already prevents replay.
|
||||
* Give terminal cleanup a small, bounded opportunity to remove that marker,
|
||||
* but never let a live state writer turn cancellation into an unbounded wait.
|
||||
*/
|
||||
export async function deleteStateJsonWithBoundedResumeCleanup({
|
||||
env,
|
||||
key,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
key: string;
|
||||
}) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), TERMINAL_RESUME_CLEANUP_TIMEOUT_MS);
|
||||
try {
|
||||
await deleteStateJson({ env, key, signal: controller.signal });
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retire a resume capability only while it is still a live, unclaimed state.
|
||||
* A consumed marker belongs to a resume that has already started its atomic
|
||||
* predecessor-to-successor handoff, so cancellation must not delete it and
|
||||
* report success while that successor remains executable.
|
||||
*/
|
||||
export async function deleteUnconsumedResumeState({
|
||||
env,
|
||||
key,
|
||||
signal = undefined,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
key: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<"deleted" | "missing" | "claimed"> {
|
||||
return withStateKeyLock({
|
||||
env,
|
||||
key,
|
||||
signal,
|
||||
task: async () => {
|
||||
let currentState: unknown;
|
||||
try {
|
||||
currentState = await readStateJson({ env, key });
|
||||
} catch (err) {
|
||||
// A corrupt state is not resumable, so explicit cancellation may still
|
||||
// remove it. This keeps the legacy workflow-alias recovery behavior.
|
||||
if (!isJsonSyntaxError(err)) throw err;
|
||||
await deleteStateJsonUnlocked({ env, key });
|
||||
return "deleted";
|
||||
}
|
||||
if (currentState === null) return "missing";
|
||||
if (isConsumedResumeState(currentState)) return "claimed";
|
||||
await deleteStateJsonUnlocked({ env, key });
|
||||
return "deleted";
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function sanitizeApprovalId(approvalId: string): string {
|
||||
return approvalId.replace(/[^a-f0-9]/g, "");
|
||||
}
|
||||
@@ -383,12 +1023,62 @@ export async function cleanupApprovalIndexByStateKey({
|
||||
}
|
||||
}
|
||||
|
||||
export async function diffAndStore({ env, key, value }) {
|
||||
const before = await readStateJson({ env, key }).catch((err) => {
|
||||
if (isJsonSyntaxError(err)) return null;
|
||||
throw err;
|
||||
export async function diffAndStore({
|
||||
env,
|
||||
key,
|
||||
value,
|
||||
signal = undefined,
|
||||
atomicWriteOptions = undefined,
|
||||
afterStore = undefined,
|
||||
}: {
|
||||
env: Record<string, string | undefined>;
|
||||
key: string;
|
||||
value: unknown;
|
||||
signal?: AbortSignal;
|
||||
atomicWriteOptions?: Omit<AtomicWriteOptions, "signal">;
|
||||
afterStore?: (snapshot: { before: unknown; after: unknown; changed: boolean }) => Promise<void>;
|
||||
}) {
|
||||
return withStateKeyLock({
|
||||
env,
|
||||
key,
|
||||
signal,
|
||||
task: async () => {
|
||||
const filePath = keyToPath(defaultStateDir(env), key);
|
||||
let beforeExists = true;
|
||||
try {
|
||||
await fsp.access(filePath);
|
||||
} catch (err: any) {
|
||||
if (err?.code !== "ENOENT") throw err;
|
||||
beforeExists = false;
|
||||
}
|
||||
const before = await readStateJson({ env, key }).catch((err) => {
|
||||
if (isJsonSyntaxError(err)) return null;
|
||||
throw err;
|
||||
});
|
||||
const changed = stableStringify(before) !== stableStringify(value);
|
||||
const snapshot = { before, after: value, changed };
|
||||
let stored = false;
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
await writeStateJsonUnlocked({ env, key, value, signal, atomicWriteOptions });
|
||||
stored = true;
|
||||
signal?.throwIfAborted();
|
||||
await afterStore?.(snapshot);
|
||||
} catch (err) {
|
||||
// A caller can publish another resource while this state lock is held.
|
||||
// If that coordinated publication fails, restore the state snapshot before
|
||||
// releasing the lock so readers never reuse a cancelled result.
|
||||
const stateWasPublished = stored || atomicWriteWasPublished(err);
|
||||
if (stateWasPublished && (signal?.aborted || afterStore || atomicWriteWasPublished(err))) {
|
||||
if (!beforeExists) {
|
||||
await deleteStateJsonUnlocked({ env, key });
|
||||
} else {
|
||||
await writeStateJsonUnlocked({ env, key, value: before });
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
return snapshot;
|
||||
},
|
||||
});
|
||||
const changed = stableStringify(before) !== stableStringify(value);
|
||||
await writeStateJson({ env, key, value });
|
||||
return { before, after: value, changed };
|
||||
}
|
||||
|
||||
+517
-171
File diff suppressed because it is too large
Load Diff
@@ -1,35 +1,17 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { runAbortableProcess } from "../abortable_process.js";
|
||||
|
||||
function runProcess(command, argv, { env, cwd }) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, argv, { env, cwd, stdio: ["ignore", "pipe", "pipe"] });
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
|
||||
child.stdout.on("data", (d) => {
|
||||
stdout += d;
|
||||
});
|
||||
child.stderr.on("data", (d) => {
|
||||
stderr += d;
|
||||
});
|
||||
|
||||
child.on("error", (err: any) => {
|
||||
if (err?.code === "ENOENT") {
|
||||
reject(new Error("gh not found on PATH (install GitHub CLI)"));
|
||||
return;
|
||||
}
|
||||
reject(err);
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
if (code === 0) return resolve({ stdout, stderr });
|
||||
reject(new Error(`gh failed (${code}): ${stderr.trim() || stdout.trim()}`));
|
||||
});
|
||||
async function runProcess(command, argv, { env, cwd, signal, forceTerminationSignal }) {
|
||||
const { stdout, stderr, code } = await runAbortableProcess({
|
||||
command,
|
||||
argv,
|
||||
env,
|
||||
cwd,
|
||||
signal,
|
||||
forceTerminationSignal,
|
||||
notFoundMessage: "gh not found on PATH (install GitHub CLI)",
|
||||
});
|
||||
if (code === 0) return { stdout, stderr };
|
||||
throw new Error(`gh failed (${code}): ${stderr.trim() || stdout.trim()}`);
|
||||
}
|
||||
|
||||
import { diffAndStore } from "../state/store.js";
|
||||
@@ -83,6 +65,7 @@ function formatPrChangeMessage({ repo, pr, changedFields, prInfo }) {
|
||||
}
|
||||
|
||||
export async function runGithubPrMonitorWorkflow({ args, ctx }) {
|
||||
ctx.signal?.throwIfAborted();
|
||||
const repo = args.repo;
|
||||
const pr = args.pr;
|
||||
if (!repo || !pr) throw new Error("github.pr.monitor requires args.repo and args.pr");
|
||||
@@ -101,7 +84,13 @@ export async function runGithubPrMonitorWorkflow({ args, ctx }) {
|
||||
"number,title,url,state,isDraft,mergeable,reviewDecision,author,baseRefName,headRefName,updatedAt",
|
||||
];
|
||||
|
||||
const { stdout } = (await runProcess("gh", argv, { env: ctx.env, cwd: process.cwd() })) as any;
|
||||
const { stdout } = (await runProcess("gh", argv, {
|
||||
env: ctx.env,
|
||||
cwd: process.cwd(),
|
||||
signal: ctx.signal,
|
||||
forceTerminationSignal: ctx.forceTerminationSignal,
|
||||
})) as any;
|
||||
ctx.signal?.throwIfAborted();
|
||||
|
||||
let current;
|
||||
try {
|
||||
@@ -110,7 +99,12 @@ export async function runGithubPrMonitorWorkflow({ args, ctx }) {
|
||||
throw new Error("gh returned non-JSON output");
|
||||
}
|
||||
|
||||
const { changed, before } = await diffAndStore({ env: ctx.env, key, value: current });
|
||||
const { changed, before } = await diffAndStore({
|
||||
env: ctx.env,
|
||||
key,
|
||||
value: current,
|
||||
signal: ctx.signal,
|
||||
});
|
||||
|
||||
if (changesOnly && !changed) {
|
||||
return {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+32
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env node
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { access } from "node:fs/promises";
|
||||
|
||||
const [runnerPath, mockGog] = process.argv.slice(2);
|
||||
const { runAbortableProcess } = await import(pathToFileURL(runnerPath).href);
|
||||
|
||||
async function waitFor(path) {
|
||||
for (let attempt = 0; attempt < 300; attempt += 1) {
|
||||
try {
|
||||
await access(path);
|
||||
return;
|
||||
} catch {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${path}`);
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const run = runAbortableProcess({
|
||||
command: process.execPath,
|
||||
argv: [mockGog, "gmail", "search"],
|
||||
env: process.env,
|
||||
signal: controller.signal,
|
||||
notFoundMessage: "mock gog not found",
|
||||
});
|
||||
|
||||
await waitFor(process.env.MOCK_GOG_SEARCH_STARTED_FILE);
|
||||
await waitFor(process.env.MOCK_GOG_DESCENDANT_STARTED_FILE);
|
||||
controller.abort(new Error("cancelled by short-lived caller"));
|
||||
await run.catch(() => undefined);
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env node
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const [runnerPath] = process.argv.slice(2);
|
||||
const { runAbortableProcess } = await import(pathToFileURL(runnerPath).href);
|
||||
|
||||
function processGroupId() {
|
||||
const stat = readFileSync("/proc/self/stat", "utf8");
|
||||
const fields = stat
|
||||
.slice(stat.lastIndexOf(")") + 1)
|
||||
.trim()
|
||||
.split(/\s+/);
|
||||
return Number(fields[2]);
|
||||
}
|
||||
|
||||
writeFileSync(process.env.LOBSTER_TERMINAL_DIRECT_GROUP_FILE, String(processGroupId()), "utf8");
|
||||
|
||||
await runAbortableProcess({
|
||||
command: process.execPath,
|
||||
argv: [
|
||||
"-e",
|
||||
`const { writeFileSync } = require("node:fs");
|
||||
writeFileSync(process.env.LOBSTER_TERMINAL_DIRECT_STARTED_FILE, String(process.pid));
|
||||
setTimeout(() => writeFileSync(process.env.LOBSTER_TERMINAL_DIRECT_COMPLETED_FILE, "completed"), 700);`,
|
||||
],
|
||||
env: process.env,
|
||||
notFoundMessage: "node missing",
|
||||
});
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env node
|
||||
import { writeFileSync } from "node:fs";
|
||||
|
||||
function mark(path, value) {
|
||||
if (path) writeFileSync(path, value, "utf8");
|
||||
}
|
||||
|
||||
process.once("SIGTERM", () => {
|
||||
mark(process.env.MOCK_GH_TERMINATED_FILE, "SIGTERM");
|
||||
const terminationDelayMs = Number(process.env.MOCK_GH_TERMINATION_DELAY_MS ?? 0);
|
||||
setTimeout(() => process.exit(143), terminationDelayMs);
|
||||
});
|
||||
mark(process.env.MOCK_GH_STARTED_FILE, String(process.pid));
|
||||
setTimeout(
|
||||
() => {
|
||||
mark(process.env.MOCK_GH_COMPLETED_FILE, "completed");
|
||||
process.stdout.write(
|
||||
JSON.stringify({
|
||||
number: 1,
|
||||
title: "Fixture PR",
|
||||
url: "https://example.invalid/pr/1",
|
||||
state: "OPEN",
|
||||
isDraft: false,
|
||||
mergeable: "MERGEABLE",
|
||||
reviewDecision: "",
|
||||
updatedAt: "2026-07-11T00:00:00Z",
|
||||
baseRefName: "main",
|
||||
headRefName: "fixture",
|
||||
}),
|
||||
);
|
||||
},
|
||||
Number(process.env.MOCK_GH_COMPLETION_DELAY_MS ?? 1200),
|
||||
);
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env node
|
||||
import { appendFileSync, writeFileSync } from "node:fs";
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
|
||||
function mark(path, value) {
|
||||
if (path) writeFileSync(path, value, "utf8");
|
||||
}
|
||||
|
||||
function startDescendant() {
|
||||
if (!process.env.MOCK_GOG_DESCENDANT_STARTED_FILE) return;
|
||||
const helper = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
"-e",
|
||||
`const { writeFileSync } = require("node:fs");
|
||||
process.once("SIGTERM", () => {});
|
||||
writeFileSync(process.env.MOCK_GOG_DESCENDANT_STARTED_FILE, String(process.pid));
|
||||
setTimeout(() => writeFileSync(process.env.MOCK_GOG_DESCENDANT_COMPLETED_FILE, "completed"), 650);`,
|
||||
],
|
||||
{ env: process.env, stdio: "ignore" },
|
||||
);
|
||||
helper.unref();
|
||||
}
|
||||
|
||||
function waitForCompletion({ startedFile, terminatedFile, completedFile, output }) {
|
||||
const terminationDelayMs = Number(process.env.MOCK_GOG_TERMINATION_DELAY_MS ?? 0);
|
||||
const completionDelayMs = Number(process.env.MOCK_GOG_COMPLETION_DELAY_MS ?? 1200);
|
||||
process.once("SIGTERM", () => {
|
||||
mark(terminatedFile, "SIGTERM");
|
||||
setTimeout(() => process.exit(143), terminationDelayMs);
|
||||
});
|
||||
mark(startedFile, String(process.pid));
|
||||
startDescendant();
|
||||
setTimeout(() => {
|
||||
mark(completedFile, "completed");
|
||||
process.stdout.write(JSON.stringify(output));
|
||||
}, completionDelayMs);
|
||||
}
|
||||
|
||||
if (argv[0] === "gmail" && argv[1] === "search") {
|
||||
waitForCompletion({
|
||||
startedFile: process.env.MOCK_GOG_SEARCH_STARTED_FILE,
|
||||
terminatedFile: process.env.MOCK_GOG_SEARCH_TERMINATED_FILE,
|
||||
completedFile: process.env.MOCK_GOG_SEARCH_COMPLETED_FILE,
|
||||
output: [{ to: "user@example.com", subject: "Reply", body: "Hello" }],
|
||||
});
|
||||
} else if (argv[0] === "gmail" && argv[1] === "send") {
|
||||
if (process.env.MOCK_GOG_SEND_INVOCATIONS_FILE) {
|
||||
appendFileSync(process.env.MOCK_GOG_SEND_INVOCATIONS_FILE, `${process.pid}\n`, "utf8");
|
||||
}
|
||||
waitForCompletion({
|
||||
startedFile: process.env.MOCK_GOG_SEND_STARTED_FILE,
|
||||
terminatedFile: process.env.MOCK_GOG_SEND_TERMINATED_FILE,
|
||||
completedFile: process.env.MOCK_GOG_SEND_COMPLETED_FILE,
|
||||
output: { ok: true },
|
||||
});
|
||||
} else {
|
||||
process.stderr.write(`mock-gog-cancellation: unsupported args: ${argv.join(" ")}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
+17
@@ -1,3 +1,5 @@
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const writeResponse = () => {
|
||||
process.stdout.write(
|
||||
JSON.stringify({
|
||||
@@ -8,6 +10,21 @@ const writeResponse = () => {
|
||||
);
|
||||
};
|
||||
|
||||
if (process.argv.includes("--spawn-descendant")) {
|
||||
const helper = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
"-e",
|
||||
`const { writeFileSync } = require("node:fs");
|
||||
process.once("SIGTERM", () => {});
|
||||
writeFileSync(process.env.MOCK_OPENCLAW_AGENT_DESCENDANT_STARTED_FILE, String(process.pid));
|
||||
setTimeout(() => writeFileSync(process.env.MOCK_OPENCLAW_AGENT_DESCENDANT_COMPLETED_FILE, "completed"), 650);`,
|
||||
],
|
||||
{ env: process.env, stdio: "ignore" },
|
||||
);
|
||||
helper.unref();
|
||||
}
|
||||
|
||||
if (process.argv.includes("--sleep")) {
|
||||
setTimeout(writeResponse, 10_000);
|
||||
} else {
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import http from "node:http";
|
||||
import { promises as fsp } from "node:fs";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { createDefaultRegistry } from "../src/commands/registry.js";
|
||||
import { keyToPath } from "../src/state/store.js";
|
||||
|
||||
function streamOf(items: any[]) {
|
||||
return (async function* () {
|
||||
@@ -167,6 +169,450 @@ test("llm.invoke uses Pi adapter over local HTTP bridge", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("llm.invoke does not retry schema validation after adapter cancellation", async () => {
|
||||
const registry = createDefaultRegistry();
|
||||
const cmd = registry.get("llm.invoke");
|
||||
assert.ok(cmd);
|
||||
const controller = new AbortController();
|
||||
let calls = 0;
|
||||
|
||||
await assert.rejects(
|
||||
cmd.run({
|
||||
input: streamOf([]),
|
||||
args: {
|
||||
_: [],
|
||||
provider: "cancel-test",
|
||||
prompt: "Decide",
|
||||
"output-schema": '{"type":"object","required":["decision"]}',
|
||||
"max-validation-retries": 2,
|
||||
},
|
||||
ctx: {
|
||||
...baseCtx({}, registry),
|
||||
signal: controller.signal,
|
||||
llmAdapters: {
|
||||
"cancel-test": {
|
||||
source: "cancel-test",
|
||||
async invoke() {
|
||||
calls += 1;
|
||||
controller.abort(new Error("adapter cancelled during validation"));
|
||||
return {
|
||||
ok: true,
|
||||
result: {
|
||||
runId: "cancelled-attempt",
|
||||
output: { data: { unexpected: true } },
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as any),
|
||||
/adapter cancelled during validation/,
|
||||
);
|
||||
assert.equal(calls, 1, "cancellation after an invalid response must suppress retries");
|
||||
});
|
||||
|
||||
test("llm.invoke aborts while waiting for its reusable run-state lock", async () => {
|
||||
const registry = createDefaultRegistry();
|
||||
const cmd = registry.get("llm.invoke");
|
||||
assert.ok(cmd);
|
||||
const cacheDir = await mkdtemp(path.join(tmpdir(), "lobster-llm-state-lock-abort-"));
|
||||
const stateDir = path.join(cacheDir, "state");
|
||||
const stateKey = "blocked-run-state";
|
||||
const lockPath = `${keyToPath(stateDir, stateKey)}.lock`;
|
||||
await fsp.mkdir(lockPath, { recursive: true });
|
||||
await fsp.writeFile(path.join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
||||
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const pending = cmd.run({
|
||||
input: streamOf([]),
|
||||
args: {
|
||||
_: [],
|
||||
provider: "state-lock-abort-test",
|
||||
prompt: "Decide",
|
||||
"state-key": stateKey,
|
||||
},
|
||||
ctx: {
|
||||
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
|
||||
signal: controller.signal,
|
||||
llmAdapters: {
|
||||
"state-lock-abort-test": {
|
||||
source: "state-lock-abort-test",
|
||||
async invoke() {
|
||||
throw new Error("adapter must not run while the state read is locked");
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as any);
|
||||
const completion = pending.then(
|
||||
() => ({ kind: "success" as const }),
|
||||
(error) => ({ kind: "error" as const, error }),
|
||||
);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
controller.abort(new Error("LLM state read cancelled"));
|
||||
const early = await Promise.race([
|
||||
completion,
|
||||
new Promise<{ kind: "timeout" }>((resolve) =>
|
||||
setTimeout(() => resolve({ kind: "timeout" }), 75),
|
||||
),
|
||||
]);
|
||||
if (early.kind === "timeout") await fsp.rm(lockPath, { recursive: true, force: true });
|
||||
const settled = early.kind === "timeout" ? await completion : early;
|
||||
assert.notEqual(
|
||||
early.kind,
|
||||
"timeout",
|
||||
"state-key reads must observe cancellation while locked",
|
||||
);
|
||||
assert.equal(settled.kind, "error");
|
||||
if (settled.kind === "error") {
|
||||
assert.match(settled.error?.message ?? "", /LLM state read cancelled/);
|
||||
}
|
||||
} finally {
|
||||
await fsp.rm(cacheDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("llm.invoke does not publish a reusable cache entry when cancellation races cache commit", async () => {
|
||||
const registry = createDefaultRegistry();
|
||||
const cmd = registry.get("llm.invoke");
|
||||
assert.ok(cmd);
|
||||
const cacheDir = await mkdtemp(path.join(tmpdir(), "lobster-cache-cancel-publication-"));
|
||||
const stateDir = path.join(cacheDir, "state");
|
||||
const controller = new AbortController();
|
||||
const originalRename = fsp.rename;
|
||||
let cacheCommitAborted = false;
|
||||
let calls = 0;
|
||||
const adapter = {
|
||||
source: "cache-cancel-test",
|
||||
async invoke() {
|
||||
calls += 1;
|
||||
return {
|
||||
ok: true,
|
||||
result: {
|
||||
runId: `call-${calls}`,
|
||||
output: { data: { call: calls } },
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
const args = {
|
||||
_: [],
|
||||
provider: "cache-cancel-test",
|
||||
prompt: "Decide",
|
||||
"state-key": "cancelled-cache-publication",
|
||||
};
|
||||
|
||||
try {
|
||||
Object.defineProperty(fsp, "rename", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
async value(from: Parameters<typeof fsp.rename>[0], to: Parameters<typeof fsp.rename>[1]) {
|
||||
const result = await originalRename(from, to);
|
||||
if (
|
||||
!cacheCommitAborted &&
|
||||
String(to).startsWith(`${path.join(cacheDir, "llm.invoke")}${path.sep}`) &&
|
||||
String(to).endsWith(".json")
|
||||
) {
|
||||
cacheCommitAborted = true;
|
||||
controller.abort(new Error("cancelled during cache publication"));
|
||||
}
|
||||
return result;
|
||||
},
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
cmd.run({
|
||||
input: streamOf([]),
|
||||
args,
|
||||
ctx: {
|
||||
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
|
||||
signal: controller.signal,
|
||||
llmAdapters: { "cache-cancel-test": adapter },
|
||||
},
|
||||
} as any),
|
||||
/cancelled during cache publication/,
|
||||
);
|
||||
} finally {
|
||||
Object.defineProperty(fsp, "rename", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: originalRename,
|
||||
});
|
||||
}
|
||||
|
||||
const retried = await cmd.run({
|
||||
input: streamOf([]),
|
||||
args,
|
||||
ctx: {
|
||||
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
|
||||
llmAdapters: { "cache-cancel-test": adapter },
|
||||
},
|
||||
} as any);
|
||||
const retriedItems = await collect(retried.output!);
|
||||
assert.equal(
|
||||
calls,
|
||||
2,
|
||||
"a cancelled invocation must not satisfy a later request from cache or run state",
|
||||
);
|
||||
assert.equal(retriedItems[0]?.source, "cache-cancel-test");
|
||||
|
||||
await rm(cacheDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("llm.invoke restores the previous cache entry when a refresh is cancelled after commit", async () => {
|
||||
const registry = createDefaultRegistry();
|
||||
const cmd = registry.get("llm.invoke");
|
||||
assert.ok(cmd);
|
||||
const cacheDir = await mkdtemp(path.join(tmpdir(), "lobster-cache-refresh-cancel-"));
|
||||
const stateDir = path.join(cacheDir, "state");
|
||||
let calls = 0;
|
||||
const adapter = {
|
||||
source: "cache-refresh-cancel-test",
|
||||
async invoke() {
|
||||
calls += 1;
|
||||
return {
|
||||
ok: true,
|
||||
result: {
|
||||
runId: `call-${calls}`,
|
||||
output: { data: { call: calls } },
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
const args = {
|
||||
_: [],
|
||||
provider: "cache-refresh-cancel-test",
|
||||
prompt: "Decide",
|
||||
"state-key": "refresh-cache-publication",
|
||||
};
|
||||
|
||||
try {
|
||||
const first = await cmd.run({
|
||||
input: streamOf([]),
|
||||
args,
|
||||
ctx: {
|
||||
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
|
||||
llmAdapters: { "cache-refresh-cancel-test": adapter },
|
||||
},
|
||||
} as any);
|
||||
assert.deepEqual((await collect(first.output!))[0]?.output.data, { call: 1 });
|
||||
|
||||
const controller = new AbortController();
|
||||
const originalRename = fsp.rename;
|
||||
let cacheCommitAborted = false;
|
||||
try {
|
||||
Object.defineProperty(fsp, "rename", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
async value(from: Parameters<typeof fsp.rename>[0], to: Parameters<typeof fsp.rename>[1]) {
|
||||
const result = await originalRename(from, to);
|
||||
if (
|
||||
!cacheCommitAborted &&
|
||||
String(to).startsWith(`${path.join(cacheDir, "llm.invoke")}${path.sep}`)
|
||||
) {
|
||||
cacheCommitAborted = true;
|
||||
controller.abort(new Error("cancelled during cache refresh publication"));
|
||||
}
|
||||
return result;
|
||||
},
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
cmd.run({
|
||||
input: streamOf([]),
|
||||
args: { ...args, refresh: true },
|
||||
ctx: {
|
||||
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
|
||||
signal: controller.signal,
|
||||
llmAdapters: { "cache-refresh-cancel-test": adapter },
|
||||
},
|
||||
} as any),
|
||||
/cancelled during cache refresh publication/,
|
||||
);
|
||||
} finally {
|
||||
Object.defineProperty(fsp, "rename", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: originalRename,
|
||||
});
|
||||
}
|
||||
|
||||
const recovered = await cmd.run({
|
||||
input: streamOf([]),
|
||||
args,
|
||||
ctx: {
|
||||
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
|
||||
llmAdapters: { "cache-refresh-cancel-test": adapter },
|
||||
},
|
||||
} as any);
|
||||
const recoveredItems = await collect(recovered.output!);
|
||||
assert.equal(calls, 2, "the cancelled refresh must restore the existing run state");
|
||||
assert.equal(recoveredItems[0]?.source, "run_state");
|
||||
assert.deepEqual(recoveredItems[0]?.output.data, { call: 1 });
|
||||
|
||||
const recoveredCache = await cmd.run({
|
||||
input: streamOf([]),
|
||||
args: { _: [], provider: "cache-refresh-cancel-test", prompt: "Decide" },
|
||||
ctx: {
|
||||
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
|
||||
llmAdapters: { "cache-refresh-cancel-test": adapter },
|
||||
},
|
||||
} as any);
|
||||
const recoveredCacheItems = await collect(recoveredCache.output!);
|
||||
assert.equal(recoveredCacheItems[0]?.source, "cache");
|
||||
assert.deepEqual(recoveredCacheItems[0]?.output.data, { call: 1 });
|
||||
} finally {
|
||||
await rm(cacheDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("llm.invoke rolls back cache and run-state publications after a cache directory sync failure", async () => {
|
||||
const registry = createDefaultRegistry();
|
||||
const cmd = registry.get("llm.invoke");
|
||||
assert.ok(cmd);
|
||||
const cacheDir = await mkdtemp(path.join(tmpdir(), "lobster-cache-dir-sync-failure-"));
|
||||
const stateDir = path.join(cacheDir, "state");
|
||||
const cacheNamespaceDir = path.join(cacheDir, "llm.invoke");
|
||||
const originalOpen = fsp.open;
|
||||
const fault = Object.assign(new Error("cache directory sync failed"), { code: "EIO" });
|
||||
let failNextCacheDirectorySync = true;
|
||||
let calls = 0;
|
||||
const adapter = {
|
||||
source: "cache-dir-sync-test",
|
||||
async invoke() {
|
||||
calls += 1;
|
||||
return { ok: true, result: { runId: `call-${calls}`, output: { data: { call: calls } } } };
|
||||
},
|
||||
};
|
||||
const args = {
|
||||
_: [],
|
||||
provider: "cache-dir-sync-test",
|
||||
prompt: "Decide",
|
||||
"state-key": "cache-directory-sync-failure",
|
||||
};
|
||||
|
||||
try {
|
||||
await fsp.mkdir(cacheNamespaceDir, { recursive: true });
|
||||
Object.defineProperty(fsp, "open", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
async value(...openArgs: any[]) {
|
||||
const handle = await (originalOpen as any)(...openArgs);
|
||||
if (
|
||||
failNextCacheDirectorySync &&
|
||||
String(openArgs[0]) === cacheNamespaceDir &&
|
||||
openArgs[1] === "r"
|
||||
) {
|
||||
failNextCacheDirectorySync = false;
|
||||
return new Proxy(handle, {
|
||||
get(target, property, receiver) {
|
||||
if (property === "sync") return async () => Promise.reject(fault);
|
||||
const value = Reflect.get(target, property, receiver);
|
||||
return typeof value === "function" ? value.bind(target) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
return handle;
|
||||
},
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
cmd.run({
|
||||
input: streamOf([]),
|
||||
args,
|
||||
ctx: {
|
||||
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
|
||||
llmAdapters: { "cache-dir-sync-test": adapter },
|
||||
},
|
||||
} as any),
|
||||
/cache directory sync failed/,
|
||||
);
|
||||
} finally {
|
||||
Object.defineProperty(fsp, "open", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: originalOpen,
|
||||
});
|
||||
}
|
||||
|
||||
const retried = await cmd.run({
|
||||
input: streamOf([]),
|
||||
args,
|
||||
ctx: {
|
||||
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir, LOBSTER_STATE_DIR: stateDir }, registry),
|
||||
llmAdapters: { "cache-dir-sync-test": adapter },
|
||||
},
|
||||
} as any);
|
||||
const items = await collect(retried.output!);
|
||||
assert.equal(calls, 2, "a failed publication must not be reused from cache or run state");
|
||||
assert.equal(items[0]?.source, "cache-dir-sync-test");
|
||||
await rm(cacheDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("llm.invoke reads a populated cache when lock creation is forbidden", async () => {
|
||||
const registry = createDefaultRegistry();
|
||||
const cmd = registry.get("llm.invoke");
|
||||
assert.ok(cmd);
|
||||
const cacheDir = await mkdtemp(path.join(tmpdir(), "lobster-readonly-cache-"));
|
||||
const originalMkdir = fsp.mkdir;
|
||||
let calls = 0;
|
||||
const adapter = {
|
||||
source: "readonly-cache-test",
|
||||
async invoke() {
|
||||
calls += 1;
|
||||
return { ok: true, result: { runId: `call-${calls}`, output: { data: { call: calls } } } };
|
||||
},
|
||||
};
|
||||
const args = { _: [], provider: "readonly-cache-test", prompt: "Decide" };
|
||||
|
||||
try {
|
||||
const first = await cmd.run({
|
||||
input: streamOf([]),
|
||||
args,
|
||||
ctx: {
|
||||
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir }, registry),
|
||||
llmAdapters: { "readonly-cache-test": adapter },
|
||||
},
|
||||
} as any);
|
||||
await collect(first.output!);
|
||||
|
||||
Object.defineProperty(fsp, "mkdir", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
async value(
|
||||
filePath: Parameters<typeof fsp.mkdir>[0],
|
||||
options?: Parameters<typeof fsp.mkdir>[1],
|
||||
) {
|
||||
if (String(filePath).endsWith(".lock")) {
|
||||
throw Object.assign(new Error("read-only cache directory"), { code: "EACCES" });
|
||||
}
|
||||
return originalMkdir(filePath, options);
|
||||
},
|
||||
});
|
||||
|
||||
const cached = await cmd.run({
|
||||
input: streamOf([]),
|
||||
args,
|
||||
ctx: {
|
||||
...baseCtx({ LOBSTER_CACHE_DIR: cacheDir }, registry),
|
||||
llmAdapters: { "readonly-cache-test": adapter },
|
||||
},
|
||||
} as any);
|
||||
const items = await collect(cached.output!);
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(items[0]?.source, "cache");
|
||||
} finally {
|
||||
Object.defineProperty(fsp, "mkdir", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: originalMkdir,
|
||||
});
|
||||
await rm(cacheDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function baseCtx(envOverrides: Record<string, string>, registry?: any) {
|
||||
return {
|
||||
stdin: process.stdin,
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { access, mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
createOpenClawAgentCommand,
|
||||
@@ -12,6 +15,34 @@ function streamOf(items: unknown[]) {
|
||||
})();
|
||||
}
|
||||
|
||||
async function fileExists(filePath: string) {
|
||||
try {
|
||||
await access(filePath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForFile(filePath: string, timeoutMs = 2000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (await fileExists(filePath)) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${filePath}`);
|
||||
}
|
||||
|
||||
function processIsRunning(pid: number) {
|
||||
try {
|
||||
const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
|
||||
const state = stat.slice(stat.lastIndexOf(")") + 2, stat.lastIndexOf(")") + 3);
|
||||
return state !== "Z";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
test("openclaw.agent delegates agent, session, and model selection to OpenClaw", async () => {
|
||||
const calls: Array<Record<string, unknown>> = [];
|
||||
const cmd = createOpenClawAgentCommand(async (params) => {
|
||||
@@ -103,6 +134,18 @@ test("OpenClaw CLI runner parses structured JSON output", async () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("OpenClaw CLI runner preserves the 10 MiB output limit", async () => {
|
||||
await assert.rejects(
|
||||
runOpenClawAgentCli({
|
||||
executable: process.execPath,
|
||||
argv: ["-e", "process.stdout.write('x'.repeat(10 * 1024 * 1024 + 1))"],
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
}),
|
||||
/openclaw\.agent output exceeded 10485760 bytes/,
|
||||
);
|
||||
});
|
||||
|
||||
test("OpenClaw CLI runner preserves workflow cancellation", async () => {
|
||||
const fixturePath = path.join(process.cwd(), "test", "fixtures", "mock-openclaw-agent.mjs");
|
||||
const controller = new AbortController();
|
||||
@@ -117,3 +160,42 @@ test("OpenClaw CLI runner preserves workflow cancellation", async () => {
|
||||
|
||||
await assert.rejects(pending, (error: Error) => error.name === "AbortError");
|
||||
});
|
||||
|
||||
test(
|
||||
"OpenClaw CLI runner terminates descendant processes on cancellation",
|
||||
{ skip: process.platform === "win32" },
|
||||
async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "lobster-openclaw-agent-cancel-"));
|
||||
try {
|
||||
const fixturePath = path.join(process.cwd(), "test", "fixtures", "mock-openclaw-agent.mjs");
|
||||
const descendantStarted = path.join(dir, "descendant-started");
|
||||
const descendantCompleted = path.join(dir, "descendant-completed");
|
||||
const controller = new AbortController();
|
||||
const pending = runOpenClawAgentCli({
|
||||
executable: process.execPath,
|
||||
argv: [fixturePath, "--spawn-descendant", "--sleep"],
|
||||
cwd: process.cwd(),
|
||||
env: {
|
||||
...process.env,
|
||||
MOCK_OPENCLAW_AGENT_DESCENDANT_STARTED_FILE: descendantStarted,
|
||||
MOCK_OPENCLAW_AGENT_DESCENDANT_COMPLETED_FILE: descendantCompleted,
|
||||
},
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
await waitForFile(descendantStarted);
|
||||
const descendantPid = Number(await readFile(descendantStarted, "utf8"));
|
||||
controller.abort();
|
||||
await assert.rejects(pending, (error: Error) => error.name === "AbortError");
|
||||
await new Promise((resolve) => setTimeout(resolve, 700));
|
||||
assert.equal(processIsRunning(descendantPid), false);
|
||||
assert.equal(
|
||||
await fileExists(descendantCompleted),
|
||||
false,
|
||||
"the OpenClaw child process must not outlive cancellation",
|
||||
);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import { once } from "node:events";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { PassThrough } from "node:stream";
|
||||
|
||||
import { readLineFromStream } from "../src/read_line.js";
|
||||
@@ -14,6 +17,55 @@ test("readLineFromStream resolves on newline", async () => {
|
||||
assert.equal(value, "yes");
|
||||
});
|
||||
|
||||
test("readLineFromStream accepts sequential reads from one stream", async () => {
|
||||
const input = new PassThrough();
|
||||
const first = readLineFromStream(input);
|
||||
input.write("yes\n");
|
||||
assert.equal(await first, "yes");
|
||||
|
||||
const second = readLineFromStream(input, { timeoutMs: 50 });
|
||||
input.write("no\n");
|
||||
assert.equal(await second, "no");
|
||||
input.end();
|
||||
});
|
||||
|
||||
test("readLineFromStream preserves the next line from a combined input chunk", async () => {
|
||||
const input = new PassThrough();
|
||||
const first = readLineFromStream(input);
|
||||
input.end("yes\nno\n");
|
||||
assert.equal(await first, "yes");
|
||||
assert.equal(await readLineFromStream(input), "no");
|
||||
});
|
||||
|
||||
test("readLineFromStream returns buffered input after a child pipe reaches EOF", async () => {
|
||||
const child = spawn(process.execPath, ["-e", "process.stdout.write('yes\\npartial')"], {
|
||||
stdio: ["ignore", "pipe", "inherit"],
|
||||
});
|
||||
assert.ok(child.stdout);
|
||||
|
||||
const first = readLineFromStream(child.stdout);
|
||||
assert.equal(await first, "yes");
|
||||
await once(child, "close");
|
||||
assert.equal(await readLineFromStream(child.stdout, { timeoutMs: 50 }), "partial");
|
||||
});
|
||||
|
||||
test("readLineFromStream drains a buffer that remains readable after EOF", async () => {
|
||||
let buffered: Buffer | null = Buffer.from("partial");
|
||||
const input = Object.assign(new EventEmitter(), {
|
||||
readableEnded: true,
|
||||
closed: true,
|
||||
read() {
|
||||
const value = buffered;
|
||||
buffered = null;
|
||||
return value;
|
||||
},
|
||||
pause() {},
|
||||
resume() {},
|
||||
}) as unknown as NodeJS.ReadableStream;
|
||||
|
||||
assert.equal(await readLineFromStream(input), "partial");
|
||||
});
|
||||
|
||||
test("readLineFromStream resolves on end without newline", async () => {
|
||||
const input = new PassThrough();
|
||||
const promise = readLineFromStream(input);
|
||||
@@ -31,3 +83,14 @@ test("readLineFromStream times out when no input arrives", async () => {
|
||||
/Timed out waiting for input/,
|
||||
);
|
||||
});
|
||||
|
||||
test("readLineFromStream rejects when its signal is aborted", async () => {
|
||||
const input = new PassThrough();
|
||||
const controller = new AbortController();
|
||||
const promise = readLineFromStream(input, { signal: controller.signal });
|
||||
controller.abort(new Error("input cancelled"));
|
||||
|
||||
await assert.rejects(() => promise, /input cancelled/);
|
||||
assert.equal(input.readableFlowing, false);
|
||||
input.end();
|
||||
});
|
||||
|
||||
@@ -8,7 +8,7 @@ import path from "node:path";
|
||||
import { resumeToolRequest, runToolRequest } from "../src/core/tool_runtime.js";
|
||||
import { runPipeline } from "../src/runtime.js";
|
||||
import { decodeResumeToken } from "../src/resume.js";
|
||||
import { readStateJson, writeStateJson } from "../src/state/store.js";
|
||||
import { readStateJsonWithLock as readStateJson, writeStateJson } from "../src/state/store.js";
|
||||
|
||||
const responseSchema = {
|
||||
type: "object",
|
||||
|
||||
+603
-1
@@ -9,9 +9,13 @@ import { diffLast, diffAndStoreValue } from "../src/sdk/primitives/diff.js";
|
||||
import { stateSet, readState, writeState } from "../src/sdk/primitives/state.js";
|
||||
import {
|
||||
createApprovalIndex,
|
||||
consumeResumeState,
|
||||
deleteResumeStateWithRollback,
|
||||
diffAndStore,
|
||||
keyToPath,
|
||||
withFileLock,
|
||||
writeStateJson,
|
||||
readStateJson,
|
||||
readStateJsonWithLock as readStateJson,
|
||||
writeFileAtomic,
|
||||
writeFileAtomicExclusive,
|
||||
} from "../src/state/store.js";
|
||||
@@ -84,6 +88,52 @@ test("state.get returns null for missing key", async () => {
|
||||
assert.deepEqual(output.items, [null]);
|
||||
});
|
||||
|
||||
test("ordinary state reads work when creating a coordination lock is forbidden", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-readonly-state-"));
|
||||
const env = { ...process.env, LOBSTER_STATE_DIR: tmp };
|
||||
const key = "demo";
|
||||
const value = { readable: true };
|
||||
const statePath = keyToPath(tmp, key);
|
||||
const lockPath = `${statePath}.lock`;
|
||||
const originalMkdir = fsp.mkdir;
|
||||
await fsp.writeFile(statePath, JSON.stringify(value), "utf8");
|
||||
Object.defineProperty(fsp, "mkdir", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
async value(
|
||||
filePath: Parameters<typeof fsp.mkdir>[0],
|
||||
options?: Parameters<typeof fsp.mkdir>[1],
|
||||
) {
|
||||
if (String(filePath) === lockPath) {
|
||||
throw Object.assign(new Error("read-only state directory"), { code: "EACCES" });
|
||||
}
|
||||
return originalMkdir(filePath, options);
|
||||
},
|
||||
});
|
||||
try {
|
||||
assert.deepEqual(await readState(key, { env }), value);
|
||||
const registry = createDefaultRegistry();
|
||||
const output = await runPipeline({
|
||||
pipeline: [{ name: "state.get", args: { _: [key] }, raw: `state.get ${key}` }],
|
||||
registry,
|
||||
input: [],
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
});
|
||||
assert.deepEqual(output.items, [value]);
|
||||
} finally {
|
||||
Object.defineProperty(fsp, "mkdir", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: originalMkdir,
|
||||
});
|
||||
await fsp.rm(tmp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Atomic-write behavior proofs (issues #108, #109) ---
|
||||
//
|
||||
// Plain fsp.writeFile truncates the target before writing, so a concurrent
|
||||
@@ -191,6 +241,94 @@ test("writeFileAtomic propagates parent directory sync failures", async () => {
|
||||
assert.deepEqual(leftovers, []);
|
||||
});
|
||||
|
||||
test("consumeResumeState restores a published marker when parent sync fails", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-consume-dir-sync-"));
|
||||
const env = { LOBSTER_STATE_DIR: tmp };
|
||||
const expectedState = { haltType: "approval_request", pipeline: [] };
|
||||
const fault = Object.assign(new Error("dir sync failed after resume claim"), { code: "EIO" });
|
||||
const originalOpen = fsp.open;
|
||||
let failNextDirectorySync = true;
|
||||
|
||||
await writeStateJson({ env, key: "resume", value: expectedState });
|
||||
Object.defineProperty(fsp, "open", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
async value(...args: any[]) {
|
||||
const handle = await (originalOpen as any)(...args);
|
||||
if (failNextDirectorySync && String(args[0]) === tmp && args[1] === "r") {
|
||||
failNextDirectorySync = false;
|
||||
return new Proxy(handle, {
|
||||
get(target, property) {
|
||||
if (property === "sync") return async () => Promise.reject(fault);
|
||||
const value = Reflect.get(target, property);
|
||||
return typeof value === "function" ? value.bind(target) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
return handle;
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await assert.rejects(
|
||||
() => consumeResumeState({ env, key: "resume", expectedState }),
|
||||
(err: NodeJS.ErrnoException) => err?.code === "EIO",
|
||||
);
|
||||
assert.deepEqual(await readStateJson({ env, key: "resume" }), expectedState);
|
||||
} finally {
|
||||
Object.defineProperty(fsp, "open", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: originalOpen,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("deleteResumeStateWithRollback restores a published marker when parent sync fails", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-delete-resume-dir-sync-"));
|
||||
const env = { LOBSTER_STATE_DIR: tmp };
|
||||
const expectedState = { haltType: "input_request", pipeline: [] };
|
||||
const fault = Object.assign(new Error("dir sync failed after terminal resume claim"), {
|
||||
code: "EIO",
|
||||
});
|
||||
const originalOpen = fsp.open;
|
||||
let failNextDirectorySync = true;
|
||||
|
||||
await writeStateJson({ env, key: "resume", value: expectedState });
|
||||
Object.defineProperty(fsp, "open", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
async value(...args: any[]) {
|
||||
const handle = await (originalOpen as any)(...args);
|
||||
if (failNextDirectorySync && String(args[0]) === tmp && args[1] === "r") {
|
||||
failNextDirectorySync = false;
|
||||
return new Proxy(handle, {
|
||||
get(target, property) {
|
||||
if (property === "sync") return async () => Promise.reject(fault);
|
||||
const value = Reflect.get(target, property);
|
||||
return typeof value === "function" ? value.bind(target) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
return handle;
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await assert.rejects(
|
||||
() => deleteResumeStateWithRollback({ env, key: "resume", expectedState }),
|
||||
(err: NodeJS.ErrnoException) => err?.code === "EIO",
|
||||
);
|
||||
assert.deepEqual(await readStateJson({ env, key: "resume" }), expectedState);
|
||||
} finally {
|
||||
Object.defineProperty(fsp, "open", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: originalOpen,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("readStateJson surfaces malformed authoritative state", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-state-corrupt-"));
|
||||
const env = { LOBSTER_STATE_DIR: tmp };
|
||||
@@ -333,6 +471,470 @@ test("diffAndStore treats corrupt previous state as a miss and rewrites atomical
|
||||
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { ok: true });
|
||||
});
|
||||
|
||||
test("diffAndStore rolls back a state publication when its parent-directory sync fails", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-dir-sync-"));
|
||||
const env = { LOBSTER_STATE_DIR: tmp };
|
||||
await writeStateJson({ env, key: "snapshot", value: { version: "before" } });
|
||||
const fault = Object.assign(new Error("dir sync failed after state publication"), {
|
||||
code: "EIO",
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
diffAndStore({
|
||||
env,
|
||||
key: "snapshot",
|
||||
value: { version: "after" },
|
||||
atomicWriteOptions: {
|
||||
async syncParentDir() {
|
||||
throw fault;
|
||||
},
|
||||
},
|
||||
}),
|
||||
(err: NodeJS.ErrnoException) => err?.code === "EIO",
|
||||
);
|
||||
|
||||
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { version: "before" });
|
||||
await fsp.rm(tmp, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("diffAndStore does not publish a snapshot after cancellation before atomic replace", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-cancel-publish-"));
|
||||
const env = { LOBSTER_STATE_DIR: tmp };
|
||||
await writeStateJson({ env, key: "snapshot", value: { version: "before" } });
|
||||
|
||||
const controller = new AbortController();
|
||||
const signal = controller.signal;
|
||||
const throwIfAborted = signal.throwIfAborted.bind(signal);
|
||||
let signalChecks = 0;
|
||||
Object.defineProperty(signal, "throwIfAborted", {
|
||||
value() {
|
||||
signalChecks += 1;
|
||||
if (signalChecks === 2) controller.abort(new Error("abort before state publish"));
|
||||
throwIfAborted();
|
||||
},
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
() => diffAndStore({ env, key: "snapshot", value: { version: "after" }, signal }),
|
||||
/abort before state publish/,
|
||||
);
|
||||
assert.equal(signalChecks, 2);
|
||||
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { version: "before" });
|
||||
const leftovers = (await fsp.readdir(tmp)).filter((file) => file.includes(".tmp"));
|
||||
assert.deepEqual(leftovers, []);
|
||||
});
|
||||
|
||||
test("diffAndStore restores the previous snapshot when cancellation arrives during atomic rename", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-cancel-rename-"));
|
||||
const env = { LOBSTER_STATE_DIR: tmp };
|
||||
await writeStateJson({ env, key: "snapshot", value: { version: "before" } });
|
||||
|
||||
const controller = new AbortController();
|
||||
await assert.rejects(
|
||||
() =>
|
||||
diffAndStore({
|
||||
env,
|
||||
key: "snapshot",
|
||||
value: { version: "after" },
|
||||
signal: controller.signal,
|
||||
atomicWriteOptions: {
|
||||
async renameFile(from, to) {
|
||||
await fsp.rename(from, to);
|
||||
controller.abort(new Error("abort during atomic rename"));
|
||||
},
|
||||
},
|
||||
}),
|
||||
/abort during atomic rename/,
|
||||
);
|
||||
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { version: "before" });
|
||||
const leftovers = (await fsp.readdir(tmp)).filter((file) => file.includes(".tmp"));
|
||||
assert.deepEqual(leftovers, []);
|
||||
});
|
||||
|
||||
test("diffAndStore removes a newly published snapshot when cancellation arrives during atomic rename", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-cancel-new-rename-"));
|
||||
const env = { LOBSTER_STATE_DIR: tmp };
|
||||
const controller = new AbortController();
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
diffAndStore({
|
||||
env,
|
||||
key: "snapshot",
|
||||
value: { version: "after" },
|
||||
signal: controller.signal,
|
||||
atomicWriteOptions: {
|
||||
async renameFile(from, to) {
|
||||
await fsp.rename(from, to);
|
||||
controller.abort(new Error("abort during initial atomic rename"));
|
||||
},
|
||||
},
|
||||
}),
|
||||
/abort during initial atomic rename/,
|
||||
);
|
||||
assert.equal(await readStateJson({ env, key: "snapshot" }), null);
|
||||
const leftovers = (await fsp.readdir(tmp)).filter((file) => file.includes(".tmp"));
|
||||
assert.deepEqual(leftovers, []);
|
||||
});
|
||||
|
||||
test("diffAndStore restores an existing null snapshot when cancellation arrives during atomic rename", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-cancel-null-rename-"));
|
||||
const env = { LOBSTER_STATE_DIR: tmp };
|
||||
await writeStateJson({ env, key: "snapshot", value: null });
|
||||
const snapshotPath = keyToPath(tmp, "snapshot");
|
||||
|
||||
const controller = new AbortController();
|
||||
await assert.rejects(
|
||||
() =>
|
||||
diffAndStore({
|
||||
env,
|
||||
key: "snapshot",
|
||||
value: { version: "after" },
|
||||
signal: controller.signal,
|
||||
atomicWriteOptions: {
|
||||
async renameFile(from, to) {
|
||||
await fsp.rename(from, to);
|
||||
controller.abort(new Error("abort during null snapshot rename"));
|
||||
},
|
||||
},
|
||||
}),
|
||||
/abort during null snapshot rename/,
|
||||
);
|
||||
assert.equal(await readStateJson({ env, key: "snapshot" }), null);
|
||||
assert.equal(await fsp.readFile(snapshotPath, "utf8"), "null\n");
|
||||
});
|
||||
|
||||
test("diffAndStore serializes cancellation rollback before a concurrent snapshot update", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-cancel-concurrent-"));
|
||||
const env = { LOBSTER_STATE_DIR: tmp };
|
||||
await writeStateJson({ env, key: "snapshot", value: { version: "before" } });
|
||||
|
||||
const controller = new AbortController();
|
||||
let publishCancelledSnapshot!: () => void;
|
||||
const cancelledSnapshotPublished = new Promise<void>((resolve) => {
|
||||
publishCancelledSnapshot = resolve;
|
||||
});
|
||||
let allowCancellation!: () => void;
|
||||
const waitForCancellation = new Promise<void>((resolve) => {
|
||||
allowCancellation = resolve;
|
||||
});
|
||||
const cancelled = diffAndStore({
|
||||
env,
|
||||
key: "snapshot",
|
||||
value: { version: "cancelled-A" },
|
||||
signal: controller.signal,
|
||||
atomicWriteOptions: {
|
||||
async renameFile(from, to) {
|
||||
await fsp.rename(from, to);
|
||||
publishCancelledSnapshot();
|
||||
await waitForCancellation;
|
||||
},
|
||||
},
|
||||
});
|
||||
await cancelledSnapshotPublished;
|
||||
|
||||
let successfulSnapshotPublished = false;
|
||||
const successful = writeState("snapshot", { version: "successful-B" }, { env }).then(() => {
|
||||
successfulSnapshotPublished = true;
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
assert.equal(successfulSnapshotPublished, false, "the next writer must wait for rollback");
|
||||
|
||||
controller.abort(new Error("abort during concurrent atomic rename"));
|
||||
allowCancellation();
|
||||
await assert.rejects(cancelled, /abort during concurrent atomic rename/);
|
||||
await successful;
|
||||
assert.equal(successfulSnapshotPublished, true);
|
||||
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { version: "successful-B" });
|
||||
const leftovers = (await fsp.readdir(tmp)).filter(
|
||||
(file) => file.includes(".tmp") || file.endsWith(".lock"),
|
||||
);
|
||||
assert.deepEqual(leftovers, []);
|
||||
});
|
||||
|
||||
test("state.get waits for a diff publication to commit or roll back", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-state-read-transaction-"));
|
||||
const env = { ...process.env, LOBSTER_STATE_DIR: tmp };
|
||||
await writeStateJson({ env, key: "snapshot", value: { version: "before" } });
|
||||
let markPublished!: () => void;
|
||||
const published = new Promise<void>((resolve) => {
|
||||
markPublished = resolve;
|
||||
});
|
||||
let release!: () => void;
|
||||
const releasePublication = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const transaction = diffAndStore({
|
||||
env,
|
||||
key: "snapshot",
|
||||
value: { version: "new" },
|
||||
afterStore: async () => {
|
||||
markPublished();
|
||||
await releasePublication;
|
||||
throw new Error("paired publication failed");
|
||||
},
|
||||
});
|
||||
await published;
|
||||
|
||||
const getCmd = createDefaultRegistry().get("state.get");
|
||||
const pendingRead = getCmd.run({
|
||||
input: streamOf([]),
|
||||
args: { _: ["snapshot"] },
|
||||
ctx: { stdin: process.stdin, stdout: process.stdout, stderr: process.stderr, env },
|
||||
});
|
||||
const early = await Promise.race([
|
||||
pendingRead.then(() => "settled" as const),
|
||||
new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 25)),
|
||||
]);
|
||||
assert.equal(early, "pending", "state.get must not expose a snapshot pending rollback");
|
||||
|
||||
release();
|
||||
await assert.rejects(transaction, /paired publication failed/);
|
||||
const result = await pendingRead;
|
||||
const items = [];
|
||||
for await (const item of result.output) items.push(item);
|
||||
assert.deepEqual(items, [{ version: "before" }]);
|
||||
await fsp.rm(tmp, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("state.set stops waiting for a live state lock when its signal is aborted", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-state-lock-abort-"));
|
||||
const env = { LOBSTER_STATE_DIR: tmp };
|
||||
const key = "blocked";
|
||||
const lockPath = `${keyToPath(tmp, key)}.lock`;
|
||||
await fsp.mkdir(lockPath);
|
||||
await fsp.writeFile(path.join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
||||
|
||||
const controller = new AbortController();
|
||||
const stateSet = createDefaultRegistry().get("state.set");
|
||||
const pending = stateSet.run({
|
||||
input: streamOf([{ value: true }]),
|
||||
args: { _: [key] },
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
signal: controller.signal,
|
||||
},
|
||||
});
|
||||
const completion = pending.then(
|
||||
() => ({ kind: "success" as const }),
|
||||
(error) => ({ kind: "error" as const, error }),
|
||||
);
|
||||
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
controller.abort(new Error("state lock cancelled"));
|
||||
const early = await Promise.race([
|
||||
completion,
|
||||
new Promise<{ kind: "timeout" }>((resolve) =>
|
||||
setTimeout(() => resolve({ kind: "timeout" }), 75),
|
||||
),
|
||||
]);
|
||||
if (early.kind === "timeout") await fsp.rm(lockPath, { recursive: true, force: true });
|
||||
const settled = early.kind === "timeout" ? await completion : early;
|
||||
|
||||
assert.notEqual(early.kind, "timeout", "state.set must not remain blocked after cancellation");
|
||||
assert.equal(settled.kind, "error");
|
||||
if (settled.kind === "error") assert.match(settled.error?.message ?? "", /state lock cancelled/);
|
||||
await fsp.rm(lockPath, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("diffAndStore does not reclaim a live fallback lock after a short heartbeat gap", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-state-lock-lease-"));
|
||||
const env = { LOBSTER_STATE_DIR: tmp };
|
||||
const key = "snapshot";
|
||||
const lockPath = `${keyToPath(tmp, key)}.lock`;
|
||||
const ownerPath = path.join(lockPath, "owner");
|
||||
await fsp.mkdir(lockPath);
|
||||
await fsp.writeFile(ownerPath, `${process.pid}::live-writer\n`, "utf8");
|
||||
const briefGap = new Date(Date.now() - 2_000);
|
||||
await fsp.utimes(lockPath, briefGap, briefGap);
|
||||
await fsp.utimes(ownerPath, briefGap, briefGap);
|
||||
|
||||
const controller = new AbortController();
|
||||
const abort = setTimeout(
|
||||
() => controller.abort(new Error("live fallback lock remained held")),
|
||||
100,
|
||||
);
|
||||
try {
|
||||
await assert.rejects(
|
||||
() => diffAndStore({ env, key, value: { version: "new" }, signal: controller.signal }),
|
||||
/live fallback lock remained held/,
|
||||
);
|
||||
} finally {
|
||||
clearTimeout(abort);
|
||||
await fsp.rm(lockPath, { recursive: true, force: true });
|
||||
}
|
||||
assert.equal(await readStateJson({ env, key }), null);
|
||||
});
|
||||
|
||||
test("diffAndStore reclaims an old lock with a malformed owner", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-malformed-lock-"));
|
||||
const env = { LOBSTER_STATE_DIR: tmp };
|
||||
const lockPath = `${keyToPath(tmp, "snapshot")}.lock`;
|
||||
await fsp.mkdir(lockPath);
|
||||
await fsp.writeFile(path.join(lockPath, "owner"), "\n", "utf8");
|
||||
const staleAt = new Date(Date.now() - 10_000);
|
||||
await fsp.utimes(lockPath, staleAt, staleAt);
|
||||
|
||||
await diffAndStore({ env, key: "snapshot", value: { version: "recovered" } });
|
||||
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { version: "recovered" });
|
||||
await assert.rejects(fsp.access(lockPath));
|
||||
});
|
||||
|
||||
test(
|
||||
"diffAndStore reclaims an old lock after its owner PID is reused",
|
||||
{ skip: process.platform !== "linux" },
|
||||
async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-diff-reused-pid-lock-"));
|
||||
const env = { LOBSTER_STATE_DIR: tmp };
|
||||
const lockPath = `${keyToPath(tmp, "snapshot")}.lock`;
|
||||
const ownerPath = path.join(lockPath, "owner");
|
||||
await fsp.mkdir(lockPath);
|
||||
await fsp.writeFile(ownerPath, `${process.pid}:0:stale-owner\n`, "utf8");
|
||||
const staleAt = new Date(Date.now() - 10_000);
|
||||
await fsp.utimes(lockPath, staleAt, staleAt);
|
||||
await fsp.utimes(ownerPath, staleAt, staleAt);
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(
|
||||
() => controller.abort(new Error("reused lock was not reclaimed")),
|
||||
250,
|
||||
);
|
||||
try {
|
||||
await diffAndStore({
|
||||
env,
|
||||
key: "snapshot",
|
||||
value: { version: "recovered" },
|
||||
signal: controller.signal,
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
assert.equal(controller.signal.aborted, false);
|
||||
assert.deepEqual(await readStateJson({ env, key: "snapshot" }), { version: "recovered" });
|
||||
await assert.rejects(fsp.access(lockPath));
|
||||
},
|
||||
);
|
||||
|
||||
test("withFileLock does not reclaim a replacement lock after observing a stale one", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-state-lock-replacement-"));
|
||||
const filePath = path.join(tmp, "snapshot.json");
|
||||
const lockPath = `${filePath}.lock`;
|
||||
await fsp.mkdir(lockPath);
|
||||
await fsp.writeFile(path.join(lockPath, "owner"), `${process.pid}:0:stale-owner\n`, "utf8");
|
||||
const staleAt = new Date(Date.now() - 10_000);
|
||||
await fsp.utimes(lockPath, staleAt, staleAt);
|
||||
await fsp.utimes(path.join(lockPath, "owner"), staleAt, staleAt);
|
||||
|
||||
const originalReadFile = fsp.readFile;
|
||||
let replaced = false;
|
||||
let replacementActive = false;
|
||||
let overlap = false;
|
||||
let replacement: Promise<void> | undefined;
|
||||
let releaseReplacement!: () => void;
|
||||
const replacementReleased = new Promise<void>((resolve) => {
|
||||
releaseReplacement = resolve;
|
||||
});
|
||||
let replacementStarted!: () => void;
|
||||
const replacementEntered = new Promise<void>((resolve) => {
|
||||
replacementStarted = resolve;
|
||||
});
|
||||
|
||||
Object.defineProperty(fsp, "readFile", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
async value(
|
||||
filePathArg: Parameters<typeof fsp.readFile>[0],
|
||||
options?: Parameters<typeof fsp.readFile>[1],
|
||||
) {
|
||||
const result = await originalReadFile(filePathArg, options);
|
||||
if (!replaced && String(filePathArg) === path.join(lockPath, "owner")) {
|
||||
replaced = true;
|
||||
await fsp.rm(lockPath, { recursive: true, force: true });
|
||||
replacement = withFileLock({
|
||||
filePath,
|
||||
task: async () => {
|
||||
replacementActive = true;
|
||||
replacementStarted();
|
||||
await replacementReleased;
|
||||
replacementActive = false;
|
||||
},
|
||||
});
|
||||
await replacementEntered;
|
||||
}
|
||||
return result;
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const original = withFileLock({
|
||||
filePath,
|
||||
task: async () => {
|
||||
if (replacementActive) overlap = true;
|
||||
},
|
||||
});
|
||||
await replacementEntered;
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
assert.equal(overlap, false, "the stale reclaimer must not enter beside the replacement");
|
||||
releaseReplacement();
|
||||
if (!replacement) throw new Error("replacement lock did not start");
|
||||
await Promise.all([original, replacement]);
|
||||
} finally {
|
||||
Object.defineProperty(fsp, "readFile", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: originalReadFile,
|
||||
});
|
||||
await fsp.rm(tmp, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
assert.equal(replaced, true);
|
||||
assert.equal(overlap, false);
|
||||
});
|
||||
|
||||
test("withFileLock releases its key when best-effort cleanup cannot remove the detached lock", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-state-lock-release-failure-"));
|
||||
const filePath = path.join(tmp, "snapshot.json");
|
||||
const lockPath = `${filePath}.lock`;
|
||||
const originalRm = fsp.rm;
|
||||
let failReleaseOnce = true;
|
||||
|
||||
Object.defineProperty(fsp, "rm", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
async value(filePathArg: Parameters<typeof fsp.rm>[0], options?: Parameters<typeof fsp.rm>[1]) {
|
||||
if (failReleaseOnce && String(filePathArg).startsWith(lockPath)) {
|
||||
failReleaseOnce = false;
|
||||
throw Object.assign(new Error("simulated lock cleanup failure"), { code: "EIO" });
|
||||
}
|
||||
return originalRm(filePathArg, options);
|
||||
},
|
||||
});
|
||||
|
||||
let second: Promise<string> | undefined;
|
||||
try {
|
||||
await withFileLock({ filePath, task: async () => {} });
|
||||
second = withFileLock({ filePath, task: async () => "reacquired" });
|
||||
const settled = await Promise.race([
|
||||
second,
|
||||
new Promise<"timed out">((resolve) => setTimeout(() => resolve("timed out"), 500)),
|
||||
]);
|
||||
if (settled === "timed out") await originalRm(lockPath, { recursive: true, force: true });
|
||||
assert.equal(settled, "reacquired", "a failed cleanup must not poison the live state lock");
|
||||
await second;
|
||||
} finally {
|
||||
Object.defineProperty(fsp, "rm", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: originalRm,
|
||||
});
|
||||
await originalRm(tmp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("SDK diff primitives treat corrupt previous state as a miss (#112)", async () => {
|
||||
const tmp = mkdtempSync(path.join(os.tmpdir(), "lobster-sdk-diff-corrupt-"));
|
||||
const ctx = { env: { LOBSTER_STATE_DIR: tmp } };
|
||||
|
||||
+683
-1
@@ -8,7 +8,7 @@ import os from "node:os";
|
||||
import { createDefaultRegistry } from "../src/commands/registry.js";
|
||||
import { runWorkflowFile } from "../src/workflows/file.js";
|
||||
import { decodeResumeToken } from "../src/resume.js";
|
||||
import { readStateJson } from "../src/state/store.js";
|
||||
import { keyToPath, readStateJsonWithLock as readStateJson } from "../src/state/store.js";
|
||||
|
||||
function streamOf(items: unknown[]) {
|
||||
return (async function* () {
|
||||
@@ -92,6 +92,61 @@ test("workflow file runs with approval and resume", async () => {
|
||||
assert.deepEqual(resumeStateFiles, []);
|
||||
});
|
||||
|
||||
test("sequential workflow approvals reclaim superseded resume markers", async () => {
|
||||
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-sequential-approval-"));
|
||||
try {
|
||||
const stateDir = path.join(tmpDir, "state");
|
||||
const filePath = path.join(tmpDir, "workflow.lobster");
|
||||
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
||||
await fsp.writeFile(
|
||||
filePath,
|
||||
JSON.stringify({
|
||||
steps: [
|
||||
{ id: "first", approval: "First?" },
|
||||
{ id: "second", approval: "Second?" },
|
||||
{ id: "finish", command: "node -e \"process.stdout.write('{}')\"" },
|
||||
],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const ctx = {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool" as const,
|
||||
};
|
||||
const first = await runWorkflowFile({ filePath, ctx });
|
||||
assert.equal(first.status, "needs_approval");
|
||||
const firstPayload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
|
||||
assert.equal(firstPayload.kind, "workflow-file");
|
||||
const second = await runWorkflowFile({
|
||||
filePath,
|
||||
ctx,
|
||||
resume: firstPayload,
|
||||
approved: true,
|
||||
});
|
||||
assert.equal(second.status, "needs_approval");
|
||||
const secondPayload = decodeResumeToken(second.requiresApproval?.resumeToken ?? "");
|
||||
assert.equal(secondPayload.kind, "workflow-file");
|
||||
const completed = await runWorkflowFile({
|
||||
filePath,
|
||||
ctx,
|
||||
resume: secondPayload,
|
||||
approved: true,
|
||||
});
|
||||
assert.equal(completed.status, "ok");
|
||||
const stateFiles = await fsp.readdir(stateDir);
|
||||
assert.deepEqual(
|
||||
stateFiles.filter((name) => name.startsWith("workflow_resume_")),
|
||||
[],
|
||||
);
|
||||
} finally {
|
||||
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("workflow resume cancellation cleans up resume state", async () => {
|
||||
const workflow = {
|
||||
steps: [
|
||||
@@ -154,6 +209,562 @@ test("workflow resume cancellation cleans up resume state", async () => {
|
||||
assert.deepEqual(resumeStateFiles, []);
|
||||
});
|
||||
|
||||
test("direct workflow resume consumes its capability after cancellation starts an effect", async () => {
|
||||
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-direct-cancel-"));
|
||||
const stateDir = path.join(tmpDir, "state");
|
||||
const effectPath = path.join(tmpDir, "effects.log");
|
||||
const filePath = path.join(tmpDir, "workflow.lobster");
|
||||
await fsp.writeFile(
|
||||
filePath,
|
||||
JSON.stringify({
|
||||
steps: [
|
||||
{
|
||||
id: "approve",
|
||||
command:
|
||||
"node -e \"process.stdout.write(JSON.stringify({requiresApproval:{prompt:'Proceed?',items:[{id:1}]}}))\"",
|
||||
approval: "required",
|
||||
},
|
||||
{
|
||||
id: "effect",
|
||||
run: `node -e "require('fs').appendFileSync(process.argv[1], 'run\\n'); setInterval(() => {}, 1000)" ${JSON.stringify(effectPath)}`,
|
||||
condition: "$approve.approved",
|
||||
},
|
||||
],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
||||
const first = await runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
});
|
||||
assert.equal(first.status, "needs_approval");
|
||||
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
|
||||
assert.equal(payload.kind, "workflow-file");
|
||||
assert.ok(payload.stateKey);
|
||||
|
||||
const controller = new AbortController();
|
||||
const resumed = runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
signal: controller.signal,
|
||||
},
|
||||
resume: payload,
|
||||
approved: true,
|
||||
});
|
||||
for (let attempt = 0; attempt < 100; attempt++) {
|
||||
try {
|
||||
await fsp.access(effectPath);
|
||||
break;
|
||||
} catch {
|
||||
if (attempt === 99) throw new Error("workflow effect did not start");
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
}
|
||||
controller.abort(new Error("cancel after dispatch"));
|
||||
await assert.rejects(() => resumed, /cancel after dispatch/);
|
||||
assert.equal(await readStateJson({ env, key: payload.stateKey! }), null);
|
||||
const stateFiles = await fsp.readdir(stateDir);
|
||||
assert.equal(
|
||||
stateFiles.some((file) => file.startsWith("approval_")),
|
||||
false,
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
resume: payload,
|
||||
approved: true,
|
||||
}),
|
||||
/Workflow resume state not found/,
|
||||
);
|
||||
assert.equal((await fsp.readFile(effectPath, "utf8")).trim().split(/\r?\n/).length, 1);
|
||||
});
|
||||
|
||||
test("workflow template setup failure preserves an unstarted approval capability", async () => {
|
||||
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-template-resume-"));
|
||||
try {
|
||||
const stateDir = path.join(tmpDir, "state");
|
||||
const filePath = path.join(tmpDir, "workflow.lobster");
|
||||
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
||||
const broken = {
|
||||
steps: [
|
||||
{ id: "approve", approval: "Continue?" },
|
||||
{
|
||||
id: "effect",
|
||||
run: "node -e \"process.stdout.write('should not run')\"",
|
||||
stdin: "$missing.stdout",
|
||||
},
|
||||
],
|
||||
};
|
||||
await fsp.writeFile(filePath, JSON.stringify(broken), "utf8");
|
||||
|
||||
const first = await runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
});
|
||||
assert.equal(first.status, "needs_approval");
|
||||
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
|
||||
assert.equal(payload.kind, "workflow-file");
|
||||
assert.ok(payload.stateKey);
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
resume: payload,
|
||||
approved: true,
|
||||
}),
|
||||
/Unknown step reference: missing\.stdout/,
|
||||
);
|
||||
assert.notEqual(await readStateJson({ env, key: payload.stateKey! }), null);
|
||||
|
||||
await fsp.writeFile(
|
||||
filePath,
|
||||
JSON.stringify({
|
||||
steps: [
|
||||
{ id: "approve", approval: "Continue?" },
|
||||
{ id: "effect", run: "node -e \"process.stdout.write('ok')\"" },
|
||||
],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const retried = await runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
resume: payload,
|
||||
approved: true,
|
||||
});
|
||||
assert.equal(retried.status, "ok");
|
||||
assert.deepEqual(retried.output, ["ok"]);
|
||||
} finally {
|
||||
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("workflow resume consumes its capability after a step timeout starts an effect", async () => {
|
||||
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-step-timeout-"));
|
||||
const stateDir = path.join(tmpDir, "state");
|
||||
const effectPath = path.join(tmpDir, "effects.log");
|
||||
const filePath = path.join(tmpDir, "workflow.lobster");
|
||||
await fsp.writeFile(
|
||||
filePath,
|
||||
JSON.stringify({
|
||||
steps: [
|
||||
{
|
||||
id: "approve",
|
||||
command:
|
||||
"node -e \"process.stdout.write(JSON.stringify({requiresApproval:{prompt:'Proceed?',items:[{id:1}]}}))\"",
|
||||
approval: "required",
|
||||
},
|
||||
{
|
||||
id: "effect",
|
||||
run: `node -e "require('fs').appendFileSync(process.argv[1], 'run\\n'); setInterval(() => {}, 1000)" ${JSON.stringify(effectPath)}`,
|
||||
condition: "$approve.approved",
|
||||
timeout_ms: 1500,
|
||||
},
|
||||
],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
||||
const first = await runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
});
|
||||
assert.equal(first.status, "needs_approval");
|
||||
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
|
||||
assert.equal(payload.kind, "workflow-file");
|
||||
assert.ok(payload.stateKey);
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
resume: payload,
|
||||
approved: true,
|
||||
}),
|
||||
/timed out|timeout|abort|cancel/i,
|
||||
);
|
||||
assert.equal(await readStateJson({ env, key: payload.stateKey! }), null);
|
||||
assert.equal((await fsp.readFile(effectPath, "utf8")).trim().split(/\r?\n/).length, 1);
|
||||
await assert.rejects(
|
||||
() =>
|
||||
runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
resume: payload,
|
||||
approved: true,
|
||||
}),
|
||||
/Workflow resume state not found/,
|
||||
);
|
||||
assert.equal((await fsp.readFile(effectPath, "utf8")).trim().split(/\r?\n/).length, 1);
|
||||
});
|
||||
|
||||
test("workflow resume applies on_error after a timed-out effect", async () => {
|
||||
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-timeout-on-error-"));
|
||||
const stateDir = path.join(tmpDir, "state");
|
||||
const effectPath = path.join(tmpDir, "effects.log");
|
||||
const filePath = path.join(tmpDir, "workflow.lobster");
|
||||
await fsp.writeFile(
|
||||
filePath,
|
||||
JSON.stringify({
|
||||
steps: [
|
||||
{
|
||||
id: "approve",
|
||||
command:
|
||||
"node -e \"process.stdout.write(JSON.stringify({requiresApproval:{prompt:'Proceed?',items:[{id:1}]}}))\"",
|
||||
approval: "required",
|
||||
},
|
||||
{
|
||||
id: "effect",
|
||||
run: `node -e "require('fs').appendFileSync(process.argv[1], 'run\\n'); setInterval(() => {}, 1000)" ${JSON.stringify(effectPath)}`,
|
||||
condition: "$approve.approved",
|
||||
timeout_ms: 1500,
|
||||
on_error: "continue",
|
||||
},
|
||||
{ id: "after", run: "echo continued" },
|
||||
],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
||||
const first = await runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
});
|
||||
assert.equal(first.status, "needs_approval");
|
||||
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
|
||||
assert.equal(payload.kind, "workflow-file");
|
||||
assert.ok(payload.stateKey);
|
||||
|
||||
const resumed = await runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
resume: payload,
|
||||
approved: true,
|
||||
});
|
||||
assert.equal(resumed.status, "ok");
|
||||
assert.deepEqual(resumed.output, ["continued\n"]);
|
||||
assert.equal(await readStateJson({ env, key: payload.stateKey! }), null);
|
||||
assert.equal((await fsp.readFile(effectPath, "utf8")).trim().split(/\r?\n/).length, 1);
|
||||
});
|
||||
|
||||
test("workflow resume retries a timed-out effect before consuming its capability", async () => {
|
||||
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-timeout-retry-"));
|
||||
const stateDir = path.join(tmpDir, "state");
|
||||
const attemptsPath = path.join(tmpDir, "attempts");
|
||||
const filePath = path.join(tmpDir, "workflow.lobster");
|
||||
await fsp.writeFile(
|
||||
filePath,
|
||||
JSON.stringify({
|
||||
steps: [
|
||||
{
|
||||
id: "approve",
|
||||
command:
|
||||
"node -e \"process.stdout.write(JSON.stringify({requiresApproval:{prompt:'Proceed?',items:[{id:1}]}}))\"",
|
||||
approval: "required",
|
||||
},
|
||||
{
|
||||
id: "effect",
|
||||
run: `node -e "const fs=require('fs'); const file=process.argv[1]; const attempt=fs.existsSync(file) ? Number(fs.readFileSync(file, 'utf8')) : 0; fs.writeFileSync(file, String(attempt + 1)); if (attempt === 0) setInterval(() => {}, 1000); else process.stdout.write('retried');" ${JSON.stringify(attemptsPath)}`,
|
||||
condition: "$approve.approved",
|
||||
timeout_ms: 1500,
|
||||
retry: { max: 2, delay_ms: 10 },
|
||||
},
|
||||
],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
||||
const first = await runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
});
|
||||
assert.equal(first.status, "needs_approval");
|
||||
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
|
||||
assert.equal(payload.kind, "workflow-file");
|
||||
assert.ok(payload.stateKey);
|
||||
|
||||
const resumed = await runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
resume: payload,
|
||||
approved: true,
|
||||
});
|
||||
assert.equal(resumed.status, "ok");
|
||||
assert.deepEqual(resumed.output, ["retried"]);
|
||||
assert.equal(await fsp.readFile(attemptsPath, "utf8"), "2");
|
||||
assert.equal(await readStateJson({ env, key: payload.stateKey! }), null);
|
||||
});
|
||||
|
||||
test("workflow resume retries a claim blocked by a state lock before dispatch", async () => {
|
||||
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-claim-retry-"));
|
||||
try {
|
||||
const stateDir = path.join(tmpDir, "state");
|
||||
const filePath = path.join(tmpDir, "workflow.lobster");
|
||||
await fsp.writeFile(
|
||||
filePath,
|
||||
JSON.stringify({
|
||||
steps: [
|
||||
{ id: "approve", approval: "Proceed?" },
|
||||
{
|
||||
id: "effect",
|
||||
run: "printf ran",
|
||||
condition: "$approve.approved",
|
||||
timeout_ms: 1000,
|
||||
retry: { max: 2, delay_ms: 250 },
|
||||
},
|
||||
],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
||||
const first = await runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
});
|
||||
assert.equal(first.status, "needs_approval");
|
||||
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
|
||||
assert.equal(payload.kind, "workflow-file");
|
||||
assert.ok(payload.stateKey);
|
||||
|
||||
const lockPath = `${keyToPath(stateDir, payload.stateKey!)}.lock`;
|
||||
const originalRename = fsp.rename;
|
||||
const originalRm = fsp.rm;
|
||||
let releasedInitialReadLocks = 0;
|
||||
let lockReplaced!: () => void;
|
||||
const replacedInitialReadLocks = new Promise<void>((resolve) => {
|
||||
lockReplaced = resolve;
|
||||
});
|
||||
Object.defineProperty(fsp, "rename", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
async value(
|
||||
oldPath: Parameters<typeof fsp.rename>[0],
|
||||
newPath: Parameters<typeof fsp.rename>[1],
|
||||
) {
|
||||
const result = await originalRename(oldPath, newPath);
|
||||
if (String(oldPath) === lockPath && ++releasedInitialReadLocks === 2) {
|
||||
await fsp.mkdir(lockPath);
|
||||
await fsp.writeFile(
|
||||
path.join(lockPath, "owner"),
|
||||
`${process.pid}::live-writer\n`,
|
||||
"utf8",
|
||||
);
|
||||
lockReplaced();
|
||||
}
|
||||
return result;
|
||||
},
|
||||
});
|
||||
try {
|
||||
const resumedRun = runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
resume: payload,
|
||||
approved: true,
|
||||
});
|
||||
await replacedInitialReadLocks;
|
||||
const release = setTimeout(
|
||||
() => void originalRm(lockPath, { recursive: true, force: true }),
|
||||
1200,
|
||||
);
|
||||
const resumed = await resumedRun;
|
||||
clearTimeout(release);
|
||||
assert.equal(resumed.status, "ok");
|
||||
assert.deepEqual(resumed.output, ["ran"]);
|
||||
assert.equal(await readStateJson({ env, key: payload.stateKey! }), null);
|
||||
} finally {
|
||||
Object.defineProperty(fsp, "rename", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: originalRename,
|
||||
});
|
||||
await fsp.rm(lockPath, { recursive: true, force: true });
|
||||
}
|
||||
} finally {
|
||||
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("workflow resume consumes its capability after a parallel timeout starts an effect", async () => {
|
||||
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-workflow-parallel-timeout-"));
|
||||
const stateDir = path.join(tmpDir, "state");
|
||||
const effectPath = path.join(tmpDir, "effects.log");
|
||||
const filePath = path.join(tmpDir, "workflow.lobster");
|
||||
await fsp.writeFile(
|
||||
filePath,
|
||||
JSON.stringify({
|
||||
steps: [
|
||||
{
|
||||
id: "approve",
|
||||
command:
|
||||
"node -e \"process.stdout.write(JSON.stringify({requiresApproval:{prompt:'Proceed?',items:[{id:1}]}}))\"",
|
||||
approval: "required",
|
||||
},
|
||||
{
|
||||
id: "effect",
|
||||
condition: "$approve.approved",
|
||||
parallel: {
|
||||
timeout_ms: 1500,
|
||||
branches: [
|
||||
{
|
||||
id: "side-effect",
|
||||
run: `node -e "require('fs').appendFileSync(process.argv[1], 'run\\n'); setInterval(() => {}, 1000)" ${JSON.stringify(effectPath)}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
||||
const first = await runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
});
|
||||
assert.equal(first.status, "needs_approval");
|
||||
const payload = decodeResumeToken(first.requiresApproval?.resumeToken ?? "");
|
||||
assert.equal(payload.kind, "workflow-file");
|
||||
assert.ok(payload.stateKey);
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
resume: payload,
|
||||
approved: true,
|
||||
}),
|
||||
/timed out|timeout|abort|cancel/i,
|
||||
);
|
||||
assert.equal(await readStateJson({ env, key: payload.stateKey! }), null);
|
||||
assert.equal((await fsp.readFile(effectPath, "utf8")).trim().split(/\r?\n/).length, 1);
|
||||
await assert.rejects(
|
||||
() =>
|
||||
runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env,
|
||||
mode: "tool",
|
||||
},
|
||||
resume: payload,
|
||||
approved: true,
|
||||
}),
|
||||
/Workflow resume state not found/,
|
||||
);
|
||||
assert.equal((await fsp.readFile(effectPath, "utf8")).trim().split(/\r?\n/).length, 1);
|
||||
});
|
||||
|
||||
test("workflow resume accepts workflow-resume_ state key aliases and cleans up state", async () => {
|
||||
const workflow = {
|
||||
steps: [
|
||||
@@ -383,6 +994,7 @@ test("workflow pipeline requestInput resume invariant bypasses on_error", async
|
||||
let sideEffects = 0;
|
||||
const choose = {
|
||||
name: "choose",
|
||||
meta: { resumeSafeBeforeInput: true },
|
||||
async run({ ctx }: any) {
|
||||
calls += 1;
|
||||
if (calls > 1) return { output: streamOf([{ skipped: true }]) };
|
||||
@@ -1246,6 +1858,76 @@ test("workflow conditions reject standalone bare identifiers", async () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("parallel wait:any fails promptly when a registered loser ignores cancellation", async () => {
|
||||
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "lobster-parallel-unsettled-"));
|
||||
const filePath = path.join(tmpDir, "workflow.lobster");
|
||||
const registry = {
|
||||
get(name: string) {
|
||||
if (name === "winner") {
|
||||
return {
|
||||
name,
|
||||
help: () => "winner",
|
||||
async run() {
|
||||
return { output: [{ winner: true }] };
|
||||
},
|
||||
};
|
||||
}
|
||||
if (name === "never") {
|
||||
return {
|
||||
name,
|
||||
help: () => "never",
|
||||
async run() {
|
||||
await new Promise<void>(() => {});
|
||||
},
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await fsp.writeFile(
|
||||
filePath,
|
||||
JSON.stringify({
|
||||
steps: [
|
||||
{
|
||||
id: "parallel",
|
||||
parallel: {
|
||||
wait: "any",
|
||||
branches: [
|
||||
{ id: "winner", pipeline: "winner" },
|
||||
{ id: "never", pipeline: "never" },
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await assert.rejects(
|
||||
Promise.race([
|
||||
runWorkflowFile({
|
||||
filePath,
|
||||
ctx: {
|
||||
stdin: process.stdin,
|
||||
stdout: process.stdout,
|
||||
stderr: process.stderr,
|
||||
env: { ...process.env },
|
||||
mode: "tool",
|
||||
registry,
|
||||
},
|
||||
}),
|
||||
new Promise<never>((_resolve, reject) =>
|
||||
setTimeout(() => reject(new Error("workflow did not finish")), 1_000),
|
||||
),
|
||||
]),
|
||||
/Parallel branches did not settle after cancellation/,
|
||||
);
|
||||
} finally {
|
||||
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("workflow conditions reject unknown step refs even under negation", async () => {
|
||||
const workflow = {
|
||||
steps: [
|
||||
|
||||
Reference in New Issue
Block a user