fix: cached LLM answers are returned when temperature or max output tokens change (#130)

* fix(llm): key the response cache on sampling parameters --temperature and --max-output-tokens were sent to the adapter but left out of the cache key, so changing either one replayed the previous answer instead of calling the model. The same key drives --state-key reuse, so a resumed workflow replayed the stale answer too. Hash each parameter only when it is set, guarding on the same predicate that decides whether it goes on the wire. stableStringify sorts keys, so an invocation that sets neither serializes exactly as before and keeps its current hash - the cache is content-addressed by filename and a blanket key change would orphan every entry written by an earlier release. metadata stays out: it is set unconditionally with --metadata-json and carries correlation data rather than decode-time parameters, so hashing it would invalidate existing caches and collapse the hit rate. --schema-version already covers callers who want metadata to segment the cache. * fix(llm): version the cache identity so old entries cannot replay sampling Keying only the parameters that were supplied kept the omitted-parameter key byte-identical to what earlier releases wrote. That preserved existing caches, but it also left one collision: an entry those releases stored for a request sampled at an explicit temperature sits under the key a request that omits sampling computes, so after upgrading, an unsampled request could be served a sampled answer. Put both parameters in the payload unconditionally, with `null` as the identity of an omitted one, and add a version field to the payload. Entries written under the previous identity are unreachable rather than ambiguous: the cost is one re-invocation per prompt after upgrade, never a wrong replay. The key-stability test is replaced by its inverse — a cache entry written under the pre-upgrade key is not returned to a request that omits sampling. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01X6UVfFPP39RkoYx5jZ3KKM --------- Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
2026-08-14 00:48:09 +00:00 · 2026-08-13 12:23:12 -07:00
parent 9769237d36
commit e88e8d3970
2 changed files with 273 additions and 2 deletions
@@ -107,6 +107,8 @@ const validateResponseEnvelope = ajv.compile(responseSchema);

 const DEFAULT_MAX_VALIDATION_RETRIES = 1;
 const STATE_VERSION = 1;
+// Identity version of the response cache key. See computeCacheKey.
+const CACHE_KEY_VERSION = 2;

 type BuiltInProvider = "openclaw" | "pi" | "http";
 type SupportedProvider = BuiltInProvider | string;
@@ -389,6 +391,8 @@ async function runLlmInvoke({
 		schemaVersion,
 		artifactHashes,
 		outputSchema: userOutputSchema,
+		temperature,
+		maxOutputTokens,
 	});

 	if (stateKey && !forceRefresh) {
@@ -834,6 +838,8 @@ function computeCacheKey({
 	schemaVersion,
 	artifactHashes,
 	outputSchema,
+	temperature,
+	maxOutputTokens,
 }: {
 	provider: SupportedProvider;
 	prompt: string;
@@ -841,14 +847,26 @@ function computeCacheKey({
 	schemaVersion: string;
 	artifactHashes: string[];
 	outputSchema: any;
+	temperature: number | null;
+	maxOutputTokens: number | null;
 }) {
+	// `null` is the identity of an omitted parameter, distinct from any value a caller can
+	// pass, so an omitted request can never resolve to an entry written with an explicit one.
+	// The version separates this identity from the keys earlier releases wrote, where the two
+	// parameters were absent from the payload and a sampled answer therefore shared a key with
+	// an unsampled request. Bump it whenever the fields below change: entries under older
+	// versions become unreachable, which costs one re-invocation and never a wrong replay.
 	const payload = {
+		cacheKeyVersion: CACHE_KEY_VERSION,
 		provider,
 		prompt,
 		model: model || `${provider}-default`,
 		schemaVersion,
 		artifactHashes,
 		outputSchema: outputSchema ?? null,
+		// The same predicate that decides whether each parameter is sent to the adapter.
+		temperature: Number.isFinite(temperature ?? NaN) ? Number(temperature) : null,
+		maxOutputTokens: Number.isFinite(maxOutputTokens ?? NaN) ? Number(maxOutputTokens) : null,
 	};
 	return createHash("sha256").update(stableStringify(payload)).digest("hex");
 }
@@ -1,13 +1,14 @@
 import test from "node:test";
 import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
 import http from "node:http";
 import { promises as fsp } from "node:fs";
-import { mkdtemp, rm } from "node:fs/promises";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import path from "node:path";

 import { createDefaultRegistry } from "../src/commands/registry.js";
-import { keyToPath } from "../src/state/store.js";
+import { keyToPath, stableStringify } from "../src/state/store.js";

 function streamOf(items: any[]) {
 	return (async function* () {
@@ -91,6 +92,258 @@ test("llm.invoke auto-detects OpenClaw provider and normalizes output", async ()
 	}
 });

+test("llm.invoke re-invokes the adapter when --temperature changes", async () => {
+	const registry = createDefaultRegistry();
+	const cmd = registry.get("llm.invoke");
+	assert.ok(cmd);
+	const cacheDir = await mkdtemp(path.join(tmpdir(), "lobster-cache-"));
+
+	const requestLog: any[] = [];
+	const server = http.createServer((req, res) => {
+		if (req.method !== "POST" || req.url !== "/invoke") {
+			res.writeHead(404);
+			res.end("nope");
+			return;
+		}
+		let buf = "";
+		req.setEncoding("utf8");
+		req.on("data", (d) => (buf += d));
+		req.on("end", () => {
+			const parsed = JSON.parse(buf || "{}");
+			requestLog.push(parsed);
+			res.writeHead(200, { "content-type": "application/json" });
+			res.end(
+				JSON.stringify({
+					ok: true,
+					result: {
+						runId: `pi_${requestLog.length}`,
+						model: parsed.model,
+						prompt: parsed.prompt,
+						output: { format: "text", text: `temperature=${parsed.temperature}` },
+					},
+				}),
+			);
+		});
+	});
+
+	await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+	const addr = server.address();
+	const port = typeof addr === "object" && addr ? addr.port : 0;
+	const ctxEnv = {
+		LOBSTER_PI_LLM_ADAPTER_URL: `http://127.0.0.1:${port}`,
+		LOBSTER_CACHE_DIR: cacheDir,
+	};
+
+	try {
+		const runWith = async (temperature: number) => {
+			const result = await cmd.run({
+				input: streamOf([]),
+				args: {
+					_: [],
+					provider: "pi",
+					model: "test-model",
+					prompt: "Sampling parameters matter",
+					"schema-version": "v1",
+					temperature,
+				},
+				ctx: baseCtx(ctxEnv, registry),
+			} as any);
+			return collect(result.output!);
+		};
+
+		const cold = await runWith(0.1);
+		assert.equal(cold[0].source, "pi");
+		assert.equal(requestLog.length, 1);
+		assert.equal(requestLog[0].temperature, 0.1);
+
+		const changed = await runWith(0.9);
+		assert.equal(changed[0].source, "pi");
+		assert.equal(requestLog.length, 2);
+		assert.equal(requestLog[1].temperature, 0.9);
+		assert.notEqual(changed[0].cacheKey, cold[0].cacheKey);
+
+		const replay = await runWith(0.1);
+		assert.equal(replay[0].source, "cache");
+		assert.equal(replay[0].cacheKey, cold[0].cacheKey);
+		assert.equal(requestLog.length, 2);
+	} finally {
+		await rm(cacheDir, { recursive: true, force: true });
+		await closeServer(server);
+	}
+});
+
+test("llm.invoke re-invokes the adapter when --max-output-tokens changes", async () => {
+	const registry = createDefaultRegistry();
+	const cmd = registry.get("llm.invoke");
+	assert.ok(cmd);
+	const cacheDir = await mkdtemp(path.join(tmpdir(), "lobster-cache-"));
+
+	const requestLog: any[] = [];
+	const server = http.createServer((req, res) => {
+		if (req.method !== "POST" || req.url !== "/invoke") {
+			res.writeHead(404);
+			res.end("nope");
+			return;
+		}
+		let buf = "";
+		req.setEncoding("utf8");
+		req.on("data", (d) => (buf += d));
+		req.on("end", () => {
+			const parsed = JSON.parse(buf || "{}");
+			requestLog.push(parsed);
+			res.writeHead(200, { "content-type": "application/json" });
+			res.end(
+				JSON.stringify({
+					ok: true,
+					result: {
+						runId: `pi_${requestLog.length}`,
+						output: { format: "text", text: `budget=${parsed.maxOutputTokens}` },
+					},
+				}),
+			);
+		});
+	});
+
+	await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+	const addr = server.address();
+	const port = typeof addr === "object" && addr ? addr.port : 0;
+	const ctxEnv = {
+		LOBSTER_PI_LLM_ADAPTER_URL: `http://127.0.0.1:${port}`,
+		LOBSTER_CACHE_DIR: cacheDir,
+	};
+
+	try {
+		const runWith = async (maxOutputTokens: number) => {
+			const result = await cmd.run({
+				input: streamOf([]),
+				args: {
+					_: [],
+					provider: "pi",
+					model: "test-model",
+					prompt: "Token budget matters",
+					"schema-version": "v1",
+					"max-output-tokens": maxOutputTokens,
+				},
+				ctx: baseCtx(ctxEnv, registry),
+			} as any);
+			return collect(result.output!);
+		};
+
+		const cold = await runWith(64);
+		assert.equal(cold[0].source, "pi");
+		assert.equal(requestLog.length, 1);
+		assert.equal(requestLog[0].maxOutputTokens, 64);
+
+		const changed = await runWith(4096);
+		assert.equal(changed[0].source, "pi");
+		assert.equal(requestLog.length, 2);
+		assert.equal(requestLog[1].maxOutputTokens, 4096);
+		assert.notEqual(changed[0].cacheKey, cold[0].cacheKey);
+	} finally {
+		await rm(cacheDir, { recursive: true, force: true });
+		await closeServer(server);
+	}
+});
+
+test("llm.invoke does not replay a cache entry written before sampling was keyed", async () => {
+	const registry = createDefaultRegistry();
+	const cmd = registry.get("llm.invoke");
+	assert.ok(cmd);
+	const cacheDir = await mkdtemp(path.join(tmpdir(), "lobster-cache-"));
+	const args = {
+		_: [],
+		provider: "pi",
+		model: "test-model",
+		prompt: "Cache key stability",
+		"schema-version": "v1",
+	};
+
+	// The identity earlier releases hashed: sampling parameters were absent from the payload,
+	// so an answer sampled at temperature 0.9 was stored under the same key a request that
+	// omits sampling computes. Reading that entry back would serve sampling nobody asked for.
+	const legacyKey = createHash("sha256")
+		.update(
+			stableStringify({
+				provider: "pi",
+				prompt: "Cache key stability",
+				model: "test-model",
+				schemaVersion: "v1",
+				artifactHashes: [],
+				outputSchema: null,
+			}),
+		)
+		.digest("hex");
+
+	const requestLog: any[] = [];
+	const server = http.createServer((req, res) => {
+		if (req.method !== "POST" || req.url !== "/invoke") {
+			res.writeHead(404);
+			res.end("nope");
+			return;
+		}
+		let buf = "";
+		req.setEncoding("utf8");
+		req.on("data", (d) => (buf += d));
+		req.on("end", () => {
+			requestLog.push(JSON.parse(buf || "{}"));
+			res.writeHead(200, { "content-type": "application/json" });
+			res.end(
+				JSON.stringify({
+					ok: true,
+					result: { runId: "pi_1", output: { format: "text", text: "fresh" } },
+				}),
+			);
+		});
+	});
+
+	await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+	const addr = server.address();
+	const port = typeof addr === "object" && addr ? addr.port : 0;
+
+	try {
+		await mkdir(path.join(cacheDir, "llm.invoke"), { recursive: true });
+		await writeFile(
+			path.join(cacheDir, "llm.invoke", `${legacyKey}.json`),
+			JSON.stringify({
+				cacheKey: legacyKey,
+				storedAt: "2026-08-01T00:00:00.000Z",
+				items: [
+					{
+						kind: "llm.invoke",
+						cacheKey: legacyKey,
+						status: "completed",
+						source: "pi",
+						cached: false,
+						output: { format: "text", text: "sampled at 0.9" },
+					},
+				],
+			}),
+			"utf8",
+		);
+
+		const result = await cmd.run({
+			input: streamOf([]),
+			args,
+			ctx: baseCtx(
+				{
+					LOBSTER_PI_LLM_ADAPTER_URL: `http://127.0.0.1:${port}`,
+					LOBSTER_CACHE_DIR: cacheDir,
+				},
+				registry,
+			),
+		} as any);
+
+		const items = await collect(result.output!);
+		assert.equal(requestLog.length, 1);
+		assert.equal(items[0].source, "pi");
+		assert.equal(items[0].output.text, "fresh");
+		assert.notEqual(items[0].cacheKey, legacyKey);
+	} finally {
+		await rm(cacheDir, { recursive: true, force: true });
+		await closeServer(server);
+	}
+});
+
 test("llm.invoke uses Pi adapter over local HTTP bridge", async () => {
 	const registry = createDefaultRegistry();
 	const cmd = registry.get("llm.invoke");