Compare commits

...
Author SHA1 Message Date
2de17dd3ee feat(ai): prompt caching for OpenRouter Anthropic routes (#1987)
Rework of #1988 against the post-#2981 cache-breakpoint placement.

- ChatTouchpoint.supports_prompt_cache may now be function-valued
  (per-model-id); OpenRouter scopes it to anthropic/claude-* routes, so
  those stop classifying as degraded:no_caching. capabilities.ts +
  gateway's supportsCache gate handle both forms (fail-closed).
- The system-block cache breakpoint additionally rides
  providerOptions.openaiCompatible on openai-compatible recipes (the
  anthropic key never reaches the compat wire); a new chat-scoped
  recipe fetch shim (compat.chatFetch) lifts the marker into
  OpenRouter's documented content-part cache_control shape before the
  request leaves the process. chatFetch deliberately does NOT displace
  the embedding path's asymmetric input_type shim the way compat.fetch
  would.

Co-authored-by: tmchow <tmchow@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 14:53:41 -07:00
81529ca25c feat(zhipu): add chat + expansion touchpoints to the zhipu recipe
Takeover of #1618, reduced to its still-missing half: the expand()
openai-compat fix from that PR is superseded by the #2372 fallback
(previous commit); the zhipu chat/expansion touchpoint recipe addition
was never picked up anywhere and lands here unchanged.

Co-authored-by: punksterlabs <punksterlabs@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 14:45:19 -07:00
692556236f fix(ai/gateway): structured-output opt-in + capability-aware expansion fallback (#2372)
Takeover of #2373 (mechanical rebase onto master; drift from #2857/#2981).

- expand() no longer calls generateObject unconditionally: openai-compatible
  backends without declared structured-output support go through
  generateText + tolerant JSON parse (prompt now pins the 'queries' key),
  so expansion stops silently degrading to [query].
- New opt-in ChatTouchpoint.supports_structured_outputs threads
  supportsStructuredOutputs into createOpenAICompatible at the chat +
  expansion build sites; opted-in recipes get strict json_schema with a
  call-time text fallback.
- parseLlmJson extracted to leaf src/core/llm-json.ts; re-export preserved
  in conversation-parser/llm-base.ts for existing importers.

Co-authored-by: brettdavies <brettdavies@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 14:44:46 -07:00
12 changed files with 515 additions and 64 deletions
+4 -1
View File
@@ -90,7 +90,10 @@ export function getProviderCapabilities(modelString: string): ProviderCapabiliti
return {
supportsToolCalling: chat.supports_tools === true,
supportsPromptCaching: chat.supports_prompt_cache === true,
// #1987: may be per-model-id (OpenRouter caches anthropic/claude-* routes).
supportsPromptCaching: typeof chat.supports_prompt_cache === 'function'
? chat.supports_prompt_cache(parsed.modelId)
: chat.supports_prompt_cache === true,
// No recipe exposes parallel-tools-specifically yet; gate on supports_tools.
// Subsequent waves can split this into its own recipe field if a provider
// ever supports tools without parallel dispatch.
+125 -19
View File
@@ -48,6 +48,7 @@ import type {
} from './types.ts';
import { resolveRecipe, assertTouchpoint, parseModelId } from './model-resolver.ts';
import { resolveModel, TIER_DEFAULTS } from '../model-config.ts';
import { parseLlmJson } from '../llm-json.ts';
import type { BrainEngine } from '../engine.ts';
import { dimsProviderOptions } from './dims.ts';
import { hasAnthropicKey } from './anthropic-key.ts';
@@ -407,6 +408,34 @@ export function applyOpenAICompatConfig(
return { baseURL, fetch: recipe.compat?.fetch };
}
/**
* Whether an openai-compatible recipe's backend honors OpenAI structured
* outputs. Threaded into `createOpenAICompatible`'s `supportsStructuredOutputs`
* at the chat + expansion build sites, and consulted by `expand()` to pick the
* strict `generateObject` path over the schemaless text path. Single source of
* truth read from the chat touchpoint: the backend serves both chat and
* expansion, so the capability is declared once.
*
* @internal exported for tests.
*/
export function recipeSupportsStructuredOutputs(recipe: Recipe): boolean {
return recipe.touchpoints.chat?.supports_structured_outputs === true;
}
/**
* #1987: `supports_prompt_cache` may be a per-model-id function on
* openai-compatible aggregators (OpenRouter caches `anthropic/claude-*`
* routes but not every routed family). Fail-closed: anything not strictly
* `true` (or a function returning true) means no caching.
*
* @internal exported for tests.
*/
export function chatSupportsPromptCache(recipe: Recipe, modelId: string): boolean {
const support = recipe.touchpoints.chat?.supports_prompt_cache;
if (typeof support === 'function') return support(modelId);
return support === true;
}
/**
* #1250: native providers (anthropic/openai) are instantiated as
* `create<Provider>({ apiKey })` with NO explicit baseURL, so the AI SDK reads
@@ -2259,11 +2288,15 @@ function instantiateExpansion(recipe: Recipe, modelId: string, cfg: AIGatewayCon
const auth = applyResolveAuth(recipe, cfg, 'expansion');
// v0.32: env-templated base URL + optional fetch wrapper.
const compat = applyOpenAICompatConfig(recipe, cfg);
// #1987: chat/expansion-scoped fetch wrapper (does not displace the
// embedding path's asymmetric shim). Recipe-wide fetch wins when both set.
const chatFetch = compat.fetch ?? recipe.compat?.chatFetch;
return createOpenAICompatible({
name: recipe.id,
baseURL: compat.baseURL,
...(compat.fetch ? { fetch: compat.fetch } : {}),
...(chatFetch ? { fetch: chatFetch } : {}),
...auth,
supportsStructuredOutputs: recipeSupportsStructuredOutputs(recipe),
}).languageModel(modelId);
}
}
@@ -2273,6 +2306,20 @@ const ExpansionSchema = z.object({
queries: z.array(z.string()).min(1).max(5),
});
/**
* Recover expansion queries from a schemaless model response. Used by the
* openai-compatible expansion paths: a tolerant JSON decode plus schema
* validation pulls the `queries` array out of the model's text (the prompt
* pins it to a bare JSON object). Returns null when the text carries no valid
* `{ queries: string[] }` object.
*
* @internal exported for tests.
*/
export function parseExpansionResponse(text: string): string[] | null {
const parsed = ExpansionSchema.safeParse(parseLlmJson<unknown>(text));
return parsed.success ? parsed.data.queries : null;
}
/**
* Expand a search query into up to 4 related queries.
* Returns the original query PLUS expansions. On failure, returns just the original.
@@ -2289,24 +2336,65 @@ export async function expand(query: string): Promise<string[]> {
metadata: { query_chars: query.length },
});
const expansionPrompt = [
'Rewrite the search query below into 3-4 different, related queries that would help find relevant documents. Respond with a JSON object in exactly this shape: {"queries": ["rewrite1", "rewrite2", "rewrite3"]}. The JSON key MUST be exactly "queries" (not "rewrites" or any other variation).',
'Return ONLY the JSON object. Do NOT include the original query in the result.',
'Each rewrite should emphasize different aspects, synonyms, or framings.',
'',
`Query: ${query}`,
].join('\n');
try {
const { model, recipe, modelId } = await resolveExpansionProvider(getExpansionModel());
const result = await generateObject({
model,
schema: ExpansionSchema,
// v0.42.20.0 (codex P0) — expansion had NO abortSignal; same stalled-socket
// class as chat. Default the chat timeout.
abortSignal: withDefaultTimeout(undefined, AI_CHAT_TIMEOUT_MS),
prompt: [
'Rewrite the search query below into 3-4 different, related queries that would help find relevant documents.',
'Return ONLY the JSON object. Do NOT include the original query in the result.',
'Each rewrite should emphasize different aspects, synonyms, or framings.',
'',
`Query: ${query}`,
].join('\n'),
});
const expansions = result.object?.queries ?? [];
let expansions: string[];
// Schemaless text path for openai-compatible backends whose structured-output
// support is unknown: the AI SDK can't send a json_schema response_format
// there, so generateObject would warn and silently degrade. generateText + a
// tolerant parse recovers the queries instead. Fresh abortSignal per call.
const viaText = async (): Promise<string[]> => {
const { text } = await generateText({
model,
// v0.42.20.0 (codex P0) — expansion had NO abortSignal; same
// stalled-socket class as chat. Default the chat timeout.
abortSignal: withDefaultTimeout(undefined, AI_CHAT_TIMEOUT_MS),
prompt: expansionPrompt,
});
return parseExpansionResponse(text) ?? [];
};
if (recipe.implementation !== 'openai-compatible') {
// Native providers (Anthropic, OpenAI, Google) support generateObject's
// structured output natively — unchanged path.
const result = await generateObject({
model,
schema: ExpansionSchema,
abortSignal: withDefaultTimeout(undefined, AI_CHAT_TIMEOUT_MS),
prompt: expansionPrompt,
});
expansions = result.object?.queries ?? [];
} else if (recipeSupportsStructuredOutputs(recipe)) {
// openai-compatible backend that honors strict json_schema: request the
// schema (strict validation), and fall back to the text path if it is
// rejected at call time so a mis-declared capability never drops expansion.
try {
const result = await generateObject({
model,
schema: ExpansionSchema,
abortSignal: withDefaultTimeout(undefined, AI_CHAT_TIMEOUT_MS),
prompt: expansionPrompt,
});
expansions = result.object?.queries ?? [];
} catch {
expansions = await viaText();
}
} else {
// openai-compatible backend, structured-output support unknown: skip the
// json_schema attempt entirely (no SDK warning, no silent degradation).
expansions = await viaText();
}
// Deduplicate + include the original query
const seen = new Set<string>();
const all = [query, ...expansions].filter(q => {
@@ -2747,11 +2835,15 @@ function instantiateChat(recipe: Recipe, modelId: string, cfg: AIGatewayConfig):
const auth = applyResolveAuth(recipe, cfg, 'chat');
// v0.32: env-templated base URL + optional fetch wrapper.
const compat = applyOpenAICompatConfig(recipe, cfg);
// #1987: chat/expansion-scoped fetch wrapper (does not displace the
// embedding path's asymmetric shim). Recipe-wide fetch wins when both set.
const chatFetch = compat.fetch ?? recipe.compat?.chatFetch;
return createOpenAICompatible({
name: recipe.id,
baseURL: compat.baseURL,
...(compat.fetch ? { fetch: compat.fetch } : {}),
...(chatFetch ? { fetch: chatFetch } : {}),
...auth,
supportsStructuredOutputs: recipeSupportsStructuredOutputs(recipe),
}).languageModel(modelId);
}
default:
@@ -3030,7 +3122,7 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
const { model, recipe, modelId } = await resolveChatProvider(modelStr);
const cfg = requireConfig();
const supportsCache = recipe.touchpoints.chat?.supports_prompt_cache === true;
const supportsCache = chatSupportsPromptCache(recipe, modelId);
const useCache = !!opts.cacheSystem && supportsCache;
const tools = toAISDKTools(opts.tools);
@@ -3129,7 +3221,21 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
? {
role: 'system' as const,
content: opts.system,
providerOptions: { anthropic: { cacheControl: cacheControlValue } },
providerOptions: {
anthropic: { cacheControl: cacheControlValue },
// #1987: OpenRouter Anthropic routes ride the openai-compatible
// wire, where `providerOptions.anthropic` never leaves the process.
// The openai-compatible adapter spreads message-level
// `openaiCompatible` metadata onto the outgoing system message; the
// recipe's chatFetch shim (liftMessageCacheControl) then lifts the
// marker into OpenRouter's documented content-part cache_control
// shape. Only reachable when chatSupportsPromptCache passed (i.e.
// anthropic/claude-* via openrouter), so no other compat recipe
// ever sees the marker.
...(recipe.implementation === 'openai-compatible'
? { openaiCompatible: { cache_control: cacheControlValue } }
: {}),
},
}
: opts.system;
+73 -1
View File
@@ -1,5 +1,72 @@
import type { Recipe } from '../types.ts';
/**
* OpenRouter prompt caching (#1987): OpenRouter forwards Anthropic
* `cache_control` breakpoints on Claude routes. Family-scoped (not "every
* anthropic/* model forever") so capability classification stays honest for
* routed models with no documented cache support.
*
* @internal exported for tests.
*/
export function openrouterSupportsPromptCache(modelId: string): boolean {
return modelId.trim().toLowerCase().startsWith('anthropic/claude-');
}
/**
* Lift message-level `cache_control` markers into OpenRouter's documented
* shape: a multipart content array whose text part carries the marker
* (OpenRouter only reads cache_control inside content parts). The gateway
* plants the message-level marker via the system block's
* `providerOptions.openaiCompatible` (the openai-compatible adapter spreads
* that metadata onto the outgoing message); this rewrite runs just before
* the request leaves the process. Mutates `body`; returns true when
* something changed.
*
* @internal exported for tests.
*/
export function liftMessageCacheControl(body: unknown): boolean {
if (!body || typeof body !== 'object') return false;
const messages = (body as Record<string, unknown>).messages;
if (!Array.isArray(messages)) return false;
let modified = false;
for (const msg of messages) {
if (!msg || typeof msg !== 'object') continue;
const m = msg as Record<string, unknown>;
if (!m.cache_control || typeof m.cache_control !== 'object') continue;
if (typeof m.content !== 'string') continue;
m.content = [{ type: 'text', text: m.content, cache_control: m.cache_control }];
delete m.cache_control;
modified = true;
}
return modified;
}
/**
* Chat-path fetch shim: rewrites outbound chat/completions bodies via
* `liftMessageCacheControl`. Fail-open — any parse error passes the original
* request through untouched. Installed as `compat.chatFetch` so the embedding
* path keeps the gateway's asymmetric input_type shim.
*/
export const openrouterCacheControlFetch = (async (
input: RequestInfo | URL,
init?: RequestInit,
): Promise<Response> => {
if (init?.body && typeof init.body === 'string') {
try {
const body = JSON.parse(init.body);
if (liftMessageCacheControl(body)) {
// Drop Content-Length so fetch recomputes it from the new body.
const headers = new Headers(init.headers ?? {});
headers.delete('content-length');
init = { ...init, body: JSON.stringify(body), headers };
}
} catch {
// Non-JSON body: pass through untouched.
}
}
return fetch(input as any, init as any);
}) as unknown as typeof fetch;
/**
* OpenRouter — single-key fan-out to OpenAI, Anthropic, Google, DeepSeek, and
* dozens of other providers via a single OpenAI-compatible endpoint at
@@ -93,13 +160,18 @@ export const openrouter: Recipe = {
supports_tools: true,
// Informational only — real gate is isAnthropicProvider() upstream.
supports_subagent_loop: false,
supports_prompt_cache: false,
// #1987: per-model-family — OpenRouter forwards Anthropic cache_control
// on Claude routes; other routed families stay uncached.
supports_prompt_cache: openrouterSupportsPromptCache,
// No max_context_tokens: catalog spans 128K to 1M+; a single recipe-wide
// value is either unsafe for smaller models or wasteful for larger ones.
// Let upstream errors surface per-model.
price_last_verified: '2026-05-20',
},
},
// #1987: chat-path-only shim (see chatFetch doc in types.ts) that lifts the
// gateway's message-level cache marker into OpenRouter's content-part shape.
compat: { chatFetch: openrouterCacheControlFetch },
setup_hint:
'Get an API key at https://openrouter.ai/settings/keys, then `export OPENROUTER_API_KEY=...` and use `openrouter:<provider>/<model>`. Optional overrides: OPENROUTER_BASE_URL (proxy), OPENROUTER_REFERER (attribution URL), OPENROUTER_TITLE (attribution name).',
};
+16 -1
View File
@@ -34,7 +34,22 @@ export const zhipu: Recipe = {
max_batch_tokens: 8192,
chars_per_token: 2,
},
expansion: {
models: ['glm-4.7', 'glm-4.7-flash'],
cost_per_1m_tokens_usd: 0.15,
price_last_verified: '2026-05-29',
},
chat: {
models: ['glm-4.7', 'glm-4.7-flash', 'glm-5.1', 'glm-5-turbo'],
supports_tools: true,
supports_subagent_loop: false,
supports_prompt_cache: false,
max_context_tokens: 200000,
cost_per_1m_input_usd: 0.5,
cost_per_1m_output_usd: 2.0,
price_last_verified: '2026-05-29',
},
},
setup_hint:
'Get an API key at https://open.bigmodel.cn/, then `export ZHIPUAI_API_KEY=...`',
'Get an API key at https://open.bigmodel.cn/ or https://z.ai/, then `export ZHIPUAI_API_KEY=...`',
};
+27 -2
View File
@@ -222,8 +222,24 @@ export interface ChatTouchpoint {
* Strictly stronger than supports_tools.
*/
supports_subagent_loop: boolean;
/** Anthropic-style ephemeral prompt cache markers honored. */
supports_prompt_cache?: boolean;
/**
* Anthropic-style ephemeral prompt cache markers honored. Static booleans
* cover native providers; openai-compatible aggregators may decide per
* model id (OpenRouter forwards Anthropic cache_control on
* `anthropic/claude-*` routes but not for every routed model family).
*/
supports_prompt_cache?: boolean | ((modelId: string) => boolean);
/**
* Backend honors OpenAI structured outputs (a strict `json_schema`
* response_format). Threaded into `createOpenAICompatible`'s
* `supportsStructuredOutputs` so query expansion's `generateObject` sends a
* real schema (strict validation) instead of degrading to schemaless JSON.
* Default false: an openai-compatible recipe may front arbitrary backends,
* most of which lack strict json_schema support, so `expand()` routes them
* through the schemaless text path. Opt in per recipe when the backend is
* known to honor it.
*/
supports_structured_outputs?: boolean;
max_context_tokens?: number;
cost_per_1m_input_usd?: number;
cost_per_1m_output_usd?: number;
@@ -337,6 +353,15 @@ export interface Recipe {
*/
compat?: {
fetch?: typeof fetch;
/**
* Chat/expansion-only fetch wrapper. Unlike `fetch`, this does NOT
* displace the embedding path's asymmetric input_type shim
* (`openAICompatAsymmetricFetch`) — use it when only the chat wire
* shape needs rewriting (OpenRouter's cache_control lift). A recipe
* `fetch` (or `resolveOpenAICompatConfig` fetch) takes precedence on
* the chat path when both are present.
*/
chatFetch?: typeof fetch;
};
/**
* v0.32 (D13=A): optional runtime readiness check for local-server
+4 -38
View File
@@ -290,41 +290,7 @@ function splitCacheKey(key: string): [string?, string?, string?] {
return [shape, model, sha];
}
/**
* 4-strategy JSON repair (lifted from `eval/longmemeval/extract.ts:50`
* for object-shaped output; the original was array-shaped). Caller's
* `parse` function uses this for tolerant LLM-output decoding.
*
* Strategies:
* 1. Strip ```json...``` fences if present, then JSON.parse.
* 2. Direct JSON.parse.
* 3. Find first {...} substring (or [...] if array=true) and parse.
* 4. Return null.
*
* Adversarial input throws caught by caller's try/catch (parse returns
* null upstream).
*/
export function parseLlmJson<T>(raw: string, opts: { array?: boolean } = {}): T | null {
if (typeof raw !== 'string' || !raw.trim()) return null;
const fenceMatch = raw.match(/```(?:json)?\s*\n?([\s\S]*?)```/i);
const cleaned = (fenceMatch ? fenceMatch[1] : raw).trim();
try {
const direct = JSON.parse(cleaned);
if (opts.array && Array.isArray(direct)) return direct as T;
if (!opts.array && direct !== null && typeof direct === 'object') return direct as T;
} catch {
// fall through
}
const pattern = opts.array ? /\[[\s\S]*\]/ : /\{[\s\S]*\}/;
const match = cleaned.match(pattern);
if (match) {
try {
const second = JSON.parse(match[0]);
if (opts.array && Array.isArray(second)) return second as T;
if (!opts.array && second !== null && typeof second === 'object') return second as T;
} catch {
// fall through
}
}
return null;
}
// Tolerant LLM-output JSON decoder. Re-exported from the leaf util so existing
// importers (llm-fallback, llm-polish) keep their import path while the gateway
// can reuse it without a dependency cycle.
export { parseLlmJson } from '../llm-json.ts';
+37
View File
@@ -0,0 +1,37 @@
/**
* Tolerant decode of a JSON object (or array) embedded in LLM output. A leaf
* util with no provider/gateway imports so any layer can reuse it without a
* dependency cycle.
*
* Strategies, in order:
* 1. Strip ```json...``` fences if present, then JSON.parse.
* 2. Direct JSON.parse.
* 3. Find the first {...} substring (or [...] when array=true) and parse.
* 4. Return null.
*
* Adversarial input throws are swallowed; callers get null on any failure.
*/
export function parseLlmJson<T>(raw: string, opts: { array?: boolean } = {}): T | null {
if (typeof raw !== 'string' || !raw.trim()) return null;
const fenceMatch = raw.match(/```(?:json)?\s*\n?([\s\S]*?)```/i);
const cleaned = (fenceMatch ? fenceMatch[1] : raw).trim();
try {
const direct = JSON.parse(cleaned);
if (opts.array && Array.isArray(direct)) return direct as T;
if (!opts.array && direct !== null && typeof direct === 'object') return direct as T;
} catch {
// fall through
}
const pattern = opts.array ? /\[[\s\S]*\]/ : /\{[\s\S]*\}/;
const match = cleaned.match(pattern);
if (match) {
try {
const second = JSON.parse(match[0]);
if (opts.array && Array.isArray(second)) return second as T;
if (!opts.array && second !== null && typeof second === 'object') return second as T;
} catch {
// fall through
}
}
return null;
}
+16
View File
@@ -24,6 +24,17 @@ describe('getProviderCapabilities (v0.38 Slice 1 — D6/D7 recipe-driven capabil
expect(caps.maxContext).toBe(1000000); // Gemini 1.5 Pro
});
it('marks OpenRouter Anthropic routes as cache-capable (#1987 function-valued capability)', () => {
const caps = getProviderCapabilities('openrouter:anthropic/claude-sonnet-4.6');
expect(caps.supportsToolCalling).toBe(true);
expect(caps.supportsPromptCaching).toBe(true);
});
it('does not mark every OpenRouter route as cache-capable', () => {
const caps = getProviderCapabilities('openrouter:deepseek/deepseek-chat');
expect(caps.supportsPromptCaching).toBe(false);
});
it('honors Anthropic alias (undated → dated)', () => {
const caps = getProviderCapabilities('anthropic:claude-haiku-4-5');
expect(caps.supportsToolCalling).toBe(true);
@@ -54,6 +65,11 @@ describe('classifyCapabilities (D6 — three-tier capability verdict)', () => {
expect(classifyCapabilities('openai:gpt-5.2')).toBe('degraded:no_caching');
});
it('returns ok for OpenRouter Anthropic routes; other routes stay degraded:no_caching (#1987)', () => {
expect(classifyCapabilities('openrouter:anthropic/claude-sonnet-4.6')).toBe('ok');
expect(classifyCapabilities('openrouter:deepseek/deepseek-chat')).toBe('degraded:no_caching');
});
it('returns degraded:no_caching for Google Gemini', () => {
expect(classifyCapabilities('google:gemini-1.5-pro')).toBe('degraded:no_caching');
});
+53 -1
View File
@@ -29,7 +29,7 @@
* the bug made you believe was sufficient.
*/
import { describe, test, expect, beforeEach } from 'bun:test';
import { describe, test, expect, beforeEach, afterAll } from 'bun:test';
import {
chat,
configureGateway,
@@ -43,6 +43,11 @@ describe('gbrain#2490 — Anthropic cache breakpoint placement', () => {
__setGenerateTextTransportForTests(null);
});
afterAll(() => {
resetGateway();
__setGenerateTextTransportForTests(null);
});
async function captureTransportArgs(
opts: Partial<Parameters<typeof chat>[0]> = {},
): Promise<any> {
@@ -192,4 +197,51 @@ describe('gbrain#2490 — Anthropic cache breakpoint placement', () => {
expect((captured.system as any)?.providerOptions?.anthropic?.cacheControl).toEqual(expected);
expect(captured.tools?.search?.providerOptions?.anthropic?.cacheControl).toEqual(expected);
});
// #1987 — OpenRouter Anthropic routes support prompt caching. The
// capability is per-model-family (function-valued supports_prompt_cache),
// and the system-block marker must ALSO ride `openaiCompatible` metadata:
// `providerOptions.anthropic` never leaves the process on the
// openai-compatible wire, so without the extra key no cache_control could
// ever reach OpenRouter.
async function captureOpenRouterArgs(model: string): Promise<any> {
let captured: any;
__setGenerateTextTransportForTests(async (args: any) => {
captured = args;
return {
content: [{ type: 'text', text: 'ok' }],
finishReason: 'stop',
usage: { inputTokens: 1, outputTokens: 1 },
} as any;
});
configureGateway({
chat_model: model,
env: { OPENROUTER_API_KEY: 'fake' },
});
await chat({
model,
system: 'SYS',
cacheSystem: true,
messages: [{ role: 'user', content: 'hello' }],
});
return captured;
}
test('cacheSystem:true on openrouter:anthropic/claude-* plants anthropic AND openaiCompatible markers on the system block (#1987)', async () => {
const args = await captureOpenRouterArgs('openrouter:anthropic/claude-sonnet-4.6');
expect(args.system).toEqual({
role: 'system',
content: 'SYS',
providerOptions: {
anthropic: { cacheControl: { type: 'ephemeral' } },
openaiCompatible: { cache_control: { type: 'ephemeral' } },
},
});
});
test('cacheSystem:true on a non-Claude OpenRouter route stays uncached (per-model-family gate)', async () => {
const args = await captureOpenRouterArgs('openrouter:deepseek/deepseek-chat');
expect(args.system).toBe('SYS');
expect(args.providerOptions?.anthropic).toBeUndefined();
});
});
+57 -1
View File
@@ -25,10 +25,13 @@ import {
getChatFallbackChain,
chat,
__setGenerateTextTransportForTests,
recipeSupportsStructuredOutputs,
parseExpansionResponse,
} from '../../src/core/ai/gateway.ts';
import { parseModelId, resolveRecipe, assertTouchpoint } from '../../src/core/ai/model-resolver.ts';
import { AIConfigError } from '../../src/core/ai/errors.ts';
import { listRecipes, getRecipe } from '../../src/core/ai/recipes/index.ts';
import type { Recipe } from '../../src/core/ai/types.ts';
describe('chat touchpoint — recipe registry', () => {
test('all six chat-capable providers ship a chat touchpoint with supports_subagent_loop', () => {
@@ -42,11 +45,14 @@ describe('chat touchpoint — recipe registry', () => {
}
});
test('only Anthropic claims supports_prompt_cache=true', () => {
test('only Anthropic and model-family-gated OpenRouter claim supports_prompt_cache', () => {
for (const r of listRecipes()) {
if (!r.touchpoints.chat) continue;
if (r.id === 'anthropic') {
expect(r.touchpoints.chat.supports_prompt_cache).toBe(true);
} else if (r.id === 'openrouter') {
// #1987: per-model-family (anthropic/claude-* routes only).
expect(typeof r.touchpoints.chat.supports_prompt_cache).toBe('function');
} else {
expect(r.touchpoints.chat.supports_prompt_cache ?? false).toBe(false);
}
@@ -65,6 +71,56 @@ describe('chat touchpoint — recipe registry', () => {
});
});
describe('expansion — structured-output capability gating', () => {
test('openai-compat chat recipes default to no structured-output support', () => {
// The capability is opt-in per recipe: an openai-compatible recipe may front
// arbitrary backends, so expand() routes the default through the schemaless
// text path rather than requesting a json_schema the backend may reject.
for (const id of ['deepseek', 'groq', 'together']) {
expect(recipeSupportsStructuredOutputs(getRecipe(id)!)).toBe(false);
}
});
test('recipeSupportsStructuredOutputs is false when no chat touchpoint exists', () => {
// Embedding-only recipes have no chat touchpoint; the helper must not throw.
expect(recipeSupportsStructuredOutputs(getRecipe('voyage')!)).toBe(false);
});
test('recipeSupportsStructuredOutputs is true when a recipe opts in', () => {
const optedIn = {
id: 'synthetic',
touchpoints: { chat: { models: [], supports_tools: true, supports_subagent_loop: true, supports_structured_outputs: true } },
} as unknown as Recipe;
expect(recipeSupportsStructuredOutputs(optedIn)).toBe(true);
});
});
describe('expansion — schemaless recovery (parseExpansionResponse)', () => {
// The openai-compat expansion paths recover queries from raw model text. This
// is the testable seam both the default and the strict-fallback paths share.
test('recovers queries from clean JSON', () => {
expect(parseExpansionResponse('{"queries":["a","b","c"]}')).toEqual(['a', 'b', 'c']);
});
test('recovers queries from fenced JSON', () => {
expect(parseExpansionResponse('```json\n{"queries":["a","b"]}\n```')).toEqual(['a', 'b']);
});
test('recovers queries from prose-wrapped JSON', () => {
expect(parseExpansionResponse('Here you go: {"queries":["a"]} done')).toEqual(['a']);
});
test('returns null for non-JSON so the caller can drop expansion cleanly', () => {
expect(parseExpansionResponse('I cannot help with that.')).toBeNull();
});
test('returns null when the JSON violates the schema', () => {
expect(parseExpansionResponse('{"queries":[]}')).toBeNull(); // min(1)
expect(parseExpansionResponse('{"rewrites":["a"]}')).toBeNull(); // wrong key
expect(parseExpansionResponse('{"queries":[1,2]}')).toBeNull(); // wrong item type
});
});
describe('chat touchpoint — model resolver + aliases (Codex F-OV-5)', () => {
test('parseModelId handles dated and undated forms identically at parse time', () => {
expect(parseModelId('anthropic:claude-sonnet-4-6')).toEqual({
+88
View File
@@ -11,6 +11,11 @@
import { describe, expect, test } from 'bun:test';
import { getRecipe } from '../../src/core/ai/recipes/index.ts';
import {
openrouterSupportsPromptCache,
liftMessageCacheControl,
openrouterCacheControlFetch,
} from '../../src/core/ai/recipes/openrouter.ts';
import { defaultResolveAuth } from '../../src/core/ai/gateway.ts';
import { assertTouchpoint } from '../../src/core/ai/model-resolver.ts';
import { AIConfigError } from '../../src/core/ai/errors.ts';
@@ -135,4 +140,87 @@ describe('recipe: openrouter', () => {
expect(r.setup_hint).toContain('OPENROUTER_REFERER');
expect(r.setup_hint).toContain('OPENROUTER_TITLE');
});
test('12. prompt cache capability is scoped to anthropic/claude-* routes (#1987)', () => {
expect(openrouterSupportsPromptCache('anthropic/claude-sonnet-4.6')).toBe(true);
expect(openrouterSupportsPromptCache('anthropic/claude-opus-4.7')).toBe(true);
expect(openrouterSupportsPromptCache('ANTHROPIC/Claude-Haiku-4.5')).toBe(true); // case-insensitive
expect(openrouterSupportsPromptCache('openai/gpt-5.2')).toBe(false);
expect(openrouterSupportsPromptCache('deepseek/deepseek-chat')).toBe(false);
expect(openrouterSupportsPromptCache('google/gemini-3-flash-preview')).toBe(false);
});
test('13. liftMessageCacheControl moves a message-level marker into the content part', () => {
const body: any = {
model: 'anthropic/claude-sonnet-4.6',
messages: [
{ role: 'system', content: 'SYS', cache_control: { type: 'ephemeral' } },
{ role: 'user', content: 'hello' },
],
};
expect(liftMessageCacheControl(body)).toBe(true);
expect(body.messages[0]).toEqual({
role: 'system',
content: [{ type: 'text', text: 'SYS', cache_control: { type: 'ephemeral' } }],
});
// The user message (no marker) is untouched.
expect(body.messages[1]).toEqual({ role: 'user', content: 'hello' });
});
test('14. liftMessageCacheControl is a no-op when no marker rides the body', () => {
const body = {
model: 'openai/gpt-5.2',
messages: [
{ role: 'system', content: 'SYS' },
{ role: 'user', content: 'hello' },
],
};
expect(liftMessageCacheControl(body)).toBe(false);
expect(body.messages[0]).toEqual({ role: 'system', content: 'SYS' });
expect(liftMessageCacheControl(undefined)).toBe(false);
expect(liftMessageCacheControl({ input: 'embedding body, no messages' })).toBe(false);
});
test('15. cache fetch shim rewrites the outbound body and recomputes content-length', async () => {
const originalFetch = globalThis.fetch;
const calls: Array<{ init?: RequestInit }> = [];
globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => {
calls.push({ init });
return new Response('{}', { status: 200 });
}) as typeof fetch;
try {
await openrouterCacheControlFetch('https://openrouter.ai/api/v1/chat/completions', {
method: 'POST',
headers: { 'content-length': '999', 'content-type': 'application/json' },
body: JSON.stringify({
model: 'anthropic/claude-sonnet-4.6',
messages: [{ role: 'system', content: 'SYS', cache_control: { type: 'ephemeral' } }],
}),
});
const rewritten = JSON.parse(calls[0].init!.body as string);
expect(rewritten.messages[0].content).toEqual([
{ type: 'text', text: 'SYS', cache_control: { type: 'ephemeral' } },
]);
expect(rewritten.messages[0].cache_control).toBeUndefined();
expect(new Headers(calls[0].init!.headers).get('content-length')).toBeNull();
// Marker-free body passes through byte-identical (fail-open contract).
const plain = JSON.stringify({ model: 'openai/gpt-5.2', messages: [{ role: 'user', content: 'hi' }] });
await openrouterCacheControlFetch('https://openrouter.ai/api/v1/chat/completions', {
method: 'POST',
body: plain,
});
expect(calls[1].init!.body).toBe(plain);
} finally {
globalThis.fetch = originalFetch;
}
});
test('16. cache shim is installed as compat.chatFetch (chat path only, embedding shim preserved)', () => {
const r = getRecipe('openrouter')!;
expect(r.compat?.chatFetch).toBe(openrouterCacheControlFetch);
// Deliberately NOT compat.fetch: that would displace the gateway's
// asymmetric input_type shim on the embedding path.
expect(r.compat?.fetch).toBeUndefined();
});
});
+15
View File
@@ -42,6 +42,21 @@ describe('recipe: zhipu', () => {
);
});
test('chat + expansion touchpoints declared (GLM via openai-compat)', () => {
const r = getRecipe('zhipu')!;
expect(r.touchpoints.chat).toBeDefined();
expect(r.touchpoints.chat!.models).toContain('glm-4.7');
expect(r.touchpoints.chat!.supports_tools).toBe(true);
// Subagent loops stay Anthropic-direct (isAnthropicProvider gate).
expect(r.touchpoints.chat!.supports_subagent_loop).toBe(false);
expect(r.touchpoints.chat!.supports_prompt_cache ?? false).toBe(false);
// No structured-output opt-in: expand() must route zhipu through the
// schemaless generateText path (#2372).
expect(r.touchpoints.chat!.supports_structured_outputs ?? false).toBe(false);
expect(r.touchpoints.expansion).toBeDefined();
expect(r.touchpoints.expansion!.models).toContain('glm-4.7-flash');
});
test('default auth: ZHIPUAI_API_KEY set → "Bearer <key>"', () => {
const r = getRecipe('zhipu')!;
const auth = defaultResolveAuth(r, { ZHIPUAI_API_KEY: 'fake-zhipu-key' }, 'embedding');