mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 08:53:22 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b584b3c0c2 | ||
|
|
f45d4a6345 | ||
|
|
140509be75 |
@@ -83,7 +83,7 @@ per-release `**vX.Y.Z:**` narration — CI enforces this
|
||||
- `src/core/eval/cycle-default.ts` — single source of truth for the eval cycle-count default. Exports `DEFAULT_CYCLES_TTY = 3`, `DEFAULT_CYCLES_NONTTY = 1`, `resolveCycleDefault(explicit, isTty): {cycles, usedNonTtyDefault}`, and `cycleDefaultSuffix(r)` (returns ` (non-interactive default; --cycles N for more)` only when the non-TTY default was applied, else `''`). Consumed by `eval-cross-modal.ts`, `eval-takes-quality.ts` (run + regress), and `takes-quality-eval/runner.ts` (core uses only the constant — library stays TTY-agnostic; the CLI owns the TTY=3 upgrade + banner annotation). `eval-suspected-contradictions.ts` applies the same transparency to its `$5`/`$1` budget default via a `budgetUsdExplicit` flag (the budget is overwritten in-place so explicitness can't be inferred post-hoc). Not shared with `resolveWorkersWithClamp` (different domain, no engine, no dedup). Pinned by `test/eval/cycle-default.test.ts`, `test/eval-suspected-contradictions-budget-default.test.ts`.
|
||||
- `src/core/cross-modal-eval/json-repair.ts` — `parseModelJSON(raw)` named export with a 4-strategy fallback chain (direct parse → fence-strip → trailing-comma + single-quote + embedded-newline repair → regex nuclear option). Adversarial input throws rather than fabricating scores — the aggregator treats a throw as "this model contributed nothing this cycle" so the gate stays correct at >=2/3 successes.
|
||||
- `src/core/cross-modal-eval/aggregate.ts` — pure verdict logic. Pass criterion: `(successes >= 2) AND (every dim mean >= 7) AND (every dim min across models >= 5)`. Inconclusive when <2/3 models returned parseable scores (regression guard for the v1 `Object.values({}).every(...) === true` empty-array PASS bug).
|
||||
- `src/core/cross-modal-eval/runner.ts` — orchestrator. Each cycle runs `Promise.allSettled([gwChat(slotA), gwChat(slotB), gwChat(slotC)])` (bare allSettled, no rate-leases for the CLI path). Stops early on PASS or INCONCLUSIVE; runs up to 3 cycles. Default slots: `openai:gpt-4o` / `anthropic:claude-opus-4-7` / `google:gemini-1.5-pro`. `estimateCost()` exports a small per-model pricing table (drifts; refresh alongside model-family bumps).
|
||||
- `src/core/cross-modal-eval/runner.ts` — orchestrator. Each cycle runs `Promise.allSettled([gwChat(slotA), gwChat(slotB), gwChat(slotC)])` (bare allSettled, no rate-leases for the CLI path). Stops early on PASS or INCONCLUSIVE; runs up to 3 cycles. Default slots are alias names (`gpt` / `opus` / `gemini`) resolved to full `provider:model` ids at run time via `resolveSlots()` → `resolveAlias`/`DEFAULT_ALIASES` (`src/core/model-config.ts`) — one maintenance point for model-family bumps instead of hardcoded ids that rot. `estimateCost()` prices resolved slot ids from the canonical table via `canonicalLookup`.
|
||||
- `src/core/cross-modal-eval/receipt-name.ts` — receipt filename binds (slug, SKILL.md sha-8). `findReceiptForSkill(skillPath, receiptDir)` returns `'found' | 'stale' | 'missing'`. Skillify-check surfaces the status as informational; the audit does NOT fail on missing/stale receipts.
|
||||
- `src/core/cross-modal-eval/receipt-write.ts` — wraps `fs.writeFileSync` with `mkdirSync({recursive:true})` ahead of every write (`gbrainPath()` does NOT auto-mkdir).
|
||||
- `src/commands/eval-export.ts` — streams `eval_candidates` rows as NDJSON to stdout with `schema_version: 1` prefix on every line. EPIPE-safe, progress heartbeats on stderr, stable id-desc tiebreaker so `--since` windows never dupe/miss rows.
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
DEFAULT_DIMENSIONS,
|
||||
DEFAULT_SLOTS,
|
||||
estimateCost,
|
||||
resolveSlots,
|
||||
runEval,
|
||||
} from '../core/cross-modal-eval/runner.ts';
|
||||
import type {
|
||||
@@ -76,9 +77,9 @@ FLAGS:
|
||||
dimensions (goal, depth, sourcing, specificity, useful).
|
||||
--cycles N 1-3. Default: 3 in TTY, 1 in non-TTY (T11). Each
|
||||
cycle is 3 model calls; verdict aggregates over them.
|
||||
--slot-a-model <id> Override default 'openai:gpt-4o'.
|
||||
--slot-b-model <id> Override default 'anthropic:claude-opus-4-7'.
|
||||
--slot-c-model <id> Override default 'google:gemini-1.5-pro'.
|
||||
--slot-a-model <id> Override default 'gpt' alias (resolved via models.aliases).
|
||||
--slot-b-model <id> Override default 'opus' alias.
|
||||
--slot-c-model <id> Override default 'gemini' alias.
|
||||
--receipt-dir <path> Default: gbrainPath('eval-receipts').
|
||||
--max-tokens N Output token budget per call. Default: 4000.
|
||||
--json Emit final aggregate as JSON to stdout (progress to stderr).
|
||||
@@ -353,11 +354,13 @@ export async function runEvalCrossModal(args: string[], opts: RunCrossModalOpts
|
||||
const receiptDir = parsed.receiptDir ?? gbrainPath('eval-receipts');
|
||||
const maxTokens = parsed.maxTokens ?? 4000;
|
||||
|
||||
const slots: SlotConfig[] = [
|
||||
// #1270: resolve alias-form defaults (gpt/opus/gemini) to full ids here so
|
||||
// the cost banner and receipts show real model ids.
|
||||
const slots: SlotConfig[] = await resolveSlots([
|
||||
{ id: 'A', model: parsed.slotAModel ?? DEFAULT_SLOTS[0]!.model },
|
||||
{ id: 'B', model: parsed.slotBModel ?? DEFAULT_SLOTS[1]!.model },
|
||||
{ id: 'C', model: parsed.slotCModel ?? DEFAULT_SLOTS[2]!.model },
|
||||
];
|
||||
]);
|
||||
|
||||
// Configure the AI gateway. Without this, every chat() call throws
|
||||
// "AI gateway is not configured" because the cli.ts no-DB branch skips
|
||||
@@ -615,11 +618,12 @@ async function runBatchMode(parsed: ParsedArgs, opts: RunCrossModalOpts): Promis
|
||||
const maxTokens = parsed.maxTokens ?? 4000;
|
||||
const maxUsd = parsed.maxUsd ?? 5.0;
|
||||
|
||||
const slots: SlotConfig[] = [
|
||||
// #1270: resolve alias-form defaults (gpt/opus/gemini) to full ids.
|
||||
const slots: SlotConfig[] = await resolveSlots([
|
||||
{ id: 'A', model: parsed.slotAModel ?? DEFAULT_SLOTS[0]!.model },
|
||||
{ id: 'B', model: parsed.slotBModel ?? DEFAULT_SLOTS[1]!.model },
|
||||
{ id: 'C', model: parsed.slotCModel ?? DEFAULT_SLOTS[2]!.model },
|
||||
];
|
||||
]);
|
||||
|
||||
// v0.40.1.0 Track D (codex CDX-2): --limit must be >= 1. Passing
|
||||
// --limit 0 would let an empty result fall through to PASS with
|
||||
|
||||
@@ -18,6 +18,9 @@ import { importFromContent } from '../core/import-file.ts';
|
||||
import { hybridSearch } from '../core/search/hybrid.ts';
|
||||
import { expandQuery } from '../core/search/expansion.ts';
|
||||
import { resolveModel } from '../core/model-config.ts';
|
||||
import { splitProviderModelId } from '../core/model-id.ts';
|
||||
import { configureGatewayIfUninitialized } from '../core/ai/gateway.ts';
|
||||
import { __thinkAdapter } from '../core/think/index.ts';
|
||||
import type { ThinkLLMClient } from '../core/think/index.ts';
|
||||
import { createProgress } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
@@ -54,6 +57,8 @@ interface ParsedArgs {
|
||||
datasetPath?: string;
|
||||
limit?: number;
|
||||
model?: string;
|
||||
/** #2099 — override the trajectory claim-extractor model. */
|
||||
extractorModel?: string;
|
||||
retrievalOnly: boolean;
|
||||
keywordOnly: boolean;
|
||||
expansion: boolean;
|
||||
@@ -109,6 +114,7 @@ function parseArgs(args: string[]): ParsedArgs {
|
||||
if (a === '--no-trajectory') { out.noTrajectory = true; continue; }
|
||||
if (a === '--limit') { out.limit = Number(args[++i]); continue; }
|
||||
if (a === '--model') { out.model = args[++i]; continue; }
|
||||
if (a === '--extractor-model') { out.extractorModel = args[++i]; continue; }
|
||||
if (a === '--top-k') { out.topK = Number(args[++i]); continue; }
|
||||
if (a === '--output') { out.outputPath = args[++i]; continue; }
|
||||
if (a === '--resume-from') { out.resumeFromPath = args[++i]; continue; }
|
||||
@@ -147,6 +153,8 @@ function printHelp(): void {
|
||||
`Options:\n` +
|
||||
` --limit N Run only the first N questions.\n` +
|
||||
` --model M Override answer-generation model (default: resolveModel).\n` +
|
||||
` --extractor-model M Override the trajectory claim-extractor model (default:\n` +
|
||||
` tier-utility via resolveModel).\n` +
|
||||
` --retrieval-only Skip LLM answer generation; emit retrieved sessions instead.\n` +
|
||||
` --keyword-only Skip vector embedding; pure keyword retrieval.\n` +
|
||||
` --expansion Enable multi-query expansion (off by default for benchmarks).\n` +
|
||||
@@ -363,6 +371,34 @@ async function generateAnswer(
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* #2099 — strip the `provider:` prefix off params.model at the raw-SDK
|
||||
* boundary. resolveModel returns provider-prefixed ids; the raw Anthropic SDK
|
||||
* wants bare model ids and 404s on prefixed ones. Exported for tests.
|
||||
*/
|
||||
export function toRawSdkParams(
|
||||
params: Anthropic.MessageCreateParamsNonStreaming,
|
||||
): Anthropic.MessageCreateParamsNonStreaming {
|
||||
return { ...params, model: splitProviderModelId(String(params.model)).model };
|
||||
}
|
||||
|
||||
/**
|
||||
* #2099 — build a ThinkLLMClient for `modelStr`: gateway-backed when the
|
||||
* gateway can serve the model's provider (any recipe, not just Anthropic),
|
||||
* raw Anthropic SDK with a prefix-stripped model id otherwise. `explicit`
|
||||
* makes an unusable user-typed --model a hard error instead of a silent
|
||||
* fallback (same contract as think's tryBuildGatewayClient).
|
||||
*/
|
||||
export async function buildLLMClient(modelStr: string, explicit: boolean): Promise<ThinkLLMClient> {
|
||||
configureGatewayIfUninitialized();
|
||||
const gatewayClient = await __thinkAdapter.tryBuildGatewayClient(modelStr, { explicitModel: explicit });
|
||||
if (gatewayClient) return gatewayClient;
|
||||
const real = new Anthropic();
|
||||
return {
|
||||
create: (params, callOpts) => real.messages.create(toRawSdkParams(params), callOpts),
|
||||
};
|
||||
}
|
||||
|
||||
export interface RunOpts {
|
||||
/** Inject an Anthropic client for tests; defaults to a fresh SDK client. */
|
||||
client?: ThinkLLMClient;
|
||||
@@ -468,25 +504,26 @@ export async function runEvalLongMemEval(args: string[], runOpts: RunOpts = {}):
|
||||
fallback: 'sonnet',
|
||||
});
|
||||
|
||||
// Wrap Anthropic SDK so its `.messages.create` shape matches ThinkLLMClient.
|
||||
// Same pattern as src/core/think/index.ts:247-249.
|
||||
const realClient = new Anthropic();
|
||||
const client: ThinkLLMClient = runOpts.client ?? {
|
||||
create: (params, callOpts) => realClient.messages.create(params, callOpts),
|
||||
};
|
||||
// v0.40.2.0 — separate extractor client (defaults to same SDK).
|
||||
const extractorClient: ThinkLLMClient = runOpts.extractorClient ?? {
|
||||
create: (params, callOpts) => realClient.messages.create(params, callOpts),
|
||||
};
|
||||
const trajectoryEnabled = !opts.noTrajectory;
|
||||
const extractorModel = trajectoryEnabled
|
||||
? await resolveModel(null, {
|
||||
cliFlag: runOpts.extractorModel,
|
||||
cliFlag: opts.extractorModel ?? runOpts.extractorModel,
|
||||
tier: 'utility',
|
||||
fallback: 'haiku',
|
||||
})
|
||||
: '';
|
||||
|
||||
// #2099: resolveModel returns provider-prefixed ids ('anthropic:claude-...');
|
||||
// feeding those verbatim into the raw Anthropic SDK 404s every call. Route
|
||||
// through the gateway client seam think/synthesize already use; when the
|
||||
// gateway can't serve the model (e.g. missing key for the resolved
|
||||
// provider), fall back to the raw SDK with the provider prefix stripped so
|
||||
// the pre-gateway behavior (clear auth error at call time) is preserved.
|
||||
const client: ThinkLLMClient = runOpts.client ?? (await buildLLMClient(model, !!opts.model));
|
||||
// v0.40.2.0 — separate extractor client (defaults to the same seam).
|
||||
const extractorClient: ThinkLLMClient = runOpts.extractorClient ??
|
||||
(trajectoryEnabled ? await buildLLMClient(extractorModel, !!(opts.extractorModel ?? runOpts.extractorModel)) : client);
|
||||
|
||||
process.stderr.write(`[longmemeval] estimated 20-60 minutes for ${questions.length} questions; use --limit N for shorter runs\n`);
|
||||
process.stderr.write(`[longmemeval] connecting in-memory brain...\n`);
|
||||
process.stderr.write(`[longmemeval] starting (questions: ${questions.length}, model: ${model}, expansion: ${opts.expansion ? 'on' : 'off'}${opts.mode ? `, mode: ${opts.mode}` : ''}, trajectory: ${trajectoryEnabled ? 'on' : 'off'}${trajectoryEnabled ? `, extractor: ${extractorModel}` : ''})\n`);
|
||||
|
||||
@@ -18,19 +18,30 @@ export const google: Recipe = {
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
expansion: {
|
||||
models: ['gemini-2.0-flash', 'gemini-2.0-flash-lite'],
|
||||
// #2507: gemini-2.0-* ids are retired server-side (HTTP 404 "no longer
|
||||
// available"). Current GA ids + the rolling -latest alias Google serves.
|
||||
models: ['gemini-2.5-flash', 'gemini-2.5-flash-lite', 'gemini-flash-latest'],
|
||||
cost_per_1m_tokens_usd: 0.10,
|
||||
price_last_verified: '2026-04-20',
|
||||
price_last_verified: '2026-07-21',
|
||||
},
|
||||
chat: {
|
||||
models: ['gemini-2.0-flash-exp', 'gemini-2.0-flash', 'gemini-1.5-pro'],
|
||||
// #2507: refreshed from retired gemini-2.0-flash* / gemini-1.5-pro to
|
||||
// the current GA family. The -latest ids are rolling aliases Google
|
||||
// serves directly on v1beta.
|
||||
models: [
|
||||
'gemini-2.5-pro',
|
||||
'gemini-2.5-flash',
|
||||
'gemini-2.5-flash-lite',
|
||||
'gemini-pro-latest',
|
||||
'gemini-flash-latest',
|
||||
],
|
||||
supports_tools: true,
|
||||
supports_subagent_loop: true,
|
||||
supports_prompt_cache: false,
|
||||
max_context_tokens: 1000000, // Gemini 1.5 Pro
|
||||
cost_per_1m_input_usd: 0.30,
|
||||
cost_per_1m_output_usd: 1.20,
|
||||
price_last_verified: '2026-04-20',
|
||||
max_context_tokens: 1000000, // Gemini 2.5 Pro
|
||||
cost_per_1m_input_usd: 0.30, // gemini-2.5-flash baseline
|
||||
cost_per_1m_output_usd: 2.50,
|
||||
price_last_verified: '2026-07-21',
|
||||
},
|
||||
},
|
||||
setup_hint: 'Get an API key at https://aistudio.google.com/apikey, then `export GOOGLE_GENERATIVE_AI_API_KEY=...`',
|
||||
|
||||
@@ -30,7 +30,9 @@ export const openai: Recipe = {
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
chat: {
|
||||
models: ['gpt-5.2', 'gpt-4o-mini'],
|
||||
// gpt-5 listed so DEFAULT_ALIASES.gpt ('openai:gpt-5') passes the
|
||||
// native-recipe allowlist for ad-hoc uses (#1270 cross-modal slots).
|
||||
models: ['gpt-5.2', 'gpt-5', 'gpt-4o-mini'],
|
||||
supports_tools: true,
|
||||
supports_subagent_loop: true,
|
||||
supports_prompt_cache: false,
|
||||
|
||||
@@ -23,6 +23,7 @@ import { parseModelJSON } from './json-repair.ts';
|
||||
import { receiptName, sha8 } from './receipt-name.ts';
|
||||
import { writeReceipt } from './receipt-write.ts';
|
||||
import { canonicalLookup } from '../model-pricing.ts';
|
||||
import { resolveAlias } from '../model-config.ts';
|
||||
|
||||
export const RECEIPT_SCHEMA_VERSION = 1;
|
||||
|
||||
@@ -36,19 +37,32 @@ export const DEFAULT_DIMENSIONS: string[] = [
|
||||
];
|
||||
|
||||
/**
|
||||
* Default 3-provider slot configuration. Implementer should refresh the
|
||||
* model strings alongside model-family bumps in CLAUDE.md.
|
||||
* Default 3-provider slot configuration.
|
||||
*
|
||||
* The model strings here resolve through `src/core/ai/recipes/`. Each slot
|
||||
* uses a distinct family so blind spots don't correlate. Override via
|
||||
* `--slot-a-model`, `--slot-b-model`, `--slot-c-model` on the CLI.
|
||||
* #1270: slots are alias names (`gpt` / `opus` / `gemini`), resolved through
|
||||
* `resolveAlias` (DEFAULT_ALIASES in src/core/model-config.ts) at run time —
|
||||
* ONE maintenance point for model-family bumps instead of hardcoded ids that
|
||||
* rot silently. Each slot uses a distinct family so blind spots don't
|
||||
* correlate. Override via `--slot-a-model`, `--slot-b-model`,
|
||||
* `--slot-c-model` on the CLI (full `provider:model` ids pass through
|
||||
* resolveAlias unchanged).
|
||||
*/
|
||||
export const DEFAULT_SLOTS: SlotConfig[] = [
|
||||
{ id: 'A', model: 'openai:gpt-4o' },
|
||||
{ id: 'B', model: 'anthropic:claude-opus-4-7' },
|
||||
{ id: 'C', model: 'google:gemini-1.5-pro' },
|
||||
{ id: 'A', model: 'gpt' },
|
||||
{ id: 'B', model: 'opus' },
|
||||
{ id: 'C', model: 'gemini' },
|
||||
];
|
||||
|
||||
/**
|
||||
* Resolve slot model aliases to full `provider:model` ids. Idempotent for
|
||||
* already-full ids (resolveAlias passes unknown names through unchanged).
|
||||
*/
|
||||
export async function resolveSlots(slots: SlotConfig[]): Promise<SlotConfig[]> {
|
||||
return Promise.all(
|
||||
slots.map(async s => ({ ...s, model: await resolveAlias(null, s.model) })),
|
||||
);
|
||||
}
|
||||
|
||||
export interface SlotConfig {
|
||||
id: string;
|
||||
/** "<provider>:<modelId>" string consumed by gateway.ts:resolveChatProvider. */
|
||||
@@ -117,7 +131,7 @@ export interface RunEvalResult {
|
||||
/** Run up to `cycles` cycles. Stops early on PASS. */
|
||||
export async function runEval(opts: RunEvalOpts): Promise<RunEvalResult> {
|
||||
const dimensions = opts.dimensions ?? DEFAULT_DIMENSIONS;
|
||||
const slots = opts.slots ?? DEFAULT_SLOTS;
|
||||
const slots = await resolveSlots(opts.slots ?? DEFAULT_SLOTS);
|
||||
const cycles = clampCycles(opts.cycles);
|
||||
const slug = opts.slug ?? `eval-${sha8(opts.output).slice(0, 6)}`;
|
||||
|
||||
|
||||
@@ -32,10 +32,44 @@
|
||||
|
||||
import { BudgetMeter, type SubmitEstimate, type BudgetCheckResult } from './budget-meter.ts';
|
||||
import { sourceScopeOpts, type OperationContext } from '../operations.ts';
|
||||
import { getChatModel } from '../ai/gateway.ts';
|
||||
import { resolveModel, TIER_DEFAULTS } from '../model-config.ts';
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
import type { CyclePhase, PhaseResult, PhaseStatus, PhaseError } from '../cycle.ts';
|
||||
import type { ProgressReporter } from '../progress.ts';
|
||||
|
||||
/**
|
||||
* Resolve the chat model for a dream phase (#2516, takeover — originally by
|
||||
* @ervza). Precedence:
|
||||
*
|
||||
* explicit opts.model → per-phase config key (`models.dream.<phase>`) →
|
||||
* `models.default` → GBRAIN_MODEL env → gateway chat model
|
||||
* (getChatModel — reflects `models.chat` / `models.tier.reasoning` /
|
||||
* config-file `chat_model` after reconfigureGatewayWithEngine) →
|
||||
* TIER_DEFAULTS.reasoning.
|
||||
*
|
||||
* This replaces per-phase hardcoded 'claude-sonnet-4-6' defaults so
|
||||
* non-Anthropic stacks (ollama, openrouter, litellm, openai-compat) route the
|
||||
* dream phases through the user's configured provider. The gateway fallback
|
||||
* is wrapped in try/catch so phases running without a configured gateway
|
||||
* (unit tests with injected judges/extractors) still resolve a model id for
|
||||
* budget accounting instead of throwing.
|
||||
*/
|
||||
export async function resolvePhaseChatModel(
|
||||
engine: BrainEngine,
|
||||
configKey: string,
|
||||
explicit?: string,
|
||||
): Promise<string> {
|
||||
if (explicit && explicit.trim()) return explicit;
|
||||
const configured = await resolveModel(engine, { configKey, fallback: '' });
|
||||
if (configured && configured.trim()) return configured;
|
||||
try {
|
||||
return getChatModel();
|
||||
} catch {
|
||||
return TIER_DEFAULTS.reasoning;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Source-scoped read options threaded through every engine call inside a
|
||||
* BaseCyclePhase. The base class produces these via `this.scope()`; subclasses
|
||||
|
||||
@@ -25,9 +25,8 @@
|
||||
* profiles per source for the same holder.
|
||||
*/
|
||||
|
||||
import { BaseCyclePhase, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts';
|
||||
import { BaseCyclePhase, resolvePhaseChatModel, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts';
|
||||
import { chat as gatewayChat } from '../ai/gateway.ts';
|
||||
import { TIER_DEFAULTS } from '../model-config.ts';
|
||||
import { gateVoice, type VoiceGateGenerator, type VoiceGateJudge } from '../calibration/voice-gate.ts';
|
||||
import { patternStatementTemplate, type PatternStatementSlots } from '../calibration/templates.ts';
|
||||
// v0.41 T10 — domain widening. The aggregator module resolves the active
|
||||
@@ -90,6 +89,8 @@ export type PatternStatementsGenerator = (input: {
|
||||
holder: string;
|
||||
attempt: number;
|
||||
feedback?: string;
|
||||
/** #2516 — resolved phase model; defaultPatternsGenerator routes its chat call through it. */
|
||||
modelHint?: string;
|
||||
}) => Promise<string[]>;
|
||||
|
||||
/** Generator function for bias tags (test seam). */
|
||||
@@ -229,7 +230,9 @@ class CalibrationProfilePhase extends BaseCyclePhase {
|
||||
): Promise<{ summary: string; details: Record<string, unknown>; status?: PhaseStatus }> {
|
||||
const holder = opts.holder ?? 'garry';
|
||||
const promptVersion = opts.promptVersion ?? CALIBRATION_PROFILE_PROMPT_VERSION;
|
||||
const modelId = opts.model ?? TIER_DEFAULTS.reasoning;
|
||||
// #2516: per-phase config key + tier resolver instead of the static
|
||||
// TIER_DEFAULTS.reasoning, so non-Anthropic stacks route this phase.
|
||||
const modelId = await resolvePhaseChatModel(engine, 'models.dream.calibration_profile', opts.model);
|
||||
const gradeCompletion = opts.gradeCompletion ?? 1.0;
|
||||
const patternsGenerator = opts.patternsGenerator ?? defaultPatternsGenerator;
|
||||
const biasTagsGenerator = opts.biasTagsGenerator ?? defaultBiasTagsGenerator;
|
||||
@@ -266,6 +269,10 @@ class CalibrationProfilePhase extends BaseCyclePhase {
|
||||
holder,
|
||||
attempt,
|
||||
...(feedback !== undefined ? { feedback } : {}),
|
||||
// #2516: without this, the resolved model is priced by the budget
|
||||
// check and persisted to model_id but the actual chat call silently
|
||||
// uses the gateway default — the exact divergence this fix removes.
|
||||
modelHint: modelId,
|
||||
});
|
||||
return lines.join('\n');
|
||||
};
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import { BaseCyclePhase, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts';
|
||||
import { BaseCyclePhase, resolvePhaseChatModel, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts';
|
||||
import { chat as gatewayChat } from '../ai/gateway.ts';
|
||||
import { GBrainError } from '../types.ts';
|
||||
import type { OperationContext } from '../operations.ts';
|
||||
@@ -395,7 +395,9 @@ class GradeTakesPhase extends BaseCyclePhase {
|
||||
const autoResolve = opts.autoResolve ?? false; // D17 default OFF
|
||||
const autoResolveThreshold = opts.autoResolveThreshold ?? 0.95; // D12 conservative
|
||||
const resolvedByLabel = opts.resolvedByLabel ?? 'gbrain:grade_takes';
|
||||
const judgeModelId = opts.model ?? 'claude-sonnet-4-6';
|
||||
// #2516: per-phase config key + tier resolver instead of a hardcoded
|
||||
// Anthropic id, so non-Anthropic stacks route the judge correctly.
|
||||
const judgeModelId = await resolvePhaseChatModel(engine, 'models.dream.grade_takes', opts.model);
|
||||
|
||||
const useEnsemble = opts.useEnsemble ?? false;
|
||||
const ensembleThreshold = opts.ensembleThreshold ?? 0.85;
|
||||
@@ -468,7 +470,9 @@ class GradeTakesPhase extends BaseCyclePhase {
|
||||
// Call the single-model judge. Errors on a single take log warning + continue.
|
||||
let verdict: JudgeVerdict;
|
||||
try {
|
||||
verdict = await judge({ take, evidence, modelHint: opts.model });
|
||||
// #2516: pass the RESOLVED model so the judge's chat call uses the
|
||||
// same model the budget check priced and the cache key records.
|
||||
verdict = await judge({ take, evidence, modelHint: judgeModelId });
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
result.warnings.push(`judge failed on take ${take.id}: ${msg}`);
|
||||
|
||||
@@ -38,8 +38,8 @@
|
||||
*/
|
||||
|
||||
import { randomUUID, createHash } from 'node:crypto';
|
||||
import { BaseCyclePhase, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts';
|
||||
import { chat as gatewayChat, getChatModel } from '../ai/gateway.ts';
|
||||
import { BaseCyclePhase, resolvePhaseChatModel, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts';
|
||||
import { chat as gatewayChat } from '../ai/gateway.ts';
|
||||
import { writeReceipt } from '../extract/receipt-writer.ts';
|
||||
import { upsertExtractRollup } from '../extract/rollup-writer.ts';
|
||||
import { GBrainError } from '../types.ts';
|
||||
@@ -330,7 +330,10 @@ class ProposeTakesPhase extends BaseCyclePhase {
|
||||
opts.reporter.start('propose_takes.pages' as never, pages.length);
|
||||
}
|
||||
|
||||
const modelId = opts.model ?? getChatModel();
|
||||
// #2516: per-phase config key + tier resolver instead of the gateway
|
||||
// default alone, so `gbrain config set models.dream.propose_takes ...`
|
||||
// (or models.default / GBRAIN_MODEL) routes this phase.
|
||||
const modelId = await resolvePhaseChatModel(engine, 'models.dream.propose_takes', opts.model);
|
||||
|
||||
for (const page of pages) {
|
||||
result.pages_scanned += 1;
|
||||
@@ -380,7 +383,9 @@ class ProposeTakesPhase extends BaseCyclePhase {
|
||||
pagePath: page.slug,
|
||||
pageBody: body,
|
||||
existingTakes,
|
||||
modelHint: opts.model,
|
||||
// #2516: pass the RESOLVED model so the extractor's chat call uses
|
||||
// the same model the budget check priced and the row records.
|
||||
modelHint: modelId,
|
||||
});
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
|
||||
@@ -58,7 +58,10 @@ export const DEFAULT_ALIASES: Record<string, string> = {
|
||||
opus: 'anthropic:claude-opus-4-7',
|
||||
sonnet: 'anthropic:claude-sonnet-4-6',
|
||||
haiku: 'anthropic:claude-haiku-4-5-20251001',
|
||||
gemini: 'google:gemini-3-pro',
|
||||
// #2507: gemini-3-pro 404s on the Generative Language API (v1beta); point
|
||||
// the alias at a model that exists today AND is in the google recipe
|
||||
// allowlist so ad-hoc (non-config-registered) uses pass assertTouchpoint.
|
||||
gemini: 'google:gemini-2.5-pro',
|
||||
gpt: 'openai:gpt-5',
|
||||
};
|
||||
|
||||
|
||||
@@ -78,6 +78,12 @@ export const CANONICAL_PRICING: Record<string, ModelPricing> = {
|
||||
'openai:gpt-5.5': { input: 4.00, output: 16.00 },
|
||||
|
||||
// ── Google ─────────────────────────────────────────────────────────────
|
||||
// Gemini 2.5 family (verified 2026-07-21). 2.5 Pro rates are the <=200K
|
||||
// prompt tier; long-context surcharge deliberately not modeled (same
|
||||
// convention as the Anthropic entries above).
|
||||
'google:gemini-2.5-pro': { input: 1.25, output: 10.00 },
|
||||
'google:gemini-2.5-flash': { input: 0.30, output: 2.50 },
|
||||
'google:gemini-2.5-flash-lite': { input: 0.10, output: 0.40 },
|
||||
'google:gemini-1.5-pro': { input: 1.25, output: 5.00 },
|
||||
// Gemini 2.0 Flash: $0.10 in / $0.40 out (verified 2026-06-03). Reconciled
|
||||
// from a stale $0.30/$1.20 entry that had drifted in takes-quality-eval.
|
||||
|
||||
@@ -105,7 +105,7 @@ describe('chat touchpoint — model resolver + aliases (Codex F-OV-5)', () => {
|
||||
test('assertTouchpoint accepts chat for chat-capable native + openai-compat providers', () => {
|
||||
expect(() => assertTouchpoint(getRecipe('anthropic')!, 'chat', 'claude-opus-4-7')).not.toThrow();
|
||||
expect(() => assertTouchpoint(getRecipe('openai')!, 'chat', 'gpt-5.2')).not.toThrow();
|
||||
expect(() => assertTouchpoint(getRecipe('google')!, 'chat', 'gemini-2.0-flash')).not.toThrow();
|
||||
expect(() => assertTouchpoint(getRecipe('google')!, 'chat', 'gemini-2.5-pro')).not.toThrow();
|
||||
expect(() => assertTouchpoint(getRecipe('deepseek')!, 'chat', 'deepseek-chat')).not.toThrow();
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
/**
|
||||
* #2507 / #1270 — provider-recipe + default-alias rot guards.
|
||||
*
|
||||
* The Google recipe shipped retired model ids (gemini-2.0-flash*,
|
||||
* gemini-1.5-pro) while DEFAULT_ALIASES.gemini pointed at a model that 404s
|
||||
* on the Generative Language API, and the cross-modal DEFAULT_SLOTS
|
||||
* hardcoded ids outside every recipe allowlist. These tests pin the repaired
|
||||
* state AND add the structural anti-rot invariant: every default alias (and
|
||||
* every default cross-modal slot, which resolves through the aliases) must
|
||||
* name a chat model its recipe's allowlist accepts — so a future model-family
|
||||
* bump that touches DEFAULT_ALIASES without refreshing the recipe fails CI
|
||||
* instead of degrading `think` / cross-modal runs at call time.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { getRecipe } from '../../src/core/ai/recipes/index.ts';
|
||||
import { assertTouchpoint, resolveRecipe } from '../../src/core/ai/model-resolver.ts';
|
||||
import { DEFAULT_ALIASES } from '../../src/core/model-config.ts';
|
||||
import { DEFAULT_SLOTS, resolveSlots } from '../../src/core/cross-modal-eval/runner.ts';
|
||||
|
||||
describe('google recipe — current model ids (#2507)', () => {
|
||||
test('retired gemini ids are gone from the chat allowlist', () => {
|
||||
const chat = getRecipe('google')!.touchpoints.chat!;
|
||||
for (const retired of ['gemini-2.0-flash', 'gemini-2.0-flash-exp', 'gemini-1.5-pro']) {
|
||||
expect(chat.models).not.toContain(retired);
|
||||
}
|
||||
});
|
||||
|
||||
test('current GA ids pass assertTouchpoint for chat', () => {
|
||||
for (const id of ['gemini-2.5-pro', 'gemini-2.5-flash', 'gemini-2.5-flash-lite', 'gemini-pro-latest', 'gemini-flash-latest']) {
|
||||
expect(() => assertTouchpoint(getRecipe('google')!, 'chat', id)).not.toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test('current flash ids pass assertTouchpoint for expansion', () => {
|
||||
for (const id of ['gemini-2.5-flash', 'gemini-2.5-flash-lite', 'gemini-flash-latest']) {
|
||||
expect(() => assertTouchpoint(getRecipe('google')!, 'expansion', id)).not.toThrow();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('DEFAULT_ALIASES stay recipe-valid (anti-rot, #2507)', () => {
|
||||
test('every default alias resolves to a chat model its recipe allows', () => {
|
||||
for (const [alias, full] of Object.entries(DEFAULT_ALIASES)) {
|
||||
const { parsed, recipe } = resolveRecipe(full);
|
||||
expect(
|
||||
() => assertTouchpoint(recipe, 'chat', parsed.modelId),
|
||||
`alias "${alias}" → "${full}" must be in the ${recipe.id} recipe chat allowlist`,
|
||||
).not.toThrow();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('cross-modal DEFAULT_SLOTS (#1270)', () => {
|
||||
test('default slots resolve via aliases to recipe-allowlisted chat models', async () => {
|
||||
const resolved = await resolveSlots(DEFAULT_SLOTS);
|
||||
expect(resolved).toHaveLength(3);
|
||||
for (const slot of resolved) {
|
||||
expect(slot.model).toContain(':'); // full provider:model id after resolution
|
||||
const { parsed, recipe } = resolveRecipe(slot.model);
|
||||
expect(
|
||||
() => assertTouchpoint(recipe, 'chat', parsed.modelId),
|
||||
`slot ${slot.id} → "${slot.model}" must be in the ${recipe.id} recipe chat allowlist`,
|
||||
).not.toThrow();
|
||||
}
|
||||
// Three distinct provider families so blind spots don't correlate.
|
||||
expect(new Set(resolved.map(s => s.model.split(':')[0]))).toEqual(
|
||||
new Set(['openai', 'anthropic', 'google']),
|
||||
);
|
||||
});
|
||||
|
||||
test('resolveSlots passes full provider:model ids through unchanged', async () => {
|
||||
const custom = [{ id: 'A', model: 'ollama:llama3' }];
|
||||
expect(await resolveSlots(custom)).toEqual(custom);
|
||||
});
|
||||
});
|
||||
@@ -39,6 +39,11 @@ function buildMockEngine(opts: { scorecard: TakesScorecard }): {
|
||||
const captured: CapturedSql[] = [];
|
||||
const engine = {
|
||||
kind: 'pglite',
|
||||
// #2516: resolvePhaseChatModel consults engine config keys; the mock
|
||||
// answers null so the model falls through to the static default.
|
||||
async getConfig() {
|
||||
return null;
|
||||
},
|
||||
async getScorecard() {
|
||||
return opts.scorecard;
|
||||
},
|
||||
@@ -331,3 +336,29 @@ describe('runPhaseCalibrationProfile — phase integration', () => {
|
||||
expect(insert!.params[0]).toBe('tenant-b');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── #2516: per-phase model config key ──────────────────────────────
|
||||
|
||||
describe('models.dream.calibration_profile config key (#2516)', () => {
|
||||
test('config-selected model is used and persisted to model_id', async () => {
|
||||
const { engine, captured } = buildMockEngine({ scorecard: ENOUGH_RESOLVED_SCORECARD });
|
||||
(engine as unknown as { getConfig: (k: string) => Promise<string | null> }).getConfig =
|
||||
async (key: string) => (key === 'models.dream.calibration_profile' ? 'openai:gpt-4o-mini' : null);
|
||||
const hints: Array<string | undefined> = [];
|
||||
await runPhaseCalibrationProfile(buildCtx(engine), {
|
||||
patternsGenerator: async ({ modelHint }) => {
|
||||
hints.push(modelHint);
|
||||
return ['You call early-stage tactics well — 8 of 10 held up.'];
|
||||
},
|
||||
biasTagsGenerator: async () => [],
|
||||
voiceGateJudge: passJudge,
|
||||
});
|
||||
// The resolved model must reach the generator's chat call, not just the
|
||||
// budget check and the persisted row — otherwise the row lies about
|
||||
// which model actually generated the patterns.
|
||||
expect(hints).toEqual(['openai:gpt-4o-mini']);
|
||||
const insert = captured.find(c => c.sql.includes('INSERT INTO calibration_profiles'));
|
||||
expect(insert).toBeDefined();
|
||||
expect(insert!.params).toContain('openai:gpt-4o-mini');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -45,7 +45,7 @@ afterEach(() => {
|
||||
|
||||
function makeChatStub(scoresBySlot: Record<string, number[]>) {
|
||||
let callIdx = 0;
|
||||
const order = ['openai:gpt-4o', 'anthropic:claude-opus-4-7', 'google:gemini-1.5-pro'];
|
||||
const order = ['openai:gpt-5', 'anthropic:claude-opus-4-7', 'google:gemini-2.5-pro'];
|
||||
return mock(async (opts: { model?: string }) => {
|
||||
const model = opts.model ?? '';
|
||||
callIdx++;
|
||||
@@ -74,9 +74,9 @@ function makeChatStub(scoresBySlot: Record<string, number[]>) {
|
||||
describe('gbrain eval cross-modal — runner verdict contract', () => {
|
||||
test('PASS: 3 happy responses, all dims >=7', async () => {
|
||||
const chatStub = makeChatStub({
|
||||
'openai:gpt-4o': [9, 8],
|
||||
'openai:gpt-5': [9, 8],
|
||||
'anthropic:claude-opus-4-7': [8, 7],
|
||||
'google:gemini-1.5-pro': [8, 8],
|
||||
'google:gemini-2.5-pro': [8, 8],
|
||||
});
|
||||
mock.module('../../src/core/ai/gateway.ts', () => ({
|
||||
chat: chatStub,
|
||||
@@ -105,9 +105,9 @@ describe('gbrain eval cross-modal — runner verdict contract', () => {
|
||||
|
||||
test('FAIL: one dim mean below 7', async () => {
|
||||
const chatStub = makeChatStub({
|
||||
'openai:gpt-4o': [9, 6],
|
||||
'openai:gpt-5': [9, 6],
|
||||
'anthropic:claude-opus-4-7': [8, 6],
|
||||
'google:gemini-1.5-pro': [8, 6],
|
||||
'google:gemini-2.5-pro': [8, 6],
|
||||
});
|
||||
mock.module('../../src/core/ai/gateway.ts', () => ({
|
||||
chat: chatStub,
|
||||
@@ -130,9 +130,9 @@ describe('gbrain eval cross-modal — runner verdict contract', () => {
|
||||
|
||||
test('FAIL: min-score floor caught when one model scores <5 (Q2)', async () => {
|
||||
const chatStub = makeChatStub({
|
||||
'openai:gpt-4o': [9, 8],
|
||||
'openai:gpt-5': [9, 8],
|
||||
'anthropic:claude-opus-4-7': [8, 8],
|
||||
'google:gemini-1.5-pro': [4, 8], // goal=4 trips the floor
|
||||
'google:gemini-2.5-pro': [4, 8], // goal=4 trips the floor
|
||||
});
|
||||
mock.module('../../src/core/ai/gateway.ts', () => ({
|
||||
chat: chatStub,
|
||||
@@ -155,7 +155,7 @@ describe('gbrain eval cross-modal — runner verdict contract', () => {
|
||||
|
||||
test('INCONCLUSIVE: 2 of 3 mock 5xx -> exit 2 contract (Q3)', async () => {
|
||||
const chatStub = mock(async (opts: { model?: string }) => {
|
||||
if (opts.model === 'openai:gpt-4o') {
|
||||
if (opts.model === 'openai:gpt-5') {
|
||||
return {
|
||||
text: JSON.stringify({
|
||||
scores: { goal: { score: 8 } },
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* #2099 — eval longmemeval's LLM clients must not feed provider-prefixed
|
||||
* model ids ('anthropic:claude-sonnet-4-6', the shape resolveModel returns)
|
||||
* into the raw Anthropic SDK: the API 404s on them, so every out-of-the-box
|
||||
* run died at the first answer-generation call and the trajectory extractor
|
||||
* failed open.
|
||||
*
|
||||
* Fix under test: clients are built via buildLLMClient — gateway-backed when
|
||||
* the gateway can serve the model's provider (any recipe, not just
|
||||
* Anthropic), raw SDK with a prefix-stripped id otherwise (toRawSdkParams).
|
||||
*/
|
||||
|
||||
import { describe, test, expect, afterAll } from 'bun:test';
|
||||
import { buildLLMClient, toRawSdkParams } from '../src/commands/eval-longmemeval.ts';
|
||||
import {
|
||||
configureGateway,
|
||||
resetGateway,
|
||||
__setChatTransportForTests,
|
||||
} from '../src/core/ai/gateway.ts';
|
||||
import type { ChatOpts, ChatResult } from '../src/core/ai/gateway.ts';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
|
||||
afterAll(() => {
|
||||
__setChatTransportForTests(null);
|
||||
resetGateway();
|
||||
});
|
||||
|
||||
describe('toRawSdkParams (#2099)', () => {
|
||||
test('strips the provider prefix off params.model at the raw-SDK boundary', () => {
|
||||
const out = toRawSdkParams({
|
||||
model: 'anthropic:claude-sonnet-4-6',
|
||||
max_tokens: 512,
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
});
|
||||
expect(out.model).toBe('claude-sonnet-4-6');
|
||||
expect(out.max_tokens).toBe(512);
|
||||
});
|
||||
|
||||
test('bare model ids pass through unchanged', () => {
|
||||
const out = toRawSdkParams({
|
||||
model: 'claude-haiku-4-5-20251001',
|
||||
max_tokens: 16,
|
||||
messages: [],
|
||||
});
|
||||
expect(out.model).toBe('claude-haiku-4-5-20251001');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildLLMClient (#2099)', () => {
|
||||
test('routes through the gateway (not the raw SDK) when the gateway can serve the model', async () => {
|
||||
const seen: string[] = [];
|
||||
// probeChatModel's Anthropic branch reads process.env (hasAnthropicKey),
|
||||
// not the gateway cfg.env — set both so the test is hermetic on CI.
|
||||
await withEnv({ ANTHROPIC_API_KEY: 'sk-ant-test' }, async () => {
|
||||
configureGateway({
|
||||
chat_model: 'anthropic:claude-sonnet-4-6',
|
||||
env: { ANTHROPIC_API_KEY: 'sk-ant-test' },
|
||||
} as never);
|
||||
__setChatTransportForTests(async (opts: ChatOpts): Promise<ChatResult> => {
|
||||
seen.push(opts.model ?? '<none>');
|
||||
return {
|
||||
text: 'stubbed answer',
|
||||
blocks: [],
|
||||
stopReason: 'end',
|
||||
usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 },
|
||||
model: opts.model ?? '',
|
||||
providerId: 'anthropic',
|
||||
} as ChatResult;
|
||||
});
|
||||
|
||||
const client = await buildLLMClient('anthropic:claude-sonnet-4-6', false);
|
||||
const msg = await client.create({
|
||||
model: 'anthropic:claude-sonnet-4-6',
|
||||
max_tokens: 32,
|
||||
messages: [{ role: 'user', content: 'q' }],
|
||||
});
|
||||
|
||||
// Pre-#2099 this path constructed `new Anthropic()` and sent the
|
||||
// prefixed id verbatim → HTTP 404. Now the gateway serves it.
|
||||
expect(seen).toEqual(['anthropic:claude-sonnet-4-6']);
|
||||
const first = msg.content[0];
|
||||
expect(first && first.type === 'text' ? first.text : '').toBe('stubbed answer');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -48,6 +48,11 @@ function buildMockEngine(opts: { takes: Take[] }): {
|
||||
const resolves: CapturedResolve[] = [];
|
||||
const engine = {
|
||||
kind: 'pglite',
|
||||
// #2516: resolvePhaseChatModel consults engine config keys; the mock
|
||||
// answers null so the model falls through to the static default.
|
||||
async getConfig() {
|
||||
return null;
|
||||
},
|
||||
async listTakes() {
|
||||
return opts.takes;
|
||||
},
|
||||
@@ -383,6 +388,7 @@ describe('runPhaseGradeTakes ensemble — auto-apply rules', () => {
|
||||
judge,
|
||||
useEnsemble: true,
|
||||
ensembleJudges: [],
|
||||
model: 'claude-sonnet-4-6',
|
||||
});
|
||||
const insert = captured.find(c => c.sql.includes('INSERT INTO take_grade_cache'));
|
||||
expect(insert!.params[2]).toBe('claude-sonnet-4-6'); // single-judge model id
|
||||
|
||||
@@ -53,6 +53,11 @@ function buildMockEngine(opts: {
|
||||
|
||||
const engine = {
|
||||
kind: 'pglite',
|
||||
// #2516: resolvePhaseChatModel consults engine config keys; the mock
|
||||
// answers null so the model falls through to the static default.
|
||||
async getConfig() {
|
||||
return null;
|
||||
},
|
||||
async listTakes() {
|
||||
return opts.takes;
|
||||
},
|
||||
@@ -286,7 +291,7 @@ describe('runPhaseGradeTakes — phase integration', () => {
|
||||
return { verdict: 'correct', confidence: 0.9, reasoning: 'x' };
|
||||
};
|
||||
const evidenceRetriever: EvidenceRetrieverFn = async () => 'mock evidence body';
|
||||
const result = await runPhaseGradeTakes(buildCtx(engine), { judge, evidenceRetriever });
|
||||
const result = await runPhaseGradeTakes(buildCtx(engine), { judge, evidenceRetriever, model: 'claude-sonnet-4-6' });
|
||||
expect(judgeCalls).toBe(0);
|
||||
const details = result.details as Record<string, unknown>;
|
||||
expect(details.cache_hits).toBe(1);
|
||||
@@ -328,3 +333,25 @@ describe('runPhaseGradeTakes — phase integration', () => {
|
||||
expect((details.warnings as string[])[0]).toContain('judge timeout');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── #2516: per-phase model config key ──────────────────────────────
|
||||
|
||||
describe('models.dream.grade_takes config key (#2516)', () => {
|
||||
test('config-selected model reaches the judge AND the cache row', async () => {
|
||||
const takes = [buildTake({ id: 1, sinceDate: '2023-01-01' })];
|
||||
const { engine, captured } = buildMockEngine({ takes });
|
||||
(engine as unknown as { getConfig: (k: string) => Promise<string | null> }).getConfig =
|
||||
async (key: string) => (key === 'models.dream.grade_takes' ? 'openai:gpt-4o-mini' : null);
|
||||
const hints: Array<string | undefined> = [];
|
||||
const judge: JudgeFn = async ({ modelHint }) => {
|
||||
hints.push(modelHint);
|
||||
return { verdict: 'correct', confidence: 0.9, reasoning: 'x' };
|
||||
};
|
||||
await runPhaseGradeTakes(buildCtx(engine), { judge });
|
||||
// Pre-#2516 the judge received modelHint=undefined (silently used the
|
||||
// gateway default) while the cache row recorded a hardcoded Anthropic id.
|
||||
expect(hints).toEqual(['openai:gpt-4o-mini']);
|
||||
const insert = captured.find(c => c.sql.includes('INSERT INTO take_grade_cache'));
|
||||
expect(insert!.params[2]).toBe('openai:gpt-4o-mini');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -118,10 +118,15 @@ describe('DRIFT GUARD — derived views stay equal to canonical (re-hardcode tri
|
||||
for (const id of [
|
||||
'openai:gpt-4o',
|
||||
'openai:gpt-4o-mini',
|
||||
'openai:gpt-5',
|
||||
'anthropic:claude-opus-4-7',
|
||||
'anthropic:claude-sonnet-4-6',
|
||||
'google:gemini-1.5-pro',
|
||||
'google:gemini-2.0-flash',
|
||||
// #1270: models the alias-form DEFAULT_SLOTS resolve to today.
|
||||
'google:gemini-2.5-pro',
|
||||
'google:gemini-2.5-flash',
|
||||
'google:gemini-2.5-flash-lite',
|
||||
'together:meta-llama/Llama-3.3-70B-Instruct-Turbo',
|
||||
'deepseek:deepseek-chat',
|
||||
]) {
|
||||
|
||||
@@ -47,6 +47,11 @@ function buildMockEngine(opts: {
|
||||
|
||||
const engine = {
|
||||
kind: 'pglite',
|
||||
// #2516: resolvePhaseChatModel consults engine config keys; the mock
|
||||
// answers null so the model falls through to the static default.
|
||||
async getConfig() {
|
||||
return null;
|
||||
},
|
||||
async listPages() {
|
||||
return opts.pages;
|
||||
},
|
||||
@@ -436,3 +441,26 @@ New prose appended here.`;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ─── #2516: per-phase model config key ──────────────────────────────
|
||||
|
||||
describe('models.dream.propose_takes config key (#2516)', () => {
|
||||
test('config-selected model reaches the extractor AND the proposal row', async () => {
|
||||
const { engine, captured } = buildMockEngine({
|
||||
pages: [buildPage({ slug: 'notes/one', body: 'I bet acme-example wins the market.' })],
|
||||
});
|
||||
(engine as unknown as { getConfig: (k: string) => Promise<string | null> }).getConfig =
|
||||
async (key: string) => (key === 'models.dream.propose_takes' ? 'openai:gpt-4o-mini' : null);
|
||||
const hints: Array<string | undefined> = [];
|
||||
const extractor: ProposeTakesExtractor = async ({ modelHint }) => {
|
||||
hints.push(modelHint);
|
||||
return [{ claim_text: 'acme wins', kind: 'take', holder: 'brain', weight: 0.7 }];
|
||||
};
|
||||
await runPhaseProposeTakes(buildCtx(engine), { extractor });
|
||||
// Pre-#2516 the extractor received modelHint=undefined while the row
|
||||
// recorded whatever the gateway default was — the two could disagree.
|
||||
expect(hints).toEqual(['openai:gpt-4o-mini']);
|
||||
const insert = captured.find(c => c.sql.includes('INSERT INTO take_proposals'));
|
||||
expect(insert!.params[11]).toBe('openai:gpt-4o-mini');
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user