mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 00:48:18 +00:00
Time Attakc
335e470394
Adversarial review: survived a hostile reviewer plus two independent refuters, each told to assume the PR was broken and to default to refuting when uncertain. 19 of 62 PRs cleared that bar. dashscope and google keys now fold into the gateway env — verified live end-to-end — and the retired gemini-1.5-pro default is swept from 9 files. Reverting the change fails 11 of 98 tests at fixed seams, and the budget-cap claim in the description reproduced. Landing first in the gateway/config-key seam, so #3648 rebases onto it. Verified before merge: the PR's own tests fail when the production change is reverted; typecheck clean; MERGEABLE/CLEAN on the current base after batch 1 landed, not a stale one. Known gap, recorded rather than hidden: no live provider call was made; the gemini retirement was taken from issue history rather than a vendor check. Minor follow-up to file: deriveEnvKey('google_api_key') yields a dead GOOGLE_API_KEY in the minion shell-inherit path.
49 lines
2.3 KiB
TypeScript
49 lines
2.3 KiB
TypeScript
/**
|
|
* Consistency guard for the takes-quality DEFAULT_MODEL_PANEL — the sibling
|
|
* of test/cross-modal-default-slots.test.ts (#3510).
|
|
*
|
|
* `google:gemini-1.5-pro` sat in the panel after Google retired it, and
|
|
* `openai:gpt-4o` after the OpenAI recipe dropped it from its chat list —
|
|
* either way the gateway rejects the slot on every default run. The guard
|
|
* only works if recipes list LIVE models: removing a dead model from its
|
|
* recipe makes every hardcoded default that still names it fail here at
|
|
* once. Do not re-add retired models to a recipe to quiet this test.
|
|
*/
|
|
import { describe, expect, test } from 'bun:test';
|
|
|
|
import { DEFAULT_MODEL_PANEL } from '../src/core/takes-quality-eval/runner.ts';
|
|
import { getPricing } from '../src/core/takes-quality-eval/pricing.ts';
|
|
import { getRecipe } from '../src/core/ai/recipes/index.ts';
|
|
import { splitProviderModelId } from '../src/core/model-id.ts';
|
|
import { canonicalLookup } from '../src/core/model-pricing.ts';
|
|
|
|
describe('takes-quality DEFAULT_MODEL_PANEL ↔ recipe consistency', () => {
|
|
test('every default panel model is listed in its recipe chat touchpoint', () => {
|
|
for (const id of DEFAULT_MODEL_PANEL) {
|
|
const { provider, model } = splitProviderModelId(id);
|
|
expect(provider).not.toBeNull();
|
|
const recipe = getRecipe(provider!);
|
|
expect(recipe, `unknown recipe "${provider}"`).toBeDefined();
|
|
expect(
|
|
recipe!.touchpoints.chat?.models ?? [],
|
|
`"${model}" not listed for ${provider} chat — the default panel can never run`,
|
|
).toContain(model);
|
|
}
|
|
});
|
|
|
|
test('every default panel model prices via canonical AND the takes-quality allowlist', () => {
|
|
for (const id of DEFAULT_MODEL_PANEL) {
|
|
expect(canonicalLookup(id), `"${id}" missing from CANONICAL_PRICING`).toBeDefined();
|
|
// getPricing throws PricingNotFoundError if the model is missing from
|
|
// SUPPORTED_MODELS — a default that can't be budget-gated aborts every
|
|
// `--budget-usd` run before the first call.
|
|
expect(getPricing(id)).toBeDefined();
|
|
}
|
|
});
|
|
|
|
test('panel spans three distinct providers (uncorrelated blind spots)', () => {
|
|
const providers = new Set(DEFAULT_MODEL_PANEL.map((id) => splitProviderModelId(id).provider));
|
|
expect(providers.size).toBe(3);
|
|
});
|
|
});
|