mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 00:48:18 +00:00
b91350d778
* fix(autopilot,eval): nightly quality probe enable path works end-to-end + wire conversation-parser probe Takeover of #2629 and #2630 (rebased onto master; dropped the test/engine-find-trajectory.test.ts hunk both PRs carried — master already ships the equivalent gateway-dims fix). #2629 — nightly quality probe enable path: - autopilot + doctor read the probe flag dual-plane (DB config row from 'gbrain config set' wins, ~/.gbrain/config.json fallback) via new resolveProbeEnabled/resolveProbeMaxUsd helpers - resolveRepoRoot prefers the gbrain package root where the committed fixture lives, not the brain repoPath - rate_limited skips no longer write an audit row every autopilot cycle - eval-longmemeval strips 'provider:' recipe ids before raw Anthropic SDK calls and emits the gold answer for downstream judges - cross-modal batch folds the gold answer into the judge task; probe passes QA-shaped dimensions instead of the agent-response rubric - DEFAULT_SLOTS slot A moves to openai:gpt-5.2 (gpt-4o left the recipe); new consistency test pins every default slot to its recipe #2630 — conversation-parser nightly probe wire-up: - autopilot step 4.6 invokes runConversationParserNightlyProbe (dual-plane flag + D10 tokenmax mode-gate, package-root fixtures, 24h gate, audit trail via new src/core/audit-parser-probe.ts) - doctor's conversation_parser_probe_health replaces the hardcoded 'Skipped' stub with a real pure-function check over the audit trail Co-authored-by: p3ob7o <p3ob7o@users.noreply.github.com> Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(pricing): add openai:gpt-5.2 canonical entry for the new default slot A DEFAULT_SLOTS slot A moved to openai:gpt-5.2, which had no CANONICAL_PRICING entry — estimateCost silently dropped slot A from the --max-usd pre-flight and est_cost_usd audit rows (~1/3 under-count on the default panel). Rates from the OpenAI recipe chat touchpoint (verified 2026-04-20). Also refresh the --slot-a-model help text default and pin a pricing-presence assertion in the DEFAULT_SLOTS consistency test. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Sinabina <sinabina@Sinabinas-MacBook-Pro-4.local> Co-authored-by: p3ob7o <p3ob7o@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Garry Tan <garrytan@gmail.com>
75 lines
3.1 KiB
TypeScript
75 lines
3.1 KiB
TypeScript
// Regression test for the nightly-quality-probe config-plane split-brain.
|
|
//
|
|
// The doctor check prints a paste-ready enable hint — `gbrain config set
|
|
// autopilot.nightly_quality_probe.enabled true` — which writes the DB config
|
|
// plane. But both the autopilot gate and the doctor check used to read ONLY
|
|
// the file plane (~/.gbrain/config.json via loadConfig), so following the
|
|
// hint was a silent no-op: the probe never ran and doctor kept reporting
|
|
// "disabled (opt-in)".
|
|
//
|
|
// resolveProbeEnabled / resolveProbeMaxUsd pin the dual-plane rule (same
|
|
// precedent as `mcp.publish_skills` in serve-http.ts): DB row wins when
|
|
// present, file plane is the fallback.
|
|
import { describe, expect, test } from 'bun:test';
|
|
|
|
import {
|
|
resolveProbeEnabled,
|
|
resolveProbeMaxUsd,
|
|
} from '../src/core/cycle/nightly-quality-probe.ts';
|
|
|
|
describe('resolveProbeEnabled — dual-plane flag resolution', () => {
|
|
test('DB plane "true" enables regardless of file plane (the doctor hint path)', () => {
|
|
expect(resolveProbeEnabled('true', undefined)).toBe(true);
|
|
expect(resolveProbeEnabled('true', false)).toBe(true);
|
|
});
|
|
|
|
test('explicit DB "false" wins over file-plane true (config set off sticks)', () => {
|
|
expect(resolveProbeEnabled('false', true)).toBe(false);
|
|
});
|
|
|
|
test('file plane is the fallback when no DB row exists', () => {
|
|
expect(resolveProbeEnabled(null, true)).toBe(true);
|
|
expect(resolveProbeEnabled(undefined, true)).toBe(true);
|
|
expect(resolveProbeEnabled(null, undefined)).toBe(false);
|
|
expect(resolveProbeEnabled(null, false)).toBe(false);
|
|
});
|
|
|
|
test('file plane stays strict boolean — string "true" in config.json does not enable', () => {
|
|
// Matches the pre-fix autopilot gate (`=== true`); the doctor check used
|
|
// Boolean(...) and could disagree with autopilot on a string value.
|
|
// Both call sites now share this helper, so they can no longer diverge.
|
|
expect(resolveProbeEnabled(null, 'true')).toBe(false);
|
|
expect(resolveProbeEnabled(null, 1)).toBe(false);
|
|
});
|
|
|
|
test('non-"true" DB strings are off (mcp.publish_skills semantics)', () => {
|
|
expect(resolveProbeEnabled('1', true)).toBe(false);
|
|
expect(resolveProbeEnabled('yes', true)).toBe(false);
|
|
expect(resolveProbeEnabled('', true)).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('resolveProbeMaxUsd — dual-plane cost cap resolution', () => {
|
|
test('DB plane wins when parseable', () => {
|
|
expect(resolveProbeMaxUsd('2.5', 10)).toBe(2.5);
|
|
expect(resolveProbeMaxUsd('0', 10)).toBe(0);
|
|
});
|
|
|
|
test('malformed or negative DB value falls through to file plane', () => {
|
|
expect(resolveProbeMaxUsd('banana', 3)).toBe(3);
|
|
expect(resolveProbeMaxUsd('-1', 3)).toBe(3);
|
|
});
|
|
|
|
test('file plane used when no DB row; default when both absent/invalid', () => {
|
|
expect(resolveProbeMaxUsd(null, 7)).toBe(7);
|
|
expect(resolveProbeMaxUsd(null, '4')).toBe(4);
|
|
expect(resolveProbeMaxUsd(null, undefined)).toBe(5);
|
|
expect(resolveProbeMaxUsd(null, 'banana')).toBe(5);
|
|
expect(resolveProbeMaxUsd(undefined, -2)).toBe(5);
|
|
});
|
|
|
|
test('explicit fallback override is honored', () => {
|
|
expect(resolveProbeMaxUsd(null, undefined, 12)).toBe(12);
|
|
});
|
|
});
|