admin/gbrain
1
0
Fork 0
mirror of https://github.com/garrytan/gbrain.git synced 2026-08-14 00:48:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(eval): resolve BrainBench defaults from package root (#3996)

Browse Source 
Wave-assembled from PR #3996 by @philip-rossoneri.

Co-Authored-By: Philip Eriksson <philip@rossoneri.se>
This commit is contained in:
Garry Tan
2026-08-12 14:38:36 -07:00
committed by Sina Matian
co-authored by Philip Eriksson
parent 0a1890bbf8
commit 30c81b709c
2 changed files with 36 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/commands/eval-brainbench.ts
+24 -9
View File
@@ -16,13 +16,14 @@
*/
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
import { dirname } from 'node:path';
import { dirname, join } from 'node:path';
import { execSync } from 'node:child_process';
import { flushThenExit } from '../core/cli-force-exit.ts';
import { cliOptsToProgressOptions, getCliOptions } from '../core/cli-options.ts';
import { createProgress } from '../core/progress.ts';
import { buildMetricGlossaryMeta } from '../core/eval/metric-glossary.ts';
import { isAvailable } from '../core/ai/gateway.ts';
import { findGbrainRoot } from '../core/skillpack/bundle.ts';
import { FixtureValidationError, loadCorpus } from '../eval/brainbench/fixtures.ts';
import { runBrainBench } from '../eval/brainbench/harness.ts';
import {
@@ -44,9 +45,19 @@ import {
type HarnessName,
} from '../eval/brainbench/types.ts';
export const DEFAULT_FIXTURES_DIR = 'evals/brainbench/fixtures';
export const DEFAULT_GOLD_DIR = 'evals/brainbench/gold';
export const DEFAULT_BASELINE_PATH = 'evals/brainbench/baselines/main.json';
const DEFAULT_FIXTURES_RELATIVE = 'evals/brainbench/fixtures';
const DEFAULT_GOLD_RELATIVE = 'evals/brainbench/gold';
const DEFAULT_BASELINE_RELATIVE = 'evals/brainbench/baselines/main.json';
const GBRAIN_ROOT = findGbrainRoot();
function resolveBundledPath(relativePath: string): string {
return GBRAIN_ROOT ? join(GBRAIN_ROOT, relativePath) : relativePath;
}
export const DEFAULT_FIXTURES_DIR = resolveBundledPath(DEFAULT_FIXTURES_RELATIVE);
export const DEFAULT_GOLD_DIR = resolveBundledPath(DEFAULT_GOLD_RELATIVE);
export const DEFAULT_BASELINE_PATH = resolveBundledPath(DEFAULT_BASELINE_RELATIVE);
const DEFAULT_LLM_BUDGET_USD = 5;
function usage(): void {
@@ -55,8 +66,8 @@ function usage(): void {
`Cross-harness memory conformance suite. Hermetic by default: in-memory\n` +
`PGLite, no API keys, no LLM calls. See docs/eval/BRAINBENCH.md.\n\n` +
`Options:\n` +
` --fixtures DIR Fixture corpus (default: ${DEFAULT_FIXTURES_DIR}).\n` +
` --gold DIR Sealed gold dir (default: ${DEFAULT_GOLD_DIR}).\n` +
` --fixtures DIR Fixture corpus (default: bundled ${DEFAULT_FIXTURES_RELATIVE}).\n` +
` --gold DIR Sealed gold dir (default: bundled ${DEFAULT_GOLD_RELATIVE}).\n` +
` --harness a,b | all Harness seams to grade (default: all).\n` +
` --suite a,b | all Suites to run (default: all).\n` +
` --include-holdout Score holdout fixtures too (published-run mode).\n` +
@@ -64,8 +75,8 @@ function usage(): void {
` --out FILE Write the full JSON result to FILE (canonical CI artifact).\n` +
` --compare BASE [CURRENT] Gate against BASE baseline. With CURRENT: pure\n` +
` file-vs-file diff, no run. CI passes MAIN's baseline\n` +
` as BASE (git show origin/master:${DEFAULT_BASELINE_PATH}).\n` +
` --committed-baseline FILE Bless-mode verification target (default: ${DEFAULT_BASELINE_PATH}).\n` +
` as BASE (git show origin/master:${DEFAULT_BASELINE_RELATIVE}).\n` +
` --committed-baseline FILE Bless-mode verification target (default: bundled ${DEFAULT_BASELINE_RELATIVE}).\n` +
` --update-baseline [FILE] Write this run as the canonical committed baseline.\n` +
` --justification "reason" Recorded in the baseline written by --update-baseline\n` +
` (REQUIRED by the gate when blessing a regression).\n` +
@@ -211,7 +222,11 @@ function parseArgs(argv: string[]): Args | { usageError: string } {
function gitHeadSha(): string {
try {
return execSync('git rev-parse HEAD', { encoding: 'utf-8' }).trim();
return execSync('git rev-parse HEAD', {
cwd: GBRAIN_ROOT ?? undefined,
encoding: 'utf-8',
stdio: ['ignore', 'pipe', 'ignore'],
}).trim();
} catch {
return 'unknown';
}
test/eval-brainbench-e2e.test.ts
+12 -1
View File
@@ -12,6 +12,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
const REPO = process.cwd();
const CLI = join(REPO, 'src', 'cli.ts');
let root: string;
let fixtures: string;
let gold: string;
@@ -23,7 +24,7 @@ function run(args: string[], cwd = REPO): { exitCode: number; stdout: string; st
const full = args.includes('--committed-baseline')
? args
: [...args, '--committed-baseline', join(root, 'no-committed-baseline.json')];
const proc = Bun.spawnSync(['bun', 'src/cli.ts', 'eval', 'brainbench', ...full], {
const proc = Bun.spawnSync(['bun', CLI, 'eval', 'brainbench', ...full], {
cwd,
env: { ...process.env, GBRAIN_QUIET: '1' },
stdout: 'pipe',
@@ -60,6 +61,16 @@ beforeAll(() => {
}, 120_000);
describe('exit contract over a multi-brain run (PGLite exitCode-hijack guard)', () => {
test('bundled defaults resolve outside the package working directory', () => {
const outside = mkdtempSync(join(tmpdir(), 'bb-outside-cwd-'));
const r = run(['--harness', 'openclaw', '--suite', 'know-to-ask', '--json'], outside);
expect(r.exitCode).toBe(0);
const doc = JSON.parse(r.stdout);
expect(doc.cells.length).toBeGreaterThan(0);
expect(doc.seed_failures).toEqual([]);
expect(r.stderr).not.toContain('not a git repository');
}, 60_000);
test('clean run: exit 0, --out is complete valid JSON with the glossary block', () => {
const out = join(root, 'r1.json');
const r = run(['--fixtures', fixtures, '--gold', gold, '--harness', 'openclaw', '--out', out]);