mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 00:48:18 +00:00
Garry Tan
75fae742d5
Public, reproducible, cross-harness benchmark for the four ways agent memory fails (know-to-ask, push precision/recall, write-back fidelity, cross-session continuity). Hermetic PGLite, sealed gold, deterministic corpus, CI gate vs master's committed baseline, published JSON-schema foreign-runner contract. See docs/eval/BRAINBENCH.md.
27 lines
834 B
JSON
27 lines
834 B
JSON
{
|
|
"name": "brainbench-corpus",
|
|
"version": 1,
|
|
"seed": 42,
|
|
"generated_fixtures": 135,
|
|
"holdout_fixtures": 23,
|
|
"gold_turns": 241,
|
|
"categories": {
|
|
"kta-pos": 25,
|
|
"kta-neg": 15,
|
|
"push": 20,
|
|
"write-back": 20,
|
|
"continuity_pairs": 15,
|
|
"multi-source": 10,
|
|
"adversarial": 15
|
|
},
|
|
"prose": "template-synthesized, PRNG-varied; deliberately no LLM pass (controlled difficulty; see README)",
|
|
"rebuild": "bun evals/brainbench/generator/gen.ts",
|
|
"generation_cost_usd": 0,
|
|
"gold_validation": {
|
|
"method": "blind double-label, stratified 10% sample (14 fixtures, 28 labeled items)",
|
|
"date": "2026-06-12",
|
|
"agreement": 0.964,
|
|
"findings": "continuity-writer rationale-clause drift (5 gold files) fixed in this corpus version; wb-001 MRR fact added; conventions documented in README"
|
|
}
|
|
}
|