mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 08:53:22 +00:00
Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Garry Tan
|
e22b8fb555 | ||
|
Garry Tan
|
2a9feb859f | ||
|
Garry Tan
|
15b9316dbf | ||
|
Garry TanandClaude Opus 4.7
|
f739de5521 | ||
|
Garry TanandClaude Opus 4.7
|
02d585c0a4 | ||
|
Garry Tan
|
e573fa6988 | ||
|
Garry Tan
|
ff6320e552 | ||
|
Garry TanandClaude Opus 4.7
|
36c750bbec | ||
|
Garry TanandClaude Opus 4.7
|
7f2c81f929 | ||
|
Garry TanandClaude Opus 4.7
|
1353366b5f | ||
|
Garry TanandClaude Opus 4.7
|
93ae40dd3a | ||
|
Garry TanandClaude Opus 4.7
|
8fcd2737bf | ||
|
Garry TanandClaude Opus 4.7
|
b23f24f91b | ||
|
Garry TanandClaude Opus 4.7
|
10d96545a4 | ||
|
root
|
6b2f3bc321 |
@@ -44,10 +44,7 @@ jobs:
|
||||
tier2:
|
||||
name: Tier 2 (LLM Skills)
|
||||
runs-on: ubuntu-latest
|
||||
# Runs on every push/PR now (promoted from schedule-only in v0.19.0).
|
||||
# Tier 1 must pass first; Tier 2 uses OPENAI_API_KEY + ANTHROPIC_API_KEY
|
||||
# from repo/org secrets. Nightly + manual triggers still supported via
|
||||
# the workflow-level `on:` list.
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
needs: tier1
|
||||
services:
|
||||
postgres:
|
||||
@@ -88,13 +85,8 @@ jobs:
|
||||
}
|
||||
EOF
|
||||
- name: Run Tier 2 skill tests
|
||||
run: bun test test/e2e/skills.test.ts test/e2e/zeroentropy-live.test.ts
|
||||
run: bun test test/e2e/skills.test.ts
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gbrain_test
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
# v0.33.3.0: ZE live API tests skip gracefully when this is unset,
|
||||
# so forks without the secret stay green. The test exercises the
|
||||
# zeroEntropyCompatFetch response-rewriter + URL rewrite + flexible
|
||||
# dim handling + gateway.rerank against the real provider.
|
||||
ZEROENTROPY_API_KEY: ${{ secrets.ZEROENTROPY_API_KEY }}
|
||||
|
||||
@@ -37,12 +37,6 @@ jobs:
|
||||
- run: bun install
|
||||
- name: Pre-test gates (shard 1 only — they're not test files)
|
||||
if: matrix.shard == 1
|
||||
run: bun run verify
|
||||
run: scripts/check-jsonb-pattern.sh && scripts/check-progress-to-stdout.sh && scripts/check-wasm-embedded.sh && bun run typecheck
|
||||
- name: Run test shard ${{ matrix.shard }}/4
|
||||
run: scripts/test-shard.sh ${{ matrix.shard }} 4
|
||||
- name: Run *.serial.test.ts at --max-concurrency=1 (shard 1 only)
|
||||
# Serial files share file-wide state (top-level mock.module, module
|
||||
# singletons) that leaks across files in the same bun-test process.
|
||||
# test-shard.sh excludes them; this step runs them at concurrency=1.
|
||||
if: matrix.shard == 1
|
||||
run: bun run test:serial
|
||||
|
||||
-23
@@ -11,33 +11,10 @@ bin/
|
||||
.gstack/
|
||||
supabase/.temp/
|
||||
.claude/skills/
|
||||
# admin/dist/ is the React SPA bundle. CLAUDE.md says it's committed for
|
||||
# self-contained binaries (the bun --compile path embeds it via
|
||||
# `import path from 'admin/dist/index.html' with { type: 'file' }`).
|
||||
# Build via: cd admin && bun install && bun run build.
|
||||
admin/node_modules/
|
||||
.idea
|
||||
eval/reports/
|
||||
eval/data/world-v1/world.html
|
||||
|
||||
# BrainBench amara-life-v1 Opus cache (regenerate via eval:generate-amara-life)
|
||||
eval/data/amara-life-v1/_cache/
|
||||
|
||||
# claw-test E2E build cache (shim + scratch outputs)
|
||||
test/.cache/
|
||||
|
||||
.claude/
|
||||
export/
|
||||
|
||||
# Conductor workspace-local agent artifacts: plans, todos, run-unit-parallel
|
||||
# failure logs and per-shard test output. v0.26.4 (run-unit-parallel.sh)
|
||||
# writes .context/test-failures.log + .context/test-summary.txt +
|
||||
# .context/test-shards/. Workspace-local by design — never committed.
|
||||
.context/
|
||||
|
||||
# Tier 3 PGLite snapshot fixture (built on demand by build:pglite-snapshot)
|
||||
test/fixtures/pglite-snapshot.tar
|
||||
test/fixtures/pglite-snapshot.version
|
||||
|
||||
# Private brain reports — never check these in (per CLAUDE.md privacy rule)
|
||||
reports/network-intelligence/
|
||||
|
||||
@@ -10,15 +10,7 @@ start here.
|
||||
2. Install: `bun install`
|
||||
3. Init the brain: `gbrain init` (defaults to PGLite, zero-config). For 1000+ files or
|
||||
multi-machine sync, init suggests Postgres + pgvector via Supabase.
|
||||
4. **STOP — ask the user about search mode.** `gbrain init` auto-applied a
|
||||
default but printed a 9-cell cost matrix (mode × downstream model)
|
||||
preceded by `[AGENT]` markers. You MUST relay the matrix to the operator
|
||||
and confirm their choice before continuing. Cost spread between corners
|
||||
is 25x — silent acceptance is the wrong default. See
|
||||
[`./INSTALL_FOR_AGENTS.md`](./INSTALL_FOR_AGENTS.md) Step 3.5 for the
|
||||
exact ask-the-user protocol. Same banner fires on `gbrain post-upgrade`
|
||||
for existing users (search modes were added in v0.32.3).
|
||||
5. Read [`./INSTALL_FOR_AGENTS.md`](./INSTALL_FOR_AGENTS.md) for the full 9-step flow
|
||||
4. Read [`./INSTALL_FOR_AGENTS.md`](./INSTALL_FOR_AGENTS.md) for the full 9-step flow
|
||||
(API keys, identity, cron, verification).
|
||||
|
||||
## Read this order
|
||||
@@ -26,13 +18,7 @@ start here.
|
||||
1. `./AGENTS.md` (this file) — install + operating protocol.
|
||||
2. [`./CLAUDE.md`](./CLAUDE.md) — architecture reference, key files, trust boundaries,
|
||||
test layout.
|
||||
3. [`./docs/architecture/brains-and-sources.md`](./docs/architecture/brains-and-sources.md)
|
||||
— the two-axis mental model (brain = which DB, source = which repo in the DB). Every
|
||||
query routes on both axes. Read before writing anything that touches brain ops.
|
||||
4. [`./skills/conventions/brain-routing.md`](./skills/conventions/brain-routing.md) —
|
||||
agent-facing decision table: when to switch brain, when to switch source, how
|
||||
cross-brain federation works (latent-space only; the agent decides).
|
||||
5. [`./skills/RESOLVER.md`](./skills/RESOLVER.md) — skill dispatcher. Read before any task.
|
||||
3. [`./skills/RESOLVER.md`](./skills/RESOLVER.md) — skill dispatcher. Read before any task.
|
||||
|
||||
## Trust boundary (critical)
|
||||
|
||||
@@ -51,38 +37,15 @@ writing or reviewing an operation, consult `src/core/operations.ts` for the cont
|
||||
[`docs/guides/minions-fix.md`](./docs/guides/minions-fix.md), `gbrain doctor --fix`.
|
||||
- **Migrate:** [`docs/UPGRADING_DOWNSTREAM_AGENTS.md`](./docs/UPGRADING_DOWNSTREAM_AGENTS.md),
|
||||
[`skills/migrations/`](./skills/migrations/), `gbrain apply-migrations`.
|
||||
- **Eval retrieval changes:** capture is off by default. To benchmark a
|
||||
retrieval change against real captured queries, set
|
||||
`GBRAIN_CONTRIBUTOR_MODE=1`, then `gbrain eval export --since 7d > base.ndjson`
|
||||
and `gbrain eval replay --against base.ndjson`. For public benchmark
|
||||
coverage (LongMemEval, ground-truth scoring), `gbrain eval longmemeval
|
||||
<dataset.jsonl>` (v0.28.8) runs against an isolated in-memory PGLite
|
||||
per question — your `~/.gbrain` is never opened. Full guide:
|
||||
[`docs/eval-bench.md`](./docs/eval-bench.md).
|
||||
- **Track a founder/company over time (v0.35.7):** when an entity has
|
||||
typed metric claims in its `## Facts` fence (`metric: mrr`, `value: 50000`,
|
||||
`unit: USD`, `period: monthly` columns), run
|
||||
`gbrain eval trajectory <entity-slug>` for the chronological history
|
||||
with regressions auto-flagged, or `gbrain founder scorecard <entity-slug>`
|
||||
for a four-signal JSON rollup (claim_accuracy / consistency /
|
||||
growth_trajectory / red_flags). MCP op `find_trajectory` exposes the
|
||||
same data — read scope, visibility-filtered for remote callers.
|
||||
- **Everything else:** [`./llms.txt`](./llms.txt) is the full documentation map.
|
||||
[`./llms-full.txt`](./llms-full.txt) is the same map with core docs inlined for
|
||||
single-fetch ingestion.
|
||||
|
||||
## Before shipping
|
||||
|
||||
Easiest path: `bun run ci:local` runs the full CI gate inside Docker (gitleaks,
|
||||
unit tests with `DATABASE_URL` unset, then all 29 E2E files sequentially against a
|
||||
fresh pgvector container) and tears down. Use `bun run ci:local:diff` for the
|
||||
diff-aware subset during fast iteration on a focused branch. Requires Docker
|
||||
(Docker Desktop / OrbStack / Colima) and `gitleaks` (`brew install gitleaks`).
|
||||
|
||||
Manual path: `bun test` plus the E2E lifecycle described in `./CLAUDE.md` (spin
|
||||
up the test Postgres container, run `bun run test:e2e`, tear it down).
|
||||
|
||||
Ship via the `/ship` skill, not by hand.
|
||||
Run `bun test` plus the E2E lifecycle described in `./CLAUDE.md` (spin up the test
|
||||
Postgres container, run `bun run test:e2e`, tear it down). Ship via the `/ship` skill,
|
||||
not by hand.
|
||||
|
||||
## Privacy
|
||||
|
||||
|
||||
+9
-7217
File diff suppressed because it is too large
Load Diff
+2
-187
@@ -52,22 +52,10 @@ docs/ Architecture docs
|
||||
## Running tests
|
||||
|
||||
```bash
|
||||
# Inner edit loop (~85s on a Mac dev box, 3700+ unit tests)
|
||||
bun run test # parallel 8-shard fan-out + serial post-pass
|
||||
bun test # all tests (unit + E2E skipped without DB)
|
||||
bun test test/markdown.test.ts # specific unit test
|
||||
|
||||
# Pre-push gate (matches what CI runs on shard 1 + typecheck)
|
||||
bun run verify # privacy + jsonb + progress + test-isolation + wasm + admin-build + typecheck
|
||||
|
||||
# Pre-merge sanity (everything CI runs)
|
||||
bun run test:full # verify + parallel unit + slow + smart e2e
|
||||
|
||||
# Slow / serial / e2e in isolation
|
||||
bun run test:slow # *.slow.test.ts only (cold-path correctness)
|
||||
bun run test:serial # *.serial.test.ts only (--max-concurrency=1)
|
||||
bun run test:e2e # real-Postgres E2E (requires DATABASE_URL)
|
||||
|
||||
# E2E setup (Postgres with pgvector)
|
||||
# E2E tests (requires Postgres with pgvector)
|
||||
docker compose -f docker-compose.test.yml up -d
|
||||
DATABASE_URL=postgresql://postgres:postgres@localhost:5434/gbrain_test bun run test:e2e
|
||||
|
||||
@@ -75,91 +63,6 @@ DATABASE_URL=postgresql://postgres:postgres@localhost:5434/gbrain_test bun run t
|
||||
DATABASE_URL=postgresql://... bun run test:e2e
|
||||
```
|
||||
|
||||
Use `bun run verify` before pushing. The guard chain catches: banned fork-name
|
||||
leaks (`scripts/check-privacy.sh`), `JSON.stringify(x)::jsonb` interpolation
|
||||
patterns (`scripts/check-jsonb-pattern.sh`), `\r` progress bleed to stdout
|
||||
(`scripts/check-progress-to-stdout.sh`), test-isolation rule violations
|
||||
(`scripts/check-test-isolation.sh` — see "Writing tests that survive the parallel
|
||||
loop" below), silent fallback to recursive chunking in the compiled binary
|
||||
(`scripts/check-wasm-embedded.sh`), and stale admin-dashboard build artifacts
|
||||
(`scripts/check-admin-build.sh`). `bun run check:all` runs the full historical
|
||||
sweep including the trailing-newline and exports-count checks.
|
||||
|
||||
### Writing tests that survive the parallel loop
|
||||
|
||||
`bun run test` shards 92+ unit-test files across 8 worker processes. Files in the
|
||||
same shard share a process, so process-global state leaks between them. Four
|
||||
lint rules (`scripts/check-test-isolation.sh`, R1-R4) enforce isolation:
|
||||
|
||||
| Rule | What it bans | Fix |
|
||||
|---|---|---|
|
||||
| **R1** | Direct `process.env.X = ...` mutation | Use `withEnv()` from `test/helpers/with-env.ts`, or rename to `*.serial.test.ts` |
|
||||
| **R2** | `mock.module(...)` anywhere in the file | Rename to `*.serial.test.ts` |
|
||||
| **R3** | `new PGLiteEngine(` outside ~50 lines after `beforeAll(` | Use the canonical PGLite block (see below) |
|
||||
| **R4** | `new PGLiteEngine(` without paired `afterAll(disconnect)` | Add the `afterAll(() => engine.disconnect())` |
|
||||
|
||||
Canonical PGLite block (R3 + R4 compliant — paste this verbatim):
|
||||
|
||||
```ts
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
afterAll(async () => { await engine.disconnect(); });
|
||||
beforeEach(async () => { await resetPgliteState(engine); });
|
||||
```
|
||||
|
||||
Env-touching tests:
|
||||
|
||||
```ts
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
|
||||
test('reads OPENAI_API_KEY', async () => {
|
||||
await withEnv({ OPENAI_API_KEY: 'sk-test' }, async () => {
|
||||
expect(loadConfig().openai_key).toBe('sk-test');
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
`withEnv` saves and restores keys via try/finally including when the callback
|
||||
throws. Cross-test safe; **NOT** intra-file concurrent-safe (`process.env` is
|
||||
process-global). Files using `withEnv` stay outside the future
|
||||
`test.concurrent()` codemod's eligibility filter.
|
||||
|
||||
When to quarantine instead of fix: rename to `*.serial.test.ts` if the file
|
||||
uses `mock.module(...)`, is genuinely env-coupled (module-load env readers +
|
||||
ESM caching defeat dynamic-import-after-env tricks), or intentionally shares
|
||||
state across `it()` boundaries. Quarantine count cap: 10 (informational).
|
||||
|
||||
Files that violated these rules at the v0.26.7 baseline are listed in
|
||||
`scripts/check-test-isolation.allowlist`. **The allow-list MUST shrink over
|
||||
time** ... never add new entries. v0.26.8 (env sweep) and v0.26.9 (PGLite sweep
|
||||
+ codemod) remove entries as files get fixed.
|
||||
|
||||
### Local CI gate (recommended before pushing, v0.23.1+)
|
||||
|
||||
```bash
|
||||
bun run ci:local # full gate: gitleaks + unit + ALL 29 E2E files (sequential)
|
||||
bun run ci:local:diff # gate with diff-aware E2E selector
|
||||
bun run ci:select-e2e # print which E2E files the selector would run
|
||||
```
|
||||
|
||||
`ci:local` spins up `pgvector/pgvector:pg16` + `oven/bun:1` via
|
||||
`docker-compose.ci.yml`, runs everything PR CI runs plus the full E2E suite, then
|
||||
tears down. Named volumes keep the install warm across runs (~16-20 min sequential
|
||||
E2E after the first cold pull). Requires Docker (Docker Desktop, OrbStack, or
|
||||
Colima) and `gitleaks` on host (`brew install gitleaks`). Override the postgres
|
||||
host port with `GBRAIN_CI_PG_PORT=5435 bun run ci:local` if 5434 collides.
|
||||
|
||||
Fail-closed selector: an unmapped `src/` change runs all 29 E2E files. Hand-tune
|
||||
narrower mappings via `scripts/e2e-test-map.ts`.
|
||||
|
||||
## Building
|
||||
|
||||
```bash
|
||||
@@ -192,94 +95,6 @@ See `docs/ENGINES.md` for the full guide. In short:
|
||||
|
||||
The SQLite engine is designed and ready for implementation. See `docs/SQLITE_ENGINE.md`.
|
||||
|
||||
## CONTRIBUTOR_MODE — turn on the dev loop
|
||||
|
||||
gbrain captures retrieval traffic so you can replay real queries against
|
||||
your code changes before merging. **This is off by default** (production
|
||||
users get a quiet brain, no surprise data accumulation). Contributors turn
|
||||
it on with one shell rc line:
|
||||
|
||||
```bash
|
||||
# In ~/.zshrc or ~/.bashrc:
|
||||
export GBRAIN_CONTRIBUTOR_MODE=1
|
||||
```
|
||||
|
||||
That's it. Every `query` / `search` you (or agents pointed at your dev
|
||||
brain) run from that shell now writes a row to `eval_candidates`, and the
|
||||
[replay tool](#running-real-world-eval-benchmarks-touching-retrieval-code)
|
||||
has data to work against.
|
||||
|
||||
What CONTRIBUTOR_MODE actually does:
|
||||
|
||||
- Turns on `query`/`search` capture into the local `eval_candidates` table.
|
||||
Without it the gate is closed and capture is a no-op.
|
||||
- That's all. PII scrubbing, retention, and replay are independent.
|
||||
|
||||
Resolution order (most explicit wins):
|
||||
|
||||
1. `eval.capture: true` in `~/.gbrain/config.json` → on
|
||||
2. `eval.capture: false` in `~/.gbrain/config.json` → off
|
||||
3. `GBRAIN_CONTRIBUTOR_MODE=1` → on
|
||||
4. otherwise → off
|
||||
|
||||
Quick check that capture is actually running:
|
||||
|
||||
```bash
|
||||
gbrain query "anything" >/dev/null
|
||||
psql $DATABASE_URL -c 'SELECT count(*) FROM eval_candidates'
|
||||
# (or `gbrain doctor` — surfaces silent capture failures cross-process)
|
||||
```
|
||||
|
||||
To disable capture even with the env var set, write
|
||||
`{"eval": {"capture": false}}` to `~/.gbrain/config.json` — explicit config
|
||||
beats the env var both directions.
|
||||
|
||||
## Running real-world eval benchmarks (touching retrieval code)
|
||||
|
||||
If your PR touches retrieval — search ranking, RRF fusion, embeddings,
|
||||
intent classification, query expansion, source boost, or the `query` /
|
||||
`search` op handlers — run `gbrain eval replay` against a snapshot of
|
||||
real traffic before merging. Requires `CONTRIBUTOR_MODE` (above) so you
|
||||
have captured rows to replay against.
|
||||
|
||||
Quick loop:
|
||||
|
||||
```bash
|
||||
gbrain eval export --since 7d > baseline.ndjson # snapshot before your change
|
||||
# ... make your change ...
|
||||
gbrain eval replay --against baseline.ndjson # diff retrieval, get Jaccard@k
|
||||
```
|
||||
|
||||
Three numbers come back: mean Jaccard@k between captured and current slug
|
||||
sets, top-1 stability, and mean latency Δ. The replay tool flags the worst
|
||||
regressions so you can eyeball whether the change is hurting real queries.
|
||||
|
||||
Trigger paths (rerun if your diff touches any of these):
|
||||
|
||||
- `src/core/search/hybrid.ts`
|
||||
- `src/core/search/source-boost.ts`, `sql-ranking.ts`
|
||||
- `src/core/search/intent.ts`, `expansion.ts`, `dedup.ts`
|
||||
- `src/core/embedding.ts`
|
||||
- `src/core/operations.ts` (query / search handlers)
|
||||
- `src/core/postgres-engine.ts` / `pglite-engine.ts` (searchKeyword /
|
||||
searchVector SQL)
|
||||
|
||||
See [`docs/eval-bench.md`](./docs/eval-bench.md) for the full guide
|
||||
including CI integration, hand-crafted NDJSON corpora (so a fresh checkout
|
||||
without captured data can still replay), and cost considerations. The
|
||||
NDJSON wire format is documented in
|
||||
[`docs/eval-capture.md`](./docs/eval-capture.md).
|
||||
|
||||
For public benchmark coverage on top of replay, `gbrain eval longmemeval
|
||||
<dataset.jsonl>` (v0.28.1) runs LongMemEval against gbrain's hybrid
|
||||
retrieval. One in-memory PGLite per question, runtime-enumerated
|
||||
`TRUNCATE` between questions, ground-truth scoring via LongMemEval's
|
||||
published `evaluate_qa.py`. Use it alongside replay when changes affect
|
||||
retrieval quality on long-context conversational data — replay catches
|
||||
regressions on YOUR queries, LongMemEval catches them on a public set the
|
||||
benchmark community already cites. See the "Public benchmarks: LongMemEval"
|
||||
section in [`docs/eval-bench.md`](./docs/eval-bench.md).
|
||||
|
||||
## Welcome PRs
|
||||
|
||||
- SQLite engine implementation
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
# DESIGN.md
|
||||
|
||||
The design system source of truth for gbrain. Born from the de facto tokens
|
||||
that landed in `admin/src/index.css` during the v0.26.0 admin SPA work and
|
||||
formalized during the v0.36.1.0 Hindsight calibration wave's design review.
|
||||
|
||||
This doc is the calibration target for `/plan-design-review` and `/design-review`.
|
||||
When a question is "does this UI fit the system?", the answer is here.
|
||||
|
||||
## Voice
|
||||
|
||||
GBrain talks like a smart friend who knows your past, not a clinical scoring
|
||||
system. Every user-facing string passes through this filter:
|
||||
|
||||
- Second person, contractions allowed.
|
||||
- Grounded in concrete data the user can verify ("2 of 3 missed" beats
|
||||
"Brier 0.31").
|
||||
- Never preachy. Never "we recommend." Never "according to your data."
|
||||
- Short. Under 25 words for narrative; under one line for status.
|
||||
- Numbers grounded in real outcomes, never abstract metrics without
|
||||
translation.
|
||||
|
||||
Five surfaces use this voice (v0.36.1.0+):
|
||||
`pattern_statement`, `nudge`, `forecast_blurb`, `dashboard_caption`,
|
||||
`morning_pulse`. All five pass through `gateVoice()` in
|
||||
`src/core/calibration/voice-gate.ts` with mode-specific rubrics. A Haiku
|
||||
judge rejects academic-sounding candidates; up to 2 regens; then fall
|
||||
back to a hand-written template from `src/core/calibration/templates.ts`.
|
||||
|
||||
## Color tokens
|
||||
|
||||
CSS variables in `admin/src/index.css`. SVG renderer inlines literals
|
||||
matching these tokens (`src/core/calibration/svg-renderer.ts`).
|
||||
|
||||
| Token | Value | Use |
|
||||
|--------------------|-----------|-------------------------------------------|
|
||||
| `--bg-primary` | `#0a0a0f` | Page background |
|
||||
| `--bg-secondary` | `#14141f` | Sidebar, cards |
|
||||
| `--bg-tertiary` | `#1e1e2e` | Subtle surfaces, borders |
|
||||
| `--text-primary` | `#e0e0e0` | Body text |
|
||||
| `--text-secondary` | `#888` | Headings, labels |
|
||||
| `--text-muted` | `#777` | Tertiary text — TD2 bumped from #555 for WCAG AA contrast (~5.5:1) |
|
||||
| `--accent` | `#3b82f6` | Active states, links, primary CTAs |
|
||||
| `--success` | `#22c55e` | Healthy / ok status |
|
||||
| `--warning` | `#f59e0b` | Doctor warnings |
|
||||
| `--error` | `#ef4444` | Failures, destructive confirmations |
|
||||
|
||||
Dark theme is the only theme. No light mode toggle planned — admin is an
|
||||
operator tool, not a marketing surface. Users live in the terminal with a
|
||||
dark theme already.
|
||||
|
||||
WCAG contrast:
|
||||
- Body text (#e0e0e0 on #0a0a0f) → ~14:1, AAA
|
||||
- Muted text (#777 on #0a0a0f) → ~5.5:1, AA (was 4.0 / fail before TD2)
|
||||
- Accent links (#3b82f6 on #0a0a0f) → ~5.7:1, AA
|
||||
|
||||
## Typography
|
||||
|
||||
| Variable | Value | Use |
|
||||
|--------------------|-----------------------------|---------------------------------|
|
||||
| `--font-sans` | `Inter, system-ui, sans-serif` | UI text, headings, body |
|
||||
| `--font-mono` | `JetBrains Mono, monospace` | Numbers, slugs, code, terminal-ish data |
|
||||
|
||||
Type scale (de facto, not formalized yet):
|
||||
- 18px: sidebar logo / page title
|
||||
- 14px: body
|
||||
- 13px: nav items
|
||||
- 12px: chart captions, secondary labels
|
||||
- 11px: tertiary labels in dense charts
|
||||
|
||||
Numbers in tables and metrics use JetBrains Mono so column alignment is
|
||||
mechanical. Avoid mixing Inter and JetBrains Mono in the same line.
|
||||
|
||||
## Spacing scale
|
||||
|
||||
4 / 8 / 16 / 24 / 32px. Linear-app-style density: 24-32px between major
|
||||
sections, 16px between row groups, 8px within a row. The Calibration tab
|
||||
(approved variant-B mockup) is the canonical example.
|
||||
|
||||
## Layout
|
||||
|
||||
- Sidebar 200px on the left. Active item gets a 3px left-border in `--accent`.
|
||||
- Main content area uses the remaining width.
|
||||
- Max content width: 720px for text-heavy pages (Calibration), 960px for
|
||||
data tables (Request Log).
|
||||
- No 3-column feature grids. No icons in colored circles. No decorative blobs.
|
||||
- Cards earn their existence — heading + content works without a card frame
|
||||
in most cases.
|
||||
|
||||
## Charts
|
||||
|
||||
Server-rendered SVG via `src/core/calibration/svg-renderer.ts`. Pure
|
||||
functions: data → SVG string. No DOM, no React component, no chart library.
|
||||
|
||||
XSS posture: server-side `escapeXml()` on every caller-controlled string.
|
||||
Numeric inputs `.toFixed()`-coerced. Admin SPA renders via
|
||||
`<TrustedSVG>` wrapper with `dangerouslySetInnerHTML`. Endpoint gated by
|
||||
`requireAdmin` middleware.
|
||||
|
||||
Why server-rendered SVG (per D23):
|
||||
- Chart logic stays close to the data math.
|
||||
- Zero new client-side chart-library dep.
|
||||
- SVG is accessible (text labels), scalable, copy-paste-friendly to PR
|
||||
descriptions and docs.
|
||||
- Sets the precedent for future admin charts (contradictions trend, takes
|
||||
scorecard, etc.).
|
||||
|
||||
Four chart renderers in v0.36.1.0:
|
||||
- `renderBrierTrend({ series })` — sparkline + baseline reference at 0.25
|
||||
- `renderDomainBars({ bars })` — horizontal accuracy bars
|
||||
- `renderAbandonedThreadsCard(threads)` — text rows + "revisit now" links
|
||||
- `renderPatternStatementsCard(statements)` — clickable drill-down anchors
|
||||
|
||||
## Interaction patterns
|
||||
|
||||
- Keyboard navigation is REQUIRED for all CLI interaction surfaces. The
|
||||
propose-queue review uses J/K/space/u/q shortcuts (gmail-style).
|
||||
- Loading states: "Loading...". Don't show spinners on sub-200ms operations.
|
||||
- Empty states ARE features: warmth + primary action + context. Cold-brain
|
||||
Calibration page tells the user EXACTLY how to build a profile, not
|
||||
"no data available."
|
||||
- Error states: name what failed + name the next step. Never "an error
|
||||
occurred — please try again."
|
||||
|
||||
## What's NOT here yet (v0.37+ roadmap)
|
||||
|
||||
- Type scale formalization (current values are de facto, not enforced)
|
||||
- Animation tokens (admin SPA has zero animations on purpose; v0.37 may
|
||||
add subtle progress / loading transitions)
|
||||
- Print stylesheet
|
||||
- Light mode (NOT planned — see "Dark theme is the only theme" above)
|
||||
- Component library extraction (the React components live inline in admin/src/pages/;
|
||||
no `<Button>` / `<Card>` abstraction layer yet)
|
||||
|
||||
## How to use this document
|
||||
|
||||
When adding a new UI surface to gbrain:
|
||||
|
||||
1. Pick existing tokens before introducing new ones. New tokens go through
|
||||
`/plan-design-review`.
|
||||
2. Match the voice rules. Run candidates through `gateVoice()` before
|
||||
shipping any user-facing string in the calibration surfaces.
|
||||
3. Match the spacing scale and density. Linear-calm-clarity over
|
||||
dashboard-card-mosaic.
|
||||
4. Match the typography: Inter for UI, JetBrains Mono for numbers.
|
||||
|
||||
When updating this document: it's a living target, not a frozen spec.
|
||||
Major changes go through `/plan-design-review` to keep the system coherent.
|
||||
+2
-69
@@ -61,63 +61,6 @@ Read `~/gbrain/docs/GBRAIN_RECOMMENDED_SCHEMA.md` and set up the MECE directory
|
||||
structure (people/, companies/, concepts/, etc.) inside the user's brain repo,
|
||||
NOT inside ~/gbrain.
|
||||
|
||||
## Step 3.5: Confirm search mode with the user (DO NOT SKIP)
|
||||
|
||||
`gbrain init` auto-applied a default search mode (`tokenmax` unless your subagent
|
||||
tier is Haiku-class or no OpenAI key is configured). The init output included the
|
||||
cost matrix below preceded by `[AGENT]` markers. You must NOT silently accept the
|
||||
default. Stop and ask the operator.
|
||||
|
||||
**Present this matrix verbatim:**
|
||||
|
||||
```
|
||||
Per-query cost @ 10K queries/mo (typical single-user volume):
|
||||
|
||||
Haiku 4.5 Sonnet 4.6 Opus 4.7
|
||||
($1/M) ($3/M) ($5/M)
|
||||
conservative $40/mo $120/mo $200/mo
|
||||
balanced $100/mo $300/mo $500/mo
|
||||
tokenmax $200/mo $600/mo $1,000/mo
|
||||
|
||||
(scales linearly: ×10 for 100K/mo, ÷10 for 1K. 25x corner-to-corner spread.
|
||||
Natural diagonal pairings — cheap/cheap → frontier/frontier — span ~4x.)
|
||||
```
|
||||
|
||||
**Ask the operator (paraphrase if needed):**
|
||||
|
||||
> Your gbrain just installed with search mode `<auto-applied default>`. This is
|
||||
> a one-time setup decision that controls retrieval payload size. Which mode
|
||||
> do you want?
|
||||
>
|
||||
> 1) conservative — tight 4K budget, no LLM expansion, 10 chunks max.
|
||||
> Best for Haiku subagents, cost-sensitive setups, high-volume loops.
|
||||
>
|
||||
> 2) balanced — 12K budget, no expansion, 25 chunks. Sonnet-tier sweet spot.
|
||||
>
|
||||
> 3) tokenmax (recommended default — preserves v0.31.x retrieval shape) —
|
||||
> no budget, LLM expansion ON, 50 chunks. Best for Opus/frontier models.
|
||||
>
|
||||
> Cost depends on BOTH the mode AND the downstream model you run. See the
|
||||
> matrix above for the 9-cell breakdown.
|
||||
|
||||
If the operator picks a non-default mode, run:
|
||||
```bash
|
||||
gbrain config set search.mode <mode>
|
||||
```
|
||||
|
||||
If they pick tokenmax AND want to preserve the literal v0.31.x default
|
||||
(limit=20 instead of tokenmax's 50), also run:
|
||||
```bash
|
||||
gbrain config set search.searchLimit 20
|
||||
```
|
||||
|
||||
Verify the choice with `gbrain search modes` before continuing.
|
||||
|
||||
**Why this matters:** the cost spread between corners of the matrix is 25x.
|
||||
An agent that silently accepts the default and starts running queries against
|
||||
a user who didn't expect tokenmax-class context loads can rack up surprise
|
||||
spend. Confirm before continuing.
|
||||
|
||||
## Step 4: Import and Index
|
||||
|
||||
```bash
|
||||
@@ -186,9 +129,8 @@ Set up using your platform's scheduler (OpenClaw cron, Railway cron, crontab):
|
||||
- **Live sync** (every 15 min): `gbrain sync --repo ~/brain && gbrain embed --stale`
|
||||
- **Auto-update** (daily): `gbrain check-update --json` (tell user, never auto-install)
|
||||
- **Dream cycle** (nightly): read `docs/guides/cron-schedule.md` for the full protocol.
|
||||
Entity sweep, citation fixes, memory consolidation, plus (v0.23+) overnight conversation
|
||||
synthesis and cross-session pattern detection. 8 phases, one cron-friendly command. This
|
||||
is what makes the brain compound. Do not skip it.
|
||||
Entity sweep, citation fixes, memory consolidation. This is what makes the brain
|
||||
compound. Do not skip it.
|
||||
- **Weekly**: `gbrain doctor --json && gbrain embed --stale`
|
||||
|
||||
## Step 8: Integrations
|
||||
@@ -216,15 +158,6 @@ Then read `~/gbrain/skills/migrations/v<NEW_VERSION>.md` (and any intermediate
|
||||
versions you skipped) and run any backfill or verification steps it lists. Skipping
|
||||
this is how features ship in the binary but stay dormant in the user's brain.
|
||||
|
||||
**v0.32.3 search modes (one-time upgrade prompt):** if the user's brain was
|
||||
created before v0.32.3, `gbrain post-upgrade` prints a banner including the
|
||||
9-cell cost matrix (mode × downstream model) preceded by `[AGENT]` markers.
|
||||
**Do NOT silently move past the banner.** Present the matrix to the operator
|
||||
verbatim, ask which mode they want (recommended default: `tokenmax` to preserve
|
||||
v0.31.x retrieval shape), then run `gbrain config set search.mode <mode>`. See
|
||||
Step 3.5 above for the full ask-the-user protocol — the upgrade path uses the
|
||||
same matrix and same default.
|
||||
|
||||
For v0.12.0+ specifically: if your brain was created before v0.12.0, run
|
||||
`gbrain extract links --source db && gbrain extract timeline --source db` to
|
||||
backfill the new graph layer (see Step 4.5 above).
|
||||
|
||||
+3
-26
@@ -68,16 +68,6 @@ The built-in HTTP transport ships with several layers of hardening on by
|
||||
default. All env vars below are optional; the defaults are intentionally
|
||||
conservative.
|
||||
|
||||
### Bind address (v0.34: loopback by default)
|
||||
|
||||
`gbrain serve --http` listens on `127.0.0.1` by default. Personal-laptop
|
||||
installs cannot accidentally publish the brain to the LAN. Self-hosted
|
||||
deployments that need remote access pass `--bind 0.0.0.0` (all
|
||||
interfaces) or `--bind <interface-ip>` (specific NIC). A stderr WARN
|
||||
fires when `--public-url` is set without `--bind` so the operator sees
|
||||
the binding before the first request — common cause of "ngrok forwards
|
||||
to me but the agent can't reach the upstream" misconfigurations.
|
||||
|
||||
### Postgres-only
|
||||
|
||||
`gbrain serve --http` requires a Postgres engine. PGLite is local-only by
|
||||
@@ -135,11 +125,9 @@ GBRAIN_HTTP_TRUST_PROXY=1 gbrain serve --http --port 8787
|
||||
**both** of these are true:
|
||||
|
||||
1. gbrain is reachable only via a trusted reverse proxy (not directly
|
||||
exposed to the internet on the configured port). As of v0.34
|
||||
`gbrain serve --http` binds `127.0.0.1` by default, so the
|
||||
reverse-proxy-only posture is the out-of-the-box shape; only
|
||||
override with `--bind 0.0.0.0` (or a specific interface IP) when
|
||||
gbrain itself needs to accept remote connections directly.
|
||||
exposed to the internet on the configured port). The simplest
|
||||
guarantee is to bind gbrain to `127.0.0.1` or a private interface
|
||||
and have the proxy forward to it.
|
||||
2. The proxy strips any client-supplied `X-Forwarded-For` and `X-Real-IP`
|
||||
headers, then sets them itself. (nginx with `proxy_set_header
|
||||
X-Forwarded-For $remote_addr` does this; Cloudflare and most cloud
|
||||
@@ -178,14 +166,3 @@ psql "$DATABASE_URL" -c \
|
||||
`body_too_large`, `parse_error`, `unknown_method`. Failed-auth rows have
|
||||
`token_name = NULL`. Inserts are fire-and-forget so audit failures
|
||||
never block requests.
|
||||
|
||||
**v0.26.9 redaction default.** The `params` column now stores
|
||||
`{redacted, kind, declared_keys, unknown_key_count, approx_bytes}` instead
|
||||
of raw JSON-RPC payloads. Declared keys (intersected against the operation's
|
||||
spec) preserve for debug visibility; unknown keys are counted but never
|
||||
named so attackers can't probe key existence; byte sizes bucket to 1KB so
|
||||
content sizes can't be binary-searched. The same shape is broadcast on the
|
||||
admin SSE feed at `/admin/events`. Operators on a personal laptop who want
|
||||
raw payloads back can pass `gbrain serve --http --log-full-params` (loud
|
||||
stderr warning at startup). Multi-tenant deployments should leave it
|
||||
on the redacted default.
|
||||
|
||||
-158
@@ -1,158 +0,0 @@
|
||||
# Design System — GBrain Admin Dashboard
|
||||
|
||||
## Product Context
|
||||
- **What this is:** Admin dashboard for GBrain MCP server — manage OAuth agents, API keys, monitor requests
|
||||
- **Who it's for:** GBrain operators managing multi-agent access to their brain
|
||||
- **Space/industry:** Developer infrastructure (peers: Supabase dashboard, Vercel, Railway)
|
||||
- **Project type:** Dense utilitarian admin panel — Steve Krug "Don't Make Me Think"
|
||||
|
||||
## Aesthetic Direction
|
||||
- **Direction:** Industrial/Utilitarian — function-first, data-dense, zero decoration
|
||||
- **Decoration level:** None — every pixel earns its place with information
|
||||
- **Mood:** Ops dashboard for someone who builds. Not a marketing site. Not a consumer app. A cockpit.
|
||||
- **Reference:** Supabase dashboard (dark + dense), Linear (restrained), Grafana (data-forward)
|
||||
|
||||
## Alignment
|
||||
- **Text alignment:** Left-align everything. No centered text in tables, cards, forms, or labels.
|
||||
- **Headings:** Left-aligned
|
||||
- **Table data:** Left-aligned (including numbers — contextual readability over columnar alignment)
|
||||
- **Form labels:** Left-aligned above inputs
|
||||
- **Buttons in forms:** Right-aligned (action flows left-to-right: Cancel → Submit)
|
||||
- **Modal titles:** Left-aligned
|
||||
- **Page titles:** Left-aligned
|
||||
- **Only exception:** Empty states and the login page lock icon can center for visual weight
|
||||
|
||||
## Typography
|
||||
- **Display/Headings:** Inter (Semibold 600) — clean, neutral, disappears into the content
|
||||
- **Body/UI:** Inter (Regular 400 / Medium 500)
|
||||
- **Data/Tables/Code:** JetBrains Mono (Regular 400 / Medium 500) — monospace for anything the user might copy, any ID, any token, any technical value
|
||||
- **Loading:** Google Fonts. `display=swap`.
|
||||
- **Scale:**
|
||||
- Page title: 24px / Inter Semibold
|
||||
- Section title: 14px / Inter Semibold, uppercase, letter-spacing 0.5px
|
||||
- Table header: 12px / Inter Medium, uppercase, letter-spacing 1px, muted color
|
||||
- Body: 14px / Inter Regular
|
||||
- Small/Caption: 13px
|
||||
- Micro: 12px (badges, timestamps)
|
||||
- Code/Data: 13px / JetBrains Mono
|
||||
|
||||
## Color
|
||||
- **Approach:** Monochrome base + semantic color only. No primary brand color. Color means something.
|
||||
- **Background:**
|
||||
- Base: #0a0a0f (near-black with blue undertone)
|
||||
- Surface/cards: #12121a
|
||||
- Hover: #1a1a2a
|
||||
- Input/code blocks: #0f0f1a
|
||||
- **Borders:** #1e1e2e (default), #3a3a5a (hover/active)
|
||||
- **Text:**
|
||||
- Primary: #e0e0e0
|
||||
- Secondary: #888888
|
||||
- Muted: #555555
|
||||
- Link: #88aaff
|
||||
- **Semantic (badges only):**
|
||||
- Success/active: #34a853
|
||||
- Error/danger: #ff6b6b
|
||||
- Warning: #f5a623
|
||||
- Read scope: #3b82f6
|
||||
- Write scope: #f59e0b
|
||||
- Admin scope: #ef4444
|
||||
- **No accent color.** The data IS the interface. Badges carry all the color.
|
||||
|
||||
## Spacing
|
||||
- **Base unit:** 4px
|
||||
- **Density:** Dense — this is an ops tool, not a landing page
|
||||
- **Scale:** 4px, 8px, 12px, 16px, 20px, 24px, 32px, 48px
|
||||
- **Table row padding:** 10px 16px
|
||||
- **Card padding:** 24px
|
||||
- **Modal padding:** 24px
|
||||
- **Section gaps:** 24px between sections, 12px between related elements
|
||||
|
||||
## Layout
|
||||
- **Sidebar:** Fixed left, 200px wide, dark (#0a0a0f)
|
||||
- **Main content:** Fluid, max-width none (fills available space)
|
||||
- **Grid:** Single column for tables (full width), 2-column for stats cards
|
||||
- **Border radius:**
|
||||
- Cards/panels: 16px
|
||||
- Buttons/inputs: 8px
|
||||
- Badges: 9999px (pill)
|
||||
- Tables: 0 (sharp edges — data is rectangular)
|
||||
|
||||
## Components
|
||||
|
||||
### Tables
|
||||
- Full-width, no outer border
|
||||
- Header row: uppercase, letter-spaced, muted color, no background
|
||||
- Data rows: subtle hover (#1a1a2a), pointer cursor when clickable
|
||||
- All text left-aligned
|
||||
- Monospace for IDs, tokens, latency values
|
||||
|
||||
### Badges
|
||||
- Pill shape (border-radius: 9999px)
|
||||
- Padding: 2px 8px
|
||||
- Font: 12px
|
||||
- Scoped to semantic meaning: `success`, `danger`, `read`, `write`, `admin`
|
||||
|
||||
### Buttons
|
||||
- Primary: white text on #3a3a5a, hover brightens
|
||||
- Secondary: muted text on transparent, border #1e1e2e
|
||||
- Danger: white text on #ff6b6b background
|
||||
- Size: 13px font, 6px 14px padding
|
||||
|
||||
### Modals
|
||||
- Overlay: rgba(0,0,0,0.7)
|
||||
- Card: #12121a, border #1e1e2e, border-radius 16px, max-width 480px
|
||||
- Title: 18px Semibold, left-aligned
|
||||
- Close: top-right ✕ button
|
||||
|
||||
### Drawers
|
||||
- Right-side panel, 400px wide
|
||||
- Slide in from right
|
||||
- Dark overlay behind
|
||||
- Close button top-right
|
||||
- Sections separated by section titles (uppercase, muted)
|
||||
|
||||
### Tabs
|
||||
- Inline horizontal, wrapping allowed
|
||||
- Active: white text, bottom border
|
||||
- Inactive: muted text, no border
|
||||
- No background color on tabs
|
||||
|
||||
### Code blocks
|
||||
- Background: rgba(0,0,0,0.3)
|
||||
- Border-radius: 8px
|
||||
- Padding: 10px 14px
|
||||
- Font: JetBrains Mono 12px
|
||||
- Copy button: right-aligned, subtle
|
||||
|
||||
### Empty states
|
||||
- Centered text (only exception to left-align rule)
|
||||
- Muted color
|
||||
- Suggest next action
|
||||
|
||||
## Motion
|
||||
- **Approach:** Minimal — transitions for hover states only
|
||||
- **Duration:** 150ms for hovers, 200ms for drawer slide
|
||||
- **No loading spinners** — show stale data until fresh arrives
|
||||
- **SSE live feed:** Real-time, no animation on new entries (just prepend)
|
||||
|
||||
## Anti-Patterns (do NOT do these)
|
||||
- ❌ Center-aligned table data
|
||||
- ❌ Center-aligned headings or labels (except empty states)
|
||||
- ❌ Gradient backgrounds
|
||||
- ❌ Shadows (the dark theme IS the depth model)
|
||||
- ❌ Rounded table corners
|
||||
- ❌ Icons as navigation (use text labels)
|
||||
- ❌ Loading skeletons (show real data or nothing)
|
||||
- ❌ Confirmation toasts (action → result is immediate and visible)
|
||||
- ❌ Color for decoration (every color means something)
|
||||
|
||||
## Decisions Log
|
||||
| Date | Decision | Rationale |
|
||||
|------|----------|-----------|
|
||||
| 2026-05-01 | Dark theme only | Ops dashboard. No light mode needed. |
|
||||
| 2026-05-01 | Steve Krug lens | Zero happy talk, mindless choices, scannable tables, billboard-speed comprehension. |
|
||||
| 2026-05-01 | JetBrains Mono for data | Anything copyable or technical should be monospace. |
|
||||
| 2026-05-03 | Left-align everything | Garry preference. Centered text is a design crutch. Left-align forces hierarchy through typography weight and spacing, not position. |
|
||||
| 2026-05-03 | Incorporate GStack design DNA | Same family: Inter + JetBrains Mono, dark base, semantic-only color. Diverges on accent (GStack: amber; GBrain: none — data is the color). |
|
||||
| 2026-05-03 | Per-client config export tabs | Claude Code, ChatGPT, Claude.ai, Cursor, Perplexity, JSON. Every agent has a copy-paste setup path. |
|
||||
| 2026-05-03 | Magic link auth | Login page tells you to ask your agent. No pasting hex strings into forms. |
|
||||
-257
@@ -1,257 +0,0 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"configVersion": 1,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "gbrain-admin",
|
||||
"dependencies": {
|
||||
"react": "^19.1.0",
|
||||
"react-dom": "^19.1.0",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.1.2",
|
||||
"@types/react-dom": "^19.1.2",
|
||||
"@vitejs/plugin-react": "^4.4.1",
|
||||
"typescript": "^5.8.3",
|
||||
"vite": "^6.3.3",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@babel/code-frame": ["@babel/code-frame@7.29.0", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw=="],
|
||||
|
||||
"@babel/compat-data": ["@babel/compat-data@7.29.0", "", {}, "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg=="],
|
||||
|
||||
"@babel/core": ["@babel/core@7.29.0", "", { "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", "@babel/helper-compilation-targets": "^7.28.6", "@babel/helper-module-transforms": "^7.28.6", "@babel/helpers": "^7.28.6", "@babel/parser": "^7.29.0", "@babel/template": "^7.28.6", "@babel/traverse": "^7.29.0", "@babel/types": "^7.29.0", "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", "json5": "^2.2.3", "semver": "^6.3.1" } }, "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA=="],
|
||||
|
||||
"@babel/generator": ["@babel/generator@7.29.1", "", { "dependencies": { "@babel/parser": "^7.29.0", "@babel/types": "^7.29.0", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" } }, "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw=="],
|
||||
|
||||
"@babel/helper-compilation-targets": ["@babel/helper-compilation-targets@7.28.6", "", { "dependencies": { "@babel/compat-data": "^7.28.6", "@babel/helper-validator-option": "^7.27.1", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" } }, "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA=="],
|
||||
|
||||
"@babel/helper-globals": ["@babel/helper-globals@7.28.0", "", {}, "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw=="],
|
||||
|
||||
"@babel/helper-module-imports": ["@babel/helper-module-imports@7.28.6", "", { "dependencies": { "@babel/traverse": "^7.28.6", "@babel/types": "^7.28.6" } }, "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw=="],
|
||||
|
||||
"@babel/helper-module-transforms": ["@babel/helper-module-transforms@7.28.6", "", { "dependencies": { "@babel/helper-module-imports": "^7.28.6", "@babel/helper-validator-identifier": "^7.28.5", "@babel/traverse": "^7.28.6" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA=="],
|
||||
|
||||
"@babel/helper-plugin-utils": ["@babel/helper-plugin-utils@7.28.6", "", {}, "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug=="],
|
||||
|
||||
"@babel/helper-string-parser": ["@babel/helper-string-parser@7.27.1", "", {}, "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="],
|
||||
|
||||
"@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="],
|
||||
|
||||
"@babel/helper-validator-option": ["@babel/helper-validator-option@7.27.1", "", {}, "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg=="],
|
||||
|
||||
"@babel/helpers": ["@babel/helpers@7.29.2", "", { "dependencies": { "@babel/template": "^7.28.6", "@babel/types": "^7.29.0" } }, "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw=="],
|
||||
|
||||
"@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="],
|
||||
|
||||
"@babel/plugin-transform-react-jsx-self": ["@babel/plugin-transform-react-jsx-self@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw=="],
|
||||
|
||||
"@babel/plugin-transform-react-jsx-source": ["@babel/plugin-transform-react-jsx-source@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw=="],
|
||||
|
||||
"@babel/template": ["@babel/template@7.28.6", "", { "dependencies": { "@babel/code-frame": "^7.28.6", "@babel/parser": "^7.28.6", "@babel/types": "^7.28.6" } }, "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ=="],
|
||||
|
||||
"@babel/traverse": ["@babel/traverse@7.29.0", "", { "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", "@babel/helper-globals": "^7.28.0", "@babel/parser": "^7.29.0", "@babel/template": "^7.28.6", "@babel/types": "^7.29.0", "debug": "^4.3.1" } }, "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA=="],
|
||||
|
||||
"@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="],
|
||||
|
||||
"@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="],
|
||||
|
||||
"@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="],
|
||||
|
||||
"@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.12", "", { "os": "android", "cpu": "arm64" }, "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg=="],
|
||||
|
||||
"@esbuild/android-x64": ["@esbuild/android-x64@0.25.12", "", { "os": "android", "cpu": "x64" }, "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg=="],
|
||||
|
||||
"@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg=="],
|
||||
|
||||
"@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA=="],
|
||||
|
||||
"@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.12", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg=="],
|
||||
|
||||
"@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.12", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ=="],
|
||||
|
||||
"@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.12", "", { "os": "linux", "cpu": "arm" }, "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw=="],
|
||||
|
||||
"@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ=="],
|
||||
|
||||
"@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.12", "", { "os": "linux", "cpu": "ia32" }, "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA=="],
|
||||
|
||||
"@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng=="],
|
||||
|
||||
"@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw=="],
|
||||
|
||||
"@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.12", "", { "os": "linux", "cpu": "ppc64" }, "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA=="],
|
||||
|
||||
"@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w=="],
|
||||
|
||||
"@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.12", "", { "os": "linux", "cpu": "s390x" }, "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg=="],
|
||||
|
||||
"@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.12", "", { "os": "linux", "cpu": "x64" }, "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw=="],
|
||||
|
||||
"@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg=="],
|
||||
|
||||
"@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.12", "", { "os": "none", "cpu": "x64" }, "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ=="],
|
||||
|
||||
"@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.12", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A=="],
|
||||
|
||||
"@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.12", "", { "os": "openbsd", "cpu": "x64" }, "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw=="],
|
||||
|
||||
"@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg=="],
|
||||
|
||||
"@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.12", "", { "os": "sunos", "cpu": "x64" }, "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w=="],
|
||||
|
||||
"@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg=="],
|
||||
|
||||
"@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.12", "", { "os": "win32", "cpu": "ia32" }, "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ=="],
|
||||
|
||||
"@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="],
|
||||
|
||||
"@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="],
|
||||
|
||||
"@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="],
|
||||
|
||||
"@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="],
|
||||
|
||||
"@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="],
|
||||
|
||||
"@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="],
|
||||
|
||||
"@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-beta.27", "", {}, "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA=="],
|
||||
|
||||
"@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.60.1", "", { "os": "android", "cpu": "arm" }, "sha512-d6FinEBLdIiK+1uACUttJKfgZREXrF0Qc2SmLII7W2AD8FfiZ9Wjd+rD/iRuf5s5dWrr1GgwXCvPqOuDquOowA=="],
|
||||
|
||||
"@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.60.1", "", { "os": "android", "cpu": "arm64" }, "sha512-YjG/EwIDvvYI1YvYbHvDz/BYHtkY4ygUIXHnTdLhG+hKIQFBiosfWiACWortsKPKU/+dUwQQCKQM3qrDe8c9BA=="],
|
||||
|
||||
"@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.60.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-mjCpF7GmkRtSJwon+Rq1N8+pI+8l7w5g9Z3vWj4T7abguC4Czwi3Yu/pFaLvA3TTeMVjnu3ctigusqWUfjZzvw=="],
|
||||
|
||||
"@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.60.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-haZ7hJ1JT4e9hqkoT9R/19XW2QKqjfJVv+i5AGg57S+nLk9lQnJ1F/eZloRO3o9Scy9CM3wQ9l+dkXtcBgN5Ew=="],
|
||||
|
||||
"@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.60.1", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-czw90wpQq3ZsAVBlinZjAYTKduOjTywlG7fEeWKUA7oCmpA8xdTkxZZlwNJKWqILlq0wehoZcJYfBvOyhPTQ6w=="],
|
||||
|
||||
"@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.60.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-KVB2rqsxTHuBtfOeySEyzEOB7ltlB/ux38iu2rBQzkjbwRVlkhAGIEDiiYnO2kFOkJp+Z7pUXKyrRRFuFUKt+g=="],
|
||||
|
||||
"@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.60.1", "", { "os": "linux", "cpu": "arm" }, "sha512-L+34Qqil+v5uC0zEubW7uByo78WOCIrBvci69E7sFASRl0X7b/MB6Cqd1lky/CtcSVTydWa2WZwFuWexjS5o6g=="],
|
||||
|
||||
"@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.60.1", "", { "os": "linux", "cpu": "arm" }, "sha512-n83O8rt4v34hgFzlkb1ycniJh7IR5RCIqt6mz1VRJD6pmhRi0CXdmfnLu9dIUS6buzh60IvACM842Ffb3xd6Gg=="],
|
||||
|
||||
"@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.60.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-Nql7sTeAzhTAja3QXeAI48+/+GjBJ+QmAH13snn0AJSNL50JsDqotyudHyMbO2RbJkskbMbFJfIJKWA6R1LCJQ=="],
|
||||
|
||||
"@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.60.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-+pUymDhd0ys9GcKZPPWlFiZ67sTWV5UU6zOJat02M1+PiuSGDziyRuI/pPue3hoUwm2uGfxdL+trT6Z9rxnlMA=="],
|
||||
|
||||
"@rollup/rollup-linux-loong64-gnu": ["@rollup/rollup-linux-loong64-gnu@4.60.1", "", { "os": "linux", "cpu": "none" }, "sha512-VSvgvQeIcsEvY4bKDHEDWcpW4Yw7BtlKG1GUT4FzBUlEKQK0rWHYBqQt6Fm2taXS+1bXvJT6kICu5ZwqKCnvlQ=="],
|
||||
|
||||
"@rollup/rollup-linux-loong64-musl": ["@rollup/rollup-linux-loong64-musl@4.60.1", "", { "os": "linux", "cpu": "none" }, "sha512-4LqhUomJqwe641gsPp6xLfhqWMbQV04KtPp7/dIp0nzPxAkNY1AbwL5W0MQpcalLYk07vaW9Kp1PBhdpZYYcEw=="],
|
||||
|
||||
"@rollup/rollup-linux-ppc64-gnu": ["@rollup/rollup-linux-ppc64-gnu@4.60.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-tLQQ9aPvkBxOc/EUT6j3pyeMD6Hb8QF2BTBnCQWP/uu1lhc9AIrIjKnLYMEroIz/JvtGYgI9dF3AxHZNaEH0rw=="],
|
||||
|
||||
"@rollup/rollup-linux-ppc64-musl": ["@rollup/rollup-linux-ppc64-musl@4.60.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-RMxFhJwc9fSXP6PqmAz4cbv3kAyvD1etJFjTx4ONqFP9DkTkXsAMU4v3Vyc5BgzC+anz7nS/9tp4obsKfqkDHg=="],
|
||||
|
||||
"@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.60.1", "", { "os": "linux", "cpu": "none" }, "sha512-QKgFl+Yc1eEk6MmOBfRHYF6lTxiiiV3/z/BRrbSiW2I7AFTXoBFvdMEyglohPj//2mZS4hDOqeB0H1ACh3sBbg=="],
|
||||
|
||||
"@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.60.1", "", { "os": "linux", "cpu": "none" }, "sha512-RAjXjP/8c6ZtzatZcA1RaQr6O1TRhzC+adn8YZDnChliZHviqIjmvFwHcxi4JKPSDAt6Uhf/7vqcBzQJy0PDJg=="],
|
||||
|
||||
"@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.60.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-wcuocpaOlaL1COBYiA89O6yfjlp3RwKDeTIA0hM7OpmhR1Bjo9j31G1uQVpDlTvwxGn2nQs65fBFL5UFd76FcQ=="],
|
||||
|
||||
"@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.60.1", "", { "os": "linux", "cpu": "x64" }, "sha512-77PpsFQUCOiZR9+LQEFg9GClyfkNXj1MP6wRnzYs0EeWbPcHs02AXu4xuUbM1zhwn3wqaizle3AEYg5aeoohhg=="],
|
||||
|
||||
"@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.60.1", "", { "os": "linux", "cpu": "x64" }, "sha512-5cIATbk5vynAjqqmyBjlciMJl1+R/CwX9oLk/EyiFXDWd95KpHdrOJT//rnUl4cUcskrd0jCCw3wpZnhIHdD9w=="],
|
||||
|
||||
"@rollup/rollup-openbsd-x64": ["@rollup/rollup-openbsd-x64@4.60.1", "", { "os": "openbsd", "cpu": "x64" }, "sha512-cl0w09WsCi17mcmWqqglez9Gk8isgeWvoUZ3WiJFYSR3zjBQc2J5/ihSjpl+VLjPqjQ/1hJRcqBfLjssREQILw=="],
|
||||
|
||||
"@rollup/rollup-openharmony-arm64": ["@rollup/rollup-openharmony-arm64@4.60.1", "", { "os": "none", "cpu": "arm64" }, "sha512-4Cv23ZrONRbNtbZa37mLSueXUCtN7MXccChtKpUnQNgF010rjrjfHx3QxkS2PI7LqGT5xXyYs1a7LbzAwT0iCA=="],
|
||||
|
||||
"@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.60.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-i1okWYkA4FJICtr7KpYzFpRTHgy5jdDbZiWfvny21iIKky5YExiDXP+zbXzm3dUcFpkEeYNHgQ5fuG236JPq0g=="],
|
||||
|
||||
"@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.60.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-u09m3CuwLzShA0EYKMNiFgcjjzwqtUMLmuCJLeZWjjOYA3IT2Di09KaxGBTP9xVztWyIWjVdsB2E9goMjZvTQg=="],
|
||||
|
||||
"@rollup/rollup-win32-x64-gnu": ["@rollup/rollup-win32-x64-gnu@4.60.1", "", { "os": "win32", "cpu": "x64" }, "sha512-k+600V9Zl1CM7eZxJgMyTUzmrmhB/0XZnF4pRypKAlAgxmedUA+1v9R+XOFv56W4SlHEzfeMtzujLJD22Uz5zg=="],
|
||||
|
||||
"@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.60.1", "", { "os": "win32", "cpu": "x64" }, "sha512-lWMnixq/QzxyhTV6NjQJ4SFo1J6PvOX8vUx5Wb4bBPsEb+8xZ89Bz6kOXpfXj9ak9AHTQVQzlgzBEc1SyM27xQ=="],
|
||||
|
||||
"@types/babel__core": ["@types/babel__core@7.20.5", "", { "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", "@types/babel__generator": "*", "@types/babel__template": "*", "@types/babel__traverse": "*" } }, "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA=="],
|
||||
|
||||
"@types/babel__generator": ["@types/babel__generator@7.27.0", "", { "dependencies": { "@babel/types": "^7.0.0" } }, "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg=="],
|
||||
|
||||
"@types/babel__template": ["@types/babel__template@7.4.4", "", { "dependencies": { "@babel/parser": "^7.1.0", "@babel/types": "^7.0.0" } }, "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A=="],
|
||||
|
||||
"@types/babel__traverse": ["@types/babel__traverse@7.28.0", "", { "dependencies": { "@babel/types": "^7.28.2" } }, "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q=="],
|
||||
|
||||
"@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
|
||||
|
||||
"@types/react": ["@types/react@19.2.14", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w=="],
|
||||
|
||||
"@types/react-dom": ["@types/react-dom@19.2.3", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ=="],
|
||||
|
||||
"@vitejs/plugin-react": ["@vitejs/plugin-react@4.7.0", "", { "dependencies": { "@babel/core": "^7.28.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-beta.27", "@types/babel__core": "^7.20.5", "react-refresh": "^0.17.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA=="],
|
||||
|
||||
"baseline-browser-mapping": ["baseline-browser-mapping@2.10.18", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-VSnGQAOLtP5mib/DPyg2/t+Tlv65NTBz83BJBJvmLVHHuKJVaDOBvJJykiT5TR++em5nfAySPccDZDa4oSrn8A=="],
|
||||
|
||||
"browserslist": ["browserslist@4.28.2", "", { "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", "electron-to-chromium": "^1.5.328", "node-releases": "^2.0.36", "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg=="],
|
||||
|
||||
"caniuse-lite": ["caniuse-lite@1.0.30001788", "", {}, "sha512-6q8HFp+lOQtcf7wBK+uEenxymVWkGKkjFpCvw5W25cmMwEDU45p1xQFBQv8JDlMMry7eNxyBaR+qxgmTUZkIRQ=="],
|
||||
|
||||
"convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="],
|
||||
|
||||
"csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="],
|
||||
|
||||
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
||||
|
||||
"electron-to-chromium": ["electron-to-chromium@1.5.336", "", {}, "sha512-AbH9q9J455r/nLmdNZes0G0ZKcRX73FicwowalLs6ijwOmCJSRRrLX63lcAlzy9ux3dWK1w1+1nsBJEWN11hcQ=="],
|
||||
|
||||
"esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="],
|
||||
|
||||
"escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="],
|
||||
|
||||
"fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="],
|
||||
|
||||
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
|
||||
|
||||
"gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="],
|
||||
|
||||
"js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="],
|
||||
|
||||
"jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="],
|
||||
|
||||
"json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="],
|
||||
|
||||
"lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="],
|
||||
|
||||
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
|
||||
|
||||
"nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
|
||||
|
||||
"node-releases": ["node-releases@2.0.37", "", {}, "sha512-1h5gKZCF+pO/o3Iqt5Jp7wc9rH3eJJ0+nh/CIoiRwjRxde/hAHyLPXYN4V3CqKAbiZPSeJFSWHmJsbkicta0Eg=="],
|
||||
|
||||
"picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="],
|
||||
|
||||
"picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="],
|
||||
|
||||
"postcss": ["postcss@8.5.9", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-7a70Nsot+EMX9fFU3064K/kdHWZqGVY+BADLyXc8Dfv+mTLLVl6JzJpPaCZ2kQL9gIJvKXSLMHhqdRRjwQeFtw=="],
|
||||
|
||||
"react": ["react@19.2.5", "", {}, "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA=="],
|
||||
|
||||
"react-dom": ["react-dom@19.2.5", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.5" } }, "sha512-J5bAZz+DXMMwW/wV3xzKke59Af6CHY7G4uYLN1OvBcKEsWOs4pQExj86BBKamxl/Ik5bx9whOrvBlSDfWzgSag=="],
|
||||
|
||||
"react-refresh": ["react-refresh@0.17.0", "", {}, "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ=="],
|
||||
|
||||
"rollup": ["rollup@4.60.1", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.60.1", "@rollup/rollup-android-arm64": "4.60.1", "@rollup/rollup-darwin-arm64": "4.60.1", "@rollup/rollup-darwin-x64": "4.60.1", "@rollup/rollup-freebsd-arm64": "4.60.1", "@rollup/rollup-freebsd-x64": "4.60.1", "@rollup/rollup-linux-arm-gnueabihf": "4.60.1", "@rollup/rollup-linux-arm-musleabihf": "4.60.1", "@rollup/rollup-linux-arm64-gnu": "4.60.1", "@rollup/rollup-linux-arm64-musl": "4.60.1", "@rollup/rollup-linux-loong64-gnu": "4.60.1", "@rollup/rollup-linux-loong64-musl": "4.60.1", "@rollup/rollup-linux-ppc64-gnu": "4.60.1", "@rollup/rollup-linux-ppc64-musl": "4.60.1", "@rollup/rollup-linux-riscv64-gnu": "4.60.1", "@rollup/rollup-linux-riscv64-musl": "4.60.1", "@rollup/rollup-linux-s390x-gnu": "4.60.1", "@rollup/rollup-linux-x64-gnu": "4.60.1", "@rollup/rollup-linux-x64-musl": "4.60.1", "@rollup/rollup-openbsd-x64": "4.60.1", "@rollup/rollup-openharmony-arm64": "4.60.1", "@rollup/rollup-win32-arm64-msvc": "4.60.1", "@rollup/rollup-win32-ia32-msvc": "4.60.1", "@rollup/rollup-win32-x64-gnu": "4.60.1", "@rollup/rollup-win32-x64-msvc": "4.60.1", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-VmtB2rFU/GroZ4oL8+ZqXgSA38O6GR8KSIvWmEFv63pQ0G6KaBH9s07PO8XTXP4vI+3UJUEypOfjkGfmSBBR0w=="],
|
||||
|
||||
"scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="],
|
||||
|
||||
"semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="],
|
||||
|
||||
"source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
|
||||
|
||||
"tinyglobby": ["tinyglobby@0.2.16", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg=="],
|
||||
|
||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
"update-browserslist-db": ["update-browserslist-db@1.2.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w=="],
|
||||
|
||||
"vite": ["vite@6.4.2", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", "picomatch": "^4.0.2", "postcss": "^8.5.3", "rollup": "^4.34.9", "tinyglobby": "^0.2.13" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "jiti": ">=1.21.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ=="],
|
||||
|
||||
"yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="],
|
||||
}
|
||||
}
|
||||
Vendored
-56
File diff suppressed because one or more lines are too long
Vendored
-1
File diff suppressed because one or more lines are too long
Vendored
-16
@@ -1,16 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>GBrain Admin</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
|
||||
<script type="module" crossorigin src="/admin/assets/index-CWq369vO.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/admin/assets/index-GxkWX7v3.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,15 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>GBrain Admin</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"name": "gbrain-admin",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"react": "^19.1.0",
|
||||
"react-dom": "^19.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.1.2",
|
||||
"@types/react-dom": "^19.1.2",
|
||||
"@vitejs/plugin-react": "^4.4.1",
|
||||
"vite": "^6.3.3",
|
||||
"typescript": "^5.8.3"
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { LoginPage } from './pages/Login';
|
||||
import { DashboardPage } from './pages/Dashboard';
|
||||
import { AgentsPage } from './pages/Agents';
|
||||
import { RequestLogPage } from './pages/RequestLog';
|
||||
import { CalibrationPage } from './pages/Calibration';
|
||||
import { api } from './api';
|
||||
|
||||
type Page = 'login' | 'dashboard' | 'agents' | 'log' | 'calibration';
|
||||
|
||||
function getPage(): Page {
|
||||
const hash = window.location.hash.replace('#', '') || 'dashboard';
|
||||
if (['login', 'dashboard', 'agents', 'log', 'calibration'].includes(hash)) return hash as Page;
|
||||
return 'dashboard';
|
||||
}
|
||||
|
||||
export function App() {
|
||||
const [page, setPage] = useState<Page>(getPage);
|
||||
|
||||
useEffect(() => {
|
||||
const onHash = () => setPage(getPage());
|
||||
window.addEventListener('hashchange', onHash);
|
||||
return () => window.removeEventListener('hashchange', onHash);
|
||||
}, []);
|
||||
|
||||
const navigate = (p: Page) => {
|
||||
window.location.hash = p;
|
||||
setPage(p);
|
||||
};
|
||||
|
||||
if (page === 'login') {
|
||||
return <LoginPage onLogin={() => navigate('dashboard')} />;
|
||||
}
|
||||
|
||||
const handleSignOutEverywhere = async () => {
|
||||
if (!confirm('Sign out every active admin session, including other browsers and tabs? Each one will need to re-authenticate via a fresh magic link.')) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await api.signOutEverywhere();
|
||||
} catch {
|
||||
// Even if the call fails, push to login — cookie is likely already invalid.
|
||||
}
|
||||
navigate('login');
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="app">
|
||||
<nav className="sidebar">
|
||||
<div className="sidebar-logo">GBrain</div>
|
||||
<div className="sidebar-nav">
|
||||
<a className={`nav-item ${page === 'dashboard' ? 'active' : ''}`}
|
||||
onClick={() => navigate('dashboard')}>Dashboard</a>
|
||||
<a className={`nav-item ${page === 'agents' ? 'active' : ''}`}
|
||||
onClick={() => navigate('agents')}>Agents</a>
|
||||
<a className={`nav-item ${page === 'log' ? 'active' : ''}`}
|
||||
onClick={() => navigate('log')}>Request Log</a>
|
||||
<a className={`nav-item ${page === 'calibration' ? 'active' : ''}`}
|
||||
onClick={() => navigate('calibration')}>Calibration</a>
|
||||
</div>
|
||||
<div style={{ marginTop: 'auto', padding: '16px 12px', borderTop: '1px solid var(--border)' }}>
|
||||
<button
|
||||
onClick={handleSignOutEverywhere}
|
||||
style={{
|
||||
background: 'transparent',
|
||||
border: '1px solid var(--border)',
|
||||
color: 'var(--text-secondary)',
|
||||
padding: '6px 10px',
|
||||
borderRadius: 6,
|
||||
fontSize: 12,
|
||||
cursor: 'pointer',
|
||||
width: '100%',
|
||||
}}
|
||||
title="Revoke every active admin session — every browser, every tab"
|
||||
>
|
||||
Sign out everywhere
|
||||
</button>
|
||||
</div>
|
||||
</nav>
|
||||
<main className="main">
|
||||
{page === 'dashboard' && <DashboardPage />}
|
||||
{page === 'agents' && <AgentsPage />}
|
||||
{page === 'log' && <RequestLogPage />}
|
||||
{page === 'calibration' && <CalibrationPage />}
|
||||
</main>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
const BASE = '';
|
||||
|
||||
// v0.26.3 trust model (D11 + D12): the admin UI does NOT cache the
|
||||
// bootstrap token in browser JS state. On 401, redirect to login —
|
||||
// no auto-reauth via saved token, no localStorage/sessionStorage read.
|
||||
// The HttpOnly cookie set by /admin/login is the only session credential.
|
||||
async function apiFetch(path: string, options?: RequestInit) {
|
||||
const res = await fetch(`${BASE}${path}`, {
|
||||
...options,
|
||||
credentials: 'same-origin',
|
||||
headers: { 'Content-Type': 'application/json', ...options?.headers },
|
||||
});
|
||||
if (res.status === 401) {
|
||||
// No token cache to retry from. Redirect to login.
|
||||
window.location.hash = '#login';
|
||||
throw new Error('Unauthorized');
|
||||
}
|
||||
if (!res.ok) {
|
||||
const body = await res.json().catch(() => ({}));
|
||||
throw new Error(body.error || `HTTP ${res.status}`);
|
||||
}
|
||||
return res.json();
|
||||
}
|
||||
|
||||
// v0.36.1.0 (T15 / E6) — SVG fetch (text/plain payload, NOT JSON).
|
||||
async function apiFetchText(path: string) {
|
||||
const res = await fetch(`${BASE}${path}`, { credentials: 'same-origin' });
|
||||
if (res.status === 401) {
|
||||
window.location.hash = '#login';
|
||||
throw new Error('Unauthorized');
|
||||
}
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
return res.text();
|
||||
}
|
||||
|
||||
export const api = {
|
||||
login: (token: string) => apiFetch('/admin/login', { method: 'POST', body: JSON.stringify({ token }) }),
|
||||
signOutEverywhere: () => apiFetch('/admin/api/sign-out-everywhere', { method: 'POST' }),
|
||||
stats: () => apiFetch('/admin/api/stats'),
|
||||
health: () => apiFetch('/admin/api/health-indicators'),
|
||||
agents: () => apiFetch('/admin/api/agents'),
|
||||
requests: (page = 1, qs = '') => apiFetch(`/admin/api/requests?page=${page}${qs}`),
|
||||
apiKeys: () => apiFetch('/admin/api/api-keys'),
|
||||
createApiKey: (name: string) => apiFetch('/admin/api/api-keys', { method: 'POST', body: JSON.stringify({ name }) }),
|
||||
revokeApiKey: (name: string) => apiFetch('/admin/api/api-keys/revoke', { method: 'POST', body: JSON.stringify({ name }) }),
|
||||
updateClientTtl: (clientId: string, tokenTtl: number | null) => apiFetch('/admin/api/update-client-ttl', { method: 'POST', body: JSON.stringify({ clientId, tokenTtl }) }),
|
||||
revokeClient: (clientId: string) => apiFetch('/admin/api/revoke-client', { method: 'POST', body: JSON.stringify({ clientId }) }),
|
||||
// v0.36.1.0 (T15 / E6) — calibration endpoints.
|
||||
calibrationProfile: (holder?: string) =>
|
||||
apiFetch(`/admin/api/calibration/profile${holder ? `?holder=${encodeURIComponent(holder)}` : ''}`),
|
||||
calibrationChart: (type: string, holder?: string) =>
|
||||
apiFetchText(`/admin/api/calibration/charts/${encodeURIComponent(type)}${holder ? `?holder=${encodeURIComponent(holder)}` : ''}`),
|
||||
};
|
||||
@@ -1,359 +0,0 @@
|
||||
:root {
|
||||
--bg-primary: #0a0a0f;
|
||||
--bg-secondary: #14141f;
|
||||
--bg-tertiary: #1e1e2e;
|
||||
--text-primary: #e0e0e0;
|
||||
--text-secondary: #888;
|
||||
/* v0.36.1.0 TD2 — bumped from #555 (contrast 4.0 on #0a0a0f bg, below WCAG AA
|
||||
4.5 for body text) to #777 (contrast ~5.5, passes AA). Applies globally
|
||||
to Dashboard, Agents, RequestLog, and the new Calibration tab. */
|
||||
--text-muted: #777;
|
||||
--accent: #3b82f6;
|
||||
--success: #22c55e;
|
||||
--warning: #f59e0b;
|
||||
--error: #ef4444;
|
||||
--font-mono: 'JetBrains Mono', monospace;
|
||||
--font-sans: 'Inter', system-ui, sans-serif;
|
||||
}
|
||||
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
|
||||
body {
|
||||
font-family: var(--font-sans);
|
||||
background: var(--bg-primary);
|
||||
color: var(--text-primary);
|
||||
font-size: 14px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
/* Layout */
|
||||
.app { display: flex; min-height: 100vh; }
|
||||
|
||||
.sidebar {
|
||||
width: 200px;
|
||||
background: var(--bg-secondary);
|
||||
border-right: 1px solid #1e1e2e;
|
||||
padding: 16px 0;
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.sidebar-logo {
|
||||
font-size: 18px;
|
||||
font-weight: 600;
|
||||
padding: 0 16px 24px;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.sidebar-nav { display: flex; flex-direction: column; gap: 2px; }
|
||||
|
||||
.nav-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 8px 16px;
|
||||
color: var(--text-secondary);
|
||||
text-decoration: none;
|
||||
font-size: 13px;
|
||||
cursor: pointer;
|
||||
border-left: 3px solid transparent;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
|
||||
.nav-item:hover { background: var(--bg-tertiary); color: var(--text-primary); }
|
||||
.nav-item.active {
|
||||
border-left-color: var(--accent);
|
||||
background: var(--bg-tertiary);
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.main { flex: 1; padding: 24px 32px; overflow-y: auto; }
|
||||
|
||||
.page-title {
|
||||
font-size: 24px;
|
||||
font-weight: 600;
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
|
||||
/* Metrics bar */
|
||||
.metrics { display: flex; gap: 16px; margin-bottom: 24px; }
|
||||
.metric {
|
||||
background: var(--bg-secondary);
|
||||
padding: 16px 20px;
|
||||
border-radius: 6px;
|
||||
min-width: 140px;
|
||||
}
|
||||
.metric-value {
|
||||
font-family: var(--font-mono);
|
||||
font-size: 28px;
|
||||
font-weight: 500;
|
||||
}
|
||||
.metric-label { font-size: 12px; color: var(--text-secondary); margin-top: 4px; }
|
||||
|
||||
/* Tables */
|
||||
table { width: 100%; border-collapse: collapse; }
|
||||
th {
|
||||
text-align: left;
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
color: var(--text-muted);
|
||||
padding: 8px 12px;
|
||||
font-weight: 500;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
td {
|
||||
padding: 10px 12px;
|
||||
font-size: 13px;
|
||||
border-top: 1px solid #1a1a2a;
|
||||
}
|
||||
tr:hover td { background: var(--bg-tertiary); }
|
||||
|
||||
/* Badges */
|
||||
.badge {
|
||||
display: inline-block;
|
||||
padding: 2px 8px;
|
||||
border-radius: 10px;
|
||||
font-size: 11px;
|
||||
font-weight: 500;
|
||||
}
|
||||
.badge-read { background: rgba(59,130,246,0.15); color: var(--accent); }
|
||||
.badge-write { background: rgba(245,158,11,0.15); color: var(--warning); }
|
||||
.badge-admin { background: rgba(239,68,68,0.15); color: var(--error); }
|
||||
.badge-success { background: rgba(34,197,94,0.15); color: var(--success); }
|
||||
.badge-error { background: rgba(239,68,68,0.15); color: var(--error); }
|
||||
|
||||
/* Status dots */
|
||||
.status-dot {
|
||||
display: inline-block;
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
margin-right: 6px;
|
||||
}
|
||||
.status-active { background: var(--success); }
|
||||
.status-warning { background: var(--warning); }
|
||||
.status-inactive { background: var(--text-muted); }
|
||||
|
||||
/* Buttons */
|
||||
.btn {
|
||||
padding: 8px 16px;
|
||||
border-radius: 6px;
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
border: none;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
.btn-primary { background: var(--accent); color: white; }
|
||||
.btn-primary:hover { background: #2563eb; }
|
||||
.btn-secondary { background: transparent; color: var(--text-secondary); border: 1px solid #333; }
|
||||
.btn-secondary:hover { border-color: var(--text-secondary); color: var(--text-primary); }
|
||||
.btn-danger { background: transparent; color: var(--error); border: 1px solid var(--error); }
|
||||
.btn-danger:hover { background: rgba(239,68,68,0.1); }
|
||||
|
||||
/* Forms */
|
||||
input, select {
|
||||
background: var(--bg-primary);
|
||||
border: 1px solid #333;
|
||||
color: var(--text-primary);
|
||||
padding: 8px 12px;
|
||||
border-radius: 6px;
|
||||
font-size: 13px;
|
||||
font-family: var(--font-sans);
|
||||
width: 100%;
|
||||
}
|
||||
input:focus, select:focus {
|
||||
outline: none;
|
||||
border-color: var(--accent);
|
||||
box-shadow: 0 0 0 2px rgba(59,130,246,0.2);
|
||||
}
|
||||
input::placeholder { color: var(--text-muted); }
|
||||
label { display: block; font-size: 13px; font-weight: 500; margin-bottom: 6px; }
|
||||
|
||||
/* Modal */
|
||||
.modal-overlay {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0,0,0,0.7);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 100;
|
||||
}
|
||||
.modal {
|
||||
background: var(--bg-secondary);
|
||||
border-radius: 8px;
|
||||
padding: 24px;
|
||||
min-width: 420px;
|
||||
max-width: 520px;
|
||||
}
|
||||
.modal-title { font-size: 18px; font-weight: 600; margin-bottom: 20px; }
|
||||
|
||||
/* Drawer */
|
||||
.drawer-overlay {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0,0,0,0.5);
|
||||
z-index: 90;
|
||||
}
|
||||
.drawer {
|
||||
position: fixed;
|
||||
right: 0;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
width: 420px;
|
||||
background: var(--bg-secondary);
|
||||
border-left: 1px solid var(--accent);
|
||||
padding: 24px;
|
||||
z-index: 91;
|
||||
overflow-y: auto;
|
||||
}
|
||||
.drawer-close {
|
||||
position: absolute;
|
||||
top: 16px;
|
||||
right: 16px;
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--text-muted);
|
||||
font-size: 18px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* Section headers */
|
||||
.section-title {
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
color: var(--text-muted);
|
||||
letter-spacing: 0.5px;
|
||||
margin: 20px 0 12px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
/* Health panel */
|
||||
.health-panel {
|
||||
background: var(--bg-secondary);
|
||||
border-radius: 6px;
|
||||
padding: 16px;
|
||||
}
|
||||
.health-row {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
padding: 6px 0;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
/* Code block */
|
||||
.code-block {
|
||||
background: var(--bg-primary);
|
||||
border-radius: 6px;
|
||||
padding: 12px;
|
||||
font-family: var(--font-mono);
|
||||
font-size: 12px;
|
||||
overflow-x: auto;
|
||||
position: relative;
|
||||
}
|
||||
.code-block .copy-btn {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 8px;
|
||||
background: var(--accent);
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 4px 10px;
|
||||
border-radius: 4px;
|
||||
font-size: 11px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* Activity feed */
|
||||
.feed { max-height: 400px; overflow-y: auto; }
|
||||
.feed-empty {
|
||||
color: var(--text-muted);
|
||||
text-align: center;
|
||||
padding: 32px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
/* Sparkline */
|
||||
.sparkline { display: inline-block; vertical-align: middle; }
|
||||
|
||||
/* Filter bar */
|
||||
.filter-bar { display: flex; gap: 12px; margin-bottom: 16px; align-items: center; }
|
||||
.filter-bar select { width: auto; min-width: 140px; }
|
||||
|
||||
/* Pagination */
|
||||
.pagination {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 12px 0;
|
||||
font-size: 13px;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
.pagination button {
|
||||
background: var(--bg-secondary);
|
||||
border: 1px solid #333;
|
||||
color: var(--text-primary);
|
||||
padding: 6px 12px;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
font-size: 12px;
|
||||
}
|
||||
.pagination button:disabled { opacity: 0.3; cursor: default; }
|
||||
|
||||
/* Warning bar */
|
||||
.warning-bar {
|
||||
background: rgba(245,158,11,0.15);
|
||||
border: 1px solid var(--warning);
|
||||
color: var(--warning);
|
||||
padding: 10px 16px;
|
||||
border-radius: 6px;
|
||||
font-size: 13px;
|
||||
margin: 12px 0;
|
||||
}
|
||||
|
||||
/* Checkbox */
|
||||
.checkbox-group { display: flex; gap: 16px; flex-wrap: wrap; }
|
||||
.checkbox-label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-size: 13px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* Tabs */
|
||||
.tabs { display: flex; gap: 0; margin-bottom: 12px; }
|
||||
.tab {
|
||||
padding: 6px 12px;
|
||||
font-size: 13px;
|
||||
color: var(--text-secondary);
|
||||
cursor: pointer;
|
||||
border-bottom: 2px solid transparent;
|
||||
}
|
||||
.tab.active { color: var(--accent); border-bottom-color: var(--accent); }
|
||||
|
||||
/* Login page */
|
||||
.login-page {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
min-height: 100vh;
|
||||
background: var(--bg-primary);
|
||||
}
|
||||
.login-box { text-align: left; width: 340px; }
|
||||
.login-logo { font-size: 32px; font-weight: 600; margin-bottom: 32px; }
|
||||
.login-hint { color: var(--text-muted); font-size: 12px; margin-top: 12px; }
|
||||
.login-error { color: var(--error); font-size: 13px; margin-top: 8px; }
|
||||
|
||||
/* Monospace data */
|
||||
.mono { font-family: var(--font-mono); font-size: 12px; }
|
||||
|
||||
/* Responsive */
|
||||
@media (max-width: 768px) {
|
||||
.sidebar { display: none; }
|
||||
.main { padding: 16px; }
|
||||
.metrics { flex-wrap: wrap; }
|
||||
.drawer { width: 100%; }
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
/**
|
||||
* Admin SPA scope constants — HAND-MAINTAINED MIRROR of src/core/scope.ts.
|
||||
*
|
||||
* The admin tsconfig.json scopes `include: ['src']` to admin/src/, so we
|
||||
* cannot directly import from ../../src/core/scope.ts without breaking the
|
||||
* SPA's compile boundary. Instead, this file is a hand-maintained duplicate;
|
||||
* scripts/check-admin-scope-drift.sh fails the build if the two lists drift.
|
||||
*
|
||||
* If you change ALLOWED_SCOPES in src/core/scope.ts, update this file too,
|
||||
* or `bun run verify` will reject the change.
|
||||
*/
|
||||
|
||||
export type Scope = 'read' | 'write' | 'admin' | 'sources_admin' | 'users_admin';
|
||||
|
||||
// MIRROR OF src/core/scope.ts ALLOWED_SCOPES_LIST — keep alphabetically sorted.
|
||||
export const ALLOWED_SCOPES_LIST: ReadonlyArray<Scope> = [
|
||||
'admin',
|
||||
'read',
|
||||
'sources_admin',
|
||||
'users_admin',
|
||||
'write',
|
||||
];
|
||||
@@ -1,10 +0,0 @@
|
||||
import React from 'react';
|
||||
import ReactDOM from 'react-dom/client';
|
||||
import { App } from './App';
|
||||
import './index.css';
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||
<React.StrictMode>
|
||||
<App />
|
||||
</React.StrictMode>,
|
||||
);
|
||||
@@ -1,633 +0,0 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { api } from '../api';
|
||||
import { ALLOWED_SCOPES_LIST, type Scope } from '../lib/scope-constants';
|
||||
|
||||
function timeAgo(date: Date): string {
|
||||
const s = Math.floor((Date.now() - date.getTime()) / 1000);
|
||||
if (s < 60) return 'just now';
|
||||
if (s < 3600) return `${Math.floor(s / 60)}m ago`;
|
||||
if (s < 86400) return `${Math.floor(s / 3600)}h ago`;
|
||||
return `${Math.floor(s / 86400)}d ago`;
|
||||
}
|
||||
|
||||
interface Agent {
|
||||
id: string;
|
||||
name: string;
|
||||
auth_type: 'oauth' | 'api_key';
|
||||
client_id?: string; // compat
|
||||
client_name?: string; // compat
|
||||
grant_types: string[];
|
||||
scope: string;
|
||||
created_at: string;
|
||||
last_used_at: string | null;
|
||||
total_requests: number;
|
||||
requests_today: number;
|
||||
token_ttl: number | null;
|
||||
status: 'active' | 'revoked';
|
||||
}
|
||||
|
||||
interface ApiKey {
|
||||
id: string;
|
||||
name: string;
|
||||
created_at: string;
|
||||
last_used_at: string | null;
|
||||
status: 'active' | 'revoked';
|
||||
}
|
||||
|
||||
export function AgentsPage() {
|
||||
const [agents, setAgents] = useState<Agent[]>([]);
|
||||
const [hideRevoked, setHideRevoked] = useState(true);
|
||||
const [showRegister, setShowRegister] = useState(false);
|
||||
const [showCredentials, setShowCredentials] = useState<{ clientId: string; clientSecret: string; name: string } | null>(null);
|
||||
const [showApiKeyCreate, setShowApiKeyCreate] = useState(false);
|
||||
const [showApiKeyToken, setShowApiKeyToken] = useState<{ name: string; token: string } | null>(null);
|
||||
const [selectedAgent, setSelectedAgent] = useState<Agent | null>(null);
|
||||
|
||||
useEffect(() => { loadAgents(); }, []);
|
||||
|
||||
const loadAgents = () => { api.agents().then(setAgents).catch(() => {}); };
|
||||
|
||||
return (
|
||||
<>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', marginBottom: 24 }}>
|
||||
<h1 className="page-title" style={{ marginBottom: 0 }}>Agents</h1>
|
||||
<div style={{ display: 'flex', gap: 8, alignItems: 'center' }}>
|
||||
<label style={{ fontSize: 13, color: 'var(--text-secondary)', display: 'flex', alignItems: 'center', gap: 6, cursor: 'pointer' }}>
|
||||
<input type="checkbox" checked={hideRevoked} onChange={e => setHideRevoked(e.target.checked)} /> Hide revoked
|
||||
</label>
|
||||
<button className="btn btn-secondary" onClick={() => setShowApiKeyCreate(true)}>+ API Key</button>
|
||||
<button className="btn btn-primary" onClick={() => setShowRegister(true)}>+ OAuth Client</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{(() => {
|
||||
// Filter once and reuse, so the empty-state guard sees the same
|
||||
// rows the table renders. Pre-fix: agents.length === 0 used the
|
||||
// unfiltered array, so an all-revoked dataset with hideRevoked=on
|
||||
// showed a header-only table with no placeholder.
|
||||
const visibleAgents = agents.filter(a => !hideRevoked || a.status !== 'revoked');
|
||||
if (agents.length === 0) {
|
||||
return (
|
||||
<div style={{ textAlign: 'center', padding: 48, color: 'var(--text-muted)' }}>
|
||||
No agents registered. Register your first agent to get started.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (visibleAgents.length === 0) {
|
||||
return (
|
||||
<div style={{ textAlign: 'center', padding: 48, color: 'var(--text-muted)' }}>
|
||||
All agents are revoked. Uncheck "Hide revoked" to view them.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Type</th>
|
||||
<th>Scopes</th>
|
||||
<th>Status</th>
|
||||
<th>Requests</th>
|
||||
<th>Last Used</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{visibleAgents.map(a => (
|
||||
<tr key={a.id} onClick={() => setSelectedAgent(a)}
|
||||
style={{ cursor: 'pointer' }}>
|
||||
<td style={{ fontWeight: 500 }}>{a.name || a.client_name}</td>
|
||||
<td>
|
||||
<span className={`badge ${a.auth_type === 'oauth' ? 'badge-read' : 'badge-write'}`} style={{ fontSize: 11 }}>
|
||||
{a.auth_type === 'oauth' ? 'OAuth' : 'API Key'}
|
||||
</span>
|
||||
</td>
|
||||
<td>
|
||||
{(a.scope || '').split(' ').filter(Boolean).map(s => (
|
||||
<span key={s} className={`badge badge-${s}`} style={{ marginRight: 4 }}>{s}</span>
|
||||
))}
|
||||
</td>
|
||||
<td>
|
||||
<span className={`badge ${a.status === 'active' ? 'badge-success' : 'badge-danger'}`}>{a.status}</span>
|
||||
</td>
|
||||
<td>
|
||||
<span style={{ fontWeight: 500 }}>{a.requests_today || 0}</span>
|
||||
<span style={{ color: 'var(--text-muted)', fontSize: 12 }}> / {a.total_requests || 0}</span>
|
||||
</td>
|
||||
<td style={{ color: 'var(--text-secondary)' }}>
|
||||
{a.last_used_at ? timeAgo(new Date(a.last_used_at)) : 'Never'}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
<div style={{ color: 'var(--text-muted)', fontSize: 13, marginTop: 12 }}>
|
||||
{agents.filter(a => a.status === 'active').length} active / {agents.length} total
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
})()}
|
||||
|
||||
{showRegister && (
|
||||
<RegisterModal
|
||||
onClose={() => setShowRegister(false)}
|
||||
onRegistered={(creds) => { setShowRegister(false); setShowCredentials(creds); loadAgents(); }}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showCredentials && (
|
||||
<CredentialsModal
|
||||
credentials={showCredentials}
|
||||
onClose={() => setShowCredentials(null)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{selectedAgent && (
|
||||
<AgentDrawer agent={selectedAgent} onClose={() => setSelectedAgent(null)} onRevoked={loadAgents} />
|
||||
)}
|
||||
|
||||
{showApiKeyCreate && (
|
||||
<ApiKeyCreateModal
|
||||
onClose={() => setShowApiKeyCreate(false)}
|
||||
onCreated={(result) => { setShowApiKeyCreate(false); setShowApiKeyToken(result); loadAgents(); }}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showApiKeyToken && (
|
||||
<ApiKeyTokenModal token={showApiKeyToken} onClose={() => setShowApiKeyToken(null)} />
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function ApiKeyCreateModal({ onClose, onCreated }: {
|
||||
onClose: () => void;
|
||||
onCreated: (result: { name: string; token: string }) => void;
|
||||
}) {
|
||||
const [name, setName] = useState('');
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState('');
|
||||
|
||||
const handleSubmit = async (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
if (!name.trim()) { setError('Name required'); return; }
|
||||
setLoading(true);
|
||||
try {
|
||||
const data = await api.createApiKey(name.trim());
|
||||
onCreated({ name: data.name, token: data.token });
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Failed');
|
||||
} finally { setLoading(false); }
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="modal-overlay" onClick={onClose}>
|
||||
<form className="modal" onClick={e => e.stopPropagation()} onSubmit={handleSubmit}>
|
||||
<div className="modal-title">Create API Key</div>
|
||||
<p style={{ color: 'var(--text-secondary)', fontSize: 13, marginBottom: 16 }}>
|
||||
API keys use simple bearer token auth. They grant full read+write+admin access.
|
||||
For scoped access, use OAuth clients instead.
|
||||
</p>
|
||||
<div style={{ marginBottom: 16 }}>
|
||||
<label>Key Name</label>
|
||||
<input placeholder="e.g. claude-code-local" value={name} onChange={e => setName(e.target.value)} autoFocus />
|
||||
</div>
|
||||
{error && <div style={{ color: 'var(--error)', fontSize: 13, marginBottom: 12 }}>{error}</div>}
|
||||
<div style={{ display: 'flex', gap: 12, justifyContent: 'flex-end' }}>
|
||||
<button type="button" className="btn btn-secondary" onClick={onClose}>Cancel</button>
|
||||
<button type="submit" className="btn btn-primary" disabled={loading}>
|
||||
{loading ? 'Creating...' : 'Create Key'}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ApiKeyTokenModal({ token, onClose }: {
|
||||
token: { name: string; token: string };
|
||||
onClose: () => void;
|
||||
}) {
|
||||
const copy = (text: string) => navigator.clipboard.writeText(text);
|
||||
|
||||
return (
|
||||
<div className="modal-overlay">
|
||||
<div className="modal" style={{ maxWidth: 560 }}>
|
||||
<div style={{ textAlign: 'center', marginBottom: 16 }}>
|
||||
<div style={{ fontSize: 36, color: 'var(--success)', marginBottom: 8 }}>✓</div>
|
||||
<div style={{ fontSize: 20, fontWeight: 600 }}>API Key Created</div>
|
||||
</div>
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<label style={{ fontSize: 12 }}>Name</label>
|
||||
<div className="code-block"><span>{token.name}</span></div>
|
||||
</div>
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<label style={{ fontSize: 12 }}>Bearer Token</label>
|
||||
<div className="code-block">
|
||||
<span>{token.token}</span>
|
||||
<button className="copy-btn" onClick={() => copy(token.token)}>Copy</button>
|
||||
</div>
|
||||
</div>
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<label style={{ fontSize: 12 }}>Usage</label>
|
||||
<div className="code-block">
|
||||
<pre style={{ whiteSpace: 'pre-wrap', margin: 0, fontSize: 12 }}>{`Authorization: Bearer ${token.token}`}</pre>
|
||||
<button className="copy-btn" onClick={() => copy(`Authorization: Bearer ${token.token}`)}>Copy</button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="warning-bar">Save this token now. It will not be shown again.</div>
|
||||
<div style={{ display: 'flex', gap: 12, justifyContent: 'flex-end', marginTop: 20 }}>
|
||||
<button className="btn btn-primary" onClick={onClose}>Done</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function RegisterModal({ onClose, onRegistered }: {
|
||||
onClose: () => void;
|
||||
onRegistered: (creds: { clientId: string; clientSecret: string; name: string }) => void;
|
||||
}) {
|
||||
const [name, setName] = useState('');
|
||||
// v0.28: scope set sourced from admin/src/lib/scope-constants.ts (mirror
|
||||
// of src/core/scope.ts). CI drift check at scripts/check-admin-scope-drift.sh
|
||||
// fails the build if these diverge.
|
||||
const [scopes, setScopes] = useState<Record<Scope, boolean>>(() =>
|
||||
Object.fromEntries(ALLOWED_SCOPES_LIST.map(s => [s, s === 'read'])) as Record<Scope, boolean>,
|
||||
);
|
||||
const [ttl, setTtl] = useState('86400'); // 24h default
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState('');
|
||||
|
||||
const ttlOptions = [
|
||||
{ label: '1 hour', value: '3600' },
|
||||
{ label: '24 hours', value: '86400' },
|
||||
{ label: '7 days', value: '604800' },
|
||||
{ label: '30 days', value: '2592000' },
|
||||
{ label: '1 year', value: '31536000' },
|
||||
{ label: 'No expiry', value: '0' },
|
||||
];
|
||||
|
||||
const handleSubmit = async (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
if (!name.trim()) { setError('Name required'); return; }
|
||||
setLoading(true);
|
||||
setError('');
|
||||
try {
|
||||
// Use the CLI registration endpoint (POST to admin API)
|
||||
const selectedScopes = Object.entries(scopes).filter(([, v]) => v).map(([k]) => k).join(' ');
|
||||
const res = await fetch('/admin/api/register-client', {
|
||||
method: 'POST',
|
||||
credentials: 'same-origin',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ name: name.trim(), scopes: selectedScopes, tokenTtl: ttl === '0' ? 315360000 : Number(ttl) }),
|
||||
});
|
||||
if (!res.ok) throw new Error('Registration failed');
|
||||
const data = await res.json();
|
||||
onRegistered({ clientId: data.clientId, clientSecret: data.clientSecret, name: name.trim() });
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Registration failed');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="modal-overlay" onClick={onClose}>
|
||||
<form className="modal" onClick={e => e.stopPropagation()} onSubmit={handleSubmit}>
|
||||
<div className="modal-title">Register Agent</div>
|
||||
<div style={{ marginBottom: 16 }}>
|
||||
<label>Agent Name</label>
|
||||
<input placeholder="e.g. perplexity-production" value={name} onChange={e => setName(e.target.value)} autoFocus />
|
||||
</div>
|
||||
<div style={{ marginBottom: 16 }}>
|
||||
<label>Scopes</label>
|
||||
<div className="checkbox-group">
|
||||
{ALLOWED_SCOPES_LIST.map(s => (
|
||||
<label key={s} className="checkbox-label">
|
||||
<input type="checkbox" checked={scopes[s]} onChange={e => setScopes(p => ({ ...p, [s]: e.target.checked }))} />
|
||||
{s}
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
<div style={{ marginBottom: 20 }}>
|
||||
<label>Token Lifetime</label>
|
||||
<select value={ttl} onChange={e => setTtl(e.target.value)}
|
||||
style={{ width: '100%', background: 'var(--bg-secondary)', color: 'var(--text-primary)', border: '1px solid var(--border)', borderRadius: 6, padding: '6px 10px', fontSize: 14 }}>
|
||||
{ttlOptions.map(o => <option key={o.value} value={o.value}>{o.label}</option>)}
|
||||
</select>
|
||||
</div>
|
||||
{error && <div style={{ color: 'var(--error)', fontSize: 13, marginBottom: 12 }}>{error}</div>}
|
||||
<div style={{ display: 'flex', gap: 12, justifyContent: 'flex-end' }}>
|
||||
<button type="button" className="btn btn-secondary" onClick={onClose}>Cancel</button>
|
||||
<button type="submit" className="btn btn-primary" disabled={loading}>
|
||||
{loading ? 'Registering...' : 'Register'}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function CredentialsModal({ credentials, onClose }: {
|
||||
credentials: { clientId: string; clientSecret: string; name: string };
|
||||
onClose: () => void;
|
||||
}) {
|
||||
const copy = (text: string) => navigator.clipboard.writeText(text);
|
||||
const downloadJson = () => {
|
||||
const blob = new Blob([JSON.stringify(credentials, null, 2)], { type: 'application/json' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url; a.download = `${credentials.name}-credentials.json`; a.click();
|
||||
URL.revokeObjectURL(url);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="modal-overlay">
|
||||
<div className="modal" style={{ maxWidth: 560 }}>
|
||||
<div style={{ textAlign: 'center', marginBottom: 16 }}>
|
||||
<div style={{ fontSize: 36, color: 'var(--success)', marginBottom: 8 }}>✓</div>
|
||||
<div style={{ fontSize: 20, fontWeight: 600 }}>Agent Registered</div>
|
||||
</div>
|
||||
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<label style={{ fontSize: 12 }}>Client ID</label>
|
||||
<div className="code-block">
|
||||
<span>{credentials.clientId}</span>
|
||||
<button className="copy-btn" onClick={() => copy(credentials.clientId)}>Copy</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<label style={{ fontSize: 12 }}>Client Secret</label>
|
||||
<div className="code-block">
|
||||
<span>{credentials.clientSecret}</span>
|
||||
<button className="copy-btn" onClick={() => copy(credentials.clientSecret)}>Copy</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="warning-bar">
|
||||
Save this secret now. It will not be shown again.
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'flex', gap: 12, justifyContent: 'flex-end', marginTop: 20 }}>
|
||||
<button className="btn btn-secondary" onClick={downloadJson}>Download as JSON</button>
|
||||
<button className="btn btn-primary" onClick={onClose}>Done</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: () => void; onRevoked: () => void }) {
|
||||
const [tab, setTab] = useState<'claude-code' | 'chatgpt' | 'claude-cowork' | 'perplexity' | 'cursor' | 'json'>('claude-code');
|
||||
const copy = (text: string) => navigator.clipboard.writeText(text);
|
||||
const serverUrl = window.location.origin;
|
||||
|
||||
const cid = agent.id || agent.client_id || '';
|
||||
const isOAuth = agent.auth_type === 'oauth';
|
||||
const agentName = agent.name || agent.client_name || 'unknown';
|
||||
|
||||
// For API keys, we can't show the actual token (it was shown once at creation).
|
||||
// For OAuth, we show the client_id and tell them to use their secret.
|
||||
|
||||
const configSnippets: Record<string, string> = {
|
||||
'claude-code': isOAuth
|
||||
? [
|
||||
`The user wants to connect GBrain MCP to your context. Here's how.`,
|
||||
``,
|
||||
`# Step 1 — read the client secret from the operator without putting it in shell history.`,
|
||||
`# read -s suppresses the echo; export keeps it scoped to this shell session.`,
|
||||
`read -rs -p "Client Secret: " GBRAIN_CS && echo`,
|
||||
``,
|
||||
`# Step 2 — mint a token and register GBrain in one go.`,
|
||||
`claude mcp add --transport http gbrain ${serverUrl}/mcp \\`,
|
||||
` --header "Authorization: Bearer $(curl -s -X POST ${serverUrl}/token \\`,
|
||||
` -d 'grant_type=client_credentials' \\`,
|
||||
` -d 'client_id=${cid}' \\`,
|
||||
` --data-urlencode "client_secret=$GBRAIN_CS" \\`,
|
||||
` -d 'scope=${agent.scope || 'read write'}' | jq -r .access_token)"`,
|
||||
``,
|
||||
`# Step 3 — clear the secret from this shell.`,
|
||||
`unset GBRAIN_CS`,
|
||||
``,
|
||||
`The token will last ${agent.token_ttl ? (agent.token_ttl >= 86400 ? Math.floor(agent.token_ttl / 86400) + ' days' : Math.floor(agent.token_ttl / 3600) + ' hours') : '1 hour (default)'}.`,
|
||||
``,
|
||||
`─── Fallback: 2-step curl + paste ───`,
|
||||
``,
|
||||
`If your shell doesn't support read -s, mint the token first, then paste:`,
|
||||
``,
|
||||
`curl -s -X POST ${serverUrl}/token \\`,
|
||||
` -d 'grant_type=client_credentials' \\`,
|
||||
` -d 'client_id=${cid}' \\`,
|
||||
` -d 'client_secret=PASTE_YOUR_CLIENT_SECRET_HERE' \\`,
|
||||
` -d 'scope=${agent.scope || 'read write'}' | jq -r .access_token`,
|
||||
``,
|
||||
`claude mcp add --transport http gbrain ${serverUrl}/mcp \\`,
|
||||
` --header "Authorization: Bearer PASTE_TOKEN_FROM_ABOVE"`,
|
||||
``,
|
||||
`# Then run: history -d $((HISTCMD-1)) # zsh: print -s -- '' && fc -p`,
|
||||
].join('\n')
|
||||
: [
|
||||
`The user wants to connect GBrain MCP to your context. Here's how.`,
|
||||
``,
|
||||
`Run this command in terminal to register GBrain:`,
|
||||
``,
|
||||
`claude mcp add --transport http gbrain ${serverUrl}/mcp \\`,
|
||||
` --header "Authorization: Bearer PASTE_YOUR_API_KEY_HERE"`,
|
||||
``,
|
||||
`Replace PASTE_YOUR_API_KEY_HERE with the API key shown when "${agentName}" was created.`,
|
||||
`API keys never expire.`,
|
||||
].join('\n'),
|
||||
|
||||
'chatgpt': [
|
||||
`The user wants to connect GBrain MCP to ChatGPT.`,
|
||||
``,
|
||||
`1. Open ChatGPT > Settings > Tools & Integrations > Add MCP Server`,
|
||||
`2. Enter this URL — ChatGPT will auto-discover the OAuth configuration:`,
|
||||
``,
|
||||
` ${serverUrl}/.well-known/oauth-authorization-server`,
|
||||
``,
|
||||
`3. When prompted for credentials:`,
|
||||
` Client ID: ${cid}`,
|
||||
` Client Secret: (the secret from agent registration)`,
|
||||
` Grant Type: client_credentials`,
|
||||
` Scope: ${agent.scope || 'read write'}`,
|
||||
].join('\n'),
|
||||
|
||||
'claude-cowork': [
|
||||
`The user wants to connect GBrain MCP to Claude.ai.`,
|
||||
``,
|
||||
`1. Open claude.ai > Settings > Connected Apps > Add MCP Server`,
|
||||
`2. Server URL: ${serverUrl}/mcp`,
|
||||
`3. When prompted for auth:`,
|
||||
` Token endpoint: ${serverUrl}/token`,
|
||||
` Client ID: ${cid}`,
|
||||
` Client Secret: (the secret from agent registration)`,
|
||||
` Scope: ${agent.scope || 'read write'}`,
|
||||
``,
|
||||
`Discovery URL: ${serverUrl}/.well-known/oauth-authorization-server`,
|
||||
].join('\n'),
|
||||
|
||||
cursor: isOAuth
|
||||
? [
|
||||
`The user wants to connect GBrain MCP to Cursor.`,
|
||||
``,
|
||||
`Cursor supports OAuth for remote MCP. Add to .cursor/mcp.json:`,
|
||||
``,
|
||||
`{`,
|
||||
` "mcpServers": {`,
|
||||
` "gbrain": {`,
|
||||
` "url": "${serverUrl}/mcp",`,
|
||||
` "transport": "sse"`,
|
||||
` }`,
|
||||
` }`,
|
||||
`}`,
|
||||
``,
|
||||
`Cursor will auto-discover OAuth via:`,
|
||||
`${serverUrl}/.well-known/oauth-authorization-server`,
|
||||
``,
|
||||
`When prompted: Client ID ${cid}, use the secret from registration.`,
|
||||
].join('\n')
|
||||
: [
|
||||
`The user wants to connect GBrain MCP to Cursor.`,
|
||||
``,
|
||||
`Add to .cursor/mcp.json:`,
|
||||
``,
|
||||
`{`,
|
||||
` "mcpServers": {`,
|
||||
` "gbrain": {`,
|
||||
` "url": "${serverUrl}/mcp",`,
|
||||
` "transport": "sse",`,
|
||||
` "headers": {`,
|
||||
` "Authorization": "Bearer PASTE_YOUR_API_KEY_HERE"`,
|
||||
` }`,
|
||||
` }`,
|
||||
` }`,
|
||||
`}`,
|
||||
``,
|
||||
`Replace PASTE_YOUR_API_KEY_HERE with the API key shown when "${agentName}" was created.`,
|
||||
].join('\n'),
|
||||
|
||||
perplexity: [
|
||||
`The user wants to connect GBrain MCP to Perplexity.`,
|
||||
``,
|
||||
`1. Go to Settings > Connectors > Add MCP`,
|
||||
`2. Server URL: ${serverUrl}/mcp`,
|
||||
`3. Client ID: ${cid}`,
|
||||
`4. Client Secret: (the secret from agent registration)`,
|
||||
].join('\n'),
|
||||
|
||||
json: JSON.stringify({
|
||||
server_url: serverUrl + '/mcp',
|
||||
token_url: serverUrl + '/token',
|
||||
discovery_url: serverUrl + '/.well-known/oauth-authorization-server',
|
||||
client_id: cid,
|
||||
client_name: agentName,
|
||||
auth_type: agent.auth_type,
|
||||
scope: agent.scope,
|
||||
}, null, 2),
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="drawer-overlay" onClick={onClose} />
|
||||
<div className="drawer">
|
||||
<button className="drawer-close" onClick={onClose}>✕</button>
|
||||
<div style={{ fontSize: 18, fontWeight: 600, marginBottom: 4 }}>{agent.name || agent.client_name}</div>
|
||||
<span className={`badge ${agent.status === 'active' ? 'badge-success' : 'badge-danger'}`}>{agent.status}</span>
|
||||
|
||||
<div className="section-title">Details</div>
|
||||
<div style={{ display: 'grid', gridTemplateColumns: '100px 1fr', gap: '6px 12px', fontSize: 13 }}>
|
||||
<span style={{ color: 'var(--text-secondary)' }}>Client ID</span>
|
||||
<span className="mono">{(agent.id || agent.id || agent.client_id || '').substring(0, 24)}...</span>
|
||||
<span style={{ color: 'var(--text-secondary)' }}>Scopes</span>
|
||||
<span>{(agent.scope || '').split(' ').filter(Boolean).map(s => (
|
||||
<span key={s} className={`badge badge-${s}`} style={{ marginRight: 4 }}>{s}</span>
|
||||
))}</span>
|
||||
<span style={{ color: 'var(--text-secondary)' }}>Registered</span>
|
||||
<span>{new Date(agent.created_at).toLocaleDateString()}</span>
|
||||
<span style={{ color: 'var(--text-secondary)' }}>Token TTL</span>
|
||||
<span>{agent.token_ttl ? (agent.token_ttl >= 31536000 ? 'No expiry' : agent.token_ttl >= 86400 ? `${Math.floor(agent.token_ttl / 86400)}d` : agent.token_ttl >= 3600 ? `${Math.floor(agent.token_ttl / 3600)}h` : `${agent.token_ttl}s`) : '1h (default)'}</span>
|
||||
</div>
|
||||
|
||||
{/*
|
||||
Config Export visible for both auth_type=oauth AND auth_type=api_key.
|
||||
Claude Code + Cursor + JSON tabs render real snippets regardless
|
||||
(commit 15's snippets are auth-type-aware for those two clients;
|
||||
JSON is just structured metadata). ChatGPT, Claude.ai, and
|
||||
Perplexity tabs render an "OAuth client required" message on
|
||||
api_key agents — those MCP clients only speak OAuth 2.0
|
||||
client_credentials, not raw bearer tokens.
|
||||
|
||||
Pre-fix (Wintermute commit 16): the entire Config Export
|
||||
section was hidden for api_key agents, dropping the working
|
||||
Claude Code + Cursor snippets along with the broken ones.
|
||||
(D5=C in the eng review.)
|
||||
*/}
|
||||
<div className="section-title">Config Export</div>
|
||||
<div className="tabs" style={{ flexWrap: 'wrap' }}>
|
||||
<div className={`tab ${tab === 'claude-code' ? 'active' : ''}`} onClick={() => setTab('claude-code')}>Claude Code</div>
|
||||
<div className={`tab ${tab === 'chatgpt' ? 'active' : ''}`} onClick={() => setTab('chatgpt')}>ChatGPT</div>
|
||||
<div className={`tab ${tab === 'claude-cowork' ? 'active' : ''}`} onClick={() => setTab('claude-cowork')}>Claude.ai</div>
|
||||
<div className={`tab ${tab === 'cursor' ? 'active' : ''}`} onClick={() => setTab('cursor')}>Cursor</div>
|
||||
<div className={`tab ${tab === 'perplexity' ? 'active' : ''}`} onClick={() => setTab('perplexity')}>Perplexity</div>
|
||||
<div className={`tab ${tab === 'json' ? 'active' : ''}`} onClick={() => setTab('json')}>JSON</div>
|
||||
</div>
|
||||
{(() => {
|
||||
const oauthOnlyTabs = new Set(['chatgpt', 'claude-cowork', 'perplexity']);
|
||||
if (!isOAuth && oauthOnlyTabs.has(tab)) {
|
||||
const clientName = { chatgpt: 'ChatGPT', 'claude-cowork': 'Claude.ai', perplexity: 'Perplexity' }[tab] || tab;
|
||||
return (
|
||||
<div style={{
|
||||
background: 'rgba(255, 200, 100, 0.08)',
|
||||
border: '1px solid rgba(255, 200, 100, 0.2)',
|
||||
borderRadius: 8,
|
||||
padding: '14px 16px',
|
||||
marginTop: 12,
|
||||
fontSize: 13,
|
||||
lineHeight: 1.6,
|
||||
color: 'var(--text-secondary)',
|
||||
}}>
|
||||
<div style={{ fontWeight: 600, color: 'var(--text-primary)', marginBottom: 6 }}>
|
||||
{clientName} requires an OAuth client
|
||||
</div>
|
||||
{clientName} only supports OAuth 2.0 (client_credentials). API keys use raw bearer tokens, which {clientName} does not accept. Register a separate OAuth client and use that to connect this AI.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<div className="code-block">
|
||||
<pre style={{ whiteSpace: 'pre-wrap', margin: 0 }}>{configSnippets[tab]}</pre>
|
||||
<button className="copy-btn" onClick={() => copy(configSnippets[tab])}>Copy</button>
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
|
||||
<div style={{ marginTop: 32 }}>
|
||||
{agent.status === 'active' && (
|
||||
<button className="btn btn-danger" onClick={async () => {
|
||||
if (!confirm(`Revoke ${agent.name || agent.client_name}? All active tokens will be invalidated.`)) return;
|
||||
try {
|
||||
if (agent.auth_type === 'oauth') {
|
||||
await api.revokeClient(agent.id || agent.client_id || '');
|
||||
} else {
|
||||
await api.revokeApiKey(agent.name || '');
|
||||
}
|
||||
onRevoked();
|
||||
onClose();
|
||||
} catch (e) {
|
||||
alert('Revoke failed: ' + (e instanceof Error ? e.message : 'unknown error'));
|
||||
}
|
||||
}}>Revoke Agent</button>
|
||||
)}
|
||||
{agent.status === 'revoked' && (
|
||||
<span style={{ color: 'var(--text-muted)', fontSize: 13 }}>This agent has been revoked.</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
/**
|
||||
* v0.36.1.0 (T15 / E6) — Calibration tab.
|
||||
*
|
||||
* Fetches the active calibration profile + 4 server-rendered SVG charts.
|
||||
* Layout: Linear calm clarity (per D23 mockup variant-B) — single column,
|
||||
* generous whitespace, ONE big sparkline as hero, then patterns, then
|
||||
* domain bars, then abandoned threads.
|
||||
*
|
||||
* Per D23 — SVG markup comes from the server (image/svg+xml endpoint).
|
||||
* Admin SPA renders inside a TrustedSVG wrapper that uses
|
||||
* dangerouslySetInnerHTML. XSS posture: server-side escapeXml() on all
|
||||
* caller-controlled strings + requireAdmin middleware on the endpoint.
|
||||
*/
|
||||
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import { api } from '../api';
|
||||
|
||||
interface CalibrationProfileSummary {
|
||||
holder: string;
|
||||
source_id: string;
|
||||
generated_at: string;
|
||||
published: boolean;
|
||||
total_resolved: number;
|
||||
brier: number | null;
|
||||
accuracy: number | null;
|
||||
partial_rate: number | null;
|
||||
grade_completion: number;
|
||||
pattern_statements: string[];
|
||||
active_bias_tags: string[];
|
||||
voice_gate_passed: boolean;
|
||||
voice_gate_attempts: number;
|
||||
}
|
||||
|
||||
interface ChartSvgProps {
|
||||
type: string;
|
||||
ariaLabel: string;
|
||||
}
|
||||
|
||||
function TrustedSVG({ markup }: { markup: string }) {
|
||||
return (
|
||||
<div
|
||||
style={{ width: '100%', overflow: 'auto' }}
|
||||
// Server-rendered SVG (image/svg+xml) gated by requireAdmin middleware.
|
||||
// All caller-controlled strings pass through escapeXml() server-side.
|
||||
dangerouslySetInnerHTML={{ __html: markup }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
function ChartSvg({ type, ariaLabel }: ChartSvgProps) {
|
||||
const [markup, setMarkup] = useState<string>('');
|
||||
const [error, setError] = useState<string>('');
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
api
|
||||
.calibrationChart(type)
|
||||
.then(svg => {
|
||||
if (!cancelled) setMarkup(svg);
|
||||
})
|
||||
.catch(err => {
|
||||
if (!cancelled) setError(err.message ?? 'fetch failed');
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [type]);
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div style={{ padding: 16, color: 'var(--error)' }} role="alert">
|
||||
{ariaLabel}: {error}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (!markup) {
|
||||
return <div style={{ padding: 16, color: 'var(--text-muted)' }}>{ariaLabel} loading...</div>;
|
||||
}
|
||||
return <TrustedSVG markup={markup} />;
|
||||
}
|
||||
|
||||
export function CalibrationPage() {
|
||||
const [profile, setProfile] = useState<CalibrationProfileSummary | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string>('');
|
||||
|
||||
useEffect(() => {
|
||||
api
|
||||
.calibrationProfile()
|
||||
.then(p => {
|
||||
setProfile(p);
|
||||
setLoading(false);
|
||||
})
|
||||
.catch(err => {
|
||||
setError(err.message ?? 'fetch failed');
|
||||
setLoading(false);
|
||||
});
|
||||
}, []);
|
||||
|
||||
if (loading) {
|
||||
return <div style={{ padding: 24, color: 'var(--text-secondary)' }}>Loading calibration profile…</div>;
|
||||
}
|
||||
if (error) {
|
||||
return (
|
||||
<div style={{ padding: 24, color: 'var(--error)' }} role="alert">
|
||||
Could not load calibration profile: {error}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (!profile) {
|
||||
return (
|
||||
<div style={{ padding: 24, maxWidth: 700 }}>
|
||||
<h1 style={{ marginBottom: 16 }}>Calibration</h1>
|
||||
<p style={{ color: 'var(--text-secondary)' }}>
|
||||
No calibration profile yet. Builds after 5+ resolved takes.
|
||||
</p>
|
||||
<pre
|
||||
style={{
|
||||
background: 'var(--bg-secondary)',
|
||||
padding: 12,
|
||||
borderRadius: 4,
|
||||
color: 'var(--text-primary)',
|
||||
marginTop: 12,
|
||||
fontFamily: 'var(--font-mono)',
|
||||
}}
|
||||
>
|
||||
gbrain dream --phase calibration_profile
|
||||
</pre>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const generated = new Date(profile.generated_at);
|
||||
const generatedAgo = Math.floor((Date.now() - generated.getTime()) / (1000 * 60 * 60 * 24));
|
||||
|
||||
return (
|
||||
<div style={{ padding: 32, maxWidth: 720 }}>
|
||||
<h1 style={{ marginBottom: 8 }}>Calibration</h1>
|
||||
<div style={{ color: 'var(--text-muted)', fontSize: 13, marginBottom: 24 }}>
|
||||
Holder: {profile.holder}
|
||||
{' · '}
|
||||
Updated {generatedAgo === 0 ? 'today' : `${generatedAgo}d ago`}
|
||||
{profile.published && ' · published'}
|
||||
{profile.grade_completion < 0.9 && ` · ~${Math.round(profile.grade_completion * 100)}% graded`}
|
||||
{!profile.voice_gate_passed && ' · voice gate fell back to template'}
|
||||
</div>
|
||||
|
||||
<section style={{ marginBottom: 32 }}>
|
||||
<ChartSvg type="brier-trend" ariaLabel="Brier trend" />
|
||||
</section>
|
||||
|
||||
<section style={{ marginBottom: 32 }}>
|
||||
<h2 style={{ fontSize: 14, color: 'var(--text-secondary)', marginBottom: 12, fontWeight: 400 }}>
|
||||
Pattern statements
|
||||
</h2>
|
||||
<ChartSvg type="pattern-statements" ariaLabel="Pattern statements" />
|
||||
</section>
|
||||
|
||||
<section style={{ marginBottom: 32 }}>
|
||||
<ChartSvg type="domain-bars" ariaLabel="Per-domain accuracy" />
|
||||
</section>
|
||||
|
||||
<section style={{ marginBottom: 32 }}>
|
||||
<ChartSvg type="abandoned-threads" ariaLabel="Abandoned threads" />
|
||||
</section>
|
||||
|
||||
{profile.active_bias_tags.length > 0 && (
|
||||
<section style={{ marginBottom: 32, color: 'var(--text-muted)', fontSize: 13 }}>
|
||||
Active bias tags: {profile.active_bias_tags.join(', ')}
|
||||
</section>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
import React, { useState, useEffect, useRef } from 'react';
|
||||
import { api } from '../api';
|
||||
|
||||
interface FeedEvent {
|
||||
agent: string;
|
||||
operation: string;
|
||||
scopes: string;
|
||||
latency_ms: number;
|
||||
status: string;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export function DashboardPage() {
|
||||
const [stats, setStats] = useState({ connected_agents: 0, requests_today: 0, active_tokens: 0 });
|
||||
const [health, setHealth] = useState({ expiring_soon: 0, error_rate: '0%' });
|
||||
const [events, setEvents] = useState<FeedEvent[]>([]);
|
||||
const [sseStatus, setSseStatus] = useState<'connecting' | 'connected' | 'disconnected'>('connecting');
|
||||
const eventSourceRef = useRef<EventSource | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
api.stats().then(setStats).catch(() => {});
|
||||
api.health().then(setHealth).catch(() => {});
|
||||
|
||||
const es = new EventSource('/admin/events');
|
||||
eventSourceRef.current = es;
|
||||
es.onopen = () => setSseStatus('connected');
|
||||
es.onmessage = (e) => {
|
||||
try {
|
||||
const event = JSON.parse(e.data) as FeedEvent;
|
||||
setEvents(prev => [event, ...prev].slice(0, 50));
|
||||
} catch {}
|
||||
};
|
||||
es.onerror = () => {
|
||||
setSseStatus('disconnected');
|
||||
setTimeout(() => {
|
||||
setSseStatus('connecting');
|
||||
es.close();
|
||||
// Reconnect handled by browser EventSource auto-retry
|
||||
}, 3000);
|
||||
};
|
||||
|
||||
const interval = setInterval(() => {
|
||||
api.stats().then(setStats).catch(() => {});
|
||||
api.health().then(setHealth).catch(() => {});
|
||||
}, 30000);
|
||||
|
||||
return () => { es.close(); clearInterval(interval); };
|
||||
}, []);
|
||||
|
||||
const timeAgo = (ts: string) => {
|
||||
const diff = Date.now() - new Date(ts).getTime();
|
||||
if (diff < 60000) return `${Math.floor(diff / 1000)}s ago`;
|
||||
if (diff < 3600000) return `${Math.floor(diff / 60000)} min ago`;
|
||||
return `${Math.floor(diff / 3600000)}h ago`;
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<h1 className="page-title">Dashboard</h1>
|
||||
|
||||
<div style={{ display: 'flex', gap: 24 }}>
|
||||
<div style={{ flex: 1 }}>
|
||||
<div className="metrics">
|
||||
<div className="metric">
|
||||
<div className="metric-value">{stats.connected_agents}</div>
|
||||
<div className="metric-label">Connected Agents</div>
|
||||
</div>
|
||||
<div className="metric">
|
||||
<div className="metric-value">{stats.requests_today}</div>
|
||||
<div className="metric-label">Requests Today</div>
|
||||
</div>
|
||||
<div className="metric">
|
||||
<div className="metric-value">{stats.active_tokens}</div>
|
||||
<div className="metric-label">Active Tokens</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h2 className="section-title">
|
||||
Live Activity
|
||||
<span style={{ marginLeft: 8, fontSize: 10, color: sseStatus === 'connected' ? 'var(--success)' : sseStatus === 'connecting' ? 'var(--warning)' : 'var(--error)' }}>
|
||||
{sseStatus === 'connected' ? '● connected' : sseStatus === 'connecting' ? '● connecting...' : '● disconnected'}
|
||||
</span>
|
||||
</h2>
|
||||
|
||||
<div className="feed">
|
||||
{events.length === 0 ? (
|
||||
<div className="feed-empty">
|
||||
{sseStatus === 'connected' ? 'No requests yet. Agents will appear when they connect.' : 'Connecting...'}
|
||||
</div>
|
||||
) : (
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Agent</th>
|
||||
<th>Operation</th>
|
||||
<th>Scopes</th>
|
||||
<th>Latency</th>
|
||||
<th>Status</th>
|
||||
<th>Time</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{events.map((e, i) => (
|
||||
<tr key={i}>
|
||||
<td className="mono">{e.agent}</td>
|
||||
<td className="mono">{e.operation}</td>
|
||||
<td>{e.scopes.split(',').map(s => (
|
||||
<span key={s} className={`badge badge-${s.trim()}`} style={{ marginRight: 4 }}>{s.trim()}</span>
|
||||
))}</td>
|
||||
<td className="mono">{e.latency_ms} ms</td>
|
||||
<td><span className={`badge badge-${e.status}`}>{e.status}</span></td>
|
||||
<td style={{ color: 'var(--text-secondary)' }}>{timeAgo(e.timestamp)}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={{ width: 220 }}>
|
||||
<h2 className="section-title">Token Health</h2>
|
||||
<div className="health-panel">
|
||||
<div className="health-row">
|
||||
<span style={{ color: 'var(--warning)' }}>Expiring Soon</span>
|
||||
<span className="mono">{health.expiring_soon}</span>
|
||||
</div>
|
||||
<div className="health-row">
|
||||
<span style={{ color: 'var(--error)' }}>Error Rate</span>
|
||||
<span className="mono">{health.error_rate}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
import React, { useState } from 'react';
|
||||
import { api } from '../api';
|
||||
|
||||
// v0.26.3 trust model (D11 + D12):
|
||||
// - The bootstrap token is NEVER stored in browser JS state. No
|
||||
// localStorage, no sessionStorage, no React state beyond the form
|
||||
// submit cycle. After successful POST /admin/login the operator's
|
||||
// token only lives in the HttpOnly cookie that the server set.
|
||||
// - Magic-link URLs use single-use server-issued nonces, not the
|
||||
// bootstrap token itself (see /admin/api/issue-magic-link). The
|
||||
// bootstrap token never appears in a URL.
|
||||
// - Closing the tab ends the session client-side. Reopening the
|
||||
// dashboard 401s and shows this page again. Operator asks the agent
|
||||
// for a fresh magic link or pastes the bootstrap token from the
|
||||
// server's terminal scrollback.
|
||||
export function LoginPage({ onLogin }: { onLogin: () => void }) {
|
||||
const [token, setToken] = useState('');
|
||||
const [error, setError] = useState('');
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
const handleSubmit = async (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
setError('');
|
||||
setLoading(true);
|
||||
try {
|
||||
await api.login(token);
|
||||
// Don't persist the token. The HttpOnly cookie is the only
|
||||
// session credential after this point.
|
||||
setToken('');
|
||||
onLogin();
|
||||
} catch (err) {
|
||||
setError('Invalid token.');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="login-page">
|
||||
<div className="login-box">
|
||||
<div className="login-logo">GBrain</div>
|
||||
|
||||
<div style={{
|
||||
background: 'rgba(136, 170, 255, 0.08)',
|
||||
border: '1px solid rgba(136, 170, 255, 0.2)',
|
||||
borderRadius: 8,
|
||||
padding: '14px 16px',
|
||||
marginBottom: 20,
|
||||
fontSize: 13,
|
||||
lineHeight: 1.5,
|
||||
color: 'var(--text-secondary)',
|
||||
}}>
|
||||
<div style={{ fontWeight: 600, color: 'var(--text-primary)', marginBottom: 6 }}>
|
||||
🔒 This is a protected dashboard
|
||||
</div>
|
||||
Ask your AI agent for the admin login link:
|
||||
<div style={{
|
||||
background: 'rgba(0,0,0,0.3)',
|
||||
borderRadius: 6,
|
||||
padding: '8px 12px',
|
||||
marginTop: 8,
|
||||
fontFamily: 'var(--font-mono)',
|
||||
fontSize: 12,
|
||||
color: '#88aaff',
|
||||
wordBreak: 'break-all',
|
||||
}}>
|
||||
"Give me the GBrain admin login link"
|
||||
</div>
|
||||
<div style={{ marginTop: 8, fontSize: 12, color: 'var(--text-muted)' }}>
|
||||
Each link is single-use. Your agent generates a fresh one each time.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<details style={{ marginBottom: 16 }}>
|
||||
<summary style={{ cursor: 'pointer', fontSize: 13, color: 'var(--text-muted)' }}>
|
||||
Or paste bootstrap token manually
|
||||
</summary>
|
||||
<form onSubmit={handleSubmit} style={{ marginTop: 12 }}>
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<input
|
||||
type="password"
|
||||
placeholder="Admin Token"
|
||||
value={token}
|
||||
onChange={e => setToken(e.target.value)}
|
||||
/>
|
||||
</div>
|
||||
<button className="btn btn-primary" style={{ width: '100%' }} disabled={loading}>
|
||||
{loading ? 'Authenticating...' : 'Submit'}
|
||||
</button>
|
||||
{error && <div className="login-error">{error}</div>}
|
||||
</form>
|
||||
</details>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { api } from '../api';
|
||||
|
||||
interface LogEntry {
|
||||
id: number;
|
||||
token_name: string;
|
||||
agent_name: string;
|
||||
operation: string;
|
||||
latency_ms: number;
|
||||
status: string;
|
||||
params: Record<string, unknown> | null;
|
||||
error_message: string | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export function RequestLogPage() {
|
||||
const [data, setData] = useState<{ rows: LogEntry[]; total: number; page: number; pages: number }>({
|
||||
rows: [], total: 0, page: 1, pages: 1,
|
||||
});
|
||||
const [page, setPage] = useState(1);
|
||||
const [agentFilter, setAgentFilter] = useState('all');
|
||||
const [expandedRow, setExpandedRow] = useState<number | null>(null);
|
||||
|
||||
useEffect(() => { loadPage(page); }, [page, agentFilter]);
|
||||
|
||||
const loadPage = (p: number) => {
|
||||
const qs = agentFilter !== 'all' ? `&agent=${encodeURIComponent(agentFilter)}` : '';
|
||||
api.requests(p, qs).then(setData).catch(() => {});
|
||||
};
|
||||
|
||||
const timeAgo = (ts: string) => {
|
||||
const diff = Date.now() - new Date(ts).getTime();
|
||||
if (diff < 60000) return `${Math.floor(diff / 1000)}s ago`;
|
||||
if (diff < 3600000) return `${Math.floor(diff / 60000)} min ago`;
|
||||
if (diff < 86400000) return `${Math.floor(diff / 3600000)}h ago`;
|
||||
return new Date(ts).toLocaleDateString();
|
||||
};
|
||||
|
||||
|
||||
|
||||
const formatParams = (params: Record<string, unknown> | null) => {
|
||||
if (!params) return null;
|
||||
const { query, slug, partial, limit, ...rest } = params as any;
|
||||
const parts: string[] = [];
|
||||
if (query) parts.push(`"${query}"`);
|
||||
if (slug) parts.push(slug);
|
||||
if (partial) parts.push(`~${partial}`);
|
||||
if (limit) parts.push(`limit=${limit}`);
|
||||
if (Object.keys(rest).length > 0) parts.push(`+${Object.keys(rest).length} params`);
|
||||
return parts.join(' ');
|
||||
};
|
||||
|
||||
// Collect unique agents for filter (use name for display, token_name for value)
|
||||
const agentMap = new Map<string, string>();
|
||||
data.rows.forEach(r => { if (r.token_name) agentMap.set(r.token_name, r.agent_name || r.token_name); });
|
||||
|
||||
return (
|
||||
<>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', marginBottom: 24 }}>
|
||||
<h1 className="page-title" style={{ marginBottom: 0 }}>Request Log</h1>
|
||||
<select value={agentFilter} onChange={e => { setAgentFilter(e.target.value); setPage(1); }}
|
||||
style={{ background: 'var(--bg-secondary)', color: 'var(--text-primary)', border: '1px solid var(--border)', borderRadius: 6, padding: '4px 8px', fontSize: 13 }}>
|
||||
<option value="all">All agents</option>
|
||||
{[...agentMap.entries()].map(([id, name]) => <option key={id} value={id}>{name}</option>)}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
{data.rows.length === 0 ? (
|
||||
<div style={{ textAlign: 'center', padding: 48, color: 'var(--text-muted)' }}>
|
||||
No requests yet.
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Time</th>
|
||||
<th>Agent</th>
|
||||
<th>Operation</th>
|
||||
<th>Params</th>
|
||||
<th>Latency</th>
|
||||
<th>Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{data.rows.map(r => (
|
||||
<React.Fragment key={r.id}>
|
||||
<tr onClick={() => setExpandedRow(expandedRow === r.id ? null : r.id)}
|
||||
style={{ cursor: 'pointer' }}>
|
||||
<td style={{ color: 'var(--text-secondary)', whiteSpace: 'nowrap' }}>{timeAgo(r.created_at)}</td>
|
||||
<td>
|
||||
<a style={{ color: 'var(--text-link, #88aaff)', cursor: 'pointer', textDecoration: 'none', fontWeight: 500 }}
|
||||
onClick={(e) => { e.stopPropagation(); setAgentFilter(r.token_name); setPage(1); }}>
|
||||
{r.agent_name || r.token_name}
|
||||
</a>
|
||||
</td>
|
||||
<td className="mono">{r.operation}</td>
|
||||
<td style={{ color: 'var(--text-secondary)', fontSize: 12, maxWidth: 200, overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>
|
||||
{formatParams(r.params)}
|
||||
</td>
|
||||
<td className="mono">{r.latency_ms}ms</td>
|
||||
<td><span className={`badge badge-${r.status}`}>{r.status}</span></td>
|
||||
</tr>
|
||||
{expandedRow === r.id && (
|
||||
<tr>
|
||||
<td colSpan={6} style={{ background: 'var(--bg-secondary, #0f0f1a)', padding: 16 }}>
|
||||
<div style={{ display: 'grid', gridTemplateColumns: '100px 1fr', gap: '6px 12px', fontSize: 13 }}>
|
||||
<span style={{ color: 'var(--text-muted)' }}>Time</span>
|
||||
<span>{new Date(r.created_at).toLocaleString()}</span>
|
||||
<span style={{ color: 'var(--text-muted)' }}>Agent</span>
|
||||
<span className="mono">{r.token_name}</span>
|
||||
<span style={{ color: 'var(--text-muted)' }}>Operation</span>
|
||||
<span className="mono">{r.operation}</span>
|
||||
<span style={{ color: 'var(--text-muted)' }}>Latency</span>
|
||||
<span>{r.latency_ms}ms</span>
|
||||
{r.params && (
|
||||
<>
|
||||
<span style={{ color: 'var(--text-muted)' }}>Params</span>
|
||||
<pre className="mono" style={{ margin: 0, whiteSpace: 'pre-wrap', fontSize: 12 }}>
|
||||
{JSON.stringify(r.params, null, 2)}
|
||||
</pre>
|
||||
</>
|
||||
)}
|
||||
{r.error_message && (
|
||||
<>
|
||||
<span style={{ color: 'var(--error, #ff6b6b)' }}>Error</span>
|
||||
<span style={{ color: 'var(--error, #ff6b6b)' }}>{r.error_message}</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
)}
|
||||
</React.Fragment>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<div className="pagination">
|
||||
<span>Page {data.page} of {data.pages} ({data.total} total)</span>
|
||||
<div style={{ display: 'flex', gap: 8 }}>
|
||||
<button disabled={data.page <= 1} onClick={() => setPage(p => p - 1)}>Previous</button>
|
||||
<button disabled={data.page >= data.pages} onClick={() => setPage(p => p + 1)}>Next</button>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
Vendored
-1
@@ -1 +0,0 @@
|
||||
/// <reference types="vite/client" />
|
||||
@@ -1,17 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"useDefineForClassFields": true,
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"skipLibCheck": true,
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"isolatedModules": true,
|
||||
"moduleDetection": "force",
|
||||
"noEmit": true,
|
||||
"jsx": "react-jsx",
|
||||
"strict": true
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
import { defineConfig } from 'vite';
|
||||
import react from '@vitejs/plugin-react';
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
base: '/admin/',
|
||||
build: {
|
||||
outDir: 'dist',
|
||||
emptyOutDir: true,
|
||||
},
|
||||
});
|
||||
@@ -5,39 +5,21 @@
|
||||
"": {
|
||||
"name": "gbrain",
|
||||
"dependencies": {
|
||||
"@ai-sdk/anthropic": "^3.0.71",
|
||||
"@ai-sdk/google": "^3.0.64",
|
||||
"@ai-sdk/openai": "^3.0.53",
|
||||
"@ai-sdk/openai-compatible": "^2.0.41",
|
||||
"@anthropic-ai/sdk": "^0.30.0",
|
||||
"@aws-sdk/client-s3": "^3.1028.0",
|
||||
"@dqbd/tiktoken": "^1.0.22",
|
||||
"@electric-sql/pglite": "0.4.3",
|
||||
"@jsquash/avif": "^2.1.1",
|
||||
"@jsquash/png": "^3.1.1",
|
||||
"@modelcontextprotocol/sdk": "1.29.0",
|
||||
"ai": "^6.0.168",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cors": "^2.8.5",
|
||||
"eventsource-parser": "^3.0.8",
|
||||
"exifr": "^7.1.3",
|
||||
"express": "^5.1.0",
|
||||
"express-rate-limit": "^7.5.0",
|
||||
"@modelcontextprotocol/sdk": "^1.0.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
"heic-decode": "^2.1.0",
|
||||
"marked": "^18.0.0",
|
||||
"openai": "^4.0.0",
|
||||
"pgvector": "^0.2.0",
|
||||
"postgres": "^3.4.0",
|
||||
"tree-sitter-wasms": "0.1.13",
|
||||
"web-tree-sitter": "0.22.6",
|
||||
"zod": "^4.3.6",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"@types/cookie-parser": "^1.4.7",
|
||||
"@types/cors": "^2.8.19",
|
||||
"@types/express": "^5.0.6",
|
||||
"bun-types": "^1.3.13",
|
||||
"typescript": "^5.6.0",
|
||||
},
|
||||
@@ -47,20 +29,6 @@
|
||||
"@electric-sql/pglite",
|
||||
],
|
||||
"packages": {
|
||||
"@ai-sdk/anthropic": ["@ai-sdk/anthropic@3.0.74", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Xew9rfz9WWhDSyF8rNhjT/XWOWelNfJrMlmG0Ahw210hStisRpQZ1s+7VeI9JTJOZ5y5tXqBi5kfPwYnCfyRTA=="],
|
||||
|
||||
"@ai-sdk/gateway": ["@ai-sdk/gateway@3.0.109", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-r6dOqThjODp1vOhGRJg2OCmyB/ZOQtGx1esZ2SDvwDX5XoX8dBqYaYjLg8MPXTzMGJSgOkJyCxWgUcZtAl16pw=="],
|
||||
|
||||
"@ai-sdk/google": ["@ai-sdk/google@3.0.67", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Qeq+SidYtzMrcf0fdw3L0QLmtXK+ErwdBzbxS4+0Q/2UP85Ges8RJJcbAj7SO8e2JbeJoM35BLqkeNy1o3wJvQ=="],
|
||||
|
||||
"@ai-sdk/openai": ["@ai-sdk/openai@3.0.58", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-2+5xGMROmrBboJuoOwqLL3b/o3i56+NRdxXDNVAiTyYjLiBj6KzembeuyuBT217be1X+zkEfAqD1H0irJlGIyw=="],
|
||||
|
||||
"@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.45", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-5YBvurNL7Oj7mT3srws4Rh4cQidoorfEGObAOb5jV40eld8IC7EkXWARZjnWYqgYzabUs6Sn6muiXfQVkgOyOQ=="],
|
||||
|
||||
"@ai-sdk/provider": ["@ai-sdk/provider@3.0.10", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw=="],
|
||||
|
||||
"@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.26", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-CsKNLKsOpvPujRlIYvoz+Ybw+kGn7J4/fIZa/58+R7iWLLfwn6ifE2G6Yq8K9XvH/I/3bzaDAJ3NhRwEMsLBKQ=="],
|
||||
|
||||
"@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.30.1", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" } }, "sha512-nuKvp7wOIz6BFei8WrTdhmSsx5mwnArYyJgh4+vYu3V4J0Ltb8Xm3odPm51n1aSI0XxNCrDl7O88cxCtUdAkaw=="],
|
||||
|
||||
"@aws-crypto/crc32": ["@aws-crypto/crc32@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg=="],
|
||||
@@ -149,14 +117,8 @@
|
||||
|
||||
"@hono/node-server": ["@hono/node-server@1.19.12", "", { "peerDependencies": { "hono": "^4" } }, "sha512-txsUW4SQ1iilgE0l9/e9VQWmELXifEFvmdA1j6WFh/aFPj99hIntrSsq/if0UWyGVkmrRPKA1wCeP+UCr1B9Uw=="],
|
||||
|
||||
"@jsquash/avif": ["@jsquash/avif@2.1.1", "", { "dependencies": { "wasm-feature-detect": "^1.2.11" } }, "sha512-LMRxd0fMgfCLtobDh0/sFYJMMiRJTNYSEEWvRDKXlAeZ08t3gI5V+1thIT0XjXJ+SVG7Zug9B0XPyx0Ti5VRNA=="],
|
||||
|
||||
"@jsquash/png": ["@jsquash/png@3.1.1", "", {}, "sha512-C10pc+0H6j0h8fENOfnGOvkXCmvpSQTDGlfGd0sHphZhPSGTyLjIrHba0FaZZdsKqA/wlmhYicUHb92vfZphaw=="],
|
||||
|
||||
"@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="],
|
||||
|
||||
"@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="],
|
||||
|
||||
"@smithy/chunked-blob-reader": ["@smithy/chunked-blob-reader@5.2.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-St+kVicSyayWQca+I1rGitaOEH6uKgE8IUWoYnnEX26SWdWQcL6LvMSD19Lg+vYHKdT9B2Zuu7rd3i6Wnyb/iw=="],
|
||||
|
||||
"@smithy/chunked-blob-reader-native": ["@smithy/chunked-blob-reader-native@4.2.3", "", { "dependencies": { "@smithy/util-base64": "^4.3.2", "tslib": "^2.6.2" } }, "sha512-jA5k5Udn7Y5717L86h4EIv06wIr3xn8GM1qHRi/Nf31annXcXHJjBKvgztnbn2TxH3xWrPBfgwHsOwZf0UmQWw=="],
|
||||
@@ -257,46 +219,18 @@
|
||||
|
||||
"@smithy/uuid": ["@smithy/uuid@1.1.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-O/IEdcCUKkubz60tFbGA7ceITTAJsty+lBjNoorP4Z6XRqaFb/OjQjZODophEcuq68nKm6/0r+6/lLQ+XVpk8g=="],
|
||||
|
||||
"@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="],
|
||||
|
||||
"@types/body-parser": ["@types/body-parser@1.19.6", "", { "dependencies": { "@types/connect": "*", "@types/node": "*" } }, "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g=="],
|
||||
|
||||
"@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="],
|
||||
|
||||
"@types/connect": ["@types/connect@3.4.38", "", { "dependencies": { "@types/node": "*" } }, "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug=="],
|
||||
|
||||
"@types/cookie-parser": ["@types/cookie-parser@1.4.10", "", { "peerDependencies": { "@types/express": "*" } }, "sha512-B4xqkqfZ8Wek+rCOeRxsjMS9OgvzebEzzLYw7NHYuvzb7IdxOkI0ZHGgeEBX4PUM7QGVvNSK60T3OvWj3YfBRg=="],
|
||||
|
||||
"@types/cors": ["@types/cors@2.8.19", "", { "dependencies": { "@types/node": "*" } }, "sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg=="],
|
||||
|
||||
"@types/express": ["@types/express@5.0.6", "", { "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", "@types/serve-static": "^2" } }, "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA=="],
|
||||
|
||||
"@types/express-serve-static-core": ["@types/express-serve-static-core@5.1.1", "", { "dependencies": { "@types/node": "*", "@types/qs": "*", "@types/range-parser": "*", "@types/send": "*" } }, "sha512-v4zIMr/cX7/d2BpAEX3KNKL/JrT1s43s96lLvvdTmza1oEvDudCqK9aF/djc/SWgy8Yh0h30TZx5VpzqFCxk5A=="],
|
||||
|
||||
"@types/http-errors": ["@types/http-errors@2.0.5", "", {}, "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg=="],
|
||||
|
||||
"@types/node": ["@types/node@25.5.2", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg=="],
|
||||
|
||||
"@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="],
|
||||
|
||||
"@types/qs": ["@types/qs@6.15.0", "", {}, "sha512-JawvT8iBVWpzTrz3EGw9BTQFg3BQNmwERdKE22vlTxawwtbyUSlMppvZYKLZzB5zgACXdXxbD3m1bXaMqP/9ow=="],
|
||||
|
||||
"@types/range-parser": ["@types/range-parser@1.2.7", "", {}, "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ=="],
|
||||
|
||||
"@types/send": ["@types/send@1.2.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ=="],
|
||||
|
||||
"@types/serve-static": ["@types/serve-static@2.2.0", "", { "dependencies": { "@types/http-errors": "*", "@types/node": "*" } }, "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ=="],
|
||||
|
||||
"@vercel/oidc": ["@vercel/oidc@3.2.0", "", {}, "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug=="],
|
||||
|
||||
"abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="],
|
||||
|
||||
"accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="],
|
||||
|
||||
"agentkeepalive": ["agentkeepalive@4.6.0", "", { "dependencies": { "humanize-ms": "^1.2.1" } }, "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ=="],
|
||||
|
||||
"ai": ["ai@6.0.174", "", { "dependencies": { "@ai-sdk/gateway": "3.0.109", "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26", "@opentelemetry/api": "1.9.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-bTrfLUWHWtkjzWyCY4bmyuk4Qvmj4S4NSNsXyNSVVqkmftQNtxRj7dzUoMeQDBBwlJO6fC7m2Q/lNOPqQQfAGA=="],
|
||||
|
||||
"ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="],
|
||||
|
||||
"ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="],
|
||||
@@ -325,9 +259,7 @@
|
||||
|
||||
"cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="],
|
||||
|
||||
"cookie-parser": ["cookie-parser@1.4.7", "", { "dependencies": { "cookie": "0.7.2", "cookie-signature": "1.0.6" } }, "sha512-nGUvgXnotP3BsjiLX2ypbQnWoGUPIIfHQNZkkC668ntrzGWEZVW70HDEB1qnNGMicPje6EttlIgzo51YSwNQGw=="],
|
||||
|
||||
"cookie-signature": ["cookie-signature@1.0.6", "", {}, "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="],
|
||||
"cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="],
|
||||
|
||||
"cors": ["cors@2.8.6", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw=="],
|
||||
|
||||
@@ -363,13 +295,11 @@
|
||||
|
||||
"eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="],
|
||||
|
||||
"eventsource-parser": ["eventsource-parser@3.0.8", "", {}, "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ=="],
|
||||
|
||||
"exifr": ["exifr@7.1.3", "", {}, "sha512-g/aje2noHivrRSLbAUtBPWFbxKdKhgj/xr1vATDdUXPOFYJlQ62Ft0oy+72V6XLIpDJfHs6gXLbBLAolqOXYRw=="],
|
||||
"eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
|
||||
|
||||
"express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="],
|
||||
|
||||
"express-rate-limit": ["express-rate-limit@7.5.1", "", { "peerDependencies": { "express": ">= 4.11" } }, "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw=="],
|
||||
"express-rate-limit": ["express-rate-limit@8.3.2", "", { "dependencies": { "ip-address": "10.1.0" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-77VmFeJkO0/rvimEDuUC5H30oqUC4EyOhyGccfqoLebB0oiEYfM7nwPrsDsBL1gsTpwfzX8SFy2MT3TDyRq+bg=="],
|
||||
|
||||
"extend-shallow": ["extend-shallow@2.0.1", "", { "dependencies": { "is-extendable": "^0.1.0" } }, "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug=="],
|
||||
|
||||
@@ -409,8 +339,6 @@
|
||||
|
||||
"hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
|
||||
|
||||
"heic-decode": ["heic-decode@2.1.0", "", { "dependencies": { "libheif-js": "^1.19.8" } }, "sha512-0fB3O3WMk38+PScbHLVp66jcNhsZ/ErtQ6u2lMYu/YxXgbBtl+oKOhGQHa4RpvE68k8IzbWkABzHnyAIjR758A=="],
|
||||
|
||||
"hono": ["hono@4.12.10", "", {}, "sha512-mx/p18PLy5og9ufies2GOSUqep98Td9q4i/EF6X7yJgAiIopxqdfIO3jbqsi3jRgTgw88jMDEzVKi+V2EF+27w=="],
|
||||
|
||||
"http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="],
|
||||
@@ -435,16 +363,12 @@
|
||||
|
||||
"js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="],
|
||||
|
||||
"json-schema": ["json-schema@0.4.0", "", {}, "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="],
|
||||
|
||||
"json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="],
|
||||
|
||||
"json-schema-typed": ["json-schema-typed@8.0.2", "", {}, "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA=="],
|
||||
|
||||
"kind-of": ["kind-of@6.0.3", "", {}, "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw=="],
|
||||
|
||||
"libheif-js": ["libheif-js@1.19.8", "", {}, "sha512-vQJWusIxO7wavpON1dusciL8Go9jsIQ+EUrckauFYAiSTjcmLAsuJh3SszLpvkwPci3JcL41ek2n+LUZGFpPIQ=="],
|
||||
|
||||
"marked": ["marked@18.0.0", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-2e7Qiv/HJSXj8rDEpgTvGKsP8yYtI9xXHKDnrftrmnrJPaFNM7VRb2YCzWaX4BP1iCJ/XPduzDJZMFoqTCcIMA=="],
|
||||
|
||||
"math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
|
||||
@@ -549,8 +473,6 @@
|
||||
|
||||
"vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="],
|
||||
|
||||
"wasm-feature-detect": ["wasm-feature-detect@1.8.0", "", {}, "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ=="],
|
||||
|
||||
"web-streams-polyfill": ["web-streams-polyfill@4.0.0-beta.3", "", {}, "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug=="],
|
||||
|
||||
"web-tree-sitter": ["web-tree-sitter@0.22.6", "", {}, "sha512-hS87TH71Zd6mGAmYCvlgxeGDjqd9GTeqXNqTT+u0Gs51uIozNIaaq/kUAbV/Zf56jb2ZOyG8BxZs2GG9wbLi6Q=="],
|
||||
@@ -575,16 +497,8 @@
|
||||
|
||||
"@aws-crypto/util/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
|
||||
|
||||
"@modelcontextprotocol/sdk/eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
|
||||
|
||||
"@modelcontextprotocol/sdk/express-rate-limit": ["express-rate-limit@8.3.2", "", { "dependencies": { "ip-address": "10.1.0" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-77VmFeJkO0/rvimEDuUC5H30oqUC4EyOhyGccfqoLebB0oiEYfM7nwPrsDsBL1gsTpwfzX8SFy2MT3TDyRq+bg=="],
|
||||
|
||||
"@types/bun/bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
|
||||
|
||||
"eventsource/eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
|
||||
|
||||
"express/cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="],
|
||||
|
||||
"form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="],
|
||||
|
||||
"openai/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="],
|
||||
|
||||
+9
-6
@@ -1,9 +1,12 @@
|
||||
[test]
|
||||
# PGLite WASM cold start + initSchema() runs ~5–20s on loaded machines.
|
||||
# Default 5s is too short for those tests' beforeAll hooks. 60s is the
|
||||
# empirical ceiling we observed for the slowest cold-init paths.
|
||||
# PGLite initialization can be slow under parallel test execution.
|
||||
# Default 5s is too short when many test files boot PGLite instances at once.
|
||||
# 60s is the empirical ceiling we observed before the first file's beforeAll
|
||||
# completed on a loaded machine.
|
||||
#
|
||||
# v0.26.4: scripts/run-unit-parallel.sh and scripts/run-unit-shard.sh
|
||||
# also pass `--timeout=60000` explicitly so the ceiling is consistent
|
||||
# whether tests are invoked through the wrapper or directly via bun test.
|
||||
# NOTE: this bunfig.toml `timeout` key is read by `bun test` but empirically
|
||||
# does NOT apply to beforeEach/afterEach hook timeouts under `bun run test`
|
||||
# chained behind `bun run typecheck`. The test script in package.json passes
|
||||
# `--timeout=60000` explicitly to cover both per-test and per-hook timeouts.
|
||||
# Leaving both in place as belt-and-suspenders.
|
||||
timeout = 60_000
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
# docker-compose.ci.yml
|
||||
#
|
||||
# Local CI gate with 4-way E2E sharding. Spins up 4 pgvector services + a bun
|
||||
# runner that bind-mounts the repo. Used by `bun run ci:local` and
|
||||
# `bun run ci:local:diff` (see scripts/ci-local.sh).
|
||||
#
|
||||
# All services are pulled as `image:` (no build) so `docker compose pull`
|
||||
# refreshes everything. The bun version floats with `oven/bun:1` to track CI's
|
||||
# `bun-version: latest`. Named volumes isolate the Linux container's deps from
|
||||
# the host's darwin-arm64 deps and keep bun + postgres data warm across runs.
|
||||
#
|
||||
# Why 4 postgres services: bun's E2E suite shares one DB across 36 files and
|
||||
# uses TRUNCATE CASCADE in setupDB(). Running files in parallel against ONE DB
|
||||
# races (file A's TRUNCATE clobbers file B's fixture import). 4 separate DBs
|
||||
# remove the race; we shard the file list 1/4..4/4 and run shards in parallel.
|
||||
# Within a shard, files still run sequentially. Total wall-time on a 16-core
|
||||
# host: ~6 min sequential -> ~1.5-2 min sharded.
|
||||
#
|
||||
# Postgres host ports default to 5434-5437 (avoid 5432 manual `gbrain-test-pg`
|
||||
# and 5433 sibling-project conflicts). Override BASE port with GBRAIN_CI_PG_PORT;
|
||||
# shards take BASE..BASE+3.
|
||||
|
||||
services:
|
||||
postgres-1:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- "${GBRAIN_CI_PG_PORT:-5434}:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d gbrain_test"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
volumes:
|
||||
- gbrain-ci-pg-data-1:/var/lib/postgresql/data
|
||||
|
||||
postgres-2:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- "${GBRAIN_CI_PG_PORT_2:-5435}:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d gbrain_test"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
volumes:
|
||||
- gbrain-ci-pg-data-2:/var/lib/postgresql/data
|
||||
|
||||
postgres-3:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- "${GBRAIN_CI_PG_PORT_3:-5436}:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d gbrain_test"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
volumes:
|
||||
- gbrain-ci-pg-data-3:/var/lib/postgresql/data
|
||||
|
||||
postgres-4:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- "${GBRAIN_CI_PG_PORT_4:-5437}:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d gbrain_test"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
volumes:
|
||||
- gbrain-ci-pg-data-4:/var/lib/postgresql/data
|
||||
|
||||
runner:
|
||||
image: oven/bun:1
|
||||
working_dir: /app
|
||||
depends_on:
|
||||
postgres-1:
|
||||
condition: service_healthy
|
||||
postgres-2:
|
||||
condition: service_healthy
|
||||
postgres-3:
|
||||
condition: service_healthy
|
||||
postgres-4:
|
||||
condition: service_healthy
|
||||
# No global DATABASE_URL — scripts/ci-local.sh sets per-shard URL via -e.
|
||||
# Unit phase explicitly unsets DATABASE_URL so test/e2e/* gracefully skip.
|
||||
volumes:
|
||||
- .:/app
|
||||
# Linux container's node_modules MUST be isolated from host darwin-arm64.
|
||||
# Without this, container `bun install` stomps host node_modules and
|
||||
# subsequent `bun test` on host fails with binary-incompat errors.
|
||||
- gbrain-ci-node-modules:/app/node_modules
|
||||
# Warm install cache across runs.
|
||||
- gbrain-ci-bun-cache:/root/.bun/install/cache
|
||||
|
||||
volumes:
|
||||
gbrain-ci-pg-data-1:
|
||||
gbrain-ci-pg-data-2:
|
||||
gbrain-ci-pg-data-3:
|
||||
gbrain-ci-pg-data-4:
|
||||
gbrain-ci-node-modules:
|
||||
gbrain-ci-bun-cache:
|
||||
@@ -102,16 +102,6 @@ Keeping it running and up to date.
|
||||
| [Upgrades & Auto-Update](guides/upgrades-auto-update.md) | check-update, agent notifications, migration files |
|
||||
| [Live Sync](guides/live-sync.md) | Keep the index current: cron, --watch, webhook approaches |
|
||||
|
||||
## Getting Started
|
||||
|
||||
After setup, the brain is empty. The cold-start skill sequences the highest-leverage
|
||||
data sources to populate it:
|
||||
|
||||
| Guide | What It Covers |
|
||||
|-------|---------------|
|
||||
| [Cold Start](../skills/cold-start/SKILL.md) | Day-one bootstrapping: contacts, calendar, email, conversations, social, archives. Uses ClawVisor for safe credential handling — agents never hold raw API keys. |
|
||||
| [Ask User](../skills/ask-user/SKILL.md) | Choice-gate pattern for human input at decision points. Used by cold-start and other skills. |
|
||||
|
||||
---
|
||||
|
||||
## Appendix: GBrain CLI Quick Reference
|
||||
|
||||
@@ -1,173 +0,0 @@
|
||||
# ZeroEntropy — zembed-1 + zerank-2
|
||||
|
||||
[ZeroEntropy](https://zeroentropy.dev) ships two specialized small models
|
||||
for retrieval pipelines:
|
||||
|
||||
- **`zembed-1`** — multilingual embedding distilled from zerank-2.
|
||||
Flexible Matryoshka dims (2560/1280/640/320/160/80/40), 32K context,
|
||||
asymmetric `input_type: query|document` encoding. $0.025/1M tokens
|
||||
(sale) / $0.05 regular.
|
||||
- **`zerank-2`** — SOTA multilingual cross-encoder reranker.
|
||||
$0.025/1M tokens (~50% cheaper than Cohere/Voyage rerankers).
|
||||
Plus `zerank-1` and `zerank-1-small` for legacy / open-source needs.
|
||||
|
||||
Both land in gbrain v0.35.0.0 behind the openai-compatible recipe path,
|
||||
alongside OpenAI and Voyage.
|
||||
|
||||
## Setup
|
||||
|
||||
1. Get an API key at
|
||||
[dashboard.zeroentropy.dev](https://dashboard.zeroentropy.dev).
|
||||
2. Export it:
|
||||
```bash
|
||||
export ZEROENTROPY_API_KEY=<your-key>
|
||||
```
|
||||
|
||||
## Embedding switch — zembed-1
|
||||
|
||||
**Important:** `gbrain config set embedding_model …` is NOT a live
|
||||
gateway switch. `embedding_model` and `embedding_dimensions` size the
|
||||
schema and must be stable across engine connects, so they only resolve
|
||||
from the **file plane** (`~/.gbrain/config.json`) and the **env plane**
|
||||
(`GBRAIN_EMBEDDING_MODEL` / `GBRAIN_EMBEDDING_DIMENSIONS`). The DB plane
|
||||
is intentionally ignored for these two keys (same posture as today's
|
||||
Voyage setup).
|
||||
|
||||
### Option A — file plane (recommended for stable installs)
|
||||
|
||||
Edit `~/.gbrain/config.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"embedding_model": "zeroentropyai:zembed-1",
|
||||
"embedding_dimensions": 2560
|
||||
}
|
||||
```
|
||||
|
||||
Valid dims: `2560` (default), `1280`, `640`, `320`, `160`, `80`, `40`.
|
||||
Matryoshka-style — smaller trades quality for storage monotonically.
|
||||
Pick the largest that fits your column width.
|
||||
|
||||
### Option B — env plane (CI / Docker)
|
||||
|
||||
```bash
|
||||
export GBRAIN_EMBEDDING_MODEL=zeroentropyai:zembed-1
|
||||
export GBRAIN_EMBEDDING_DIMENSIONS=2560
|
||||
```
|
||||
|
||||
### Re-embed
|
||||
|
||||
Switching embedding models invalidates the vector index. Re-embed:
|
||||
|
||||
```bash
|
||||
gbrain embed --stale --limit 50 # smoke a small batch
|
||||
gbrain embed --stale # full re-embed
|
||||
```
|
||||
|
||||
### Verify
|
||||
|
||||
```bash
|
||||
gbrain models doctor --json | jq '.probes[] | select(.touchpoint=="embedding_config")'
|
||||
```
|
||||
|
||||
Expected: `status: "ok"`. Invalid dims (e.g. `1024`, `1536`, `3072`)
|
||||
surface as `status: "config"` with a paste-ready
|
||||
`gbrain config set embedding_dimensions <one of 2560|1280|640|320|160|80|40>` fix hint.
|
||||
|
||||
## Reranker switch — zerank-2
|
||||
|
||||
The reranker is the bigger story: gbrain had no cross-encoder reranker
|
||||
stage before v0.35.0.0. It slots between RRF dedup and token-budget
|
||||
enforcement in hybrid search.
|
||||
|
||||
### Default-on with `tokenmax` mode
|
||||
|
||||
`tokenmax` mode now defaults `search.reranker.enabled = true` with
|
||||
`zerank-2`. If you already use `tokenmax` AND have `ZEROENTROPY_API_KEY`
|
||||
set, reranker fires automatically. Without the key, every rerank call
|
||||
fails-open (audit-logged) and search returns RRF order — same UX as
|
||||
before, just with an observable failure surfaced via `gbrain doctor`.
|
||||
|
||||
### Opt-in on `conservative` or `balanced` mode
|
||||
|
||||
```bash
|
||||
gbrain config set search.reranker.enabled true
|
||||
```
|
||||
|
||||
The override sits above the mode-bundle default; opt-out is one flip.
|
||||
|
||||
### Cost anchor
|
||||
|
||||
At 30 candidates × ~400 tokens/chunk × $0.025/1M = **~$0.0003/query**.
|
||||
Rounding error against the `tokenmax + Opus` pairing's ~$700/mo at
|
||||
single-user volume per the CLAUDE.md cost matrix.
|
||||
|
||||
### Verify
|
||||
|
||||
```bash
|
||||
gbrain models doctor --json | jq '.probes[] | select(.touchpoint=="reranker_config")'
|
||||
```
|
||||
|
||||
Two probes run for reranker:
|
||||
- `reranker_config` (zero-network) — validates the model resolves
|
||||
through the recipe registry and is in the touchpoint's allowlist.
|
||||
- A reachability probe sends a minimal `{query: "probe", documents:
|
||||
["probe"]}` rerank to verify auth + URL.
|
||||
|
||||
## Knobs reference
|
||||
|
||||
| Config key | Default | Notes |
|
||||
|---|---|---|
|
||||
| `search.reranker.enabled` | `true` for tokenmax, `false` for others | One-flip opt-in/out |
|
||||
| `search.reranker.model` | `zeroentropyai:zerank-2` | Try `zerank-1` (older SOTA) or `zerank-1-small` (Apache-2.0 open) |
|
||||
| `search.reranker.top_n_in` | `30` | Candidates sent to reranker (caps API spend) |
|
||||
| `search.reranker.top_n_out` | `null` (no truncate) | Truncate reranked output to this many; `null` preserves full length |
|
||||
| `search.reranker.timeout_ms` | `5000` | HTTP timeout; long stalls degrade UX worse than RRF fallback |
|
||||
|
||||
## Failure observability
|
||||
|
||||
Reranker is fail-open by construction: every error class (auth, rate-limit,
|
||||
network, timeout, payload-too-large, unknown) returns the original RRF
|
||||
order unchanged. Failures log to
|
||||
`~/.gbrain/audit/rerank-failures-YYYY-Www.jsonl` (ISO-week rotation).
|
||||
|
||||
`gbrain doctor` reads the audit and surfaces:
|
||||
- **auth failures** — any single one warns (config-time problem doctor's
|
||||
own probe should have caught)
|
||||
- **payload-too-large** — any single one warns (workload-mismatch signal)
|
||||
- **transient (network/timeout/rate_limit)** — warns at >=5 in 7 days
|
||||
|
||||
Query text is SHA-256 hashed in the audit; never logged raw.
|
||||
|
||||
## Asymmetric input_type
|
||||
|
||||
ZE zembed-1 (and Voyage v3+) use asymmetric query/document encoding for
|
||||
better retrieval. The gateway's `embedQuery(text)` companion threads
|
||||
`input_type: 'query'`; standard `embed(texts)` defaults to
|
||||
`'document'`. Hybrid search's two query-side embed sites use
|
||||
`embedQuery()` automatically; all ingest paths use `embed()`.
|
||||
|
||||
Symmetric providers (OpenAI text-embedding-3, fixed-dim Voyage models)
|
||||
ignore the field — no behavior change.
|
||||
|
||||
## Cache key versioning
|
||||
|
||||
v0.35.0.0 bumped `KNOBS_HASH_VERSION` 1 → 2 to fold reranker config into
|
||||
the `query_cache.knobs_hash` column. During a rolling deploy:
|
||||
|
||||
- Expect a temporary cache hit-rate dip (~1 hour at default
|
||||
`cache.ttl_seconds = 3600s`)
|
||||
- Hot queries may briefly double their cache row count (one row per
|
||||
version)
|
||||
|
||||
Both clear naturally; no operator action required.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Likely cause | Fix |
|
||||
|---|---|---|
|
||||
| `embedding_config` probe says invalid dim | Defaulting to 1536 (OpenAI default) | Set `embedding_dimensions` to one of 2560/1280/640/320/160/80/40 |
|
||||
| `reranker_config` probe says model not in allowlist | Typo in `search.reranker.model` | Use one of `zerank-2` / `zerank-1` / `zerank-1-small` |
|
||||
| `reranker_health` doctor warns about auth | `ZEROENTROPY_API_KEY` not set or invalid | Re-export the env var; `gbrain models doctor` to verify |
|
||||
| `reranker_health` doctor warns about transient failures | Upstream flake or rate limit | Reranker fails open to RRF; check ZE status page if persistent |
|
||||
| Cache hit rate dipped after upgrade | Expected during rolling deploy | Clears within `cache.ttl_seconds` (default 3600s) |
|
||||
@@ -1,242 +0,0 @@
|
||||
# Brains and Sources — the mental model
|
||||
|
||||
GBrain has two orthogonal axes for organizing knowledge. Users and agents both
|
||||
need to understand both of them, or queries misroute silently.
|
||||
|
||||
**TL;DR:**
|
||||
- A **brain** is a database. You can have many.
|
||||
- A **source** is a named repo of content *inside* a brain. One brain can hold many.
|
||||
- `--brain <id>` picks WHICH DATABASE.
|
||||
- `--source <id>` picks WHICH REPO WITHIN that database.
|
||||
- They're independent. You can target any combination.
|
||||
|
||||
---
|
||||
|
||||
## The two axes
|
||||
|
||||
### Brains (the DB axis)
|
||||
|
||||
A **brain** is one database — PGLite file, self-hosted Postgres, or Supabase.
|
||||
Each brain has:
|
||||
- Its own `pages` table, `chunks` table, `embeddings`, etc.
|
||||
- Its own OAuth surface if served over HTTP MCP (v0.19+, PR 2).
|
||||
- Its own separate lifecycle, backup, access control.
|
||||
|
||||
Brains are enumerated by:
|
||||
- **host** — your default brain, configured in `~/.gbrain/config.json`.
|
||||
- **mounts** — additional brains registered in `~/.gbrain/mounts.json` via
|
||||
`gbrain mounts add <id>` (v0.19+).
|
||||
|
||||
Routing: `--brain <id>`, `GBRAIN_BRAIN_ID`, `.gbrain-mount` dotfile, or
|
||||
longest-path match against registered mount paths. Falls back to `host`.
|
||||
|
||||
### Sources (the repo axis, v0.18.0+)
|
||||
|
||||
A **source** is a named content repo *inside* one brain. Every `pages` row
|
||||
carries a `source_id`. Slugs are unique per source, not globally.
|
||||
|
||||
Example: in one brain, the slug `topics/ai` can exist under `source=wiki`
|
||||
AND under `source=gstack` — they're different pages.
|
||||
|
||||
Routing: `--source <id>`, `GBRAIN_SOURCE`, `.gbrain-source` dotfile, or
|
||||
registered `local_path` match in the `sources` table.
|
||||
|
||||
### When does each axis move?
|
||||
|
||||
| You want to | Adjust |
|
||||
|---|---|
|
||||
| Work in a different repo within the same brain (wiki → gstack notes) | `--source` |
|
||||
| Query a team-published brain that isn't yours | `--brain` |
|
||||
| Isolate a topic so it never leaks into personal search | `--source` with `federated=false` |
|
||||
| Share a brain with teammates | `--brain` (mount the team brain) |
|
||||
| Add a new repo to your personal brain | `--source` via `gbrain sources add` |
|
||||
| Add a team brain | `--brain` via `gbrain mounts add` |
|
||||
|
||||
**Rule of thumb:** if the data owner changes, it's a brain boundary. If the
|
||||
data owner stays the same but the topic/repo changes, it's a source boundary.
|
||||
|
||||
---
|
||||
|
||||
## Topology: a single-person developer
|
||||
|
||||
Simplest case. One brain, one source.
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────┐
|
||||
│ host brain (~/.gbrain) │
|
||||
│ ├── source: default (federated=true) │
|
||||
│ │ └── all pages │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
`gbrain query "retry budgets"` finds everything. No `--brain`, no `--source`
|
||||
needed.
|
||||
|
||||
---
|
||||
|
||||
## Topology: a personal brain with multiple repos
|
||||
|
||||
You maintain several codebases or writing streams. Each is its own source
|
||||
inside one brain. Cross-source search is on by default so a query about
|
||||
"caching" returns hits from every repo.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ host brain (~/.gbrain) │
|
||||
│ ├── source: wiki (federated=true) │
|
||||
│ │ └── personal notes, people, companies │
|
||||
│ ├── source: gstack (federated=true) │
|
||||
│ │ └── gstack plans, learnings │
|
||||
│ ├── source: openclaw (federated=true) │
|
||||
│ │ └── openclaw docs, memos │
|
||||
│ └── source: essays (federated=false) │
|
||||
│ └── draft essays, isolated on purpose │
|
||||
└──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Inside `~/openclaw/` the `.gbrain-source` dotfile pins every command to
|
||||
`source=openclaw`. Inside `~/gstack/` the dotfile pins to `source=gstack`.
|
||||
Everything still targets one DB.
|
||||
|
||||
Use this topology when:
|
||||
- You own all the content.
|
||||
- You want cross-repo search to just work.
|
||||
- You don't need to share any of it with someone who isn't you.
|
||||
|
||||
---
|
||||
|
||||
## Topology: personal brain + one team brain
|
||||
|
||||
You're on a team that publishes a shared brain. Your personal brain stays
|
||||
as-is; you mount the team brain alongside it.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ host brain (~/.gbrain) — YOUR personal DB │
|
||||
│ ├── source: wiki │
|
||||
│ ├── source: gstack │
|
||||
│ └── ... │
|
||||
└──────────────────────────────────────────────┘
|
||||
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ mount: media-team │
|
||||
│ path: ~/team-brains/media │
|
||||
│ engine: postgres (team's Supabase) │
|
||||
│ └── sources: wiki, raw, enriched │
|
||||
└──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
`gbrain query "X"` (no flags) → runs against host (your personal brain).
|
||||
`gbrain query "X" --brain media-team` → runs against the team's DB.
|
||||
Inside `~/team-brains/media/` a `.gbrain-mount` dotfile pins brain to
|
||||
`media-team` automatically.
|
||||
|
||||
Use this topology when:
|
||||
- You're on a team and someone publishes a brain the team subscribes to.
|
||||
- You need data isolation between work and personal.
|
||||
- Different teams/orgs own different brains.
|
||||
|
||||
---
|
||||
|
||||
## Topology: a CEO-class user with multiple team memberships
|
||||
|
||||
You're senior enough to sit across multiple teams. You maintain your personal
|
||||
brain (with N sources inside) AND mount several work team brains. Each team
|
||||
brain is itself a multi-source brain in the v0.18.0 sense — organized
|
||||
internally however the team owner chose.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ host brain — YOUR personal DB │
|
||||
│ ├── source: wiki │
|
||||
│ ├── source: essays │
|
||||
│ ├── source: gstack │
|
||||
│ └── source: openclaw │
|
||||
└──────────────────────────────────────────────┘
|
||||
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ mount: media-team (your media team's brain) │
|
||||
│ └── sources: wiki, pipeline, enriched │
|
||||
└──────────────────────────────────────────────┘
|
||||
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ mount: policy-team (your policy team's) │
|
||||
│ └── sources: wiki, research, letters │
|
||||
└──────────────────────────────────────────────┘
|
||||
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ mount: portfolio (another team's) │
|
||||
│ └── sources: companies, deals, diligence │
|
||||
└──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Inside each team's checkout, a `.gbrain-mount` dotfile pins the brain. Inside
|
||||
a specific subdirectory, a `.gbrain-source` dotfile pins the source. So `cd
|
||||
~/team-brains/policy/research && gbrain query "X"` targets
|
||||
`brain=policy-team, source=research` with zero flags.
|
||||
|
||||
Use this topology when:
|
||||
- You cross-cut multiple teams.
|
||||
- Each team owns its own brain with its own access policy.
|
||||
- You need latent-space federation (agent decides when to query across
|
||||
brains), not SQL federation.
|
||||
|
||||
Cross-brain queries are **not deterministic** in v0.19. The agent sees the
|
||||
brain list and re-queries as needed. That's the feature — it keeps debugging
|
||||
sane and access control clean.
|
||||
|
||||
---
|
||||
|
||||
## Resolution precedence (one page to remember)
|
||||
|
||||
```
|
||||
WHICH BRAIN (DB)? WHICH SOURCE (repo in DB)?
|
||||
1. --brain <id> 1. --source <id>
|
||||
2. GBRAIN_BRAIN_ID env 2. GBRAIN_SOURCE env
|
||||
3. .gbrain-mount dotfile 3. .gbrain-source dotfile
|
||||
4. longest-prefix mount path match 4. longest-prefix source path match
|
||||
5. (reserved: brains.default v2) 5. sources.default config
|
||||
6. fallback: 'host' 6. fallback: 'default'
|
||||
```
|
||||
|
||||
Both axes follow the same layered pattern on purpose. If you know one, you
|
||||
know the other.
|
||||
|
||||
---
|
||||
|
||||
## For agents reading this
|
||||
|
||||
- Default assumption when the user asks a question: start in the current
|
||||
brain (resolved via the precedence above). Don't jump brains without a
|
||||
reason.
|
||||
- If the user asks a question that crosses topic areas a team might own
|
||||
(e.g. "what did Team X decide last week?"), the right move is to *query
|
||||
the team's brain explicitly* rather than searching host with "team x".
|
||||
- Cross-brain federation is YOUR JOB, not the DB's. You have the brain list
|
||||
(`gbrain mounts list`). You decide when to fan out. You synthesize
|
||||
findings. You cite `brain:source:slug`.
|
||||
- When writing a page, respect the brain boundary. A fact about a team's
|
||||
work belongs in the team's brain, not in the user's personal brain. Ask
|
||||
before writing cross-brain.
|
||||
- See `skills/conventions/brain-routing.md` for the full decision table.
|
||||
|
||||
## For users reading this
|
||||
|
||||
- **Default path:** set up your personal brain (`gbrain init`), add a source
|
||||
per repo you care about (`gbrain sources add gstack --path ~/gstack`).
|
||||
You'll almost never need `--brain`.
|
||||
- **When a team publishes a brain:** `gbrain mounts add <team-id> --path
|
||||
<clone> --db-url <url>` and the `.gbrain-mount` dotfile in that checkout
|
||||
routes queries there automatically.
|
||||
- **When you are the CEO-class user with multiple team memberships:** mount
|
||||
each team brain. Trust the resolver — inside a team's directory the
|
||||
dotfile picks the brain, inside a subdirectory the dotfile picks the
|
||||
source. The flags are for when you want to query across the boundary
|
||||
deliberately.
|
||||
|
||||
## Further reading
|
||||
|
||||
- v0.18.0 CHANGELOG — introduced `sources` primitive.
|
||||
- v0.19.0 CHANGELOG (TBD after PR 0+1+2 ship) — introduces `mounts`.
|
||||
- `docs/mounts/publishing-a-team-brain.md` (PR 2) — how to be the brain
|
||||
publisher, not just the subscriber.
|
||||
@@ -1,198 +0,0 @@
|
||||
# System of record
|
||||
|
||||
**The GitHub repo (markdown + frontmatter) is the system of record.
|
||||
The Postgres/PGLite database is a derived cache. We do not back up
|
||||
the database — we rebuild it from the repo.**
|
||||
|
||||
This document is the canonical reference for that contract. Every code
|
||||
path that writes user-knowledge state should match the pattern
|
||||
described here. The CI gate at `scripts/check-system-of-record.sh`
|
||||
enforces it programmatically.
|
||||
|
||||
## Why this matters
|
||||
|
||||
The DB is a derived index over the markdown content. It exists to make
|
||||
search fast, to dedup embedding-similar claims, to materialize the
|
||||
cross-page graph. None of that data is irreplaceable — as long as the
|
||||
markdown is intact, `gbrain sync && gbrain extract all` rebuilds the
|
||||
entire DB from scratch.
|
||||
|
||||
This means:
|
||||
|
||||
- **Disaster recovery is one command.** If your DB volume corrupts, if
|
||||
Postgres eats itself, if PGLite's WASM lock wedges — you don't need
|
||||
a backup. You wipe the DB, re-import from your brain repo, and the
|
||||
derived state regenerates. v0.32.3 ships `gbrain rebuild
|
||||
--confirm-destructive` as the documented one-liner.
|
||||
- **Multi-machine sync is git.** Your brain is a repo. Push from one
|
||||
machine, pull from another, and the second machine's DB rebuilds on
|
||||
its next sync. No "back up the database" step.
|
||||
- **Privacy is in your hands.** Sensitive entity pages can be
|
||||
gitignored (via `gbrain.yml` `db_only` paths or per-page) and they
|
||||
stay on disk but not in git. The fence respects whatever git
|
||||
tracking choice you make at the page level.
|
||||
- **Cross-agent collaboration is possible.** Multiple agents can write
|
||||
to the same brain because the fence is the merge point, not the DB.
|
||||
Git handles concurrent edits the way git handles concurrent edits.
|
||||
|
||||
## The three categories
|
||||
|
||||
Every table in the gbrain schema belongs to exactly one of three
|
||||
categories. The category determines how it gets rebuilt during
|
||||
disaster recovery.
|
||||
|
||||
### FS-canonical (markdown is the source of truth)
|
||||
|
||||
These are user-authored knowledge. The DB row is a derived index over
|
||||
the markdown — wipe the table and `gbrain extract` rebuilds it
|
||||
identically. The CI gate keeps direct DB writes from drifting away
|
||||
from the markdown contract.
|
||||
|
||||
| Category | How it's stored in markdown | Derived DB table | Reconciler |
|
||||
|---|---|---|---|
|
||||
| **Takes** (incl. hunches, bets) | `## Takes` fenced table between `<!--- gbrain:takes:begin -->` / `:end -->` markers | `takes` | `extract takes` |
|
||||
| **Facts** | `## Facts` fenced table between `<!--- gbrain:facts:begin -->` / `:end -->` markers | `facts` | `extract_facts` cycle phase |
|
||||
| **Links** | Inline `[text](slug)` / `[[slug]]` in markdown body + frontmatter `direction: incoming` | `links` | `extract links` |
|
||||
| **Timeline** | `## Timeline` section after `<!-- timeline -->` sentinel | `timeline_entries` | `extract timeline` |
|
||||
| **Tags** | Frontmatter `tags:` YAML array | `tags` | `importFromFile` (reconciles per-page on import) |
|
||||
| **emotional_weight** | Recomputed from takes + tags | `pages.emotional_weight` (signal column) | `recompute_emotional_weight` cycle phase |
|
||||
| **synthesis_evidence** | FK into `takes` rows (`slug#N`) inside synthesis pages | `synthesis_evidence` | `extract takes` (transitively) |
|
||||
|
||||
### Derived from FS but not user-authored
|
||||
|
||||
These hold derived state that's automatically reconstructible from the
|
||||
markdown but not directly authored as markdown by the user. The
|
||||
chunker + embedder rebuild these on import.
|
||||
|
||||
| Table | Source | Notes |
|
||||
|---|---|---|
|
||||
| `pages` | The markdown file as a whole | One row per file; `compiled_truth` + `frontmatter` come from parse |
|
||||
| `content_chunks` | `pages.compiled_truth` after chunker strip | Re-chunked on content_hash change; embedded via configured model |
|
||||
| `page_versions` | Each `pages` UPDATE | Audit history; rebuildable in principle but not in practice |
|
||||
|
||||
### DB-only by design (named exceptions)
|
||||
|
||||
These hold runtime / infrastructure state that's intentionally not in
|
||||
the repo. The architectural rule still holds — these aren't
|
||||
"user knowledge" — but they're DB-only by design.
|
||||
|
||||
| Category | Why it's OK to be DB-only |
|
||||
|---|---|
|
||||
| `raw_data` | Webhook/transcript sidecars; not user-authored knowledge. |
|
||||
| `subagent_messages` / `subagent_tool_executions` / `subagent_rate_leases` | Runtime job state. Replay-only, not persistent knowledge. |
|
||||
| `oauth_clients` / `oauth_tokens` / `access_tokens` | Credentials. Not in source control by definition. |
|
||||
| `mcp_request_log` | Audit trail. Volatile by design. |
|
||||
| `minion_jobs` / `minion_inbox` / `minion_attachments` | Job queue. Restarts re-enqueue or drop. |
|
||||
| `eval_candidates` / `eval_capture_failures` | Contributor-mode dev loop; opt-in capture. |
|
||||
| `dream_verdicts` | Cheap verdict cache. Rebuildable by re-running Haiku. |
|
||||
| `gbrain_cycle_locks` / migration ledger | Infrastructure. |
|
||||
| `config` (some keys) | Site-local routing config (e.g. `sync.repo_path`). |
|
||||
|
||||
A new derived table that holds user-knowledge MUST land FS-first.
|
||||
If you're tempted to add one as "DB-only for now," the structural
|
||||
question is: does it belong in this DB-only-by-design list? If not,
|
||||
it's FS-canonical and needs a fence (or frontmatter field) plus a
|
||||
reconciler.
|
||||
|
||||
## The privacy boundary
|
||||
|
||||
Private knowledge in a fence still lives in the markdown file. If the
|
||||
user commits the page to git, the private data lands in git too. This
|
||||
is the existing operational model — we don't infer git policy.
|
||||
|
||||
For untrusted readers (remote MCP, subagent), the v0.32.2 release ships
|
||||
a 3-layer strip:
|
||||
|
||||
1. **Layer A (chunker):** `src/core/chunkers/recursive.ts` calls
|
||||
`stripFactsFence({keepVisibility: ['world']})` + `stripTakesFence`
|
||||
before chunking. Private fact text never reaches
|
||||
`content_chunks.chunk_text`, embeddings, or search results.
|
||||
2. **Layer B (get_page):** when `ctx.remote === true`, the response
|
||||
body has both fences stripped (private rows from facts; entire
|
||||
takes fence). Local CLI (`ctx.remote === false`) sees the full
|
||||
fence.
|
||||
3. **Layer C (git tracking):** the user decides whether to commit the
|
||||
entity page. `gbrain.yml` `db_only` paths are gitignored
|
||||
automatically; per-page choices via the user's normal git workflow.
|
||||
|
||||
For universally-private entities (a friend's name, an investor's
|
||||
internal notes), mark the entity page's directory as `db_only` in
|
||||
`gbrain.yml`. The file stays on disk but never lands in git.
|
||||
|
||||
## The forget contract
|
||||
|
||||
`gbrain forget <id>` and the MCP `forget_fact` op rewrite the fence
|
||||
row with strikethrough + `valid_until = today` + `context: "forgotten:
|
||||
<reason>"`. The DB's `expired_at = valid_until + now()` derivation
|
||||
reconstructs the forget state on every rebuild because the fence is
|
||||
canonical.
|
||||
|
||||
Strikethrough has two semantics distinguished by context:
|
||||
|
||||
- `~~claim~~` + `context: "superseded by #N"` → row was replaced by
|
||||
a newer row in the same fence
|
||||
- `~~claim~~` + `context: "forgotten: <reason>"` → row was retracted
|
||||
via the forget op
|
||||
|
||||
Both encodings keep the row in the markdown for audit history. To
|
||||
permanently delete a fact, edit the fence directly in markdown and
|
||||
remove the row. The next `extract_facts` cycle wipes the DB row.
|
||||
|
||||
## Disaster recovery
|
||||
|
||||
The promise the rule makes:
|
||||
|
||||
```bash
|
||||
# Snapshot what's there
|
||||
gbrain stats > /tmp/before.txt
|
||||
|
||||
# Wipe and rebuild
|
||||
gbrain rebuild --confirm-destructive # v0.32.3 — deletes derived tables
|
||||
# (pages + content_chunks survive
|
||||
# the CASCADE-safe design)
|
||||
# OR manually for v0.32.2:
|
||||
psql -c 'DELETE FROM facts; DELETE FROM takes; DELETE FROM links; DELETE FROM timeline_entries;'
|
||||
gbrain sync
|
||||
gbrain extract all
|
||||
|
||||
# Counts match
|
||||
gbrain stats > /tmp/after.txt
|
||||
diff /tmp/before.txt /tmp/after.txt
|
||||
```
|
||||
|
||||
The invariant E2E test at `test/e2e/system-of-record-invariant.test.ts`
|
||||
exercises this exact flow on every CI run.
|
||||
|
||||
## Rule for new code
|
||||
|
||||
When you add a new user-knowledge category:
|
||||
|
||||
1. **Define the markdown shape.** Fence (`<!--- gbrain:NAME:begin
|
||||
--> ... :end -->` table) or frontmatter field.
|
||||
2. **Build a parser** that produces structured data from markdown.
|
||||
See `src/core/fence-shared.ts` for the shared primitives.
|
||||
3. **Build a writer** that round-trips: parse + edit + render produces
|
||||
byte-identical markdown for identical input.
|
||||
4. **Add the engine method** that takes parsed data and stamps a
|
||||
derived table. The method gets an entry in the CI gate's
|
||||
banned-direct-call list.
|
||||
5. **Add a reconciler:** a cycle phase that walks pages, parses the
|
||||
fence, and rebuilds the derived table from scratch. The reconciler
|
||||
is the only legitimate call site for the engine method;
|
||||
`// gbrain-allow-direct-insert: <reason>` annotates it explicitly.
|
||||
6. **Add a round-trip test** in `test/e2e/system-of-record-invariant.test.ts`
|
||||
that proves DELETE + reconcile rebuilds the table byte-identically.
|
||||
|
||||
The CI gate at `scripts/check-system-of-record.sh` fails any PR that
|
||||
adds a new direct call to a derived-table writer outside the
|
||||
reconciler / migration layer without the explicit allow-list comment.
|
||||
|
||||
## Related
|
||||
|
||||
- `~/.claude/plans/system-instruction-you-are-working-expressive-pony.md`
|
||||
— the v0.32.2 design plan (decisions D1-D22 + Q1-Q8, Codex round 1
|
||||
and round 2 finds)
|
||||
- `skills/migrations/v0.32.2.md` — the agent-facing migration guide
|
||||
- `CHANGELOG.md` v0.32.2 entry — the release manifesto
|
||||
- `scripts/check-system-of-record.sh` — the CI gate that enforces
|
||||
the rule
|
||||
@@ -1,367 +0,0 @@
|
||||
# GBrain Deployment Topologies
|
||||
|
||||
GBrain supports three deployment shapes. They compose: a single user can mix
|
||||
all three on the same machine without conflict, because every shape resolves
|
||||
to "which `~/.gbrain/config.json` is active right now?" and `GBRAIN_HOME`
|
||||
controls that selection.
|
||||
|
||||
This page covers the three topologies, when each fits, and concrete setup
|
||||
recipes. Pair this doc with `docs/architecture/brains-and-sources.md` (which
|
||||
covers the in-brain organization axes) — that doc is about WHICH database;
|
||||
this doc is about WHERE that database lives.
|
||||
|
||||
## Quick decision tree
|
||||
|
||||
```
|
||||
"I'm setting up gbrain..."
|
||||
│
|
||||
▼
|
||||
Just for me, on one machine? ─── yes ───▶ Topology 1 (single brain)
|
||||
│
|
||||
no
|
||||
│
|
||||
▼
|
||||
Will a remote machine host the brain
|
||||
while my agent runs locally? ──── yes ───▶ Topology 2 (cross-machine thin client)
|
||||
│
|
||||
no
|
||||
│
|
||||
▼
|
||||
Multiple Conductor worktrees that
|
||||
shouldn't share a code index? ─── yes ───▶ Topology 3 (split-engine)
|
||||
```
|
||||
|
||||
Topologies 2 and 3 stack: a thin-client install can also host per-worktree
|
||||
code engines, and a per-worktree code engine can also point its artifact
|
||||
brain at a remote server.
|
||||
|
||||
## Topology 1 — Single brain (today's default)
|
||||
|
||||
```
|
||||
┌────────────────┐
|
||||
│ one machine │
|
||||
│ ┌──────────┐ │
|
||||
│ │ gbrain │──┼──→ ~/.gbrain/ → PGLite or Supabase
|
||||
│ │ CLI │ │
|
||||
│ └──────────┘ │
|
||||
└────────────────┘
|
||||
```
|
||||
|
||||
What you get: one local DB (PGLite for small brains, Supabase for ~1000+
|
||||
files). All commands work directly against it. `gbrain serve` exposes it
|
||||
to a single agent over MCP.
|
||||
|
||||
When it fits: solo use, single machine, one agent, no Conductor parallelism.
|
||||
This is the default; `gbrain init` (no flags) gives you this.
|
||||
|
||||
Setup:
|
||||
|
||||
```
|
||||
gbrain init # interactive — defaults to PGLite
|
||||
gbrain init --pglite # explicit local
|
||||
gbrain init --supabase # remote Supabase (recommended for 1000+ files)
|
||||
```
|
||||
|
||||
Nothing else here is special. The other two topologies are variations on
|
||||
"who owns the DB" and "how does the agent talk to it."
|
||||
|
||||
## Topology 2 — Cross-machine thin client
|
||||
|
||||
```
|
||||
┌────────────┐ ┌──────────────────┐
|
||||
│ neuromancer│ │ brain-host │
|
||||
│ ┌────────┐ │ HTTP MCP / OAuth │ ┌────────────┐ │
|
||||
│ │ Hermes │─┼───────────────────→│ │ gbrain │──┼──→ Supabase
|
||||
│ │ agent │ │ │ │ serve --http│ │
|
||||
│ └────────┘ │ │ └────────────┘ │
|
||||
│ │ │ (with autopilot)│
|
||||
│ no local │ │ │
|
||||
│ gbrain DB │ │ │
|
||||
└────────────┘ └──────────────────┘
|
||||
```
|
||||
|
||||
What you get: the agent on one machine ("neuromancer") consumes a brain
|
||||
hosted on another machine ("brain-host") over HTTP MCP with OAuth. The
|
||||
agent's machine has NO local engine. All queries, searches, embeddings,
|
||||
and indexing happen on the host.
|
||||
|
||||
When it fits:
|
||||
|
||||
- Heavy brain (Supabase + autopilot) lives on a beefy machine; agents
|
||||
elsewhere just consume it.
|
||||
- You want one source of truth across many machines.
|
||||
- Spinning up a parallel local install would create source-ID contention or
|
||||
duplicate work.
|
||||
|
||||
The thin client's `~/.gbrain/config.json` carries a `remote_mcp` field
|
||||
instead of a local DB connection:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"engine": "postgres", // ignored — never used
|
||||
"remote_mcp": {
|
||||
"issuer_url": "https://brain-host.local:3001",
|
||||
"mcp_url": "https://brain-host.local:3001/mcp",
|
||||
"oauth_client_id": "neuromancer-...",
|
||||
"oauth_client_secret": "..." // or set GBRAIN_REMOTE_CLIENT_SECRET
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The CLI dispatch guard refuses any DB-bound command (`sync`, `embed`,
|
||||
`extract`, `migrate`, `apply-migrations`, `repair-jsonb`, `orphans`,
|
||||
`integrity`, `serve`) on a thin-client install with a clear error pointing
|
||||
at the remote host. `gbrain doctor` runs a dedicated thin-client check set
|
||||
(OAuth discovery, token round-trip, MCP smoke).
|
||||
|
||||
### Setup
|
||||
|
||||
**Step 1 — On the host (brain-host):**
|
||||
|
||||
```bash
|
||||
gbrain init --supabase # or --pglite, doesn't matter
|
||||
gbrain serve --http --port 3001 --bind 0.0.0.0 # v0.34: bind explicitly for remote access
|
||||
# (defaults to 127.0.0.1 since v0.34)
|
||||
gbrain auth register-client neuromancer \
|
||||
--grant-types client_credentials \
|
||||
--scopes read,write,admin # admin needed for ping/doctor
|
||||
|
||||
# v0.34: source-scoped client (write to one source, federate reads across
|
||||
# multiple sources). Omit both flags for a v0.33-compatible super-client.
|
||||
gbrain auth register-client neuromancer-dept \
|
||||
--grant-types client_credentials \
|
||||
--scopes read,write \
|
||||
--source dept-x \
|
||||
--federated-read dept-x,shared,parent-canon
|
||||
```
|
||||
|
||||
The `register-client` command prints a `client_id` and `client_secret`.
|
||||
Note both. **Scope must include `admin`** — `submit_job` (used by
|
||||
`gbrain remote ping`) and `run_doctor` (used by `gbrain remote doctor`)
|
||||
both require it.
|
||||
|
||||
**Step 2 — On the thin client (neuromancer):**
|
||||
|
||||
```bash
|
||||
gbrain init --mcp-only \
|
||||
--issuer-url https://brain-host.local:3001 \
|
||||
--mcp-url https://brain-host.local:3001/mcp \
|
||||
--oauth-client-id <id> \
|
||||
--oauth-client-secret <secret>
|
||||
```
|
||||
|
||||
Pre-flight smoke runs three probes (OAuth discovery, token round-trip,
|
||||
MCP initialize). If any fails, init exits with an actionable error. On
|
||||
success, `~/.gbrain/config.json` gets `remote_mcp` set and NO local DB
|
||||
is created.
|
||||
|
||||
**Step 3 — Configure your agent's MCP client.**
|
||||
|
||||
For Claude Desktop / Hermes / openclaw, add a single MCP server entry
|
||||
pointing at the host's `mcp_url` with the bearer token from `register-client`.
|
||||
Example for Claude Desktop's `~/.config/claude/claude_desktop_config.json`:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"mcpServers": {
|
||||
"gbrain": {
|
||||
"type": "url",
|
||||
"url": "https://brain-host.local:3001/mcp",
|
||||
"headers": { "Authorization": "Bearer <client_secret>" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Step 4 — Verify.**
|
||||
|
||||
```bash
|
||||
gbrain doctor # runs thin-client checks (no local DB needed)
|
||||
gbrain remote ping # triggers an autopilot cycle on the host (Tier B)
|
||||
gbrain remote doctor # asks the host to run its own doctor (Tier B)
|
||||
```
|
||||
|
||||
`gbrain sync` and friends will refuse with a clear thin-client error
|
||||
naming the `mcp_url`. That's the correct behavior — those commands need
|
||||
a local engine that doesn't exist here.
|
||||
|
||||
### Re-run guard
|
||||
|
||||
Running `gbrain init` (no flags) on a machine that already has thin-client
|
||||
config set refuses without `--force`. This catches the scripted-setup-loop
|
||||
friction where an orchestrator keeps trying to create a local DB. Use
|
||||
`gbrain init --mcp-only --force` to refresh thin-client config.
|
||||
|
||||
### Storing the OAuth secret
|
||||
|
||||
Three storage paths in priority order:
|
||||
|
||||
1. **`GBRAIN_REMOTE_CLIENT_SECRET` env var** (preferred for headless agents).
|
||||
When set, overrides whatever's in the config file. The init flow doesn't
|
||||
persist a config-file copy when the env var was the source.
|
||||
2. **`~/.gbrain/config.json` with 0600 perms** (default for interactive
|
||||
setup; mirrors how Supabase keys are stored today).
|
||||
3. macOS Keychain integration is on the roadmap; not in v1.
|
||||
|
||||
## Topology 3 — Split-engine, per-worktree code + remote artifacts
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────┐
|
||||
│ one machine │
|
||||
│ │
|
||||
│ ┌─ worktree A ──────────────┐ │
|
||||
│ │ GBRAIN_HOME=A/.conductor │ │
|
||||
│ │ gbrain serve --port 3001 │── PGLite (code A) │
|
||||
│ └───────────────────────────┘ │
|
||||
│ │
|
||||
│ ┌─ worktree B ──────────────┐ │
|
||||
│ │ GBRAIN_HOME=B/.conductor │ │
|
||||
│ │ gbrain serve --port 3002 │── PGLite (code B) │
|
||||
│ └───────────────────────────┘ │
|
||||
│ │
|
||||
│ ┌─ default ~/.gbrain ───────┐ HTTP MCP / OAuth │
|
||||
│ │ gbrain serve --port 3000 │──────────────────────→ remote artifacts
|
||||
│ └───────────────────────────┘ (Supabase / brain-host)
|
||||
│ │
|
||||
│ Agent's MCP config (Hermes / Claude Desktop): │
|
||||
│ mcp__gbrain_code__* → http://localhost:3001 │
|
||||
│ mcp__gbrain_artifacts__* → http://brain-host/mcp │
|
||||
└──────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
What you get: each Conductor worktree has its own per-worktree code index
|
||||
(local PGLite, disposable when the worktree dies). Artifacts (plans,
|
||||
learnings, transcripts) still live in a shared brain that all worktrees
|
||||
can see and write to.
|
||||
|
||||
When it fits:
|
||||
|
||||
- Multiple Conductor worktrees on one machine, all touching the same code
|
||||
repo.
|
||||
- You don't want each worktree's code-import to clobber the others'
|
||||
`last_commit`, source IDs, or symbol tables.
|
||||
- You DO want artifacts (plans, learnings, retros, transcripts) to be
|
||||
visible across worktrees.
|
||||
|
||||
### How it works
|
||||
|
||||
`GBRAIN_HOME` selects which `~/.gbrain` directory is active. Set per worktree:
|
||||
|
||||
```bash
|
||||
export GBRAIN_HOME=/path/to/worktree-A/.conductor/gbrain
|
||||
gbrain init --pglite
|
||||
gbrain serve --http --port 3001
|
||||
```
|
||||
|
||||
Each worktree's `gbrain serve` instance binds its own port and indexes its
|
||||
own DB. Multiple `gbrain serve` processes coexist fine — they're separate
|
||||
OS processes with separate config and separate connection pools.
|
||||
|
||||
The artifact brain runs as a separate `gbrain serve` instance with the
|
||||
default `~/.gbrain` (no GBRAIN_HOME override) — or remote, in which case
|
||||
it's a Topology 2 setup.
|
||||
|
||||
The agent's MCP client config lists multiple servers, each with a unique
|
||||
alias. Tool names are namespaced as `mcp__<alias>__<tool>`, so the agent
|
||||
calls `mcp__gbrain_code__search` for code lookups and `mcp__gbrain_artifacts__search`
|
||||
for artifact lookups.
|
||||
|
||||
### CRITICAL: alias-level routing is manual
|
||||
|
||||
Topology 3 has no smart per-tool routing inside gbrain. The agent picks
|
||||
which brain to query when it picks the alias. **A wrong alias writes (or
|
||||
queries) the wrong brain silently.** This is intentional (explicit beats
|
||||
magic) but real:
|
||||
|
||||
- If the agent calls `mcp__gbrain_artifacts__put_page` with code-shaped
|
||||
content, that page lands in the artifact brain forever.
|
||||
- If the agent calls `mcp__gbrain_code__search` for a question that
|
||||
actually wants artifact context, the search comes back empty.
|
||||
|
||||
Mitigations:
|
||||
|
||||
- Name aliases clearly. `gbrain_code` vs `gbrain_artifacts` is unambiguous;
|
||||
`gbrain` vs `gbrain_local` is not.
|
||||
- Document in your agent's system prompt or rules which alias goes where.
|
||||
Be explicit about "code questions → `gbrain_code`; everything else →
|
||||
`gbrain_artifacts`."
|
||||
- Pair Topology 3 with `gstack`'s per-worktree wiring (which sets the
|
||||
alias names + agent rules consistently across worktrees).
|
||||
|
||||
### Setup (manual; gstack automates this side)
|
||||
|
||||
The gbrain side requires zero new code — `GBRAIN_HOME` and `--port` already
|
||||
exist. Setup looks like:
|
||||
|
||||
```bash
|
||||
# Start the artifact brain (default ~/.gbrain) on port 3000
|
||||
gbrain serve --http --port 3000 &
|
||||
|
||||
# Start a per-worktree code brain on port 3001
|
||||
export GBRAIN_HOME=/path/to/worktree-A/.conductor/gbrain
|
||||
gbrain init --pglite
|
||||
gbrain serve --http --port 3001 &
|
||||
unset GBRAIN_HOME
|
||||
```
|
||||
|
||||
Then configure the agent's MCP config with two entries (different aliases,
|
||||
different ports). For Claude Desktop:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"mcpServers": {
|
||||
"gbrain_artifacts": {
|
||||
"type": "url",
|
||||
"url": "http://localhost:3000/mcp",
|
||||
"headers": { "Authorization": "Bearer <token-A>" }
|
||||
},
|
||||
"gbrain_code": {
|
||||
"type": "url",
|
||||
"url": "http://localhost:3001/mcp",
|
||||
"headers": { "Authorization": "Bearer <token-B>" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The gstack-side wiring (per-worktree home setup, port allocation, automatic
|
||||
MCP config generation, gitignore for the per-worktree DB) is in the gstack
|
||||
repo's setup-gbrain skill — it composes these primitives, gbrain doesn't
|
||||
have to know about Conductor.
|
||||
|
||||
## Combining topologies
|
||||
|
||||
The three shapes compose. A single machine can run:
|
||||
|
||||
- A thin-client default config pointing at a remote artifact brain
|
||||
(Topology 2).
|
||||
- Plus per-worktree code brains under their own `GBRAIN_HOME` (Topology 3).
|
||||
- Each worktree's `gbrain serve` instance is local; the agent's MCP config
|
||||
lists them alongside the remote artifact brain.
|
||||
|
||||
`GBRAIN_HOME` controls which config file is active for any one CLI
|
||||
invocation. `gbrain serve --port` controls which port a server listens on.
|
||||
The agent's MCP client picks the alias and thus the destination per tool
|
||||
call. There's no global gbrain orchestrator that knows about all of them
|
||||
simultaneously — that's by design.
|
||||
|
||||
## When NOT to use these topologies
|
||||
|
||||
- **Don't use Topology 2 if your agent only ever runs on the same machine
|
||||
as the brain.** A local `gbrain` install + `gbrain serve` (stdio) is
|
||||
simpler and faster.
|
||||
- **Don't use Topology 3 if you only have one Conductor worktree at a
|
||||
time.** Per-worktree engines exist to prevent contention; one-at-a-time
|
||||
use has no contention.
|
||||
- **Don't use a `remote_mcp` thin client AND a local engine on the same
|
||||
machine in the same `GBRAIN_HOME`.** The dispatch guard refuses DB-bound
|
||||
commands when `remote_mcp` is set. If you genuinely want both modes on
|
||||
one machine, use `GBRAIN_HOME` to separate them (one home for the thin
|
||||
client, another for the local engine).
|
||||
|
||||
## See also
|
||||
|
||||
- `docs/architecture/brains-and-sources.md` — in-brain organization (brains
|
||||
vs sources axes).
|
||||
- `docs/mcp/CLAUDE_DESKTOP.md` and siblings — per-client MCP setup.
|
||||
- `gbrain init --help` and `gbrain auth --help` for command-level details.
|
||||
@@ -1,166 +0,0 @@
|
||||
# gbrain eval suspected-contradictions (v0.32.6)
|
||||
|
||||
The contradiction probe samples retrieval results, asks an LLM judge whether
|
||||
any pair contradicts on a factual claim relevant to the user's query, and
|
||||
aggregates into a calibrated report. The output is data — the operator
|
||||
decides what to act on. This doc covers the architecture, severity rubric,
|
||||
how to interpret the headline number, and when to act.
|
||||
|
||||
## Why this exists
|
||||
|
||||
gbrain handles contradictions for *curated* pages via compiled-truth-plus-
|
||||
timeline and source-boost: when `companies/acme.md` says MRR is $2M and a
|
||||
chat transcript from 2024 says MRR was $50K, the curated page outranks the
|
||||
chat. `takes.active` filtering hides explicitly-superseded takes. Recency
|
||||
decay biases ranking toward fresher content per source-tier.
|
||||
|
||||
What none of those mechanisms measure: how often do unmarked semantic
|
||||
contradictions actually surface in retrieval? Without a probe, every
|
||||
"should we build the bigger swing (chunk-level `revises` field + ranking
|
||||
change)" decision is vibes. The probe produces evidence.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────┐
|
||||
│ gbrain eval suspected-contradictions │
|
||||
└──────────────────┬───────────────────┘
|
||||
│
|
||||
┌──────────────────▼───────────────────┐
|
||||
│ For each query: hybridSearch top-K │
|
||||
│ → cross_slug_chunks + intra_page │
|
||||
│ chunk-vs-take pairs │
|
||||
└──────────────────┬───────────────────┘
|
||||
│
|
||||
┌──────────────────▼───────────────────┐
|
||||
│ Date pre-filter: skip pairs whose │
|
||||
│ dates are >30d apart (Codex fix: │
|
||||
│ same-paragraph-dual-date overrides) │
|
||||
└──────────────────┬───────────────────┘
|
||||
│
|
||||
┌──────────────────▼───────────────────┐
|
||||
│ Persistent cache lookup │
|
||||
│ (chunk_a_hash, chunk_b_hash, model, │
|
||||
│ prompt_version, truncation_policy) │
|
||||
└────────┬─────────┬────────────────────┘
|
||||
hit│ │miss
|
||||
│ ▼
|
||||
│ ┌─────────────────────────┐
|
||||
│ │ LLM judge call │
|
||||
│ │ → JudgeVerdict │
|
||||
│ │ confidence floor ≥ 0.7 │
|
||||
│ └─────────┬───────────────┘
|
||||
│ │
|
||||
▼ ▼
|
||||
┌──────────────────────────────────────┐
|
||||
│ Aggregate per-query + global stats │
|
||||
│ Wilson 95% CI on headline % │
|
||||
│ source-tier breakdown │
|
||||
│ hot pages + resolution proposals │
|
||||
└──────────────────┬───────────────────┘
|
||||
│
|
||||
▼
|
||||
ProbeReport JSON
|
||||
│
|
||||
┌──────────────────┼──────────────────────┬───────────────┐
|
||||
▼ ▼ ▼ ▼
|
||||
doctor (M1) MCP (M3) synthesize (M2) trend (M5)
|
||||
surfaces find_contradictions informational persistent
|
||||
findings op for agents block in prompt tracking
|
||||
```
|
||||
|
||||
## Severity rubric
|
||||
|
||||
The judge assigns severity per finding:
|
||||
|
||||
| Level | Rubric | Example |
|
||||
|---|---|---|
|
||||
| `low` | naming/format differences | "Alice Smith" vs "A. Smith" |
|
||||
| `medium` | factual values that may be stale | revenue figure, headcount, valuation |
|
||||
| `high` | identity / structural claims | founder/CEO/CFO role, company status |
|
||||
|
||||
Doctor sorts findings by severity DESC. The MCP op accepts a severity filter
|
||||
so agents can fetch just the high-priority items.
|
||||
|
||||
## How to interpret the headline number
|
||||
|
||||
The probe outputs `queries_with_contradiction / queries_evaluated` with a
|
||||
Wilson 95% confidence interval:
|
||||
|
||||
```
|
||||
Queries with >=1 contradiction: 12 / 50 (24%) Wilson CI 95%: 14–37%
|
||||
```
|
||||
|
||||
What this says: with 95% confidence, the true rate is between 14% and 37%.
|
||||
The 24% point estimate is the most-likely-value but bounded by sampling
|
||||
noise. **`small_sample_note` fires when n < 30** — at that scale the CI is
|
||||
too wide to act on.
|
||||
|
||||
Decision criteria for the bigger swing (chunk-level `revises` field):
|
||||
|
||||
| Wilson CI lower bound | What it says | Action |
|
||||
|---|---|---|
|
||||
| < 5% | Source-boost + recency-decay + curated pages handle the load | Stop here; this is the right scope |
|
||||
| 5–15% | Real but bounded | Operator decides whether the cost justifies the swing |
|
||||
| > 15% | Real and substantial | Plan the bigger swing in v0.34+ |
|
||||
|
||||
## When to act on findings
|
||||
|
||||
Each finding ships with a `resolution_command` field — paste-ready:
|
||||
|
||||
- `gbrain takes supersede <slug> --row N` — newer take should replace
|
||||
the older chunk text on the same page (intra_page kind).
|
||||
- `gbrain dream --phase synthesize --slug <slug>` — compiled_truth for
|
||||
the curated entity needs an update (cross_slug curated-vs-bulk).
|
||||
- `gbrain takes mark-debate <slug> --row N` — intentional disagreement
|
||||
(e.g., two opinions you want to keep both of).
|
||||
- `# manual review: <a> vs <b>` — judge wasn't sure; operator decides.
|
||||
|
||||
Run `gbrain eval suspected-contradictions review --severity high` to
|
||||
inspect findings without re-running the probe.
|
||||
|
||||
## Cost model
|
||||
|
||||
Default judge is `claude-haiku-4-5` at ~$1/Mtok in, $5/Mtok out. With
|
||||
the v0.32.6 truncation at 1500 chars per pair, ~500 input + 80 output
|
||||
tokens per judge call. Budget cap defaults to $5 in TTY / $1 non-TTY.
|
||||
|
||||
- ~$0.0006 per judge call
|
||||
- ~$0.005 per query (after date pre-filter + cache hits)
|
||||
- ~$0.50 per 100 queries
|
||||
|
||||
The persistent cache means nightly runs against the same query set
|
||||
pay near-zero on re-runs (until you bump PROMPT_VERSION).
|
||||
|
||||
## Trust posture
|
||||
|
||||
- Probe never mutates the brain. Runs only read pages/takes/chunks.
|
||||
Writes go only to `eval_contradictions_runs` and `eval_contradictions_cache`.
|
||||
- MCP `find_contradictions` is read-scope. NOT in the subagent allowlist —
|
||||
user-initiated only, not autonomous-action surface.
|
||||
- Build-fixture script is local-only. The redactor + `isCleanForCommit`
|
||||
gate makes accidental private-data commits hard, but the operator MUST
|
||||
inspect every redaction before commit.
|
||||
|
||||
## See also
|
||||
|
||||
- Plan: `~/.claude/plans/system-instruction-you-are-working-hashed-dewdrop.md`
|
||||
- CHANGELOG: `## [0.32.6]` entry covers the whole release.
|
||||
- Cost discipline: `docs/eval-bench.md` for the recommended nightly cadence
|
||||
+ trend-tracking workflow.
|
||||
- **Temporal axis follow-on (v0.35.3.1 + v0.35.7):** v0.35.3.1 added a
|
||||
six-member verdict enum (`no_contradiction | contradiction |
|
||||
temporal_supersession | temporal_regression | temporal_evolution |
|
||||
negation_artifact`) and threaded `pages.effective_date` into the judge
|
||||
prompt so the probe stops crying wolf on legitimate change-over-time.
|
||||
v0.35.7 lands the trajectory substrate the probe pointed at:
|
||||
`gbrain eval trajectory <entity>` shows the chronological typed-claim
|
||||
history with regressions flagged inline; `gbrain founder scorecard
|
||||
<entity>` rolls up four signals (accuracy, consistency, growth
|
||||
direction, red flags) into a stable JSON contract. MCP op
|
||||
`find_trajectory` (read scope, visibility-filtered for remote callers)
|
||||
exposes the same data to agents. The probe's `temporal_supersession`
|
||||
verdict and the consolidate phase's `valid_until` writeback both
|
||||
preserve the `auto-supersession.ts:4` "NEVER auto-applies" invariant
|
||||
— the probe still emits paste-ready commands, only `consolidate`
|
||||
writes `valid_until` (R1+R8 grep guard pins this).
|
||||
@@ -1,580 +0,0 @@
|
||||
# Embedder Shootout — May 2026 Eval Plan
|
||||
|
||||
**Status:** approved, ready to execute
|
||||
**Owner:** Garry
|
||||
**Plan source:** `~/.claude/plans/system-instruction-you-are-working-linear-origami.md` (review log)
|
||||
**Target wallclock:** ~2 weeks
|
||||
**Target API spend:** ~$525 (hard cap $700)
|
||||
|
||||
## What this is
|
||||
|
||||
A head-to-head A/B/C comparison of three embedding providers under v0.35.0.0's new
|
||||
multi-vendor gateway routing:
|
||||
|
||||
- **OpenAI** `text-embedding-3-large` @ 1536 dims
|
||||
- **Voyage** `voyage-4-large` @ 2048 dims
|
||||
- **ZeroEntropy** `zembed-1` @ 2560 dims (also 1280 in a Matryoshka ablation)
|
||||
|
||||
Each tested with and without the `zerank-2` reranker. Two corpora: public LongMemEval
|
||||
(500q) and BrainBench in-house (145 relational queries + 50 newly-curated Cat 13
|
||||
embedder-sensitive queries).
|
||||
|
||||
The goal: produce a publishable comparison report that answers "which embedder wins,
|
||||
and does zerank-2 carry the win for ZeroEntropy" with bootstrap p-values, suitable
|
||||
for a v0.35.2.0 release-note headline.
|
||||
|
||||
## Why this design
|
||||
|
||||
Locked decisions from the planning review (see plan file + `GSTACK REVIEW REPORT` at
|
||||
the bottom of the linked plan):
|
||||
|
||||
- **Synthetic-only** — LongMemEval (public) + BrainBench (in-house). No `~/.gbrain` data.
|
||||
- **Answer-gen mode** — `gbrain eval longmemeval` runs the default answer-gen path
|
||||
(Anthropic Sonnet), then feeds the resulting hypothesis JSONL to LongMemEval's
|
||||
published `evaluate_qa.py` (OpenAI gpt-4o judge) for real correctness numbers.
|
||||
`--retrieval-only` is NOT used (would produce an attackable headline; the judge
|
||||
expects answer text, not retrieval text).
|
||||
- **`tokenmax` search mode** pinned across all cells (expansion + reranker slot active).
|
||||
- **Serial execution** in one workspace. Clean rate-limit profile; first-contact run on
|
||||
ZE wants debuggable signal.
|
||||
- **7-cell matrix** (no matched-dim cross-vendor row — no shared dim exists across
|
||||
all three vendors; honest framing is "each vendor at marketed sweet spot").
|
||||
|
||||
## Architectural facts that constrain the plan
|
||||
|
||||
- `content_chunks.embedding vector(N)` dim is fixed per brain. Per-question PGLite in
|
||||
LongMemEval makes this free; BrainBench needs separate brain per cell.
|
||||
- pgvector HNSW caps at **2000 dims** (`PGVECTOR_HNSW_VECTOR_MAX_DIMS` in
|
||||
`src/core/vector-index.ts:19`). Voyage 2048 and ZE 2560 fall back to exact vector
|
||||
scan. Helps quality (no HNSW approximation) but adds latency. Footnoted in writeup.
|
||||
- Reranker disable key is **`search.reranker.enabled false`**, NOT `reranker_model none`.
|
||||
`tokenmax` mode defaults reranker=true.
|
||||
- `gbrain/ai/gateway` is NOT exported in v0.35.0.0. PR α exposes it.
|
||||
|
||||
## Matrix
|
||||
|
||||
| Cell | Embedder | Dim | HNSW | Reranker | Notes |
|
||||
|---|---|---|---|---|---|
|
||||
| A0 | `openai:text-embedding-3-large` | 1536 | yes | none | OpenAI baseline |
|
||||
| A1 | `openai:text-embedding-3-large` | 1536 | yes | `zerank-2` | mixed-vendor |
|
||||
| B0 | `voyage:voyage-4-large` | 2048 | no (exact) | none | Voyage solo |
|
||||
| B1 | `voyage:voyage-4-large` | 2048 | no (exact) | `zerank-2` | mixed-vendor |
|
||||
| C0 | `zeroentropyai:zembed-1` | 2560 | no (exact) | none | ZE embedder solo |
|
||||
| C1 | `zeroentropyai:zembed-1` | 2560 | no (exact) | `zerank-2` | **ZE full stack** |
|
||||
| C2 | `zeroentropyai:zembed-1` | 1280 | yes | `zerank-2` | ZE-Matryoshka ablation |
|
||||
|
||||
## PR structure — as few as possible
|
||||
|
||||
**PR α — gbrain repo: v0.35.1.0 infra.** All gbrain changes bundled. Lands first.
|
||||
Bisect-friendly commits inside, ship at the very end.
|
||||
|
||||
**PR β — gbrain-evals repo: adapter + smoke + curation + eval receipts + writeup.** The
|
||||
big one. Includes the full eval-run output committed alongside the code that produced
|
||||
it, plus the comparison writeup. Lands when everything is done.
|
||||
|
||||
**PR γ (optional) — gbrain repo: v0.35.2.0 release** that cross-links the gbrain-evals
|
||||
benchmark in CHANGELOG. Small commit; no code changes.
|
||||
|
||||
Total: 2 substantive PRs + 1 optional release commit. **No mid-stream ships.**
|
||||
|
||||
## Conductor sessions
|
||||
|
||||
Each section below is a self-contained brief. Copy-paste into a fresh Conductor session
|
||||
to hand off. Each session ends with a clean deliverable.
|
||||
|
||||
---
|
||||
|
||||
## Session 1 — PR α: gbrain infra (v0.35.1.0)
|
||||
|
||||
**Repo:** `/Users/garrytan/conductor/workspaces/gbrain/<NEW-WORKSPACE>` (fresh from `master`)
|
||||
**Branch:** `garrytan/v0.35.1.0-infra`
|
||||
**Wallclock:** ~2h
|
||||
**API spend:** $0
|
||||
|
||||
### What this session ships
|
||||
Three changes in one PR, bundled so the embedder shootout in gbrain-evals (PR β) has a
|
||||
clean prereq baseline:
|
||||
|
||||
1. Add `voyage:voyage-4-large` ($0.18/M) and `zeroentropyai:zembed-1` ($0.05/M) to the
|
||||
embedding pricing table. Patch the `gbrain models doctor` cost estimator + test.
|
||||
2. Expose `gbrain/ai/gateway` in `package.json` exports map so the gbrain-evals
|
||||
adapters can call `configureGateway({embedding_model, embedding_dimensions, reranker_model})`
|
||||
from outside the gbrain process.
|
||||
3. Add `--resume-from <jsonl>` to `gbrain eval longmemeval` so a mid-run abort
|
||||
(rate-limit, cost-cap, OS interrupt) doesn't lose the cells we already paid for.
|
||||
|
||||
Ships at the end as v0.35.1.0.
|
||||
|
||||
### Prereqs (verify before starting)
|
||||
- On gbrain master at v0.35.0.0 baseline. `cat VERSION` shows `0.35.0.0`.
|
||||
- `bun test` and `bun run verify` both pass on master.
|
||||
|
||||
### Commits (bisect-friendly, one feature per commit)
|
||||
|
||||
```
|
||||
1. feat(pricing): add voyage-4-large + zembed-1 to EMBEDDING_PRICING
|
||||
- src/core/embedding-pricing.ts: add both entries
|
||||
- test/embedding-pricing.test.ts: pin both with $0.18 and $0.05
|
||||
- Verify: bun test test/embedding-pricing.test.ts
|
||||
|
||||
2. feat(exports): expose gbrain/ai/gateway with canary test
|
||||
- package.json: add "./ai/gateway" to exports map
|
||||
- test/public-exports.test.ts: add canary for configureGateway + embed
|
||||
- scripts/check-exports-count.sh: 17 -> 18
|
||||
- Verify: bun run verify
|
||||
|
||||
3. feat(eval): add --resume-from <jsonl> to longmemeval
|
||||
- src/commands/eval-longmemeval.ts: parse flag, skip questions already in input JSONL
|
||||
- test/eval-longmemeval.test.ts: simulated mid-run abort + resume regression
|
||||
- Verify: bun test test/eval-longmemeval.test.ts
|
||||
|
||||
4. chore: v0.35.1.0
|
||||
- VERSION: 0.35.1.0
|
||||
- package.json: 0.35.1.0
|
||||
- CHANGELOG.md: new entry
|
||||
- bun install (refresh lockfile)
|
||||
```
|
||||
|
||||
### Verify before /ship
|
||||
```bash
|
||||
bun run typecheck
|
||||
bun run verify
|
||||
bun test test/embedding-pricing.test.ts test/public-exports.test.ts test/eval-longmemeval.test.ts
|
||||
```
|
||||
|
||||
### Ship
|
||||
```bash
|
||||
/ship
|
||||
```
|
||||
|
||||
### Deliverable
|
||||
- `master` of gbrain at v0.35.1.0
|
||||
- `gbrain/ai/gateway` reachable from external consumers (verified by canary test)
|
||||
- `git tag eval-run-v0.35.1.0-baseline` (annotated, names this exact commit)
|
||||
- `gbrain --version` prints `0.35.1.0`
|
||||
|
||||
### Hand-off to Session 2
|
||||
- gbrain-evals can now `bun update gbrain` to v0.35.1.0
|
||||
- The tag preserves the exact commit for any future reproducibility need
|
||||
|
||||
---
|
||||
|
||||
## Session 2 — PR β setup: gbrain-evals adapter + smoke + subset flag
|
||||
|
||||
**Repo:** `/Users/garrytan/git/gbrain-evals` (or a fresh Conductor workspace cloned from it)
|
||||
**Branch:** `garrytan/embedder-shootout`
|
||||
**Wallclock:** ~3-4h
|
||||
**API spend:** ~$0.10 (smoke verification calls only)
|
||||
|
||||
### What this session ships into PR β (does NOT merge yet)
|
||||
Wire the harness to drive 3 embedding providers via the newly-exposed gbrain gateway:
|
||||
|
||||
1. New typed `EvalAdapterConfig {embedder, dim, reranker?}` passed into each adapter.
|
||||
2. Rewrite `vector.ts` + `hybrid-rrf.ts` to call `configureGateway()` from
|
||||
`gbrain/ai/gateway` instead of the hardcoded `gbrain/embedding` import.
|
||||
3. Critical: hybrid adapter must also route `search.reranker.enabled` (true/false) and
|
||||
`search.mode` (tokenmax) — codex flagged that the existing hybrid never sets these.
|
||||
4. New 3-phase smoke harness: wiring (5 queries × embed roundtrip + dim check) +
|
||||
long-haystack (1 query × 50K-token synthetic haystack) + rerank-payload (1 query
|
||||
× `topNIn=30`). Exit code is the gate.
|
||||
5. New `--include-subset <name>` flag on the BrainBench runner (Cat 13 wiring; subset
|
||||
itself comes in Session 3).
|
||||
|
||||
### Prereqs
|
||||
- Session 1 done. gbrain master at v0.35.1.0.
|
||||
- API keys present: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `VOYAGE_API_KEY`,
|
||||
`ZEROENTROPY_API_KEY`. Smoke fails-loud on missing key.
|
||||
|
||||
### Commits
|
||||
|
||||
```
|
||||
1. chore(deps): bump gbrain pin to v0.35.1.0
|
||||
- package.json + bun.lock
|
||||
- Verify: bun install && bun run typecheck
|
||||
|
||||
2. feat(adapter): typed EvalAdapterConfig + gateway swap
|
||||
- NEW: eval/runner/eval-adapter-config.ts (the type)
|
||||
- eval/runner/adapters/vector.ts: constructor takes EvalAdapterConfig,
|
||||
calls configureGateway({embedding_model, embedding_dimensions})
|
||||
- Drop hardcoded gbrain/embedding import
|
||||
- Verify: existing vector adapter unit tests still pass
|
||||
|
||||
3. feat(adapter): hybrid-rrf wires reranker_enabled + search.mode
|
||||
- eval/runner/adapters/hybrid-rrf.ts: constructor takes EvalAdapterConfig,
|
||||
plumbs search.reranker.enabled + search.mode = tokenmax through
|
||||
- Verify: bun test eval/
|
||||
|
||||
4. feat(smoke): 3-phase smoke harness
|
||||
- NEW: eval/runner/smoke.ts (CLI entry: bun run eval:smoke -- --embedder X --dim Y [--reranker Z])
|
||||
- Phase 1: 5 queries × embed roundtrip, assert vector dim matches config
|
||||
- Phase 2: 1 query × synthetic 50K-token haystack, assert no token-limit error
|
||||
- Phase 3: 1 query × topNIn=30 documents, assert no 5MB payload cap hit
|
||||
- Non-zero exit on any failure
|
||||
- Verify: bun run eval:smoke -- --embedder openai:text-embedding-3-large --dim 1536
|
||||
|
||||
5. feat(runner): --include-subset flag for BrainBench
|
||||
- eval/runner/multi-adapter.ts: parse flag, filter queries by subset tag
|
||||
- Subset itself comes in next commit (Session 3)
|
||||
- Verify: bun run eval:run -- --include-subset cat13-embedder (errors politely because subset file doesn't exist yet)
|
||||
```
|
||||
|
||||
### Smoke verification (run manually before opening PR)
|
||||
```bash
|
||||
bun run eval:smoke -- --embedder openai:text-embedding-3-large --dim 1536
|
||||
bun run eval:smoke -- --embedder voyage:voyage-4-large --dim 2048
|
||||
bun run eval:smoke -- --embedder zeroentropyai:zembed-1 --dim 2560
|
||||
bun run eval:smoke -- --embedder zeroentropyai:zembed-1 --dim 2560 --reranker zeroentropyai:zerank-2
|
||||
```
|
||||
|
||||
All four MUST exit 0. Reports should print the observed vector dim, matching the
|
||||
configured dim.
|
||||
|
||||
### Open PR β
|
||||
```bash
|
||||
gh pr create --base main --title "feat: embedder shootout (adapter + smoke + Cat 13 + eval receipts)" --body "$(cat <<'EOF'
|
||||
## Summary
|
||||
v0.35.0.0 shipped ZeroEntropy zembed-1 + zerank-2 reranker support. This PR runs a head-to-head A/B/C comparison across OpenAI, Voyage, and ZeroEntropy under the new gateway routing.
|
||||
|
||||
This first commit batch lands the harness. Cat 13 curation, Phase 1+2 evals, and the
|
||||
writeup follow in subsequent commits to this same PR.
|
||||
|
||||
## Test plan
|
||||
- [x] Adapter unit tests pass
|
||||
- [x] Smoke harness exits 0 against all 3 providers
|
||||
- [ ] Cat 13 subset committed (Session 3)
|
||||
- [ ] LongMemEval x 7 cells run (Session 4)
|
||||
- [ ] BrainBench x 7 cells run (Session 5)
|
||||
- [ ] Writeup committed (Session 5)
|
||||
|
||||
🤖 Generated with [Claude Code](https://claude.com/claude-code)
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
### Deliverable
|
||||
- PR β open against gbrain-evals `main`, green CI
|
||||
- Smoke verified against all 3 providers (paste the smoke output in the PR body)
|
||||
- Branch ready for Session 3 (Cat 13 curation)
|
||||
|
||||
### Hand-off to Session 3
|
||||
- Branch `garrytan/embedder-shootout` exists on origin
|
||||
- The `--include-subset cat13-embedder` flag is wired but the subset file doesn't exist
|
||||
yet — that's Session 3
|
||||
|
||||
---
|
||||
|
||||
## Session 3 — PR β: Cat 13 conceptual-recall curation
|
||||
|
||||
**Repo:** `/Users/garrytan/git/gbrain-evals`, branch `garrytan/embedder-shootout` (same as Session 2)
|
||||
**Wallclock:** ~3-4h (heavily user-interactive; AI proposes, you review each)
|
||||
**API spend:** $0
|
||||
|
||||
### What this session ships into PR β
|
||||
Hand-curated 50 embedder-sensitive queries from BrainBench's Cat 13 (conceptual recall)
|
||||
corpus. These are the queries where a graph/keyword adapter would likely miss but a
|
||||
semantic adapter would find.
|
||||
|
||||
Codex flagged the existing 145-query relational corpus as graph/keyword-dominated and
|
||||
weak for embedder claims. Cat 13 is closer to the embedder-sensitive workload but
|
||||
needs hand-selection.
|
||||
|
||||
### Prereqs
|
||||
- Session 2 done. PR β open with adapter + smoke + subset flag.
|
||||
|
||||
### Workflow
|
||||
Interactive: Claude proposes queries in batches of 10, you accept/reject/edit each.
|
||||
|
||||
1. Claude reads the existing Cat 13 raw query pool:
|
||||
```bash
|
||||
ls eval/data/raw/ | grep -i cat13
|
||||
cat eval/data/raw/cat13-*.json | jq '.'
|
||||
```
|
||||
2. Claude proposes 10 candidate queries per batch, each tagged with the inclusion
|
||||
reasoning ("would a graph adapter miss this?")
|
||||
3. User accepts/rejects/edits inline. Target: 50 queries × ~5 batches.
|
||||
4. Claude commits to `eval/data/gold/brainbench-cat13-embedder-subset.json`:
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"subset": "cat13-embedder",
|
||||
"queries": [
|
||||
{
|
||||
"id": "cat13-emb-001",
|
||||
"query": "...",
|
||||
"relevant_chunk_ids": ["..."],
|
||||
"inclusion_reason": "paraphrase relationship; graph adapter wouldn't catch the synonym"
|
||||
}
|
||||
// ... 49 more
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Commit
|
||||
|
||||
```
|
||||
feat(eval): curate Cat 13 conceptual-recall subset (50 embedder-sensitive queries)
|
||||
- NEW: eval/data/gold/brainbench-cat13-embedder-subset.json
|
||||
- Each query tagged with inclusion_reason for future audit
|
||||
```
|
||||
|
||||
### Spot-check before commit
|
||||
- Pick 5 random queries, run them against a hypothetical graph adapter (e.g. grep on
|
||||
the relevant terms) and verify they would NOT surface the right chunk.
|
||||
- Run the same 5 against the existing hybrid adapter and verify they DO.
|
||||
|
||||
### Deliverable
|
||||
- `eval/data/gold/brainbench-cat13-embedder-subset.json` committed to PR β
|
||||
- Exactly 50 queries
|
||||
- Spot-check evidence in the commit message
|
||||
|
||||
### Hand-off to Session 4
|
||||
- PR β now has: adapter + smoke + Cat 13 subset
|
||||
- Ready for the actual eval runs
|
||||
|
||||
---
|
||||
|
||||
## Session 4 — PR β Phase 1: LongMemEval × 7 cells (overnight)
|
||||
|
||||
**Repo:** Same gbrain-evals branch
|
||||
**Wallclock:** ~10.5h (mostly hands-off, kick off and walk away)
|
||||
**API spend:** ~$476 (LongMemEval-heavy; 7 × $68/cell)
|
||||
|
||||
### What this session ships into PR β
|
||||
7 LongMemEval scored receipts (one per matrix cell). Each is a JSONL of 500
|
||||
hypotheses + a JSON file of correctness scores from `evaluate_qa.py`.
|
||||
|
||||
### Prereqs
|
||||
- Sessions 1+2+3 done. PR β has adapter + smoke + Cat 13.
|
||||
- LongMemEval dataset downloaded (gated HuggingFace; one-time setup).
|
||||
- `evaluate_qa.py` checked out somewhere (from
|
||||
https://github.com/xiaowu0162/LongMemEval) with its own venv set up.
|
||||
- API keys: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `VOYAGE_API_KEY`,
|
||||
`ZEROENTROPY_API_KEY`.
|
||||
|
||||
### Wrapper script
|
||||
Claude writes `scripts/run-shootout-phase1.sh` in the gbrain-evals branch. Single
|
||||
entry point that loops the 7 cells serially with smoke gating + cost-cap aborts.
|
||||
|
||||
```
|
||||
NEW: scripts/run-shootout-phase1.sh
|
||||
- Per cell: gbrain config set (embedder, dim, reranker, search.reranker.enabled, search.mode=tokenmax)
|
||||
- Per cell: bun run eval:smoke (abort cell on non-zero)
|
||||
- Per cell: gbrain eval longmemeval ... --output results/longmemeval-{cell}.jsonl
|
||||
- Per cell: cost-cap check ($90/cell hard stop)
|
||||
- Per cell: --resume-from existing results/longmemeval-{cell}.jsonl if present
|
||||
- Logs to results/phase1-run-log.txt
|
||||
```
|
||||
|
||||
### Run
|
||||
```bash
|
||||
# Kick off in background; check back in 10-12h
|
||||
bash scripts/run-shootout-phase1.sh 2>&1 | tee results/phase1-run-log.txt &
|
||||
```
|
||||
|
||||
Use `run_in_background: true` if running through Claude. Check back periodically.
|
||||
|
||||
### Scoring (after all 7 cells done)
|
||||
```bash
|
||||
for cell in A0 A1 B0 B1 C0 C1 C2; do
|
||||
python evaluate_qa.py \
|
||||
--input results/longmemeval-${cell}.jsonl \
|
||||
--output results/longmemeval-${cell}-scored.json
|
||||
done
|
||||
```
|
||||
|
||||
Each scored file has correctness %.
|
||||
|
||||
### Commits
|
||||
|
||||
```
|
||||
1. feat(scripts): Phase 1 LongMemEval wrapper with smoke gating + cost cap
|
||||
- NEW: scripts/run-shootout-phase1.sh
|
||||
|
||||
2. data(phase1): 7 LongMemEval cells (raw hypothesis JSONL)
|
||||
- results/longmemeval-{A0,A1,B0,B1,C0,C1,C2}.jsonl
|
||||
- results/phase1-run-log.txt (run timing + cost ledger)
|
||||
|
||||
3. data(phase1): evaluate_qa.py scoring results
|
||||
- results/longmemeval-{cell}-scored.json × 7
|
||||
```
|
||||
|
||||
### Verify
|
||||
- Each `longmemeval-{cell}.jsonl` has exactly 500 lines
|
||||
- Each `hypothesis` field is non-empty AND is actual answer text (NOT retrieval text)
|
||||
- Each `scored.json` has a `correctness_score` field
|
||||
|
||||
### Deliverable
|
||||
- 7 scored LongMemEval receipts committed to PR β
|
||||
- Real cost ledger committed alongside (compare against estimate)
|
||||
|
||||
### Hand-off to Session 5
|
||||
- Phase 1 done. Phase 2 (BrainBench, ~3.5h) and writeup remaining.
|
||||
|
||||
---
|
||||
|
||||
## Session 5 — PR β Phase 2 + writeup + ship
|
||||
|
||||
**Repo:** Same gbrain-evals branch
|
||||
**Wallclock:** ~7h (3.5h BrainBench + 3h writeup + /ship)
|
||||
**API spend:** ~$56 (BrainBench is cheap)
|
||||
|
||||
### What this session ships into PR β
|
||||
- 7 BrainBench cells (relational corpus + Cat 13 subset)
|
||||
- Final comparison writeup
|
||||
- PR β merged
|
||||
|
||||
### Prereqs
|
||||
- Session 4 done. PR β has Phase 1 receipts.
|
||||
|
||||
### Phase 2 wrapper script
|
||||
```
|
||||
NEW: scripts/run-shootout-phase2.sh
|
||||
- Per cell: configure provider (same as Phase 1)
|
||||
- Per cell: bun run eval:run -- --N 10 --include-subset cat13-embedder
|
||||
--output docs/benchmarks/2026-05-22-{cell}.md
|
||||
- Cost-cap check
|
||||
```
|
||||
|
||||
### Run
|
||||
```bash
|
||||
bash scripts/run-shootout-phase2.sh 2>&1 | tee results/phase2-run-log.txt
|
||||
```
|
||||
|
||||
### Writeup
|
||||
`docs/benchmarks/2026-05-22-embedder-shootout.md`. Structure:
|
||||
|
||||
1. **Headline table** — 7 cells × {LongMemEval correctness %, BrainBench relational MRR + P@5, Cat 13 correctness %, total cost}
|
||||
2. **Two questions answered:**
|
||||
- Which embedder wins solo? (A0 vs B0 vs C0)
|
||||
- Does zerank-2 carry ZE's win? (C0 vs C1 vs A1 vs B1)
|
||||
- Bonus: does dim matter for ZE? (C1 vs C2)
|
||||
3. **Paired-bootstrap p-values** per headline pair (methodology in
|
||||
`gbrain/docs/eval/SEARCH_MODE_METHODOLOGY.md`)
|
||||
4. **HNSW footnote** — Voyage 2048 and ZE 2560 used exact vector scan; OpenAI 1536
|
||||
and ZE 1280 used HNSW. Quality is primary, latency is secondary
|
||||
5. **What this does NOT prove** — synthetic-only, tokenmax-only, no real-brain replay
|
||||
6. **Recommendation:** explicit NON-recommendation to change `gbrain init` default;
|
||||
defer to a v0.36.x evidence pass with real-brain replay data
|
||||
|
||||
### Commits
|
||||
|
||||
```
|
||||
1. feat(scripts): Phase 2 BrainBench wrapper
|
||||
- NEW: scripts/run-shootout-phase2.sh
|
||||
|
||||
2. data(phase2): 7 BrainBench cells
|
||||
- docs/benchmarks/2026-05-22-{cell}.md × 7
|
||||
|
||||
3. docs(benchmark): embedder shootout comparison writeup
|
||||
- NEW: docs/benchmarks/2026-05-22-embedder-shootout.md
|
||||
- Bootstrap p-values, HNSW footnote, NOT-in-scope section
|
||||
```
|
||||
|
||||
### Ship
|
||||
```bash
|
||||
# Merge PR β to gbrain-evals main
|
||||
gh pr merge --squash --auto
|
||||
# Or non-auto if reviewing one more time:
|
||||
gh pr merge --squash
|
||||
```
|
||||
|
||||
### Deliverable
|
||||
- PR β merged to gbrain-evals `main`
|
||||
- Comparison report public at
|
||||
`gbrain-evals/docs/benchmarks/2026-05-22-embedder-shootout.md`
|
||||
|
||||
### Hand-off to Session 6 (optional)
|
||||
- gbrain-evals master has the full data + writeup
|
||||
- Ready for a v0.35.2.0 gbrain release that cross-links it
|
||||
|
||||
---
|
||||
|
||||
## Session 6 (optional) — PR γ: gbrain v0.35.2.0 release
|
||||
|
||||
**Repo:** `/Users/garrytan/conductor/workspaces/gbrain/<NEW-WORKSPACE>` (fresh from master)
|
||||
**Branch:** `garrytan/v0.35.2.0-benchmark-release`
|
||||
**Wallclock:** ~30min
|
||||
**API spend:** $0
|
||||
|
||||
### What this session ships
|
||||
A release-notes-only PR that bumps gbrain to v0.35.2.0 with a CHANGELOG entry
|
||||
cross-linking the embedder shootout benchmark. Optional — could be folded into the
|
||||
next routine release if no rush.
|
||||
|
||||
### Prereqs
|
||||
- Session 5 done. gbrain-evals merged with the comparison writeup.
|
||||
|
||||
### Commits
|
||||
|
||||
```
|
||||
1. docs(benchmark): mirror embedder shootout summary
|
||||
- NEW: docs/benchmarks/2026-05-22-embedder-shootout.md (slim mirror)
|
||||
- Cross-link to gbrain-evals canonical version
|
||||
|
||||
2. chore: v0.35.2.0
|
||||
- VERSION: 0.35.2.0
|
||||
- package.json: 0.35.2.0
|
||||
- CHANGELOG.md: new entry with the GStack-voice release summary
|
||||
+ "numbers that matter" table from the benchmark
|
||||
```
|
||||
|
||||
### Ship
|
||||
```bash
|
||||
/ship
|
||||
```
|
||||
|
||||
### Deliverable
|
||||
- gbrain v0.35.2.0 on master
|
||||
- CHANGELOG entry that drives the release-note headline
|
||||
|
||||
---
|
||||
|
||||
## Cost ledger (revised, post-review)
|
||||
|
||||
| Component | Per cell | × 7 cells |
|
||||
|---|---|---|
|
||||
| LongMemEval embed | <$0.05 | <$0.35 |
|
||||
| LongMemEval Sonnet answer-gen (500q × 2K tokens × $3/M) | $18 | $126 |
|
||||
| LongMemEval gpt-4o judge (500q × $0.10/q) | $50 | $350 |
|
||||
| BrainBench relational embed | $0.05-0.18 | <$1 |
|
||||
| BrainBench Cat 13 answer-gen + judge (50q × $0.14) | $7 | $49 |
|
||||
| Smoke harness (30 calls/cell) | <$0.10 | <$1 |
|
||||
| **Total** | **~$75/cell** | **~$525** |
|
||||
|
||||
**Hard cap: $700.** Per-cell hard cap: $90 (wrapper aborts cell if exceeded; partial
|
||||
JSONL preserved for resume).
|
||||
|
||||
## Failure modes and recovery
|
||||
|
||||
| Failure | Recovery |
|
||||
|---|---|
|
||||
| Voyage/ZE 429 rate-limit mid-cell | `gateway._shrinkState` halves safety_factor and retries. Cell continues. |
|
||||
| ZE 5MB rerank payload cap hit | `applyReranker` fail-opens, returns un-reranked results. Stderr warn. |
|
||||
| Mid-cell OS interrupt / cost-cap abort | Re-run with `gbrain eval longmemeval --resume-from results/longmemeval-{cell}.jsonl`. Picks up where it left off. |
|
||||
| `evaluate_qa.py` auth fail | OPENAI_API_KEY check in wrapper aborts before any spend. |
|
||||
| Adapter typo (bad dim) | `EvalAdapterConfig` runtime assertion at constructor throws AIConfigError. Cell aborts before API call. |
|
||||
|
||||
## NOT in scope (deliberate)
|
||||
|
||||
- **Real `~/.gbrain` replay** — adds 6-12h wallclock + $40-80 embed. Filed as v0.36.x.
|
||||
- **All 3 search modes** — pinned to tokenmax. `conservative` + `balanced` are v0.35.3.0
|
||||
follow-ups if reviewers push back.
|
||||
- **Matched-dim cross-vendor row** — no shared dim exists across all 3 vendors.
|
||||
Permanently out.
|
||||
- **`gbrain eval whoknows` / `cross-modal` / `takes-quality`** — embedding-invariant;
|
||||
rerunning across embedders produces noise.
|
||||
- **`gbrain eval code-retrieval`** — code corpus, separate concern.
|
||||
- **`gbrain eval suspected-contradictions`** — wants a real brain.
|
||||
- **`gbrain init --recommended` default change** — codex correctly flagged the evidence
|
||||
base as insufficient. Defer to v0.36.x with real-brain replay data.
|
||||
|
||||
## What already exists (reused, not rebuilt)
|
||||
|
||||
- `gbrain eval longmemeval` CLI (in-tree, answer-gen mode default)
|
||||
- gbrain-evals BrainBench runner (`eval:run`) — needs adapter parameterization but
|
||||
per-cell test plumbing is reused
|
||||
- Gateway routing for Voyage + ZE (shipped v0.35.0.0)
|
||||
- Reranker pipeline (`src/core/search/rerank.ts`, fail-open)
|
||||
- Pricing table (extended, not rebuilt)
|
||||
- Paired-bootstrap methodology (`docs/eval/SEARCH_MODE_METHODOLOGY.md`)
|
||||
- LongMemEval published `evaluate_qa.py` (invoked externally, not bundled)
|
||||
@@ -1,105 +0,0 @@
|
||||
# Switching embedding models or dimensions on an existing brain
|
||||
|
||||
GBrain stores embeddings in a fixed-dimension `vector(N)` column on
|
||||
`content_chunks`. If you switch to a model with a different dimension
|
||||
(e.g. `text-embedding-3-large` 1536 → `voyage-multilingual-large-2` 2048,
|
||||
or back to a smaller model like `nomic-embed-text` 768), the on-disk
|
||||
column type doesn't change automatically.
|
||||
|
||||
`gbrain init` and `gbrain doctor` both detect and refuse to silently
|
||||
proceed in this case. This doc is the recipe they point at.
|
||||
|
||||
## Why we don't do this automatically
|
||||
|
||||
Switching dimensions requires:
|
||||
|
||||
1. Dropping the HNSW vector index (pgvector won't survive an `ALTER COLUMN TYPE`).
|
||||
2. Altering the column type.
|
||||
3. Wiping every existing embedding (the old vectors are unusable in the new space).
|
||||
4. Re-embedding the entire corpus (can take hours on a 50K-page brain and costs $1-100 in API calls depending on model).
|
||||
5. Conditionally recreating the index (HNSW supports up to 2000 dimensions per pgvector; above that you must use exact scans).
|
||||
|
||||
That's not an upgrade-time auto-run. It's a deliberate, expensive
|
||||
operation. Run it when you've decided you actually want the new model.
|
||||
|
||||
## Recipe — manual `psql` against your brain
|
||||
|
||||
Replace `<NEW_DIMS>` with your target dimension count.
|
||||
|
||||
```sql
|
||||
BEGIN;
|
||||
|
||||
-- 1. Drop the HNSW index. It can't survive the column type change.
|
||||
DROP INDEX IF EXISTS idx_chunks_embedding;
|
||||
|
||||
-- 2. Alter the column type. (You can DROP COLUMN + ADD COLUMN instead
|
||||
-- if the existing data is already gone — same end state.)
|
||||
ALTER TABLE content_chunks ALTER COLUMN embedding TYPE vector(<NEW_DIMS>);
|
||||
|
||||
-- 3. Clear stale embeddings so they don't survive into the new space.
|
||||
-- Either truncate (faster, drops all chunks) or null out (preserves
|
||||
-- chunk text so re-embed regenerates without re-chunking):
|
||||
UPDATE content_chunks SET embedding = NULL, embedded_at = NULL;
|
||||
|
||||
-- 4. Recreate the HNSW index ONLY IF dims <= 2000. Above that, leave it
|
||||
-- indexless and rely on exact scans (gbrain searchVector handles this
|
||||
-- automatically — search just gets slower, not broken).
|
||||
-- For dims <= 2000 (e.g. 1024, 1536, 768):
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_embedding
|
||||
ON content_chunks USING hnsw (embedding vector_cosine_ops);
|
||||
-- For dims > 2000 (e.g. 2048 Voyage 4 Large): skip step 4.
|
||||
|
||||
COMMIT;
|
||||
```
|
||||
|
||||
Then update gbrain's config so it knows the new dim:
|
||||
|
||||
```bash
|
||||
gbrain config set embedding_model <model>
|
||||
gbrain config set embedding_dimensions <NEW_DIMS>
|
||||
```
|
||||
|
||||
And re-embed the corpus:
|
||||
|
||||
```bash
|
||||
gbrain embed --stale
|
||||
```
|
||||
|
||||
## PGLite (local brain)
|
||||
|
||||
Same recipe, but you connect to the embedded database differently:
|
||||
|
||||
```bash
|
||||
gbrain config get database_url # confirm engine: pglite
|
||||
# Open a psql-equivalent — for PGLite, the easiest path is to write a small
|
||||
# script that imports PGLiteEngine and runs the SQL via engine.executeRaw.
|
||||
# Or migrate to Postgres temporarily (gbrain migrate --to supabase) if you
|
||||
# want a real psql connection.
|
||||
```
|
||||
|
||||
For most PGLite users the simpler path is to **wipe and re-init** if your
|
||||
corpus is small enough that re-syncing is faster than hand-crafting the
|
||||
migration:
|
||||
|
||||
```bash
|
||||
mv ~/.gbrain/brain.pglite ~/.gbrain/brain.pglite.bak
|
||||
gbrain init --pglite --embedding-dimensions <NEW_DIMS>
|
||||
gbrain sync # re-imports your brain repo from disk
|
||||
```
|
||||
|
||||
## Verify
|
||||
|
||||
After the recipe lands, `gbrain doctor --fast` should report green and
|
||||
`gbrain doctor` (full) should say check 8b passes:
|
||||
|
||||
```
|
||||
✓ embedding_provider dim parity: config 768 / column vector(768) / live probe 768
|
||||
```
|
||||
|
||||
If it doesn't, file an issue with the doctor output and the SQL you ran.
|
||||
|
||||
## v0.29+ plans
|
||||
|
||||
`gbrain migrate-embedding-dim --to <N>` is a tracked TODO. It will run
|
||||
the recipe above with progress reporting + an explicit confirmation
|
||||
gate. Until that lands, this manual recipe is the canonical path.
|
||||
@@ -1,330 +0,0 @@
|
||||
# Running real-world eval benchmarks against your gbrain changes
|
||||
|
||||
Audience: gbrain maintainers and contributors. If you're touching retrieval
|
||||
(search, ranking, embeddings, intent classification, query expansion, source
|
||||
boost, hybrid fusion), this is the doc.
|
||||
|
||||
For the **NDJSON wire format** consumed by gbrain-evals, see
|
||||
[`eval-capture.md`](./eval-capture.md). This doc is the human dev loop
|
||||
that lives on top of that format.
|
||||
|
||||
## Prerequisite: turn on contributor mode
|
||||
|
||||
Capture is **off by default** for production users (privacy-positive — no
|
||||
surprise data accumulation). Contributors flip it on with one line:
|
||||
|
||||
```bash
|
||||
# In ~/.zshrc or ~/.bashrc:
|
||||
export GBRAIN_CONTRIBUTOR_MODE=1
|
||||
```
|
||||
|
||||
Verify:
|
||||
|
||||
```bash
|
||||
gbrain query "anything" >/dev/null
|
||||
psql $DATABASE_URL -c 'SELECT count(*) FROM eval_candidates' # should be > 0
|
||||
```
|
||||
|
||||
To override (force on/off regardless of env var), edit `~/.gbrain/config.json`:
|
||||
|
||||
```json
|
||||
{"eval": {"capture": true}} // force on
|
||||
{"eval": {"capture": false}} // force off
|
||||
```
|
||||
|
||||
Explicit config beats the env var both directions.
|
||||
|
||||
## The 4-command loop
|
||||
|
||||
```bash
|
||||
# ① Capture: writes to eval_candidates whenever CONTRIBUTOR_MODE is set.
|
||||
# Inspect what's been collected:
|
||||
gbrain doctor # surfaces capture failures
|
||||
psql $DATABASE_URL -c 'SELECT count(*) FROM eval_candidates'
|
||||
|
||||
# ② Snapshot: freeze a baseline before your code change.
|
||||
gbrain eval export --since 7d > baseline.ndjson
|
||||
|
||||
# ③ Code change: do whatever you want — tune RRF_K, swap embed model, edit
|
||||
# hybrid.ts, add a new boost source, change the intent classifier.
|
||||
|
||||
# ④ Replay: re-run every captured query against the current build.
|
||||
gbrain eval replay --against baseline.ndjson
|
||||
```
|
||||
|
||||
Output:
|
||||
|
||||
```
|
||||
Replaying 247 captured queries…
|
||||
...25/247
|
||||
...50/247
|
||||
...
|
||||
Replayed 247 of 247 captured queries (0 skipped, 0 errored)
|
||||
Mean Jaccard@k: 0.927
|
||||
Top-1 stability: 91.5%
|
||||
Mean latency Δ: +14ms (current vs captured)
|
||||
|
||||
Top 5 regression(s):
|
||||
jaccard=0.20 captured=12 current=3 "find every reference to widget-co"
|
||||
jaccard=0.43 captured=14 current=8 "show me everything tagged for review"
|
||||
jaccard=0.50 captured=8 current=4 "what did alice say about the spec"
|
||||
...
|
||||
```
|
||||
|
||||
Three numbers tell you whether the change is safe to land:
|
||||
|
||||
| Metric | What it means | Healthy range |
|
||||
|---|---|---|
|
||||
| **Mean Jaccard@k** | Average overlap between captured retrieved slugs and current run's slugs. 1.0 = identical sets. | ≥0.85 for "neutral" changes. <0.7 means major retrieval shift. |
|
||||
| **Top-1 stability** | Fraction of queries whose #1 result didn't change. | ≥85% for tuning passes. <70% means top-of-funnel broke. |
|
||||
| **Mean latency Δ** | Current minus captured. Positive = slower now. | Within ±50ms of captured. >2× anywhere = regression alarm. |
|
||||
|
||||
## What it actually does
|
||||
|
||||
`gbrain eval replay` reads your NDJSON snapshot and, for each row:
|
||||
|
||||
1. Re-executes the same op (`searchKeyword` for `tool_name='search'`,
|
||||
`hybridSearch` for `tool_name='query'`) with the captured `detail` and
|
||||
`expand_enabled` values threaded back in.
|
||||
2. Captures the current `retrieved_slugs` (deduped, in result order).
|
||||
3. Computes set-Jaccard between captured and current slug sets.
|
||||
4. Records top-1 match (was the #1 result the same slug?).
|
||||
5. Records latency delta vs captured `latency_ms`.
|
||||
|
||||
It does NOT compute MRR or nDCG — those need ground-truth relevance labels,
|
||||
not a baseline comparison. For metric-against-truth eval, use
|
||||
`gbrain eval --qrels <path>` (the legacy IR-eval path, still supported). The
|
||||
replay tool answers a different question: "did my code change move
|
||||
retrieval, and which queries did it move most?"
|
||||
|
||||
For a third evaluation axis — public benchmark, ground-truth labels, full
|
||||
question-answer pipeline (not just retrieval) — `gbrain eval longmemeval
|
||||
<dataset.jsonl>` (v0.28.8) runs the LongMemEval benchmark against gbrain's
|
||||
hybrid retrieval. Each question gets a clean in-memory PGLite, its haystack
|
||||
imported, the question asked, the hypothesis emitted as JSONL — exactly the
|
||||
shape LongMemEval's `evaluate_qa.py` consumes. Your `~/.gbrain` brain is
|
||||
never opened. See `## Public benchmarks: LongMemEval` below.
|
||||
|
||||
## Best-effort by design
|
||||
|
||||
Replay is not pure. Three things can drift between capture and replay:
|
||||
|
||||
1. **Brain state** — your brain probably has more pages now than when the
|
||||
snapshot was taken. Unless you explicitly seed a fixed corpus, mean
|
||||
Jaccard will drop simply because new pages are eligible.
|
||||
2. **Embedding source** — if you changed `OPENAI_API_KEY` between capture
|
||||
and replay (or the embedding model rotated), vector-path results drift
|
||||
even with identical code.
|
||||
3. **Capture cap** — captured `retrieved_slugs` is a deduped set; it doesn't
|
||||
preserve internal ranking metadata. Two tools can return the same slug
|
||||
set with different scores — Jaccard will say 1.0, but a downstream
|
||||
consumer that orders by score may behave differently.
|
||||
|
||||
The metrics are **regression alarms on real queries**, not a hash check.
|
||||
Pair them with manual inspection of the top regressions.
|
||||
|
||||
## Cost
|
||||
|
||||
Every `query` row in the snapshot embeds the query string via OpenAI to run
|
||||
the vector half of `hybridSearch`. Cost is identical to a normal `gbrain
|
||||
query` invocation — text-embedding-3-large at OpenAI list price, batched
|
||||
inside a single replay row.
|
||||
|
||||
If you're iterating locally and don't want to pay per change, use
|
||||
`--limit 50` to cap rows replayed. The 50 most recent rows are usually
|
||||
enough to catch direction; expand for the final pre-merge run.
|
||||
|
||||
```bash
|
||||
# Iteration mode — 50 most recent queries
|
||||
gbrain eval replay --against baseline.ndjson --limit 50
|
||||
|
||||
# Pre-merge — full snapshot
|
||||
gbrain eval replay --against baseline.ndjson --top-regressions 20
|
||||
```
|
||||
|
||||
## CI integration
|
||||
|
||||
```bash
|
||||
gbrain eval replay --against baseline.ndjson --json > replay.json
|
||||
jq -e '.summary.mean_jaccard >= 0.85' replay.json || exit 1
|
||||
jq -e '.summary.top1_stability_rate >= 0.85' replay.json || exit 1
|
||||
```
|
||||
|
||||
Stable JSON shape (schema_version: 1):
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"summary": {
|
||||
"rows_total": 247,
|
||||
"rows_replayed": 247,
|
||||
"rows_skipped": 0,
|
||||
"rows_errored": 0,
|
||||
"mean_jaccard": 0.927,
|
||||
"top1_stability_rate": 0.915,
|
||||
"mean_latency_delta_ms": 14,
|
||||
"rows_over_2x_latency": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`--verbose` adds a `results: [...]` array with one entry per replayed row
|
||||
(useful for piping into jq or a notebook for deeper analysis).
|
||||
|
||||
## When to run this
|
||||
|
||||
Before merging anything that touches:
|
||||
|
||||
- `src/core/search/hybrid.ts` (RRF, fusion, dedup, two-pass retrieval)
|
||||
- `src/core/search/source-boost.ts` / `sql-ranking.ts` (per-source ranking)
|
||||
- `src/core/search/intent.ts` (auto-detail classification)
|
||||
- `src/core/search/expansion.ts` (Haiku query expansion)
|
||||
- `src/core/search/dedup.ts` (cross-page result collapse)
|
||||
- `src/core/embedding.ts` or any embedding model swap
|
||||
- `src/core/operations.ts` `query` or `search` op handlers (capture surface)
|
||||
- `src/core/postgres-engine.ts` / `pglite-engine.ts` `searchKeyword` /
|
||||
`searchVector` SQL
|
||||
|
||||
Skip for: schema-only migrations, doc changes, tests-only PRs, CLI ergonomics
|
||||
that don't touch retrieval.
|
||||
|
||||
## Building your own corpus
|
||||
|
||||
If you don't have captured traffic yet (fresh install, can't dogfood for a
|
||||
week before merging), you can hand-author an NDJSON file:
|
||||
|
||||
```jsonl
|
||||
{"schema_version":1,"id":1,"tool_name":"query","query":"who is alice","retrieved_slugs":["people/alice","people/alice-bio"],"expand_enabled":false,"detail":null,"latency_ms":0,"remote":false}
|
||||
{"schema_version":1,"id":2,"tool_name":"search","query":"acme deal","retrieved_slugs":["deals/acme-seed","companies/acme"],"latency_ms":0,"remote":false}
|
||||
```
|
||||
|
||||
Then run `gbrain eval replay --against handcrafted.ndjson` to confirm the
|
||||
authoritative slugs come back. This is the seam between the BrainBench-Real
|
||||
pipeline (replay against live captures) and the BrainBench fixed-fixture
|
||||
pipeline (`gbrain eval --qrels` with the sibling
|
||||
[gbrain-evals](https://github.com/garrytan/gbrain-evals) corpus).
|
||||
|
||||
## Off-switch
|
||||
|
||||
Two ways to disable capture:
|
||||
|
||||
```bash
|
||||
unset GBRAIN_CONTRIBUTOR_MODE # easy: just unset the env var
|
||||
```
|
||||
|
||||
Or force off regardless of the env var via `~/.gbrain/config.json`:
|
||||
|
||||
```json
|
||||
{"eval": {"capture": false}}
|
||||
```
|
||||
|
||||
Existing `eval_candidates` rows stay until you `gbrain eval prune
|
||||
--older-than 0d` (or just drop the table).
|
||||
|
||||
## Failure modes
|
||||
|
||||
| What you see | What it means |
|
||||
|---|---|
|
||||
| `Mean Jaccard@k: 0.4`, top regressions all in one source dir | Source boost or hard-exclude regression on that prefix |
|
||||
| `Top-1 stability: 30%`, mean Jaccard still high | RRF tuning shifted the rank order without changing the set — re-tune `rrfK` |
|
||||
| `Mean latency Δ: +500ms`, jaccard high | Vector path got slower; check embedding API or HNSW probes |
|
||||
| `rows_errored > 0` | One or more queries threw. Inspect first 3 in human output, or `--json` to see all `error_message` fields |
|
||||
| Many `skipped: empty query` | Capture ran on rows where someone passed empty `query` — check why those were captured |
|
||||
|
||||
## Public benchmarks: LongMemEval (v0.28.8)
|
||||
|
||||
`gbrain eval longmemeval` runs the public [LongMemEval](https://huggingface.co/datasets/xiaowu0162/longmemeval)
|
||||
benchmark directly against gbrain's hybrid retrieval. Different evaluation
|
||||
axis from `eval replay`: public dataset with ground-truth labels, end-to-end
|
||||
question-answer pipeline, hermetic per-question brains.
|
||||
|
||||
```bash
|
||||
# Download the dataset (visit the HF page in a browser; gated/manual download).
|
||||
# Place longmemeval_oracle.json (or _s.json) somewhere local.
|
||||
|
||||
# Retrieval-only (no LLM answer-gen, fastest path, no Anthropic key needed):
|
||||
gbrain eval longmemeval ./longmemeval_oracle.json --limit 50 --retrieval-only \
|
||||
> /tmp/hypothesis.jsonl
|
||||
|
||||
# Full pipeline (Anthropic key required for answer-gen):
|
||||
gbrain eval longmemeval ./longmemeval_oracle.json --limit 50 \
|
||||
> /tmp/hypothesis.jsonl
|
||||
|
||||
# Score with LongMemEval's published evaluate_qa.py (not bundled — needs
|
||||
# OpenAI gpt-4o per their spec):
|
||||
python evaluate_qa.py /tmp/hypothesis.jsonl
|
||||
```
|
||||
|
||||
### Architecture (read this if you're touching the harness)
|
||||
|
||||
- One in-memory PGLite per benchmark run via `createBenchmarkBrain` +
|
||||
`withBenchmarkBrain`. Your `~/.gbrain` is never opened.
|
||||
- Between questions: `TRUNCATE` over runtime-enumerated `pg_tables`, NOT a
|
||||
hardcoded list — schema migrations don't silently leak data across
|
||||
questions. Infrastructure tables (`sources`, `config`,
|
||||
`gbrain_cycle_locks`, `subagent_rate_leases`) are preserved across resets.
|
||||
- Sanitization parity: re-uses `INJECTION_PATTERNS` from
|
||||
`src/core/think/sanitize.ts` so adding a new injection pattern
|
||||
automatically covers takes AND benchmarks. One source of truth.
|
||||
- Retrieved chat content is wrapped in `<chat_session id="..." date="...">`
|
||||
framing; the answer-gen system prompt declares the content UNTRUSTED.
|
||||
Same posture as `<take>` framing.
|
||||
- LLM injection seam: `runEvalLongMemEval(args, {client?: ThinkLLMClient})`.
|
||||
Tests stub the client so the full pipeline runs hermetically without any
|
||||
API key.
|
||||
|
||||
### Flags
|
||||
|
||||
| Flag | Default | Purpose |
|
||||
|---|---|---|
|
||||
| `--limit N` | run all | Cap question count (iterate fast) |
|
||||
| `--retrieval-only` | off | Emit retrieved chunks; no LLM answer-gen |
|
||||
| `--keyword-only` | off | Disable vector path (debug retrieval issues) |
|
||||
| `--expansion` | **off** | Multi-query expansion. Off by default for determinism (no per-query Haiku call). Pass to opt in. |
|
||||
| `--top-k K` | 10 | Retrieval depth |
|
||||
| `--model M` | resolved | Default resolves through `resolveModel()` 6-tier chain (`models.eval.longmemeval` config key) |
|
||||
| `--output FILE` | stdout | Write hypothesis JSONL to file instead of stdout |
|
||||
|
||||
### Numbers
|
||||
|
||||
p50 25.9ms / p99 30.3ms warm reset+import+search on Apple Silicon (per the
|
||||
`test/eval-longmemeval.test.ts` perf gate). Per-question cost well under the
|
||||
500ms speed gate. 500 questions = ~13s of overhead plus your retrieval and
|
||||
LLM latency.
|
||||
|
||||
## Measuring brain consistency over time (v0.32.6)
|
||||
|
||||
`gbrain eval suspected-contradictions` is a complementary measurement
|
||||
instrument: it samples retrieval results for unmarked semantic
|
||||
contradictions (e.g., compiled_truth vs chat content, intra-page chunk
|
||||
vs active take). Where LongMemEval measures retrieval correctness on a
|
||||
fixed labeled set, the contradiction probe measures how often a real
|
||||
brain surfaces conflicting answers.
|
||||
|
||||
### Recommended nightly cadence
|
||||
|
||||
```bash
|
||||
# Once a day, against your top 50 most-frequent queries:
|
||||
gbrain eval suspected-contradictions \
|
||||
--queries-file ~/.gbrain/queries.jsonl \
|
||||
--top-k 5 \
|
||||
--budget-usd 5 \
|
||||
--output ~/.gbrain/probe-runs/$(date +%Y-%m-%d).json
|
||||
```
|
||||
|
||||
Persistent cache (`eval_contradictions_cache`) makes re-runs near-zero
|
||||
cost until you bump `PROMPT_VERSION`. Trend-track via:
|
||||
|
||||
```bash
|
||||
gbrain eval suspected-contradictions trend --days 30
|
||||
```
|
||||
|
||||
The ASCII bar chart shows total flagged per day. Headline % surfaces in
|
||||
`gbrain doctor`'s `contradictions` check with paste-ready resolution
|
||||
commands per high-severity finding.
|
||||
|
||||
### See also
|
||||
|
||||
- `docs/contradictions.md` — architecture, severity rubric, action criteria.
|
||||
- CHANGELOG `## [0.32.6]` — full release notes including the bigger-swing
|
||||
decision criteria gated on Wilson CI lower-bound.
|
||||
@@ -1,160 +0,0 @@
|
||||
# Eval capture — NDJSON schema reference
|
||||
|
||||
**Status:** stable from v0.21.0. Schema versioning via `schema_version`
|
||||
on every row; additive changes increment the minor version; removals
|
||||
are breaking-schema-v2.
|
||||
|
||||
**Audience:** downstream consumers (primarily the sibling
|
||||
[gbrain-evals](https://github.com/garrytan/gbrain-evals) repo) that
|
||||
replay captured real-world queries as a BrainBench-Real fixture.
|
||||
|
||||
## The pipeline
|
||||
|
||||
```
|
||||
MCP / CLI / subagent tool-bridge caller
|
||||
│
|
||||
▼
|
||||
src/core/operations.ts — query + search op handlers
|
||||
│
|
||||
│ (hybridSearch or searchKeyword)
|
||||
│
|
||||
▼
|
||||
{results, meta: HybridSearchMeta} ┌── captureEvalCandidate
|
||||
│ │ (fire-and-forget)
|
||||
▼ │
|
||||
return to caller ▼
|
||||
scrubPii(query) ←── src/core/eval-capture-scrub.ts
|
||||
│
|
||||
▼
|
||||
buildEvalCandidateInput
|
||||
│
|
||||
▼
|
||||
engine.logEvalCandidate
|
||||
│
|
||||
┌──────────────┴──────────────┐
|
||||
│ success │ fail
|
||||
▼ ▼
|
||||
INSERT into eval_candidates engine.logEvalCaptureFailure
|
||||
(reason: db_down | rls_reject |
|
||||
check_violation |
|
||||
scrubber_exception | other)
|
||||
```
|
||||
|
||||
## `gbrain eval export` — the consumer contract
|
||||
|
||||
```sh
|
||||
gbrain eval export [--since DUR] [--limit N] [--tool query|search]
|
||||
```
|
||||
|
||||
Emits NDJSON to **stdout**. One JSON object per `\n`-terminated line.
|
||||
stderr receives progress heartbeats. Every line starts with
|
||||
`"schema_version": 1` so a forward-compat parser can fail loudly on
|
||||
schema v2 instead of silently misparsing.
|
||||
|
||||
Typical usage from gbrain-evals:
|
||||
|
||||
```sh
|
||||
# Snapshot the last week of real traffic for replay
|
||||
gbrain eval export --since 7d > brainbench-real.ndjson
|
||||
```
|
||||
|
||||
```sh
|
||||
# Stream through jq for ad-hoc analysis
|
||||
gbrain eval export --tool query | jq -c 'select(.latency_ms > 500)'
|
||||
```
|
||||
|
||||
## Row schema (v1)
|
||||
|
||||
Every exported row has this shape. Field order in JSON output is not
|
||||
guaranteed; consumers MUST key by name, not position.
|
||||
|
||||
| Field | Type | Notes |
|
||||
|---|---|---|
|
||||
| `schema_version` | number | Always `1` on v1 rows. Forward-compat gate. |
|
||||
| `id` | number | Autoincrement primary key. Stable across exports. |
|
||||
| `tool_name` | `"query"` \| `"search"` | Which MCP operation captured this row. |
|
||||
| `query` | string | **Already PII-scrubbed** by `scrubPii` unless `eval.scrub_pii: false`. Emails / phones / SSN / Luhn-verified credit cards / JWTs / bearer tokens replaced with `[REDACTED]`. Max length 50KB (CHECK-enforced). |
|
||||
| `retrieved_slugs` | string[] | Deduplicated slugs that came back in `SearchResult[]`. |
|
||||
| `retrieved_chunk_ids` | number[] | Every chunk id in result order (duplicates preserved — one per hit). |
|
||||
| `source_ids` | string[] | Distinct `sources.id` values across the result set (v0.18 multi-source). Empty for pre-v0.18 rows that lacked the column. |
|
||||
| `expand_enabled` | boolean \| null | Whether the caller **requested** Haiku expansion. `null` for `search` (no expansion concept). |
|
||||
| `detail` | `"low"` \| `"medium"` \| `"high"` \| null | Detail level the caller **requested**. `null` when omitted. |
|
||||
| `detail_resolved` | `"low"` \| `"medium"` \| `"high"` \| null | What `hybridSearch` **actually used** after auto-detect. `null` when neither caller nor heuristic classified. |
|
||||
| `vector_enabled` | boolean | True iff vector search actually ran. `false` when `OPENAI_API_KEY` was missing or the embed call failed. **Replay MUST respect this** — rows with `false` only exercised the keyword path. |
|
||||
| `expansion_applied` | boolean | True iff Haiku expansion actually produced variants (not just "was requested"). |
|
||||
| `latency_ms` | number | Wall-clock duration of the op handler (includes capture itself — negligible since it's fire-and-forget). |
|
||||
| `remote` | boolean | `true` for MCP callers (untrusted), `false` for local CLI. Partitions "real agent traffic" from "operator probing." |
|
||||
| `job_id` | number \| null | `OperationContext.jobId` when the caller was a subagent tool-bridge. Null for MCP + CLI. |
|
||||
| `subagent_id` | number \| null | `OperationContext.subagentId` for subagent-owned runs. |
|
||||
| `created_at` | string (ISO 8601) | UTC timestamp of insert. |
|
||||
|
||||
## Ordering + determinism
|
||||
|
||||
`listEvalCandidates` orders by `created_at DESC, id DESC`. Same-
|
||||
millisecond inserts tie on `created_at`; `id DESC` is the stable
|
||||
tiebreaker. Replay tools can consume rows in order and assume:
|
||||
- no duplicate rows across calls with non-overlapping `--since` windows
|
||||
- no missed rows across calls that chain `--since` windows (window end
|
||||
of run 1 is the strict upper bound, not a soft cursor)
|
||||
|
||||
## Schema versioning promise
|
||||
|
||||
- **v1 (shipped v0.21.0)** — this document. All fields listed above.
|
||||
- **Additive changes** increment gbrain minor version (v0.25.0, v0.23.0
|
||||
…) and ship with new optional fields. Consumers keyed on known fields
|
||||
ignore unknown keys and keep working.
|
||||
- **Breaking changes** (rename, type change, removal) increment
|
||||
`schema_version` to 2. Consumers MUST branch on `schema_version` to
|
||||
stay compatible.
|
||||
|
||||
## `eval_capture_failures` — companion audit table
|
||||
|
||||
Not exported by `gbrain eval export`. Surfaced via `gbrain doctor`:
|
||||
|
||||
```sh
|
||||
gbrain doctor # warns when failures in last 24h > 0
|
||||
```
|
||||
|
||||
Reason enum (stable): `db_down` | `rls_reject` | `check_violation` |
|
||||
`scrubber_exception` | `other`. Cross-process visibility is the whole
|
||||
point — `gbrain doctor` runs in its own process and reads the table
|
||||
directly, so in-process counters wouldn't work.
|
||||
|
||||
## Config + CONTRIBUTOR_MODE
|
||||
|
||||
Capture is **off by default** as of v0.25.0 (was on for everyone in
|
||||
earlier drafts). Two paths to turn it on:
|
||||
|
||||
**Path A — env var (contributor opt-in, the common case):**
|
||||
|
||||
```bash
|
||||
export GBRAIN_CONTRIBUTOR_MODE=1 # in ~/.zshrc or ~/.bashrc
|
||||
```
|
||||
|
||||
**Path B — explicit config (`~/.gbrain/config.json`, file-plane only):**
|
||||
|
||||
```json
|
||||
{
|
||||
"engine": "postgres",
|
||||
"database_url": "...",
|
||||
"eval": {
|
||||
"capture": true,
|
||||
"scrub_pii": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Resolution order (most explicit wins):
|
||||
|
||||
1. `eval.capture: true` in config → on
|
||||
2. `eval.capture: false` in config → off (overrides CONTRIBUTOR_MODE=1)
|
||||
3. `GBRAIN_CONTRIBUTOR_MODE === '1'` → on
|
||||
4. otherwise → off
|
||||
|
||||
`scrub_pii` defaults to `true` independent of capture. Set
|
||||
`eval.scrub_pii: false` to preserve raw query text (only if you control
|
||||
the brain's distribution).
|
||||
|
||||
`gbrain config set eval.capture false` does **not** work — that
|
||||
command writes the DB-plane config, and the MCP server reads the
|
||||
file-plane. Edit the JSON directly or use the env var.
|
||||
@@ -1,159 +0,0 @@
|
||||
# `gbrain eval takes-quality` — reproducible cross-modal quality eval
|
||||
|
||||
v0.32+ ships a CI-able quality gate for the takes layer. Three frontier models
|
||||
score a sample of takes against a 5-dimension rubric, the runner aggregates to
|
||||
PASS / FAIL / INCONCLUSIVE, and the receipt persists to `eval_takes_quality_runs`
|
||||
so a follow-up `trend` or `regress` can compare against history.
|
||||
|
||||
This doc is the consumer contract. The sibling [gbrain-evals](https://github.com/garrytan/gbrain-evals)
|
||||
repo and any future CI gate read receipts shaped exactly like the JSON below.
|
||||
Fields are additive-stable at `schema_version: 1`. A breaking shape change
|
||||
bumps the version.
|
||||
|
||||
## Subcommands
|
||||
|
||||
| Command | Brain required? | Exit codes |
|
||||
|---|---|---|
|
||||
| `gbrain eval takes-quality run [flags]` | yes (samples takes) | 0 PASS, 1 FAIL, 2 INCONCLUSIVE |
|
||||
| `gbrain eval takes-quality replay <receipt>` | **no** (disk-only) | 0 PASS, 1 FAIL, 2 INCONCLUSIVE |
|
||||
| `gbrain eval takes-quality trend [flags]` | yes (reads runs table) | 0 |
|
||||
| `gbrain eval takes-quality regress --against <receipt>` | yes | 0 OK, 1 regression |
|
||||
|
||||
`replay` is the only mode that runs without `DATABASE_URL` — it reads the
|
||||
receipt file from disk and re-renders it. The other modes need the brain.
|
||||
|
||||
## `run` flags
|
||||
|
||||
| Flag | Default | Notes |
|
||||
|---|---|---|
|
||||
| `--limit N` | 100 | Random sample of N takes from the brain. |
|
||||
| `--cycles N` | 3 (TTY) / 1 (non-TTY) | Up to N panel calls before giving up; early-stop on PASS or INCONCLUSIVE. |
|
||||
| `--budget-usd N` | unset | Abort before next call's projected cost would exceed cap. Models without a `pricing.ts` entry fail loud (codex #4). |
|
||||
| `--source db|fs` | `db` | `fs` is reserved for v0.33+. |
|
||||
| `--slug-prefix P` | unset | Filter takes to pages whose slug starts with P. |
|
||||
| `--models a,b,c` | `openai:gpt-4o,anthropic:claude-opus-4-7,google:gemini-1.5-pro` | Comma-separated panel. |
|
||||
| `--json` | off | Emit the full receipt to stdout. |
|
||||
|
||||
## Receipt JSON shape (`schema_version: 1`)
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"ts": "2026-05-09T22:00:00.000Z",
|
||||
"rubric_version": "v1.0",
|
||||
"rubric_sha8": "abcd1234",
|
||||
"corpus": {
|
||||
"source": "db",
|
||||
"n_takes": 100,
|
||||
"slug_prefix": null,
|
||||
"corpus_sha8": "abcd1234"
|
||||
},
|
||||
"prompt_sha8": "abcd1234",
|
||||
"models_sha8": "abcd1234",
|
||||
"models": ["openai:gpt-4o", "anthropic:claude-opus-4-7", "google:gemini-1.5-pro"],
|
||||
"cycles_run": 3,
|
||||
"successes_per_cycle": [3, 3, 2],
|
||||
"verdict": "pass",
|
||||
"scores": {
|
||||
"accuracy": { "mean": 7.8, "min": 7, "max": 9, "scores": [9,7,7], "per_model": {...} },
|
||||
"attribution": { "mean": 7.0, "min": 7, "max": 7, "scores": [7,7,7], "per_model": {...} },
|
||||
"weight_calibration": { "mean": 7.5, "min": 7, "max": 8, "scores": [8,7,7], "per_model": {...} },
|
||||
"kind_classification": { "mean": 7.2, "min": 7, "max": 8, "scores": [7,8,7], "per_model": {...} },
|
||||
"signal_density": { "mean": 7.0, "min": 6, "max": 8, "scores": [8,7,6], "per_model": {...} }
|
||||
},
|
||||
"overall_score": 7.3,
|
||||
"cost_usd": 1.85,
|
||||
"improvements": ["..."],
|
||||
"errors": [],
|
||||
"verdictMessage": "PASS: every dim mean >=7 and min >=5 ..."
|
||||
}
|
||||
```
|
||||
|
||||
### Field reference
|
||||
|
||||
- `schema_version` — locks the contract. Adding optional fields is additive
|
||||
and compatible. Renaming, removing, or changing semantics bumps the version.
|
||||
- `rubric_version` + `rubric_sha8` — segregate trend rows by rubric epoch
|
||||
(codex review #3). When the rubric definition changes, both fields update,
|
||||
and trend mode groups runs accordingly so a stricter rubric doesn't
|
||||
silently look like a quality drop.
|
||||
- `corpus.corpus_sha8` — fingerprint over the joined takes-text the judge
|
||||
saw. Determines whether two runs are over the "same" sample.
|
||||
- `models_sha8` — fingerprint over the sorted model id list. Re-ordering
|
||||
models in `--models` doesn't change the sha (sort is stable).
|
||||
- `successes_per_cycle` — count of contributing models per cycle. A model
|
||||
contributes when (a) its JSON parsed AND (b) every declared rubric dim
|
||||
has a finite score (codex review #5 — missing-dim drops the contribution).
|
||||
- `verdict` — `pass` if every dim mean >= 7 AND every dim min across
|
||||
contributing models >= 5; `fail` otherwise; `inconclusive` if fewer than
|
||||
2/3 models contributed complete scores.
|
||||
- `cost_usd` — sum of per-call cost via `pricing.ts`. Unknown models when
|
||||
`--budget-usd` is set produce a `PricingNotFoundError` before any call
|
||||
fires.
|
||||
|
||||
## Receipt persistence
|
||||
|
||||
Receipts persist to **`eval_takes_quality_runs`** (DB-authoritative per
|
||||
codex review #6) AND to disk at `~/.gbrain/eval-receipts/takes-quality-<corpus>-<prompt>-<models>-<rubric>.json`
|
||||
as a best-effort artifact. The DB row carries the full receipt JSON in the
|
||||
`receipt_json` JSONB column, so when the disk artifact is gone, `replay`
|
||||
can still reconstruct via `loadReceiptFromDb` (v0.33+ flag wiring).
|
||||
|
||||
The 4-sha primary key is unique (`UNIQUE` constraint) so re-running an
|
||||
identical eval is `INSERT ... ON CONFLICT DO NOTHING` — idempotent.
|
||||
|
||||
## Trend output
|
||||
|
||||
Plain text (default):
|
||||
|
||||
```
|
||||
ts rubric verdict overall cost corpus
|
||||
─────────────────────────────────────────────────────────────────────────────
|
||||
2026-05-09T22:00:00 v1.0 pass 7.3 $1.85 abcd1234
|
||||
2026-05-08T18:30:00 v1.0 fail 6.8 $1.92 ef567890
|
||||
```
|
||||
|
||||
JSON shape (`--json`):
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"rows": [
|
||||
{ "id": 42, "ts": "...", "rubric_version": "v1.0", "verdict": "pass",
|
||||
"overall_score": 7.3, "cost_usd": 1.85, "corpus_sha8": "abcd1234" }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Regress: gating CI on quality
|
||||
|
||||
```bash
|
||||
# Capture a baseline.
|
||||
gbrain eval takes-quality run --limit 100 --json \
|
||||
> .ci/takes-quality-baseline.json
|
||||
|
||||
# Later, after changing the extraction prompt:
|
||||
gbrain eval takes-quality regress --against .ci/takes-quality-baseline.json \
|
||||
--threshold 0.5
|
||||
# exit 0 → no regression past threshold
|
||||
# exit 1 → some dim dropped > 0.5; CI fails
|
||||
```
|
||||
|
||||
The threshold is the per-dim-mean drop counting as regression. Default 0.5.
|
||||
Regress reuses the **same** model panel + slug prefix + source as the prior
|
||||
receipt for an apples-to-apples compare. Diffs in `corpus_sha8` /
|
||||
`prompt_sha8` / `rubric_sha8` are surfaced as informational warnings (the
|
||||
runner doesn't refuse — that's the caller's call).
|
||||
|
||||
## Contract stability
|
||||
|
||||
The shape above is the read contract for downstream consumers. Anything
|
||||
not listed (e.g. internal aggregator state, gateway providerMetadata) is
|
||||
**not** in the receipt and may change without notice.
|
||||
|
||||
When you need to evolve the schema:
|
||||
1. Additive optional field → no version bump; old consumers ignore the
|
||||
new key, new consumers read it.
|
||||
2. Renamed or removed field, or changed semantics → bump
|
||||
`schema_version` to `2`; runner emits both shapes for one release as
|
||||
a deprecation runway.
|
||||
@@ -1,124 +0,0 @@
|
||||
# Evaluation Metric Glossary
|
||||
|
||||
**Auto-generated from `src/core/eval/metric-glossary.ts`. Do not edit by hand.** Run `bun run scripts/generate-metric-glossary.ts` to regenerate.
|
||||
|
||||
Every metric `gbrain eval *` and `gbrain search stats` reports has a plain-English explanation here. Industry terms are preserved verbatim so users searching the literature find what we report.
|
||||
|
||||
## Retrieval Metrics
|
||||
|
||||
### Precision at k (P@k)
|
||||
|
||||
**Key:** `precision@k`
|
||||
|
||||
**Plain English:** Of the top k results the engine returned, what fraction were actually relevant? High precision means few junk results in the top of the list.
|
||||
|
||||
**Range:** 0..1, higher is better. P@10 = 0.7 means 7 of the top 10 results were on-topic.
|
||||
|
||||
### Recall at k (R@k)
|
||||
|
||||
**Key:** `recall@k`
|
||||
|
||||
**Plain English:** Of all the relevant results that exist in the brain, what fraction did the engine find in its top k? High recall means few missed answers.
|
||||
|
||||
**Range:** 0..1, higher is better. R@10 = 0.81 means out of every 100 questions, the right answer was in the top 10 for 81 of them.
|
||||
|
||||
### Mean Reciprocal Rank (MRR)
|
||||
|
||||
**Key:** `mrr`
|
||||
|
||||
**Plain English:** On average, how far down the list is the FIRST relevant result? An MRR of 1.0 means the first hit is always right; an MRR of 0.5 means it's typically at rank 2.
|
||||
|
||||
**Range:** 0..1, higher is better. Computed as the average of 1/rank-of-first-relevant-result across all test queries.
|
||||
|
||||
### Normalized Discounted Cumulative Gain at k (nDCG@k)
|
||||
|
||||
**Key:** `ndcg@k`
|
||||
|
||||
**Plain English:** Like precision@k, but the engine gets MORE credit for putting good results near the top than near rank k. A perfect ordering scores 1.0; a totally random ordering scores near 0.
|
||||
|
||||
**Range:** 0..1, higher is better. nDCG@10 above 0.65 is the common "ship it" threshold for hybrid retrieval on technical corpora.
|
||||
|
||||
## Set-Similarity / Stability Metrics
|
||||
|
||||
### Jaccard similarity at k (set Jaccard @k)
|
||||
|
||||
**Key:** `jaccard@k`
|
||||
|
||||
**Plain English:** How much do two result lists overlap? Compare the top k slugs from the captured baseline against the current run; Jaccard@10 = 1.0 means perfect agreement, 0.0 means zero overlap.
|
||||
|
||||
**Range:** 0..1, higher = more stable. Below 0.5 on a stable corpus means retrieval changed significantly.
|
||||
|
||||
### Top-1 stability rate
|
||||
|
||||
**Key:** `top1_stability`
|
||||
|
||||
**Plain English:** Fraction of queries where the #1 result is the same between two runs. The most aggressive stability check — small ranking shifts that don't change the top answer don't hurt it.
|
||||
|
||||
**Range:** 0..1, higher = more stable. Above 0.85 typically means safe-to-merge for retrieval changes.
|
||||
|
||||
## Statistical-Significance Metrics
|
||||
|
||||
### p-value (paired bootstrap)
|
||||
|
||||
**Key:** `p_value`
|
||||
|
||||
**Plain English:** How likely the observed difference between two modes is just noise. Lower = stronger evidence the difference is real. We compute paired bootstrap with 10,000 resamples and Bonferroni correction across the 12 comparisons (3 modes × 4 metrics).
|
||||
|
||||
**Range:** 0..1, lower = stronger signal. Below 0.05 is the common "statistically significant" threshold; below 0.01 is strong evidence.
|
||||
|
||||
### 95% Confidence Interval (CI)
|
||||
|
||||
**Key:** `confidence_interval`
|
||||
|
||||
**Plain English:** The range we're 95% sure the true value falls inside, given the sample we measured. Narrower CI = more reliable estimate. Computed via bootstrap resampling.
|
||||
|
||||
**Range:** Two-tuple [low, high]. If 0 is inside the CI for a Δ, the difference isn't statistically significant.
|
||||
|
||||
## Operational / Cost Metrics
|
||||
|
||||
### Cache hit rate
|
||||
|
||||
**Key:** `cache_hit_rate`
|
||||
|
||||
**Plain English:** Fraction of searches that reused a recent cached answer instead of running fresh. Higher hit rate = lower latency + lower LLM spend, but stale results may slip through if the threshold is too loose.
|
||||
|
||||
**Range:** 0..1, higher generally better. 0.7-0.9 is the sweet spot for a busy brain; above 0.9 may indicate the similarity threshold is too loose.
|
||||
|
||||
### Average results returned
|
||||
|
||||
**Key:** `avg_results`
|
||||
|
||||
**Plain English:** Mean number of search-result rows the engine returned per call. Should be near the active mode's searchLimit unless the brain is small or the budget is dropping results.
|
||||
|
||||
**Range:** 0..searchLimit. Far below searchLimit suggests budget pressure or sparse retrieval.
|
||||
|
||||
### Average tokens delivered
|
||||
|
||||
**Key:** `avg_tokens`
|
||||
|
||||
**Plain English:** Estimated tokens (chars / 4) in the chunk text returned per search call. The direct measure of how much context an agent loop is paying for each search.
|
||||
|
||||
**Range:** 0..tokenBudget. Approximates OpenAI tiktoken count for English; off by ~5-10% for Anthropic and worse for non-English.
|
||||
|
||||
### Cost per query (USD)
|
||||
|
||||
**Key:** `cost_per_query_usd`
|
||||
|
||||
**Plain English:** Sum of LLM + embedding API charges for one search call. Includes Haiku expansion call (tokenmax mode only) + embedding cost + downstream answer-model cost if measured.
|
||||
|
||||
**Range:** 0..unbounded. Conservative mode is typically <\$0.001 per call; tokenmax with answer-gen can exceed \$0.01.
|
||||
|
||||
### p99 latency (ms)
|
||||
|
||||
**Key:** `p99_latency_ms`
|
||||
|
||||
**Plain English:** 99th percentile wall-clock time per search call. The latency that 1% of users see — long-tail experience, not the average.
|
||||
|
||||
**Range:** 0..unbounded. Warm-cache hits should be <50ms; tokenmax with expansion can exceed 200ms due to the Haiku call.
|
||||
|
||||
---
|
||||
|
||||
## Coverage
|
||||
|
||||
Every metric printed by any `gbrain eval *` or `gbrain search stats` command resolves through `getMetricGloss()` in `src/core/eval/metric-glossary.ts`. Adding a new metric to the glossary REQUIRES updating this doc; the CI guard catches drift.
|
||||
|
||||
@@ -1,285 +0,0 @@
|
||||
# Search Mode Evaluation Methodology
|
||||
|
||||
_How v0.32.3 measures the difference between `conservative`, `balanced`, and `tokenmax`. Written haters-immune: every claim is reproducible from the committed dataset + raw outputs._
|
||||
|
||||
## 1. What this measures and what it doesn't
|
||||
|
||||
**Measures:** retrieval quality and operational cost on fixed public datasets, under each named search mode, against the same brain content.
|
||||
|
||||
**Does NOT measure:**
|
||||
- Your specific brain content (this is a benchmark, not your bill).
|
||||
- Your specific query distribution.
|
||||
- End-user satisfaction or downstream task success.
|
||||
- Latency under concurrent load.
|
||||
- Production cost (the cost numbers are model-pricing estimates × dataset size, not your actual API spend).
|
||||
|
||||
If you want to know how a mode behaves on YOUR brain, run `gbrain search stats --days 30` after a real usage window, then run `gbrain search tune` for actionable recommendations.
|
||||
|
||||
## 2. Datasets and sizes
|
||||
|
||||
- **LongMemEval** — public split, `n=500` questions. Downloaded from [Hugging Face](https://huggingface.co/datasets/xiaowu0162/longmemeval). The corpus + answer keys are pinned to a specific commit; recorded in every per-run record.
|
||||
- **Replay captures** — NDJSON from the sibling `gbrain-evals` repo, `n=200` queries. Each query carries a `retrieved_slugs` baseline + a `latency_ms` measurement from the original production run.
|
||||
- **BrainBench v1** — `n=1240` documents / `n=350` qrels (binary relevance judgments). Lives in the sibling [`gbrain-evals`](https://github.com/garrytan/gbrain-evals) repo, SHA-pinned at every run.
|
||||
|
||||
No private brain content is used in any reported result. The committed NDJSON dumps under `<repo>/.gbrain-evals/` contain only the LongMemEval question IDs + the rank-ordered retrieved session IDs.
|
||||
|
||||
## 3. Sample selection
|
||||
|
||||
- **Random seed:** `42` throughout. Set via `--seed N` on `gbrain eval run-all`; recorded in every per-run record.
|
||||
- **No per-question curation.** Splits are taken whole; no question is filtered for reporting.
|
||||
- **No mode-specific tuning.** The same dataset + same seed feeds every mode. The mode is the only independent variable.
|
||||
- **Stability across re-runs:** with `--seed 42` and the same dataset SHA, two runs of the same (mode, suite) produce identical retrieval orderings (modulo the optional Haiku expansion call, which is non-deterministic). Persisted in `eval_results` so anyone can re-score from the committed dumps.
|
||||
|
||||
## 4. Run procedure
|
||||
|
||||
The command is the doc. Anyone can reproduce.
|
||||
|
||||
```bash
|
||||
# Setup: in your gbrain working tree, with OPENAI_API_KEY + ANTHROPIC_API_KEY exported.
|
||||
git rev-parse HEAD # record the commit for the methodology footer
|
||||
|
||||
# Sweep all 3 modes × 2 retrieval-focused suites with seed 42.
|
||||
gbrain eval run-all \
|
||||
--modes conservative,balanced,tokenmax \
|
||||
--suites longmemeval,replay \
|
||||
--seed 42 \
|
||||
--limit 500 \
|
||||
--budget-usd-retrieval 5 \
|
||||
--budget-usd-answer 20 \
|
||||
--output docs/eval/results/v0.32.3/
|
||||
|
||||
# Render the comparison.
|
||||
gbrain eval compare --md > docs/eval/results/v0.32.3/README.md
|
||||
gbrain eval compare --json > docs/eval/results/v0.32.3/comparison.json
|
||||
```
|
||||
|
||||
The orchestrator writes per-run records to `<repo>/.gbrain-evals/eval-results.jsonl`. Every record carries: `run_id`, `ran_at`, `suite`, `mode`, `commit`, `seed`, `limit`, `params`, `status`, `duration_ms`. The dumps under `docs/eval/results/v0.32.3/` carry the raw question-level outputs so a reviewer can re-score with their own metric implementation.
|
||||
|
||||
## 5. Threats to validity
|
||||
|
||||
Honest list. We name what would let a critic dismiss the numbers.
|
||||
|
||||
- **LongMemEval skews English + technical.** The questions are software-engineering and consumer-product flavored. Performance on a brain rich in non-English / non-technical content (writing, art history, etc.) may differ.
|
||||
- **BrainBench is small** (1240 docs) relative to a production brain (10K-100K pages). Absolute scores aren't predictive of your hit rate; the _delta_ between modes is.
|
||||
- **char/4 token heuristic.** Token-budget enforcement and cost estimates use a character-count / 4 heuristic. Accurate within ~5-10% for English with the OpenAI tiktoken family; off worse for Voyage (we don't use Voyage in chat retrieval, so it doesn't bias the reported numbers, but if you do, your budget caps will be approximate).
|
||||
- **Expansion's quality lift varies by query distribution.** The eval data shows ~97.6% relative quality with LLM expansion vs without (i.e., barely measurable lift) on the LongMemEval corpus. On rarer-entity / longer-tail queries, the lift can be larger. We report the corpus we measured; YMMV.
|
||||
- **Paired bootstrap assumes question-level independence.** Multi-hop questions within the same conversation thread aren't independent; the bootstrap CI is slightly tighter than reality.
|
||||
- **Single brain instance per benchmark.** The benchmark spins up an in-memory PGLite per question. Cache hit rate measured here doesn't reflect a long-running production brain's cache state.
|
||||
|
||||
## 6. Per-question raw outputs
|
||||
|
||||
Every reported metric is reproducible from the NDJSON dumps committed at `docs/eval/results/v0.32.3/`. The commit SHA in the methodology footer pins the code version.
|
||||
|
||||
**Examples per mode:** the auto-generated `README.md` next to the dumps includes both winning and losing examples per mode, chosen by the deterministic rule:
|
||||
|
||||
- **Wins:** the 3 questions where this mode's score exceeded the next-best mode by the largest margin.
|
||||
- **Losses:** the 3 questions where this mode's score fell short of the next-best mode by the largest margin.
|
||||
|
||||
Picked by the score delta, NOT cherry-picked by hand. The README documents the rule so a critic can verify.
|
||||
|
||||
## 7. Pre-registered expectations
|
||||
|
||||
Before running, we expect:
|
||||
|
||||
1. **tokenmax wins Recall@10** by 5-15 percentage points over conservative. LLM expansion + 50-result ceiling helps rare-entity surface forms.
|
||||
2. **conservative wins cost-per-query** by 5-15× over tokenmax. No Haiku expansion + tight 4K budget cap = single-digit-cent queries.
|
||||
3. **balanced lands within 3pp of tokenmax** on Recall@10. Intent weighting (zero-LLM cost) closes most of the expansion gap on common queries.
|
||||
4. **No mode breaks nDCG@10 ≥ 0.65** — the published "ship it" threshold for hybrid retrieval on technical corpora.
|
||||
|
||||
Then we publish whether the data agrees. **If a hypothesis fails, that's documented honestly** in the release README, not buried. Pre-registration is what makes the comparison defensible — without it, a "we expected X and got X" outcome is observation, not prediction.
|
||||
|
||||
## 8. Re-run cadence
|
||||
|
||||
This document + the eval results are regenerated on every release that touches retrieval-affecting code. The `gbrain doctor eval_drift` check surfaces changes to the curated watch-list in `src/core/eval/drift-watch.ts`:
|
||||
|
||||
- `src/core/search/**`
|
||||
- `src/core/embedding.ts`
|
||||
- `src/core/chunkers/**`
|
||||
- `src/core/ai/recipes/anthropic.ts`
|
||||
- `src/core/ai/recipes/openai.ts`
|
||||
- `src/core/operations.ts`
|
||||
|
||||
Additions to the watch-list require a CHANGELOG line.
|
||||
|
||||
## Statistical-significance discipline
|
||||
|
||||
When `gbrain eval compare --md` reports a Δ between two modes, it computes:
|
||||
|
||||
- **Paired bootstrap** with 10,000 resamples per metric. Each resample draws _question-level_ pairs (same question, mode A vs mode B), so question-level variance is differenced out.
|
||||
- **Bonferroni correction** across the 12 comparisons (3 modes × 4 metrics). The reported p-value is the comparison's raw p-value × 12 (clamped at 1.0).
|
||||
- **95% confidence intervals** computed from the bootstrap distribution.
|
||||
|
||||
If the CI for a Δ includes 0 OR the Bonferroni-adjusted p-value exceeds 0.05, the difference is **not** statistically significant. The MD report says "not significant" verbatim.
|
||||
|
||||
## Glossary
|
||||
|
||||
Every metric the report prints has a plain-English entry in `docs/eval/METRIC_GLOSSARY.md`, auto-generated from `src/core/eval/metric-glossary.ts`. The CI guard at `scripts/check-eval-glossary-fresh.sh` regenerates and diffs against the committed file on every test run; a stale doc fails the build.
|
||||
|
||||
## Cost anchors
|
||||
|
||||
The mode-picker prompt at `gbrain init` and the CLAUDE.md `## Search Mode` table both surface these rough cost anchors. Working through the math so they're auditable:
|
||||
|
||||
**Variables:**
|
||||
- `T` = avg tokens per search-result chunk. The recursive chunker targets 300 words / chunk → ~400 tokens (English, OpenAI tiktoken approx).
|
||||
- `N` = chunks delivered per query (capped by the mode's `searchLimit`).
|
||||
- `R` = downstream model input rate. Sonnet 4.6 = \$3/M. Opus 4.7 = \$5/M. Haiku 4.5 = \$1/M.
|
||||
- `Q` = queries per month.
|
||||
|
||||
**Per-query input cost** (downstream agent reads the chunks):
|
||||
|
||||
cost_per_query = T × N × R
|
||||
|
||||
| Mode | T (tokens) | N (chunks) | Sonnet (\$3/M) | Opus (\$5/M) | Haiku (\$1/M) |
|
||||
|---|---|---|---|---|---|
|
||||
| conservative (4K cap, 10 max) | ~400 | 10 (or fewer if budget hits) | \$0.012 | \$0.020 | \$0.004 |
|
||||
| balanced (12K cap, 25 max) | ~400 | ~25 | \$0.030 | \$0.050 | \$0.010 |
|
||||
| tokenmax (no cap, 50 max) | ~400 | ~50 | \$0.060 | \$0.100 | \$0.020 |
|
||||
|
||||
**Monthly cost** (Q × per-query):
|
||||
|
||||
| Mode @ Sonnet | 1K Q/mo | 10K Q/mo | 100K Q/mo |
|
||||
|---|---|---|---|
|
||||
| conservative | \$12 | \$120 | \$1,200 |
|
||||
| balanced | \$30 | \$300 | \$3,000 |
|
||||
| tokenmax | \$60 | \$600 | \$6,000 |
|
||||
|
||||
| Mode @ Opus | 1K Q/mo | 10K Q/mo | 100K Q/mo |
|
||||
|---|---|---|---|
|
||||
| conservative | \$20 | \$200 | \$2,000 |
|
||||
| balanced | \$50 | \$500 | \$5,000 |
|
||||
| tokenmax | \$100 | \$1,000 | \$10,000 |
|
||||
|
||||
**gbrain's own cost** on top:
|
||||
- Query embedding (text-embedding-3-large @ \$0.13/M tokens): ~\$0.00001 per query. Negligible at every scale.
|
||||
- Tokenmax Haiku expansion call (\$1/M input, \$5/M output, ~500 input + 200 output per call): ~\$0.0015 per query, or \$150/mo at 100K queries. Cache hits cut this in half.
|
||||
- Per-page indexing (one-time): bounded by your import volume, not query volume. Not modeled here.
|
||||
|
||||
**Cache hit adjustment.** A warmed brain typically sees 30-50% cache hits on repeat-query traffic. Cache hits skip the downstream input cost entirely (the cached result was already in the agent's context once). So real-world costs run ~50-70% of the table above on a busy brain.
|
||||
|
||||
**Why these numbers DRIFT from your actual bill:**
|
||||
- Your agent's system prompt + reasoning tokens add input that gbrain doesn't see.
|
||||
- Compaction reduces input over a long session.
|
||||
- Most agents make 1-5 searches per turn; cost-per-turn is what bills you, not cost-per-query.
|
||||
- The model price column drifts as providers reprice; pin the rate via `src/core/anthropic-pricing.ts` for a current snapshot.
|
||||
|
||||
The picker copy + CLAUDE.md table are the canonical user-facing source. Update them in lockstep when the underlying chunker size or default `searchLimit` changes.
|
||||
|
||||
## Mode × Model matrix (the 25x spread)
|
||||
|
||||
The per-query math above assumes Sonnet 4.6 downstream. In reality, the
|
||||
downstream model tier is the BIGGER cost lever. Per-query cost at 10K
|
||||
queries/month (typical single-user volume), search payload only (no cache
|
||||
savings):
|
||||
|
||||
| Mode (search tokens) | Haiku 4.5 (\$1/M) | Sonnet 4.6 (\$3/M) | Opus 4.7 (\$5/M) |
|
||||
|---|---|---|---|
|
||||
| conservative (~4K) | **\$40/mo** | \$120/mo | \$200/mo |
|
||||
| balanced (~10K) | \$100/mo | \$300/mo | \$500/mo |
|
||||
| tokenmax (~20K) | \$200/mo | \$600/mo | **\$1,000/mo** |
|
||||
|
||||
Scales linearly: multiply by 10 for 100K/mo (heavy power user / multi-user
|
||||
fleet); divide by 10 for 1K/mo (light usage).
|
||||
|
||||
**Natural pairings span ~4x** (cheap model + tight mode → frontier model + loose
|
||||
mode). **Mismatches waste capacity:**
|
||||
|
||||
- `tokenmax + Haiku`: Haiku gets 20K of search results stuffed into its
|
||||
context per query. Haiku's reasoning is weaker; more chunks = more noise,
|
||||
not more signal. You pay Haiku rates but get sub-Haiku quality. Wrong
|
||||
direction.
|
||||
- `conservative + Opus`: Opus has 200K context window and can synthesize
|
||||
across many chunks. Capping at 10 chunks / 4K tokens leaves Opus
|
||||
reasoning underfed. You pay Opus rates but get conservative-shape
|
||||
retrieval. Wasted spend.
|
||||
|
||||
**Right-sizing rule:** match the mode's `searchLimit` to the downstream
|
||||
model's "useful context depth":
|
||||
|
||||
- Haiku struggles past ~5-10 chunks of cross-referenced content → conservative
|
||||
- Sonnet handles ~25-40 chunks well → balanced
|
||||
- Opus benefits from 50+ chunks for multi-hop reasoning → tokenmax
|
||||
|
||||
## Realistic-scale anchor (single power-user agent loop)
|
||||
|
||||
The per-query math above is honest but theoretical: it treats each search as an isolated billable event. Real agent loops amortize a lot of context across turns via Anthropic prompt caching. Here's what one heavy power-user loop actually looks like in production, anonymized + scaled so the numbers represent a representative power user rather than any specific deployment.
|
||||
|
||||
**Reference shape — tokenmax in production at a single-user scale:**
|
||||
|
||||
| Quantity | Approximate value |
|
||||
|---|---|
|
||||
| 30-day total agent spend | ~\$700/mo |
|
||||
| 30-day total tokens billed | ~800M |
|
||||
| Turns per month | ~860 (~29/day; one active agent loop) |
|
||||
| Average tokens per turn | ~900K |
|
||||
| Average cost per turn | ~\$0.85 |
|
||||
| Anthropic prompt-cache hit rate | ~88% |
|
||||
|
||||
A "turn" here is one agent loop iteration: read user message, plan, execute tool calls (including gbrain searches), generate response. Each turn typically includes 2-4 gbrain searches.
|
||||
|
||||
**Per-mode scaling from the tokenmax anchor:**
|
||||
|
||||
The cost difference between modes is concentrated in the search-attributable fraction of per-turn cost. System prompt, tool definitions, conversation history, and reasoning tokens don't change with mode — only the chunks gbrain delivers do. Assume 3 searches per turn at the mode's `searchLimit`:
|
||||
|
||||
| Mode | Search tokens/turn | Search cost/turn (at \$3/M effective) | Search-attributable @ 860 turns | Δ vs tokenmax |
|
||||
|---|---|---|---|---|
|
||||
| tokenmax | ~60K (3 × 20K) | ~\$0.18 | ~\$155/mo | — |
|
||||
| balanced | ~30K (3 × 10K) | ~\$0.09 | ~\$77/mo | -\$78 |
|
||||
| conservative | ~12K (3 × 4K) | ~\$0.036 | ~\$31/mo | -\$124 |
|
||||
|
||||
**Implied total agent spend by NATURAL PAIRING** (mode + matched
|
||||
downstream model). Per-turn cost scales with the downstream model's
|
||||
per-token rate, since the cached prefix + uncached portion + reasoning
|
||||
tokens all bill at that rate:
|
||||
|
||||
| Pairing | Per-turn cost | Total @ 860 turns/mo |
|
||||
|---|---|---|
|
||||
| tokenmax + Opus (frontier, max quality) | ~\$0.85 | ~\$700/mo |
|
||||
| balanced + Sonnet (the sweet spot) | ~\$0.50 | ~\$430/mo |
|
||||
| conservative + Haiku (cost-sensitive) | ~\$0.20 | ~\$170/mo |
|
||||
|
||||
**4x spread across natural pairings.** The model tier dominates because
|
||||
the per-token rate applies to the WHOLE per-turn payload (system + tools
|
||||
+ history + reasoning + search), not just gbrain's chunks. Mode choice
|
||||
contributes ~10-20% on top of that base.
|
||||
|
||||
**Mismatched pairings push you off the curve:**
|
||||
|
||||
| Pairing | Per-turn estimate | Total @ 860 turns/mo | Compared to natural |
|
||||
|---|---|---|---|
|
||||
| tokenmax + Haiku | ~\$0.20 | ~\$170/mo | Same cost as conservative+Haiku, worse quality |
|
||||
| conservative + Opus | ~\$0.75 | ~\$640/mo | 92% of tokenmax+Opus spend, conservative-shape retrieval |
|
||||
|
||||
The mismatch math says: a tokenmax+Haiku user pays the same as
|
||||
conservative+Haiku but gets a noisier context (Haiku can't filter signal
|
||||
from 50 chunks). A conservative+Opus user pays nearly the same as
|
||||
tokenmax+Opus but starves Opus on retrieval depth. Both burn budget for
|
||||
no improvement.
|
||||
|
||||
**What this anchor tells us that the per-query math doesn't:**
|
||||
|
||||
1. **At realistic agent-loop scale with disciplined prompt caching, mode choice saves 10-20% of total agent spend** — meaningful, but smaller than the per-query 5x ratio implies. Disciplined prompt-cache layouts blunt the mode delta because most of the per-turn cost is the cached prefix, not the search payload.
|
||||
|
||||
2. **Without that prompt-cache discipline, the per-query framing reasserts itself.** Setups that churn the prompt prefix on every turn (frequent system-prompt edits, untemplated tool defs, no prompt-cache structuring) see search payload contribute a much larger fraction of total cost. Those setups should care about mode choice more, not less.
|
||||
|
||||
3. **The cache hit rate quoted here (~88%) is achievable but not automatic.** It requires structuring the prompt so the cached prefix stays stable across turns: system prompt + tool defs first, history compacted but cache-aware, retrieved chunks appended LAST (where their volatility doesn't invalidate the prefix). Agents that interleave search results inside the cached region pay the prefix-rebuild tax on every turn.
|
||||
|
||||
**Caveats stacked here:**
|
||||
|
||||
- The anchor represents ONE power-user loop. Multi-user fleets aggregate proportionally; the per-user shape doesn't change.
|
||||
- The "3 searches per turn" assumption varies wildly. A code-review agent might issue 10+ searches per turn; a chat-only loop might do 0.
|
||||
- The 88% cache hit rate is the high end of what's achievable. Half that is closer to a default agent without cache-aware prompt layout.
|
||||
- The "Δ vs tokenmax" math assumes the OTHER cost components (system, tools, history, reasoning) stay constant. In practice, conservative's smaller per-turn payload also leaves more room in the context window for history → which can change agent behavior in either direction.
|
||||
|
||||
This anchor + the per-query math both live in this doc on purpose. The per-query framing is what an isolated benchmark would measure (and what `gbrain eval run-all` will produce). The realistic-scale anchor is what an operator actually pays. Both are honest; neither is the whole truth.
|
||||
|
||||
## Reproducibility footer
|
||||
|
||||
Every release that publishes eval numbers includes a footer with:
|
||||
|
||||
- Code commit SHA
|
||||
- Dataset SHA (LongMemEval, BrainBench, Replay)
|
||||
- `--seed N`
|
||||
- Run commands verbatim
|
||||
- API model identifiers used (Anthropic + OpenAI + judge model)
|
||||
|
||||
Without these, the numbers are unfalsifiable. With them, anyone with API keys can re-score.
|
||||
@@ -34,85 +34,6 @@ docs/guides/rls-and-you.md for the GBRAIN:RLS_EXEMPT comment escape hatch.
|
||||
|
||||
99% of the time, you want the fix. Run the SQL. Re-run `gbrain doctor`. Done.
|
||||
|
||||
## v0.26.7 — auto-RLS event trigger and one-time backfill
|
||||
|
||||
Starting in v0.26.7 (migration v35), gbrain ships two changes that close the
|
||||
gap where a table could exist in your `public` schema without RLS for any
|
||||
amount of time at all.
|
||||
|
||||
**1. The event trigger.** A Postgres DDL event trigger named
|
||||
`auto_rls_on_create_table` runs `ALTER TABLE … ENABLE ROW LEVEL SECURITY`
|
||||
on every newly created `public.*` table. It covers `CREATE TABLE`,
|
||||
`CREATE TABLE AS … SELECT`, and `SELECT … INTO` — every syntax Postgres
|
||||
reports as a table-creation command. Tables created by gbrain itself, by
|
||||
your other apps sharing the same Supabase project (Baku, Hermes, anything),
|
||||
or by a human running raw SQL all get RLS enabled the moment they exist.
|
||||
Non-`public` schemas (`auth`, `storage`, `realtime`, etc.) are explicitly
|
||||
ignored — Supabase manages those, and we should not touch them.
|
||||
|
||||
**2. The one-time backfill.** When you upgrade to v0.26.7, the migration
|
||||
walks every existing `public.*` base table whose RLS is off and whose comment
|
||||
doesn't carry the `GBRAIN:RLS_EXEMPT` exemption (see below) and enables RLS
|
||||
on each. After the upgrade, `gbrain doctor`'s `rls` check should be a no-op
|
||||
on every brain.
|
||||
|
||||
### Breaking change: read this before upgrading
|
||||
|
||||
If you have public tables that are intentionally RLS-off and you want them
|
||||
to stay that way, you MUST add the `GBRAIN:RLS_EXEMPT` comment **before**
|
||||
running `gbrain upgrade` to v0.26.7. The backfill flips RLS on for any public
|
||||
table that doesn't carry the exact comment contract documented below. There
|
||||
is no `--dry-run` flag on the migration.
|
||||
|
||||
The minimum cost of getting this wrong is one round-trip: the operator runs
|
||||
the SQL to enable RLS on a table that should have been exempt, then
|
||||
`ALTER TABLE … DISABLE ROW LEVEL SECURITY` and adds the exempt comment to
|
||||
prevent a re-flip on a later doctor run. No data is lost.
|
||||
|
||||
### Cross-app implications
|
||||
|
||||
If a non-gbrain app (Baku, Hermes, a script you wrote, anything) creates
|
||||
tables in the same Supabase project, the trigger will enable RLS on those
|
||||
tables too. Two ways to handle that:
|
||||
|
||||
1. **The app's connection role has BYPASSRLS** (e.g. it's also using the
|
||||
`postgres` role). Newly created tables get RLS on but the app reads/writes
|
||||
freely because BYPASSRLS bypasses policies entirely.
|
||||
2. **The app's role does NOT have BYPASSRLS.** Then the app needs to add a
|
||||
`CREATE POLICY` immediately after creating the table, granting itself
|
||||
the read/write access it needs. The trigger does NOT add policies — it
|
||||
only enables RLS, leaving the deny-by-default posture in place until the
|
||||
app's policy lands.
|
||||
|
||||
If neither condition holds, the app will fail to read its own freshly-created
|
||||
tables. The fix is at the app side, not gbrain's: either grant BYPASSRLS or
|
||||
ship a policy.
|
||||
|
||||
### What if the trigger gets dropped?
|
||||
|
||||
`gbrain doctor` includes a new `rls_event_trigger` check that verifies the
|
||||
trigger is installed and enabled. If you drop it manually for any reason
|
||||
(debugging, migration testing, anything), doctor warns and gives you the
|
||||
recovery command:
|
||||
|
||||
```
|
||||
gbrain apply-migrations --force-retry 35
|
||||
```
|
||||
|
||||
Re-running migration v35 is idempotent — it `DROP EVENT TRIGGER IF EXISTS`
|
||||
and recreates cleanly.
|
||||
|
||||
### Why no FORCE ROW LEVEL SECURITY?
|
||||
|
||||
Postgres has two RLS dials. `ENABLE` blocks anon/authenticated; `FORCE` also
|
||||
blocks the table OWNER unless they hold BYPASSRLS. We use `ENABLE` only,
|
||||
matching the posture in `src/schema.sql`, migrations v24, and v29. `FORCE`
|
||||
would lock non-BYPASSRLS apps out of their own freshly-created tables (the
|
||||
trigger function inherits the caller's role, not the gbrain role) — which
|
||||
defeats the cross-app coexistence story above. If you want defense-in-depth
|
||||
`FORCE` on a specific gbrain-owned table, add it explicitly in your own
|
||||
migration; gbrain's auto-RLS does not opt you in by default.
|
||||
|
||||
## The 1% case: deliberate exemption
|
||||
|
||||
Sometimes a public table is supposed to be readable by the anon key. An
|
||||
|
||||
@@ -1,208 +0,0 @@
|
||||
# Skillpacks as scaffolding, not amber
|
||||
|
||||
GBrain v0.33 reshapes `gbrain skillpack` from a package manager into a
|
||||
scaffold + reference library. This guide explains the model and the
|
||||
workflow.
|
||||
|
||||
## Why we changed it
|
||||
|
||||
Pre-v0.33 (the "amber" model):
|
||||
|
||||
- `gbrain skillpack install <name>` copied bundled skills into your
|
||||
workspace AND wrote a managed-block fence into your `RESOLVER.md` /
|
||||
`AGENTS.md` with a `cumulative-slugs="..."` receipt.
|
||||
- Subsequent installs hash-checked every file and refused to overwrite
|
||||
local edits unless you passed `--overwrite-local`.
|
||||
- `gbrain skillpack uninstall` had its own data-loss safeguards (D8
|
||||
receipt gate + D11 content-hash pre-scan) and rebuilt the fence.
|
||||
|
||||
It worked, but it treated personal-AI skills like vendor packages.
|
||||
Users couldn't cleanly fork a skill without the next install fighting
|
||||
them. Every release re-litigated the same managed block. The test
|
||||
surface alone for the managed block was ~1000 lines.
|
||||
|
||||
Skills aren't vendor packages. They're first-class code in your agent
|
||||
repo. You scaffold once, you own them, you fork and edit freely. When
|
||||
gbrain ships a new version, you ask "what changed?" — the agent reads
|
||||
the diff and decides what (if anything) to integrate.
|
||||
|
||||
## The five commands
|
||||
|
||||
### `gbrain skillpack scaffold <name> [--workspace PATH]`
|
||||
|
||||
One-time, additive copy of a bundled skill into your repo. Refuses to
|
||||
overwrite any file that exists. Routing comes from each skill's
|
||||
frontmatter `triggers:` array — gbrain does NOT touch your `RESOLVER.md`
|
||||
or `AGENTS.md` (see "How agents discover scaffolded skills" below).
|
||||
|
||||
```bash
|
||||
cd ~/git/your-agent-repo
|
||||
gbrain skillpack scaffold book-mirror
|
||||
# files in skills/book-mirror/ + (if the skill declares paired source)
|
||||
# src/commands/book-mirror.ts land in your workspace
|
||||
```
|
||||
|
||||
`scaffold --all` copies every bundled skill that's missing. Never
|
||||
prunes.
|
||||
|
||||
If a skill's frontmatter declares paired source files (`sources: [...]`
|
||||
in the SKILL.md YAML head), scaffold copies them too. The partial-state
|
||||
policy handles "skill shipped earlier, gained a paired source later" —
|
||||
scaffold copies the new paired file even when the skill dir already
|
||||
exists.
|
||||
|
||||
### `gbrain skillpack reference <name> [--workspace PATH] [--apply-clean-hunks] [--json]`
|
||||
|
||||
Read-only update lens. Diffs gbrain's bundle against your local copy
|
||||
and emits per-file status (`identical` / `differs` / `missing`) plus
|
||||
unified diffs for any `differs` entries.
|
||||
|
||||
```bash
|
||||
gbrain skillpack reference book-mirror
|
||||
# These files live at <gbrain-path> as reference. Read them and
|
||||
# decide what (if anything) to integrate into your local skills/.
|
||||
# Your local edits are intentional — do not blindly overwrite.
|
||||
#
|
||||
# reference: identical:14 differs:1 missing:0
|
||||
#
|
||||
# differs /your/workspace/skills/book-mirror/SKILL.md
|
||||
# --- a/skills/book-mirror/SKILL.md
|
||||
# +++ b/skills/book-mirror/SKILL.md
|
||||
# @@ -10,3 +10,5 @@
|
||||
# ... unified diff ...
|
||||
```
|
||||
|
||||
`reference --all` sweeps the whole bundle (one-line-per-skill summary).
|
||||
|
||||
`reference <name> --apply-clean-hunks` is the auto-apply path. It
|
||||
parses the diff between gbrain's bundle and your local copy, applies
|
||||
every hunk whose pre-change context matches uniquely. **Two-way merge
|
||||
limitation**: without scaffold-time base tracking (intentionally
|
||||
out-of-scope for v0.33), this cannot distinguish "gbrain changed X"
|
||||
from "you changed X." Applied hunks align everything to gbrain. Use
|
||||
`--dry-run` first to preview, or run plain `reference` to inspect the
|
||||
diff before letting auto-apply touch anything.
|
||||
|
||||
### `gbrain skillpack migrate-fence [--workspace PATH] [--dry-run]`
|
||||
|
||||
One-shot conversion for workspaces on the pre-v0.33 managed-block
|
||||
model. Strips the `<!-- gbrain:skillpack:begin -->` / `end -->`
|
||||
markers and the manifest receipt comment from your resolver file.
|
||||
|
||||
**Preserves every row inside the fence verbatim.** Those rows become
|
||||
user-owned routing the agent can still see during the transition to
|
||||
frontmatter-based discovery.
|
||||
|
||||
```bash
|
||||
cd ~/git/your-agent-repo
|
||||
gbrain skillpack migrate-fence
|
||||
# migrate-fence: fence_stripped
|
||||
# resolver: /your/workspace/skills/RESOLVER.md
|
||||
# fenced slugs: alpha, beta, gamma
|
||||
# already present: alpha, beta
|
||||
# skills copied: gamma (additive — beta and alpha kept their local edits)
|
||||
```
|
||||
|
||||
Idempotent. Re-running after migration finds no fence and exits 0.
|
||||
|
||||
### `gbrain skillpack scrub-legacy-fence-rows [--workspace PATH] [--dry-run]`
|
||||
|
||||
Opt-in cleanup. Once you've confirmed your agent walks frontmatter
|
||||
`triggers:` for routing, this command removes the legacy rows that
|
||||
`migrate-fence` left behind.
|
||||
|
||||
**Two-condition gate** (both must hold for a row to be removed):
|
||||
|
||||
1. `skills/<slug>/` exists on host (it was a real scaffold).
|
||||
2. That skill's frontmatter declares non-empty `triggers:` (proof
|
||||
that frontmatter discovery covers this skill).
|
||||
|
||||
Rows whose slug fails either gate are preserved — user-owned routing
|
||||
the migration shouldn't touch.
|
||||
|
||||
### `gbrain skillpack harvest <slug> --from <host-repo-root> [--no-lint] [--dry-run]`
|
||||
|
||||
Inverse of scaffold: lifts a proven skill from your host repo back
|
||||
into gbrain so other clients can scaffold it. Default behavior:
|
||||
|
||||
- Symlinks in the host skill dir are rejected (canonical-path
|
||||
confinement).
|
||||
- Privacy linter scans the harvested files against
|
||||
`~/.gbrain/harvest-private-patterns.txt` plus built-in defaults
|
||||
(canonical private fork name, common email regex, Slack channel pattern). Any
|
||||
match → rollback (delete the harvested files) and exit non-zero.
|
||||
- `openclaw.plugin.json` updated with the new slug, sorted.
|
||||
- `--no-lint` bypasses the linter (after a manual editorial scrub).
|
||||
|
||||
Use the `skillpack-harvest` skill (its companion editorial workflow)
|
||||
to walk the genericization checklist before running the CLI.
|
||||
|
||||
## How agents discover scaffolded skills
|
||||
|
||||
Routing under the new model lives entirely in each skill's frontmatter:
|
||||
|
||||
```yaml
|
||||
---
|
||||
name: book-mirror
|
||||
triggers:
|
||||
- "personalized version of this book"
|
||||
- "mirror this book"
|
||||
- "two-column book analysis"
|
||||
---
|
||||
```
|
||||
|
||||
Your agent's job at runtime is to walk `skills/*/SKILL.md`, parse the
|
||||
frontmatter, and match the user's intent against every skill's
|
||||
`triggers:` array. When a match scores high enough, invoke that skill.
|
||||
|
||||
This replaces the v0.32 model where `gbrain skillpack install` wrote
|
||||
table rows into your `RESOLVER.md`. Rows are gone (or, for users
|
||||
migrating from the old model, preserved transitionally by
|
||||
`migrate-fence` until they run `scrub-legacy-fence-rows`).
|
||||
|
||||
If you're a downstream agent author updating to this model:
|
||||
|
||||
1. On startup, scan `skills/*/SKILL.md` for frontmatter.
|
||||
2. Build an in-memory routing table from each skill's `triggers:`
|
||||
array.
|
||||
3. On every user message, match against this table — either by
|
||||
substring containment, semantic similarity, or whatever your
|
||||
downstream agent already does for intent classification.
|
||||
|
||||
## Removing a scaffolded skill
|
||||
|
||||
There's no `gbrain skillpack uninstall` command in v0.33. The files
|
||||
in your `skills/<slug>/` are first-class members of your repo —
|
||||
delete them like any other code:
|
||||
|
||||
```bash
|
||||
rm -rf skills/book-mirror
|
||||
# if the skill declared paired source files:
|
||||
rm src/commands/book-mirror.ts
|
||||
# (consult the skill's frontmatter `sources:` array for the full list)
|
||||
|
||||
# if no other scaffolded skill needs them, you can also remove the
|
||||
# shared deps that scaffold drops in:
|
||||
rm skills/_brain-filing-rules.md
|
||||
rm -rf skills/conventions/
|
||||
rm skills/_output-rules.md
|
||||
```
|
||||
|
||||
You own the files. There's no manifest to update, no fence to rebuild.
|
||||
|
||||
## When to use which command (quick decision tree)
|
||||
|
||||
- **New host repo, want a gbrain skill** → `scaffold`
|
||||
- **gbrain shipped a new version, want to see what's changed**
|
||||
→ `reference` (read-only) or `reference --apply-clean-hunks` (auto)
|
||||
- **Upgrading from v0.32 or earlier** → `migrate-fence` (one-shot)
|
||||
- **Cleanup after `migrate-fence`** → `scrub-legacy-fence-rows`
|
||||
- **Lift your fork's skill back into gbrain** → `harvest` + the
|
||||
`skillpack-harvest` editorial skill
|
||||
|
||||
## What about `install` and `uninstall`?
|
||||
|
||||
Both are removed in v0.33. Running either prints an error pointing at
|
||||
the replacement command. No deprecated alias — this is a clean break.
|
||||
If you have existing scripts referencing the old names, update them
|
||||
once and move on.
|
||||
@@ -1,130 +0,0 @@
|
||||
# Embedding providers
|
||||
|
||||
GBrain ships with 14 embedding-provider recipes covering OpenAI, the major hosted alternatives, three local options, and a universal escape hatch (LiteLLM proxy). Run `gbrain providers list` to see the live registry; `gbrain providers explain --json` emits a machine-readable matrix for agents.
|
||||
|
||||
This page is the human-readable counterpart: capability per provider, env-var setup, dimensions, cost, and known constraints.
|
||||
|
||||
## Quick start
|
||||
|
||||
```
|
||||
gbrain providers list # see all providers
|
||||
gbrain providers env <provider-id> # see required env vars
|
||||
gbrain providers test --model openai:text-embedding-3-large # smoke-test
|
||||
gbrain init --pglite --model voyage # use a non-default provider
|
||||
```
|
||||
|
||||
## TL;DR table
|
||||
|
||||
| Provider | env vars | default dims | cost ($/1M tokens) | local? | multimodal? |
|
||||
|---|---|---|---|---|---|
|
||||
| `openai` | `OPENAI_API_KEY` | 1536 | 0.13 | no | no |
|
||||
| `voyage` | `VOYAGE_API_KEY` | 1024 | 0.18 | no | yes (`voyage-multimodal-3`) |
|
||||
| `google` | `GOOGLE_GENERATIVE_AI_API_KEY` | 768 | 0.025 | no | no |
|
||||
| `azure-openai` | `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENT` | 1536 | 0.13 | no | no |
|
||||
| `minimax` | `MINIMAX_API_KEY` | 1536 | 0.07 | no | no |
|
||||
| `dashscope` | `DASHSCOPE_API_KEY` | 1024 | varies | no | no |
|
||||
| `zhipu` | `ZHIPUAI_API_KEY` | 1024 | varies | no | no |
|
||||
| `ollama` | (none — runs locally) | 768 | 0 | yes | no |
|
||||
| `llama-server` | (none — runs locally) | user-set | 0 | yes | no |
|
||||
| `litellm` | `LITELLM_API_KEY` (optional) | user-set | varies | yes (proxy) | no |
|
||||
| `together` | `TOGETHER_API_KEY` | 768 | varies | no | no |
|
||||
| `anthropic` | (no embedding model — chat only) | — | — | — | — |
|
||||
| `deepseek` | (no embedding model — chat only) | — | — | — | — |
|
||||
| `groq` | (no embedding model — chat only) | — | — | — | — |
|
||||
|
||||
## Decision tree
|
||||
|
||||
- **Cost-sensitive, English-only**: Ollama (free, local) or Voyage (paid, best quality per dollar).
|
||||
- **Quality-first**: Voyage `voyage-4-large` (1024-2048 dims, ~3-4× more dense tokens than OpenAI tiktoken).
|
||||
- **Reranking pair**: Voyage (their reranker `rerank-2.5` pairs cleanly with Voyage embeddings).
|
||||
- **Enterprise compliance**: Azure OpenAI (data residency + private endpoints) or self-hosted via llama-server / Ollama.
|
||||
- **China region**: DashScope (Alibaba) or Zhipu (BigModel). DashScope's international endpoint at `dashscope-intl.aliyuncs.com`; override `provider_base_urls.dashscope` for the China endpoint.
|
||||
- **OSS local, full control**: llama-server (`llama.cpp`) for any GGUF model; Ollama for the curated catalog.
|
||||
- **Anything else**: LiteLLM proxy. Run LiteLLM in front of any provider (Bedrock, Vertex, Cohere, Jina, Fireworks, etc.) and point gbrain at it via `LITELLM_BASE_URL`.
|
||||
|
||||
## Per-provider details
|
||||
|
||||
### OpenAI
|
||||
|
||||
Default. Set `OPENAI_API_KEY`. Models: `text-embedding-3-large` (3072 max, 1536 default), `text-embedding-3-small` (1536). Matryoshka via the `dimensions` field — gbrain pins it from `embedding_dimensions` config so existing 1536-dim brains stay aligned across SDK upgrades.
|
||||
|
||||
### Voyage AI
|
||||
|
||||
Best-in-class quality on the Voyage 4 family (Jan 2026 release). Set `VOYAGE_API_KEY`. Models: `voyage-4-large`, `voyage-4`, `voyage-4-lite`, `voyage-4-nano`, `voyage-3.5`, `voyage-code-3` (code-tuned), `voyage-finance-2`, `voyage-law-2`, `voyage-multimodal-3` (text + image).
|
||||
|
||||
Voyage 4 family shares an embedding space across all variants, so you can index with `voyage-4-large` and query with `voyage-4-lite` without reindexing. Dims: 256, 512, 1024, 2048. **2048 exceeds pgvector's HNSW cap of 2000** — those brains fall back to exact vector scans (still correct, just slower).
|
||||
|
||||
### Google Gemini
|
||||
|
||||
Set `GOOGLE_GENERATIVE_AI_API_KEY` (the AI Studio public API key). Model: `gemini-embedding-001`. Default 768 dims; Matryoshka up to 3072. Cheap.
|
||||
|
||||
For GCP service-account / Vertex AI auth (production deployments), see the v0.32.x follow-up — Vertex ADC is on the roadmap.
|
||||
|
||||
### Azure OpenAI
|
||||
|
||||
Enterprise OpenAI behind Azure tenancy. Required env: `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT` (e.g. `https://my-resource.openai.azure.com`), `AZURE_OPENAI_DEPLOYMENT` (the deployment name from your Azure portal). Optional: `AZURE_OPENAI_API_VERSION` (defaults to `2024-10-21`).
|
||||
|
||||
Unlike vanilla OpenAI, Azure uses `api-key:` header (not `Authorization: Bearer`) and a templated URL with `?api-version=` query param — gbrain handles both via the recipe's resolveAuth + resolveOpenAICompatConfig overrides.
|
||||
|
||||
Models: `text-embedding-3-large`, `text-embedding-3-small`, `text-embedding-ada-002` (your Azure deployment must serve the requested model).
|
||||
|
||||
### MiniMax (海螺AI)
|
||||
|
||||
Set `MINIMAX_API_KEY`. Optional `MINIMAX_GROUP_ID` for org-scoped accounts. Model: `embo-01` (1536 dims).
|
||||
|
||||
MiniMax's API takes a `type: 'db' | 'query'` field for asymmetric retrieval. v0.32 routes everything as `type='db'` (symmetric retrieval — same vector space for indexing and queries). Asymmetric query support is a v0.32.x follow-up.
|
||||
|
||||
### DashScope (Alibaba)
|
||||
|
||||
Set `DASHSCOPE_API_KEY`. International endpoint at `dashscope-intl.aliyuncs.com` by default; override `provider_base_urls.dashscope` for the China endpoint. Models: `text-embedding-v3` (current; Matryoshka 64-1024 dims), `text-embedding-v2`.
|
||||
|
||||
CJK-dominant content tokenizes denser than OpenAI tiktoken; gbrain declares `chars_per_token: 2` so the batch pre-split leaves headroom.
|
||||
|
||||
### Zhipu AI (BigModel)
|
||||
|
||||
Set `ZHIPUAI_API_KEY`. Models: `embedding-3` (current; Matryoshka 256-2048 dims), `embedding-2`. v0.32 default is 1024 (HNSW-compatible). The 2048-dim option works but falls into the exact-scan branch (see Voyage 4 Large note above).
|
||||
|
||||
### Ollama (local)
|
||||
|
||||
No env required — Ollama runs unauthenticated locally. Optional `OLLAMA_BASE_URL` (default `http://localhost:11434/v1`) and `OLLAMA_API_KEY` (for auth-enabled deployments).
|
||||
|
||||
Recipe ships with `nomic-embed-text` (768d, recommended), `mxbai-embed-large` (1024d), `all-minilm` (384d). `gbrain providers test --model ollama:nomic-embed-text` smoke-tests the local install.
|
||||
|
||||
### llama-server (local, llama.cpp)
|
||||
|
||||
`llama.cpp`'s `llama-server --embeddings` endpoint. No env required. Optional `LLAMA_SERVER_BASE_URL` (default `http://localhost:8080/v1`) and `LLAMA_SERVER_API_KEY`.
|
||||
|
||||
User-driven models: launch llama-server with `--model <gguf-path> --embeddings`, then run `gbrain init --embedding-model llama-server:<your-id> --embedding-dimensions <N>`. The recipe refuses the implicit shorthand `--model llama-server` because there's no canonical first model.
|
||||
|
||||
### LiteLLM proxy (universal escape hatch)
|
||||
|
||||
Run [LiteLLM](https://docs.litellm.ai/docs/proxy/quick_start) in front of any provider — Bedrock, Vertex, Cohere, Jina, Fireworks, OctoAI, etc. The proxy normalizes everything to the OpenAI-compatible API; gbrain points at the proxy via `LITELLM_BASE_URL` and proxies the call.
|
||||
|
||||
This is the catch-all for "my provider isn't in the list above." Set up LiteLLM, then `gbrain init --embedding-model litellm:<your-model-id> --embedding-dimensions <N>`.
|
||||
|
||||
## Choosing dimensions
|
||||
|
||||
Three numbers matter:
|
||||
1. **Provider's native dims**: each model has a "true" output dim (e.g. OpenAI `text-embedding-3-large` is 3072 native).
|
||||
2. **Matryoshka reductions**: most modern providers let you request a smaller vector via the `dimensions` field.
|
||||
3. **HNSW cap**: pgvector's HNSW index supports up to 2000 dims. Brains above that fall back to exact vector scans (slower but correct; gbrain handles the SQL automatically via `chunkEmbeddingIndexSql` in `src/core/vector-index.ts`).
|
||||
|
||||
For most users: **stay at 1024 or 1536**. Bigger isn't better below the noise floor; smaller saves disk + RAM with marginal recall loss on Matryoshka providers.
|
||||
|
||||
## My provider isn't listed
|
||||
|
||||
Three options:
|
||||
|
||||
1. **Use LiteLLM proxy** (above) — the universal escape hatch. Works for 100+ providers.
|
||||
2. **Open a feature request** at [github.com/garrytan/gbrain/issues](https://github.com/garrytan/gbrain/issues) with the provider's API docs URL and a setup snippet. Recipes are ~30-40 lines of TypeScript.
|
||||
3. **Submit a recipe**: clone, copy `src/core/ai/recipes/voyage.ts` as the gold-standard openai-compat template, register in `src/core/ai/recipes/index.ts`, add a per-recipe smoke test under `test/ai/recipe-<name>.test.ts`. The recipe contract test (`test/ai/recipes-contract.test.ts`) and IRON RULE regression test pin the structural invariants.
|
||||
|
||||
## Switching providers on an existing brain
|
||||
|
||||
Embedding dimensions are baked into the schema at `gbrain init` time. To change providers post-init, you usually need to re-embed:
|
||||
|
||||
1. Update config: `gbrain config set embedding_model <provider>:<model>` and `embedding_dimensions <N>`.
|
||||
2. Reindex schema if dims changed: `gbrain doctor` will detect the mismatch and print the exact `ALTER TABLE` recipe.
|
||||
3. Re-embed: `gbrain embed --all` (or `--stale` for incremental).
|
||||
|
||||
`gbrain doctor` 8c "alternative_providers" surfaces unconfigured providers whose env is already set — useful when you've configured OpenAI but also have e.g. `VOYAGE_API_KEY` exported and want to know you can switch without extra setup.
|
||||
@@ -73,7 +73,7 @@ hook resumes blocking malformed pages.
|
||||
|
||||
## For downstream agent forks
|
||||
|
||||
If your OpenClaw wraps gbrain in a host repo
|
||||
If your fork (Wintermute, Hermes, OpenClaw) wraps gbrain in a host repo
|
||||
that's not the brain repo itself, you may want a separate hook strategy:
|
||||
|
||||
- **Brain repo IS the host repo** (gbrain skills + brain pages in one repo):
|
||||
|
||||
@@ -1,224 +0,0 @@
|
||||
# Doctor Auto-Heal and Scoring Improvements
|
||||
|
||||
## Summary
|
||||
|
||||
The `gbrain doctor` health score system has several false-positive patterns and missing auto-heal capabilities. After the crash classification fix (shipped in this PR), these are the remaining improvements ranked by impact.
|
||||
|
||||
---
|
||||
|
||||
## 1. Frontmatter severity levels
|
||||
|
||||
### Problem
|
||||
|
||||
`NESTED_QUOTES` warnings dominate the frontmatter check (6,900+ of ~7,100 total issues). These are cosmetic YAML style issues — values like `title: "foo"` where the quotes are technically unnecessary. They don't affect sync, search, embedding, or any functionality.
|
||||
|
||||
By counting them the same as `YAML_PARSE` (actual parse failures) or `MISSING_OPEN` (missing frontmatter delimiters), the frontmatter check is perpetually WARN and the real issues are lost.
|
||||
|
||||
### Evidence
|
||||
|
||||
```
|
||||
frontmatter_integrity: 7131 issues across 3 sources
|
||||
default: 7012 (NESTED_QUOTES=6922, YAML_PARSE=90)
|
||||
media-corpus: 16 (MISSING_OPEN=15, YAML_PARSE=1)
|
||||
zion-brain: 103 (MISSING_OPEN=14, NESTED_QUOTES=89)
|
||||
```
|
||||
|
||||
Only 280 of 7,131 issues are real problems. 96% are cosmetic noise.
|
||||
|
||||
### Proposed Fix
|
||||
|
||||
- Introduce severity levels: `error` (YAML_PARSE, MISSING_OPEN) vs `info` (NESTED_QUOTES)
|
||||
- Doctor WARN/FAIL only on error-level issues
|
||||
- Report info-level in the message text but don't affect check status
|
||||
- Optional `--pedantic` flag includes info-level in status
|
||||
|
||||
### Test Cases
|
||||
|
||||
| Frontmatter issues | Severity breakdown | Expected status |
|
||||
|---|---|---|
|
||||
| 0 issues | n/a | OK |
|
||||
| 50 NESTED_QUOTES only | 0 error, 50 info | OK (with note) |
|
||||
| 3 YAML_PARSE | 3 error | WARN |
|
||||
| 6900 NESTED_QUOTES + 3 YAML_PARSE | 3 error, 6900 info | WARN (mentions 3 errors) |
|
||||
|
||||
---
|
||||
|
||||
## 2. Temporal contradiction awareness
|
||||
|
||||
### Problem
|
||||
|
||||
The contradiction probe flags temporal evolutions as contradictions. Example:
|
||||
|
||||
- Page A (April): "Considering option X"
|
||||
- Page B (May): "Decided on option Y"
|
||||
|
||||
These aren't contradictions — they're the same topic evolving over time. The probe has no time awareness.
|
||||
|
||||
### Evidence
|
||||
|
||||
From a probe run on 50 queries with top-k=15:
|
||||
- 120 contradictions detected (112 high, 8 medium)
|
||||
- After manual review: ~60% were temporal evolutions, not real conflicts
|
||||
- Pages have `effective_date` or `created` timestamps that could disambiguate
|
||||
|
||||
### Proposed Fix
|
||||
|
||||
- Pass `effective_date` / `created` to the judge prompt
|
||||
- Add verdict: `temporal_supersession` (later claim supersedes earlier)
|
||||
- When both pages have dates and claims overlap, bias toward temporal interpretation
|
||||
- Already designed in PR #993
|
||||
|
||||
### Test Cases
|
||||
|
||||
| Page A date | Page A claim | Page B date | Page B claim | Expected verdict |
|
||||
|---|---|---|---|---|
|
||||
| 2026-04 | "Considering X" | 2026-05 | "Chose Y" | temporal_supersession |
|
||||
| 2026-04 | "Revenue is $1M" | 2026-04 | "Revenue is $500K" | contradiction |
|
||||
| null | "X is true" | null | "X is false" | contradiction |
|
||||
| 2025-01 | "CEO of Company" | 2026-01 | "Former CEO" | temporal_supersession |
|
||||
|
||||
---
|
||||
|
||||
## 3. Multi-source drift baseline
|
||||
|
||||
### Problem
|
||||
|
||||
4,791 pages show "multi-source drift" due to a pre-v0.30.3 `putPage` routing bug. These pages exist at the `default` source but should be at a named source. The `sources rehome` command to fix this hasn't shipped yet.
|
||||
|
||||
Every doctor run shows WARN for ~4,800 pages nobody can fix.
|
||||
|
||||
### Proposed Fix
|
||||
|
||||
Allow `doctor.baselines` config to acknowledge known-unfixable counts:
|
||||
|
||||
```yaml
|
||||
doctor:
|
||||
baselines:
|
||||
multi_source_drift: 4800
|
||||
```
|
||||
|
||||
When actual drift ≤ baseline: OK. When drift exceeds baseline: WARN (new drift).
|
||||
|
||||
Store in `.gbrain/doctor-baselines.json` so it works without config too:
|
||||
|
||||
```json
|
||||
{
|
||||
"multi_source_drift": { "count": 4800, "acknowledged_at": "2026-05-15", "reason": "pre-v0.30.3 putPage misroutes" }
|
||||
}
|
||||
```
|
||||
|
||||
### Test Cases
|
||||
|
||||
| Actual drift | Baseline | Expected |
|
||||
|---|---|---|
|
||||
| 4791 | 4800 | OK |
|
||||
| 4900 | 4800 | WARN ("100 new drift beyond baseline") |
|
||||
| 4791 | 0 (no baseline) | WARN (current behavior) |
|
||||
|
||||
---
|
||||
|
||||
## 4. Image assets acknowledgment
|
||||
|
||||
### Problem
|
||||
|
||||
When image files are missing from disk (stored externally, purged from git), the check permanently warns. No way to say "these are intentionally external."
|
||||
|
||||
### Proposed Fix
|
||||
|
||||
- `doctor --acknowledge image_assets` marks current missing count as accepted
|
||||
- Stored in `.gbrain/doctor-baselines.json`
|
||||
- WARN only for NEW missing images beyond acknowledged count
|
||||
- Optional `image_assets.external_storage: true` config to skip disk check entirely
|
||||
|
||||
---
|
||||
|
||||
## 5. Auto-heal mode
|
||||
|
||||
### Problem
|
||||
|
||||
Many doctor warnings have known fixes that are safe to auto-apply:
|
||||
|
||||
| Warning | Auto-fix |
|
||||
|---|---|
|
||||
| Supervisor not running | Start supervisor |
|
||||
| Stale embeddings | Submit `embed --stale` job |
|
||||
| Extract coverage < 70% | Submit `extract all --skip-existing` job |
|
||||
| Stale sync | Submit sync job |
|
||||
| Effective date drift | Run `reindex-frontmatter` |
|
||||
|
||||
### Proposed Fix
|
||||
|
||||
`doctor --auto-heal` mode:
|
||||
|
||||
1. Run all checks
|
||||
2. For fixable WARNs: submit fix as a job (not inline — via job queue)
|
||||
3. Report what was fixed vs needs manual attention
|
||||
4. Idempotent: check queue first, don't submit duplicates
|
||||
5. Safety gate: never auto-heals FAILs, only WARNs
|
||||
|
||||
Config:
|
||||
|
||||
```yaml
|
||||
doctor:
|
||||
autoHeal:
|
||||
enabled: true
|
||||
minInterval: "6h"
|
||||
skip:
|
||||
- image_assets
|
||||
- multi_source_drift
|
||||
```
|
||||
|
||||
### Test Cases
|
||||
|
||||
| Check status | Auto-heal enabled | Job already queued | Expected |
|
||||
|---|---|---|---|
|
||||
| WARN: stale embeds | yes | no | Submit embed job |
|
||||
| WARN: stale embeds | yes | yes | Skip (idempotent) |
|
||||
| FAIL: max_crashes | yes | n/a | Don't auto-fix FAILs |
|
||||
| WARN: stale embeds | no | n/a | Report only |
|
||||
| WARN: image_assets | yes (but skipped) | n/a | Report only |
|
||||
|
||||
---
|
||||
|
||||
## 6. Score delta tracking
|
||||
|
||||
### Problem
|
||||
|
||||
No history — each `doctor` run is a snapshot. Can't tell if score is improving or degrading.
|
||||
|
||||
### Proposed Fix
|
||||
|
||||
- Write each run to `.gbrain/doctor-history.jsonl`:
|
||||
```json
|
||||
{"ts":"2026-05-15T12:00:00Z","score":60,"brain_score":79,"checks":{"supervisor":"ok","embeddings":"ok",...}}
|
||||
```
|
||||
- `doctor --trend` shows last N scores with deltas
|
||||
- `doctor --json` includes `previous_score` and `delta` fields
|
||||
|
||||
---
|
||||
|
||||
## 7. Weighted scoring
|
||||
|
||||
### Problem
|
||||
|
||||
Going from 99% → 100% embed coverage weighs the same as 50% → 51%. But the last percent is the hardest (oversized pages, rate limits).
|
||||
|
||||
### Proposed Fix
|
||||
|
||||
Threshold-based scoring:
|
||||
- 100% = full points
|
||||
- ≥95% = 90% of points
|
||||
- ≥80% = 70% of points
|
||||
- <80% = proportional
|
||||
|
||||
---
|
||||
|
||||
## Priority Order
|
||||
|
||||
1. Frontmatter severity levels (highest noise reduction)
|
||||
2. Temporal contradiction awareness (highest false positive reduction, already designed)
|
||||
3. Auto-heal mode (biggest long-term value)
|
||||
4. Score delta tracking (enables monitoring)
|
||||
5. Multi-source drift baseline (quality of life)
|
||||
6. Image assets acknowledgment (quality of life)
|
||||
7. Weighted scoring (nice to have)
|
||||
@@ -1,103 +0,0 @@
|
||||
# Connect GBrain to ChatGPT
|
||||
|
||||
**Status (v0.26.0):** Unblocked. GBrain's `gbrain serve --http` ships OAuth 2.1
|
||||
with PKCE, which is the ChatGPT MCP connector's hard requirement. Before v1.0,
|
||||
this was a P0 TODO — the only major AI client that could not connect.
|
||||
|
||||
ChatGPT does not support bearer-token MCP servers. You must use the OAuth 2.1
|
||||
HTTP server.
|
||||
|
||||
## Setup
|
||||
|
||||
### 1. Start the HTTP server
|
||||
|
||||
```bash
|
||||
gbrain serve --http --port 3131
|
||||
```
|
||||
|
||||
Save the admin bootstrap token printed on stderr. Open
|
||||
`http://localhost:3131/admin` and paste it to access the dashboard.
|
||||
|
||||
### 2. Register a ChatGPT client
|
||||
|
||||
ChatGPT uses the authorization code flow with PKCE (browser-based OAuth).
|
||||
Register from the `/admin` dashboard:
|
||||
|
||||
1. Click **Register client**.
|
||||
2. Name: `chatgpt`.
|
||||
3. Grant type: `authorization_code`.
|
||||
4. Scopes: `read`, `write` (leave `admin` unchecked for ChatGPT).
|
||||
5. Redirect URI: ChatGPT's OAuth redirect (copy it from the ChatGPT
|
||||
connector setup screen — something like
|
||||
`https://chat.openai.com/connector_platform_oauth_redirect`).
|
||||
6. Hit **Register**. The credential-reveal modal shows the `client_id` once
|
||||
with Copy and Download JSON buttons. There is no client secret for
|
||||
PKCE-based public clients.
|
||||
|
||||
Host-repo wrappers can register programmatically:
|
||||
|
||||
```ts
|
||||
await oauthProvider.registerClientManual(
|
||||
'chatgpt',
|
||||
['authorization_code'],
|
||||
'read write',
|
||||
['https://chat.openai.com/connector_platform_oauth_redirect'],
|
||||
);
|
||||
```
|
||||
|
||||
### 3. Expose the server publicly
|
||||
|
||||
```bash
|
||||
brew install ngrok
|
||||
ngrok http 3131 --url your-brain.ngrok.app
|
||||
```
|
||||
|
||||
Your OAuth issuer URL becomes `https://your-brain.ngrok.app`. ChatGPT's
|
||||
connector auto-discovers the spec-compliant endpoint at
|
||||
`/.well-known/oauth-authorization-server`.
|
||||
|
||||
### 4. Add the connector in ChatGPT
|
||||
|
||||
1. Open ChatGPT > Settings > Connectors.
|
||||
2. Click **Add connector**.
|
||||
3. MCP server URL: `https://your-brain.ngrok.app/mcp`.
|
||||
4. Client ID: the `client_id` you saved in step 2.
|
||||
5. Click **Connect**. ChatGPT opens the OAuth consent page, you approve, and
|
||||
the connector is live.
|
||||
|
||||
Start a new conversation and ask ChatGPT to search your brain. The MCP tool
|
||||
calls show up in the admin dashboard's live SSE feed in real time.
|
||||
|
||||
## Scopes
|
||||
|
||||
ChatGPT clients can request any combination of `read`, `write`, `admin`. The
|
||||
scopes granted at consent time are enforced on every tool call. Four
|
||||
operations are `localOnly` and rejected over HTTP regardless of scope:
|
||||
`sync_brain`, `file_upload`, `file_list`, `file_url`. The HTTP server fails
|
||||
closed for any attempt to reach local filesystem surface area.
|
||||
|
||||
Recommended ChatGPT scope: `read write`. Leave `admin` for your local CLI
|
||||
and the admin dashboard.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"Invalid redirect_uri" during the ChatGPT connector OAuth handshake**
|
||||
The registered `redirect-uri` must match ChatGPT's exactly. If ChatGPT
|
||||
rejects your server, check the admin dashboard's **Agents** table for the
|
||||
client, confirm the redirect URI matches what the error page shows, and
|
||||
re-register with the correct URI.
|
||||
|
||||
**ChatGPT shows an MCP connection error after approval**
|
||||
Open `/admin`, watch the SSE feed, and try again. If no request arrives, the
|
||||
connector isn't reaching your ngrok URL. If a request arrives but fails,
|
||||
the Request Log tab shows the exact error.
|
||||
|
||||
**"Unsupported grant_type" on the token endpoint**
|
||||
ChatGPT uses `authorization_code`, which the MCP SDK supports natively.
|
||||
If you see this error, verify the client was registered with
|
||||
`--grant-types authorization_code` and not `client_credentials`.
|
||||
|
||||
## See also
|
||||
|
||||
- [DEPLOY.md](DEPLOY.md) — full OAuth 2.1 setup reference
|
||||
- [ALTERNATIVES.md](ALTERNATIVES.md) — tunnel options (ngrok, Tailscale, Fly)
|
||||
+15
-170
@@ -1,21 +1,17 @@
|
||||
# Deploy GBrain Remote MCP Server
|
||||
|
||||
> **v0.26.0+:** `gbrain serve --http` ships full OAuth 2.1 (client credentials,
|
||||
> auth code + PKCE, refresh rotation, optional DCR), an embedded React admin
|
||||
> dashboard at `/admin`, scoped operations, and a live SSE activity feed.
|
||||
> Pre-v0.26 legacy bearer tokens still work — `verifyAccessToken` falls back
|
||||
> to the `access_tokens` table and grandfathers tokens to `read+write+admin`.
|
||||
> Postgres-only for the legacy fallback (the `access_tokens` table is Postgres-only);
|
||||
> OAuth tables work on both PGLite and Postgres. See [SECURITY.md](../../SECURITY.md)
|
||||
> for env vars and tunable defaults.
|
||||
> **v0.22.7+:** Use `gbrain serve --http` for remote access. It includes built-in
|
||||
> bearer token auth, default-deny CORS, two-bucket rate limiting, body cap, and
|
||||
> per-request audit log. **Postgres-only** (PGLite is local-only by design).
|
||||
> See [SECURITY.md](../../SECURITY.md) for env vars and tunable defaults.
|
||||
|
||||
Access your brain from any device, any AI client. GBrain ships two transports:
|
||||
`gbrain serve` (stdio) for local agents, and `gbrain serve --http` (v0.26.0+)
|
||||
for remote clients over OAuth 2.1.
|
||||
Access your brain from any device, any AI client. GBrain's MCP server runs locally
|
||||
via `gbrain serve` (stdio). For remote access, expose it via the built-in HTTP
|
||||
transport behind a public tunnel.
|
||||
|
||||
## Three Paths
|
||||
## Two Paths
|
||||
|
||||
### Local stdio (zero setup)
|
||||
### Local (zero setup)
|
||||
|
||||
```bash
|
||||
gbrain serve
|
||||
@@ -24,30 +20,7 @@ gbrain serve
|
||||
Works with Claude Code, Cursor, Windsurf, and any MCP client that supports stdio.
|
||||
No server, no tunnel, no token needed. Works on both PGLite and Postgres engines.
|
||||
|
||||
### Remote over OAuth 2.1 (recommended, v0.26.0+)
|
||||
|
||||
```bash
|
||||
gbrain serve --http --port 3131
|
||||
ngrok http 3131 --url your-brain.ngrok.app
|
||||
gbrain serve --http --port 3131 --public-url https://your-brain.ngrok.app
|
||||
```
|
||||
|
||||
Built-in HTTP transport with OAuth 2.1, scoped operations, an admin dashboard
|
||||
at `/admin`, and a live SSE activity feed. Zero external dependencies. This is
|
||||
the only path that works with ChatGPT (OAuth 2.1 + PKCE is required by the
|
||||
ChatGPT MCP connector). Pass `--public-url` whenever the server is reachable
|
||||
at anything other than `http://localhost:<port>` so the OAuth issuer in
|
||||
discovery metadata matches what clients hit (RFC 8414 §3.3).
|
||||
|
||||
Supported clients:
|
||||
- **ChatGPT** — requires OAuth 2.1 + PKCE. Works natively with `--http`.
|
||||
- **Claude Desktop / Cowork** — OAuth 2.1 or legacy bearer tokens.
|
||||
- **Perplexity** — OAuth 2.1 client credentials grant.
|
||||
- **Claude Code, Cursor, Windsurf** — can use OAuth or legacy bearer.
|
||||
|
||||
See the [OAuth 2.1 setup](#oauth-21-setup-v100) section below.
|
||||
|
||||
### Remote with legacy bearer tokens (pre-v0.26 deployments) — Postgres only
|
||||
### Remote (any device, any AI client) — Postgres only
|
||||
|
||||
```
|
||||
Your AI client (Claude Desktop, Perplexity, etc.)
|
||||
@@ -63,133 +36,7 @@ This requires:
|
||||
3. A public tunnel (ngrok, Tailscale, or cloud host)
|
||||
4. A bearer token created via `gbrain auth create <name>`
|
||||
|
||||
Pre-v1.0 tokens are grandfathered as `read+write+admin` scopes when you upgrade
|
||||
to the HTTP server, so no migration is required.
|
||||
|
||||
## OAuth 2.1 Setup (v0.26.0+)
|
||||
|
||||
### 1. Start the HTTP server
|
||||
|
||||
```bash
|
||||
gbrain serve --http --port 3131
|
||||
```
|
||||
|
||||
On first start, the server prints an **admin bootstrap token** to stderr:
|
||||
|
||||
```
|
||||
Admin bootstrap token: 3a1f9c...
|
||||
Open http://localhost:3131/admin and paste it to log in.
|
||||
```
|
||||
|
||||
Save this token. Open `http://localhost:3131/admin` and paste it to access the
|
||||
dashboard. The dashboard shows live activity, registered clients, request logs,
|
||||
and per-client config export.
|
||||
|
||||
> **v0.26.9+:** `mcp_request_log.params` and the live SSE activity feed default
|
||||
> to a redacted summary `{redacted, kind, declared_keys, unknown_key_count, approx_bytes}`.
|
||||
> Declared param keys are kept (intersected against the operation's spec); unknown
|
||||
> keys are counted but never named, and byte sizes round up to 1KB so size-probe
|
||||
> attacks can't binary-search secret content. Operators on a personal laptop who
|
||||
> want raw payloads back can pass `gbrain serve --http --log-full-params` (loud
|
||||
> stderr warning fires at startup). Multi-tenant deployments should leave it on
|
||||
> the redacted default.
|
||||
|
||||
### 2. Register OAuth clients
|
||||
|
||||
Register clients from the **`/admin` dashboard**:
|
||||
|
||||
1. Click **Register client**.
|
||||
2. Enter a name (e.g. `perplexity`, `chatgpt`).
|
||||
3. Pick scopes: `read`, `write`, `admin` (checkboxes).
|
||||
4. Pick grant type: `client_credentials` for machine-to-machine (Perplexity,
|
||||
Claude Desktop bearer mode) or `authorization_code` for browser-based
|
||||
clients with PKCE (ChatGPT).
|
||||
5. For `authorization_code` clients, paste the redirect URI.
|
||||
6. Hit **Register**. The credential-reveal modal shows the `client_id` (and
|
||||
`client_secret` for confidential clients) once. Copy or Download JSON
|
||||
immediately — secrets are hashed on storage and never shown again.
|
||||
|
||||
Or from the CLI — faster for scripting:
|
||||
|
||||
```bash
|
||||
gbrain auth register-client perplexity \
|
||||
--grant-types client_credentials \
|
||||
--scopes "read write"
|
||||
```
|
||||
|
||||
**v0.34 — source-scoped clients.** Multi-source brains can scope a client's
|
||||
write authority to one source and its read scope to a curated set with the
|
||||
new `--source` and `--federated-read` flags:
|
||||
|
||||
```bash
|
||||
gbrain auth register-client dept-x-agent \
|
||||
--grant-types client_credentials \
|
||||
--scopes "read write" \
|
||||
--source dept-x \
|
||||
--federated-read dept-x,shared,parent-canon
|
||||
```
|
||||
|
||||
`--source` controls the write authority — `put_page` / `add_link` / etc only
|
||||
land in `dept-x`. `--federated-read` controls the read axis independently;
|
||||
queries return rows from any of the listed sources. Omit both flags for the
|
||||
v0.33-compatible super-client shape. Pre-v0.34 clients are backfilled to
|
||||
`source_id='default'` on `gbrain upgrade`.
|
||||
|
||||
Host-repo wrappers can register programmatically:
|
||||
|
||||
```ts
|
||||
await oauthProvider.registerClientManual(
|
||||
'perplexity',
|
||||
['client_credentials'],
|
||||
'read write',
|
||||
[], // redirect_uris, empty for CC
|
||||
);
|
||||
```
|
||||
|
||||
For self-service client registration (Dynamic Client Registration, RFC 7591),
|
||||
start the server with `--enable-dcr`. DCR is off by default.
|
||||
|
||||
### 3. Expose the server
|
||||
|
||||
**v0.34 — bind explicitly.** `gbrain serve --http` defaults to `127.0.0.1`.
|
||||
To accept connections from the ngrok tunnel (or any non-loopback source),
|
||||
restart with `--bind`:
|
||||
|
||||
```bash
|
||||
gbrain serve --http --port 3131 --bind 0.0.0.0 --public-url https://your-brain.ngrok.app
|
||||
```
|
||||
|
||||
When `--public-url` is set without `--bind`, a stderr WARN fires at
|
||||
startup so the misconfiguration ("the tunnel is up but my agent gets
|
||||
ECONNREFUSED") is loud.
|
||||
|
||||
```bash
|
||||
brew install ngrok
|
||||
ngrok config add-authtoken YOUR_TOKEN
|
||||
ngrok http 3131 --url your-brain.ngrok.app
|
||||
```
|
||||
|
||||
Your OAuth issuer URL becomes `https://your-brain.ngrok.app`. The MCP SDK's
|
||||
router exposes the spec-compliant discovery endpoint at
|
||||
`/.well-known/oauth-authorization-server`.
|
||||
|
||||
### 4. Scopes and localOnly
|
||||
|
||||
Every operation is tagged `read | write | admin`. Four operations are
|
||||
`localOnly` and rejected over HTTP regardless of scope: `sync_brain`,
|
||||
`file_upload`, `file_list`, `file_url`. Remote agents cannot reach local
|
||||
filesystem surface area.
|
||||
|
||||
| Scope | What it allows |
|
||||
|-------|---------------|
|
||||
| `read` | `search`, `query`, `get_page`, `list_pages`, graph traversal |
|
||||
| `write` | `put_page`, `delete_page`, `add_link`, `add_timeline_entry` |
|
||||
| `admin` | Client management, token revocation, sweep, local-only ops |
|
||||
|
||||
## Legacy Bearer Token Setup
|
||||
|
||||
Keep using pre-v0.26 bearer tokens if you aren't ready to migrate. They
|
||||
grandfather to `read+write+admin` scopes on the HTTP server.
|
||||
## Remote Setup
|
||||
|
||||
### 1. Set up the tunnel
|
||||
|
||||
@@ -220,7 +67,6 @@ if compromised. Tokens are stored SHA-256 hashed in your database.
|
||||
|
||||
### 3. Connect your AI client
|
||||
|
||||
- **ChatGPT:** [setup guide](CHATGPT.md) (OAuth 2.1 + PKCE, requires `gbrain serve --http`)
|
||||
- **Claude Code:** [setup guide](CLAUDE_CODE.md)
|
||||
- **Claude Desktop:** [setup guide](CLAUDE_DESKTOP.md) (must use GUI, not JSON config)
|
||||
- **Claude Cowork:** [setup guide](CLAUDE_COWORK.md)
|
||||
@@ -277,8 +123,7 @@ Remote servers must be added via Settings > Integrations, NOT
|
||||
| put_page | 100-500ms | Write + trigger search_vector update |
|
||||
| get_stats | < 100ms | Aggregate query |
|
||||
|
||||
**Note:** `gbrain serve --http` shipped in v0.26.0 with OAuth 2.1 + admin
|
||||
dashboard baked into the binary. The custom HTTP wrapper pattern (see
|
||||
[voice recipe](../../recipes/twilio-voice-brain.md)) is still supported for
|
||||
teams that need bespoke middleware, but for most remote deployments the
|
||||
built-in server is the recommended path.
|
||||
**Note:** `gbrain serve --http` (built-in HTTP transport) is planned but not yet
|
||||
implemented. Currently, remote MCP requires a custom HTTP wrapper. See the
|
||||
production deployment pattern in the [voice recipe](../../recipes/twilio-voice-brain.md)
|
||||
for a reference implementation.
|
||||
|
||||
@@ -1,213 +0,0 @@
|
||||
# Proposal: Temporal Axis for Contradiction Probe
|
||||
|
||||
**Status:** Report / RFC
|
||||
**Date:** 2026-05-14
|
||||
**Context:** A large production run of `gbrain eval suspected-contradictions` surfaced ~115 HIGH findings. Walking through them by hand exposed a structural limitation in the probe.
|
||||
|
||||
## The Problem
|
||||
|
||||
The contradiction probe (`gbrain eval suspected-contradictions`) treats all claims as timeless. When two chunks make conflicting statements, the judge flags a contradiction regardless of whether both statements were true at their respective points in time.
|
||||
|
||||
This worked fine when the brain was mostly static wiki pages. It breaks now that the brain contains:
|
||||
- Conversation transcripts with claims that were true when spoken
|
||||
- Meeting pages capturing what people said on specific dates
|
||||
- Takes that evolve (a founder's ARR claim in January vs. July)
|
||||
- Status records that supersede each other (a state moves from "trial" to "confirmed")
|
||||
|
||||
The probe can't distinguish "this changed" from "this is wrong."
|
||||
|
||||
## Bug-class examples (synthetic placeholders)
|
||||
|
||||
### 1. Temporal Evolution (False Positive)
|
||||
|
||||
```
|
||||
Finding: HIGH
|
||||
A: [daily/transcripts/2026/2026-04-28] "status: trial"
|
||||
B: [meetings/2026-05-07-session] "status: confirmed"
|
||||
Axis: Whether status is trial or confirmed
|
||||
```
|
||||
|
||||
Both are correct as of their respective dates. April 28: trial. May 7: confirmed. The probe flags this because it has no concept of "this claim was valid from X until Y." The May 7 record didn't make the April 28 transcript wrong; it recorded a change.
|
||||
|
||||
### 2. Negation Parsing (False Positive)
|
||||
|
||||
```
|
||||
Finding: HIGH
|
||||
A: [people/alice-example] "person traveled to city-a for alice-example's event — NOT bob-example's event"
|
||||
B: [meetings/2026-05-11-context] mentions of bob-example's event in city-b
|
||||
Axis: Whose event the city-a trip was for
|
||||
```
|
||||
|
||||
The disambiguation fact contains "NOT bob-example's event" as an explicit negation. The judge reads "bob-example's event" as a positive claim and flags it against the alice-example context. The data is correct; the probe can't parse negation.
|
||||
|
||||
### 3. Role Changes (True Positive That Needs Time Awareness)
|
||||
|
||||
```
|
||||
Finding: HIGH
|
||||
A: [sources/notes/2017-03-28] advisor-example: "Partner, venture-firm-a"
|
||||
B: [people/advisor-example] advisor-example: "Senior Policy Advisor, gov-org-b"
|
||||
```
|
||||
|
||||
Both true at their respective times. 2017: partner at venture-firm-a. 2025: gov-org-b advisor. The current probe correctly flags this as a contradiction, but the resolution should be "superseded by time" not "one side is wrong." The 2017 note isn't wrong; it's a historical record.
|
||||
|
||||
## Scenario #1: Founder Tracking (the big one)
|
||||
|
||||
This is the use case that makes a time axis transformative rather than incremental.
|
||||
|
||||
The brain holds hundreds of company pages and thousands of meeting pages. Founders make claims:
|
||||
|
||||
- "We're at $50K MRR" (January OH)
|
||||
- "We hit $200K MRR" (April OH)
|
||||
- "We're at $150K MRR" (July OH — what happened?)
|
||||
|
||||
Today the probe would flag January vs. April as a contradiction. The real signal is April vs. July: **a claimed metric went backwards.** That's not a data quality issue; that's intelligence.
|
||||
|
||||
What a time-aware probe could surface:
|
||||
|
||||
**Claim trajectory tracking:**
|
||||
```
|
||||
Company: Acme Corp
|
||||
2026-01: "$50K MRR" (source: OH transcript)
|
||||
2026-04: "$200K MRR" (source: OH transcript)
|
||||
2026-07: "$150K MRR" (source: OH transcript) ← REGRESSION DETECTED
|
||||
2026-07: "$2M ARR" (source: investor update) ← INCONSISTENT WITH MRR
|
||||
```
|
||||
|
||||
**Prediction vs. outcome:**
|
||||
```
|
||||
Founder: Jane Doe (Acme Corp)
|
||||
2026-01: "We'll hit $1M ARR by June" (source: batch kickoff)
|
||||
2026-06: Actual ARR: $400K (source: investor update)
|
||||
→ Prediction accuracy: 40%
|
||||
→ Pattern: consistently 2-3x optimistic on timeline
|
||||
```
|
||||
|
||||
**Narrative consistency:**
|
||||
```
|
||||
Founder: John Smith (WidgetCo)
|
||||
2026-01: "Our moat is proprietary data" (source: interview)
|
||||
2026-03: "We're pivoting to an API-first model" (source: OH)
|
||||
2026-06: "Our moat is network effects" (source: Demo Day)
|
||||
→ Moat narrative changed 3x in 6 months — flag for review
|
||||
```
|
||||
|
||||
This isn't adversarial. It's the kind of pattern an experienced operator notices intuitively across hundreds of conversations. GBrain can make it systematic.
|
||||
|
||||
## Scenario #2: Event Disambiguation
|
||||
|
||||
Two distinct events within a short window can conflate during ingestion because the probe has no temporal frame to say "event A is a different event from event B."
|
||||
|
||||
Time-aware facts would store (synthetic placeholders):
|
||||
```
|
||||
fact: "alice-example milestone" valid_from: 2026-04-15 valid_until: 2026-04-15
|
||||
fact: "alice-example event in city-a" valid_from: 2026-04-17 valid_until: 2026-04-19
|
||||
fact: "bob-example milestone" valid_from: 2026-05-04 valid_until: 2026-05-04
|
||||
fact: "bob-example event in city-b" valid_from: 2026-05-12 valid_until: 2026-05-12
|
||||
```
|
||||
|
||||
The probe should recognize these as two distinct events with non-overlapping time windows, not as contradictions about "whose event."
|
||||
|
||||
## Scenario #3: Role and Status Changes
|
||||
|
||||
People change roles. Companies change status. The brain records history. Synthetic examples representative of the cases observed in production:
|
||||
|
||||
- advisor-example: venture-firm-a partner (2019) → gov-org-b advisor (2025)
|
||||
- investor-example: fund-a partner → fund-b CEO (2023)
|
||||
- agent-fork: provider restriction event (2026-04-04) ≠ shutdown
|
||||
- fund-c: "interesting fund" (early) → "declined" (later) → "losing confidence" (latest)
|
||||
|
||||
All of these are correct historical records. The probe should classify them as **temporal supersession** rather than **contradiction.**
|
||||
|
||||
## Scenario #4: Decision Tracking
|
||||
|
||||
Multi-step decisions that supersede earlier framings example (synthetic):
|
||||
```
|
||||
2026-04-24: "status: trial" (initial framing)
|
||||
2026-04-25: "status: in progress" (confirmed, no longer "trial")
|
||||
2026-05-07: "status: finalized" (session record)
|
||||
2026-05-11: follow-up actions taken
|
||||
```
|
||||
|
||||
Each step supersedes the previous. A time-aware probe would show the **evolution chain** rather than flagging each pair as a contradiction.
|
||||
|
||||
## What Exists Today
|
||||
|
||||
The probe already has some temporal infrastructure:
|
||||
|
||||
1. **`date-filter.ts`** — `shouldSkipForDateMismatch()` pre-filters pairs, but only checks whether dates are "too far apart" (a coarse heuristic). It doesn't reason about which claim is newer or whether one supersedes the other.
|
||||
|
||||
2. **`auto-supersession.ts`** — proposes resolution commands, checks `since_date` on takes. But this is post-hoc (after the judge flags a contradiction). The judge itself doesn't see dates.
|
||||
|
||||
3. **Facts table** has `valid_from` and `valid_until` columns. These exist but are sparsely populated and not used by the probe.
|
||||
|
||||
4. **Takes table** has `since_date`. Also sparsely populated.
|
||||
|
||||
## What Would Need to Change
|
||||
|
||||
### Phase 1: Judge prompt enhancement (smallest change, biggest impact)
|
||||
|
||||
Pass the source dates to the judge. The current judge prompt shows two text chunks and asks "are these contradictory?" If it also showed:
|
||||
|
||||
```
|
||||
Statement A (from: 2026-04-28):
|
||||
"status: trial"
|
||||
|
||||
Statement B (from: 2026-05-07):
|
||||
"status: confirmed"
|
||||
```
|
||||
|
||||
The judge could output a `temporal_supersession` verdict instead of `contradiction`. New verdict taxonomy:
|
||||
|
||||
- `no_contradiction` — statements are compatible
|
||||
- `contradiction` — genuinely conflicting claims at the same point in time
|
||||
- `temporal_supersession` — newer claim updates/replaces older claim (not an error)
|
||||
- `temporal_regression` — a metric or status went backwards (potential signal)
|
||||
- `temporal_evolution` — legitimate change over time, neither supersession nor regression
|
||||
- `negation_artifact` — one side contains an explicit negation the judge misread
|
||||
|
||||
### Phase 2: Claim trajectory view (new command)
|
||||
|
||||
```bash
|
||||
gbrain eval trajectory "Acme Corp MRR"
|
||||
gbrain eval trajectory "advisor-example role"
|
||||
gbrain eval trajectory "deal-x status"
|
||||
```
|
||||
|
||||
Pull all time-stamped claims about an entity+attribute, sort chronologically, detect:
|
||||
- Regressions (metric went down)
|
||||
- Contradictions within the same time window
|
||||
- Prediction vs. outcome gaps
|
||||
- Narrative drift (moat story changed 3x)
|
||||
|
||||
### Phase 3: Automatic `valid_from`/`valid_until` population
|
||||
|
||||
During `extract_facts`, infer temporal bounds from source context:
|
||||
- Meeting page dated 2026-04-28 → claims valid_from 2026-04-28
|
||||
- Takes from transcripts → valid_from = transcript date
|
||||
- Imported notes → valid_from = note date
|
||||
- Entity pages with no date → valid_from = page created date (weakest signal)
|
||||
|
||||
### Phase 4: Founder scorecard
|
||||
|
||||
For founders specifically, a temporal probe could generate:
|
||||
- **Claim accuracy score** — what they predicted vs. what happened
|
||||
- **Consistency score** — how stable their narrative is over time
|
||||
- **Growth trajectory** — whether the numbers are actually moving
|
||||
- **Red flag detector** — metrics going backwards, story changing, timeline slipping
|
||||
|
||||
## Recommendation
|
||||
|
||||
Start with Phase 1. The judge prompt change is small. It immediately eliminates the temporal false positives (which were a majority of the residual HIGH findings in the production audit) and gives the probe a new vocabulary for time-aware reasoning.
|
||||
|
||||
Phase 2 (trajectory view) is the one that would change how operators use the brain for founder evaluation. Worth scoping as a standalone feature.
|
||||
|
||||
Phases 3–4 are downstream and can wait.
|
||||
|
||||
## Appendix: Production probe stats (2026-05-14)
|
||||
|
||||
- ~107K pages, ~257K chunks
|
||||
- Previous run: ~115 HIGH findings across 50 queries
|
||||
- After manual resolution: ~25 residual findings
|
||||
- Of those ~25: roughly two-thirds temporal false positives, the remainder probe artifacts (self-contradiction, negation parsing)
|
||||
- 0 genuine data contradictions remained on the queries tested
|
||||
- Fresh targeted probe on a representative entity-role query: 0 contradictions (was 14+ before fixes)
|
||||
@@ -1,93 +0,0 @@
|
||||
# Takes vs Facts — Architectural Distinction
|
||||
|
||||
gbrain has two epistemological storage layers that serve different purposes.
|
||||
**Never conflate them.**
|
||||
|
||||
## Takes (cold storage — `takes` table)
|
||||
|
||||
The epistemological layer. WHO believes WHAT, with confidence weight and time.
|
||||
|
||||
- **Source:** Extracted from brain pages (markdown) by LLM analysis
|
||||
- **Scope:** Multi-holder — captures beliefs from *any* speaker, not just the brain owner
|
||||
- **Kinds:** `take` (opinion), `fact` (verifiable), `bet` (prediction), `hunch` (intuition)
|
||||
- **Lifecycle:** Cold storage, retrospective. Updated when pages change or re-extraction runs.
|
||||
- **Scale:** 100K+ rows across thousands of holders in a mature brain
|
||||
|
||||
**Example takes:**
|
||||
- `holder=people/garry-tan kind=bet` "AI will replace 50% of coding by 2030" (w=0.75)
|
||||
- `holder=people/jared-friedman kind=take` "Momo has strong retention" (w=0.80)
|
||||
- `holder=world kind=fact` "Clipboard raised $100M Series C" (w=1.0)
|
||||
- `holder=brain kind=hunch` "Garry has a hero/rescuer pattern" (w=0.70)
|
||||
|
||||
**Query surface:** `gbrain takes list`, `gbrain takes search`, `gbrain think`
|
||||
|
||||
## Facts (hot memory — `facts` table, v0.31)
|
||||
|
||||
Personal knowledge from the brain owner's conversations. Real-time capture.
|
||||
|
||||
- **Source:** Extracted per-turn from conversation by the facts hook (Haiku)
|
||||
- **Scope:** Single-user — only the brain owner's stated knowledge
|
||||
- **Kinds:** `event`, `preference`, `commitment`, `belief`, `fact`
|
||||
- **Lifecycle:** Hot storage, real-time. Captured as conversations happen.
|
||||
- **Bridge:** Dream cycle `consolidate` phase promotes hot facts → cold takes nightly
|
||||
|
||||
**Example facts:**
|
||||
- `kind=event` "I have a meeting with Brian tomorrow"
|
||||
- `kind=preference` "I don't drink coffee"
|
||||
- `kind=commitment` "We decided on nesting custody"
|
||||
- `kind=belief` "I think the market is overheated"
|
||||
|
||||
**Query surface:** `gbrain recall`, MCP `_meta.brain_hot_memory`
|
||||
|
||||
## The Category Error
|
||||
|
||||
**Never dump takes into the facts table.** Takes include other people's attributed
|
||||
beliefs (Jared's assessment of a company, PG's view on schools, a founder's
|
||||
revenue claims). These are NOT the brain owner's personal facts.
|
||||
|
||||
**Never dump facts into the takes table without transformation.** Facts are
|
||||
scoped to what the owner said in conversation. They become takes only through
|
||||
the dream cycle's consolidate phase, which adds proper attribution, deduplication,
|
||||
and temporal reasoning.
|
||||
|
||||
## The Bridge
|
||||
|
||||
The dream cycle's `consolidate` phase (v0.31) is the one-way bridge:
|
||||
|
||||
```
|
||||
hot facts → [dream consolidate] → cold takes
|
||||
```
|
||||
|
||||
Facts flow in ONE direction. The consolidate phase:
|
||||
1. Groups related facts by entity
|
||||
2. Deduplicates against existing takes
|
||||
3. Promotes durable facts to takes with proper holder/weight
|
||||
4. Marks consolidated facts with `consolidated_at` + `consolidated_into`
|
||||
|
||||
## Production Extraction Data (2026-05-10)
|
||||
|
||||
First full takes extraction run on a ~100K-page brain:
|
||||
- **Model:** Azure GPT-5.5 (ties Opus quality at 1/8th cost — $0.033 vs $0.260/page)
|
||||
- **Result:** 100,720 takes from 28,256 on-disk pages, $361.49, 83 errors (0.3%)
|
||||
- **Breakdown:** 70,960 takes / 24,342 facts / 2,875 bets / 2,649 hunches
|
||||
- **Holders:** 6,239 unique holders
|
||||
- **Cross-modal eval:** 6.8/10 overall (GPT-5.5 + Opus 4.6 scored independently)
|
||||
|
||||
### Eval Dimensions
|
||||
|
||||
| Dimension | Score | Notes |
|
||||
|-----------|-------|-------|
|
||||
| Accuracy | 7.5 | Claims faithfully represent sources |
|
||||
| Attribution | 6.5 | Holder/subject confusion was #1 issue |
|
||||
| Weight calibration | 7.0 | Good range usage, some false precision |
|
||||
| Kind classification | 6.5 | Occasional fact/take misclassification |
|
||||
| Signal density | 6.5 | Some trivial extractions pass through |
|
||||
|
||||
### Key Learnings for Extraction Prompts
|
||||
|
||||
1. **Holder ≠ subject.** "Garry has a hero/rescuer pattern" → holder=brain, NOT people/garry-tan
|
||||
2. **Atomic claims.** Split compound claims into separate rows
|
||||
3. **Amplification ≠ endorsement.** Retweet-only → max weight 0.55
|
||||
4. **Self-reported ≠ verified.** "Reports 7 figures" → holder=person, weight=0.75, NOT world/1.0
|
||||
5. **No false precision.** Use 0.05 increments (0.35, 0.55, 0.75), not 0.74 or 0.82
|
||||
6. **"So what" test.** Skip Twitter handles, follower counts, obvious metadata
|
||||
@@ -1,40 +0,0 @@
|
||||
{
|
||||
"version": 1,
|
||||
"description": "Embedding provider smoke test — verifies semantic search returns expected results for known brain content. Run after any embedding model change or migration.",
|
||||
"queries": [
|
||||
{
|
||||
"id": "yc-labs-strategy",
|
||||
"query": "YC Labs strategy and product team",
|
||||
"relevant": [
|
||||
"originals/yc-labs-internal-team",
|
||||
"originals/harj-yc-labs-strategy-2026-05"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "garry-tan-person",
|
||||
"query": "Who is Garry Tan",
|
||||
"relevant": [
|
||||
"people/garry-tan"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "gstack-project",
|
||||
"query": "GStack open source AI coding framework",
|
||||
"relevant": [
|
||||
"projects/gstack/gstackbrain"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "yc-carry-compensation",
|
||||
"query": "GP carry and compensation structure at YC",
|
||||
"relevant": [
|
||||
"originals/harj-yc-labs-strategy-2026-05"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "meeting-search",
|
||||
"query": "recent office hours meeting notes",
|
||||
"relevant": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
# Per-run output JSONLs land here; only baseline-runs/<date>-<model>.jsonl is canonical.
|
||||
run-*.jsonl
|
||||
@@ -1,189 +0,0 @@
|
||||
# functional-area-resolver A/B eval
|
||||
|
||||
Maintainer-side eval evidence for the `functional-area-resolver` skill. Lives
|
||||
outside `skills/` deliberately — the skillpack bundler walks `skills/<skill>/`
|
||||
recursively, so an eval surface in there would ship to every downstream
|
||||
`gbrain skillpack install`. This directory is NOT bundled. The pattern (in
|
||||
SKILL.md) ships everywhere; the eval evidence stays in the gbrain repo where
|
||||
maintainers can re-baseline.
|
||||
|
||||
## What this proves
|
||||
|
||||
Three resolver shapes tested across three Anthropic frontier models. The
|
||||
pattern in `skills/functional-area-resolver/SKILL.md` (functional-area
|
||||
dispatchers with `(dispatcher for: ...)` clauses) **beats the verbose
|
||||
bullet-list baseline by +13 to +17pp on training while shipping at 48% the
|
||||
size**, and **catastrophically beats compression without the dispatcher
|
||||
clause** on Sonnet (100% vs 41.7% training, lenient).
|
||||
|
||||
## Methodology
|
||||
|
||||
### Variants
|
||||
|
||||
- `variants/baseline.md` — the verbose 270-row bullet-list shape extracted
|
||||
from a real production AGENTS.md at git commit `93848ff3b^` (pre-compression
|
||||
state), with owner PII scrubbed. ~25KB.
|
||||
- `variants/functional-areas.md` — the dispatcher pattern at git commit
|
||||
`93848ff3b` (the commit titled "AGENTS.md: functional-area resolver —
|
||||
25KB→13KB, 100% routing accuracy"). ~13KB.
|
||||
- `variants/resolver-of-resolvers.md` — derived mechanically from
|
||||
functional-areas by stripping `(dispatcher for: ...)` clauses. The ablation
|
||||
case: same structure, no sub-skill visibility. ~10KB.
|
||||
|
||||
### Corpora
|
||||
|
||||
- `fixtures.jsonl` — 20 hand-authored training fixtures used to develop the
|
||||
variants. Headline accuracy on training is informative but not the claim
|
||||
(same-author overfitting risk).
|
||||
- `fixtures-held-out.jsonl` — 5 fixtures authored BEFORE the variants and
|
||||
not adjusted afterward. Held-out is the canonical claim, but small n means
|
||||
it saturates near 100% for most cells.
|
||||
|
||||
### Scoring
|
||||
|
||||
Every output row carries two scores:
|
||||
|
||||
- **STRICT** (`correct`) — predicted slug equals expected exactly.
|
||||
- **LENIENT** (`correct_lenient`) — predicted is in the same dispatcher area
|
||||
as expected per the variant's `(dispatcher for: ...)` clauses. For variants
|
||||
without dispatcher clauses (baseline, resolver-of-resolvers), LENIENT
|
||||
collapses to STRICT.
|
||||
|
||||
Both matter:
|
||||
- STRICT measures "does the LLM return the exact slug?"
|
||||
- LENIENT measures "does the LLM land in the right area, even if it picks a
|
||||
more-specific sub-skill?" This reflects production agent behavior — landing
|
||||
in `gmail` for an email intent succeeds even if the resolver wrote
|
||||
`executive-assistant`.
|
||||
|
||||
### Repeats + statistics
|
||||
|
||||
- n=3 seeded repeats per (fixture, variant, model).
|
||||
- 95% confidence interval via t-distribution across the 3 seeded means
|
||||
(t-critical=4.303 for df=2).
|
||||
- Models: `claude-opus-4-7`, `claude-sonnet-4-6`, `claude-haiku-4-5-20251001`.
|
||||
|
||||
### Receipt format
|
||||
|
||||
Each run writes one JSONL with:
|
||||
- Header row: `{kind:'receipt', model, prompt_template_hash, fixtures_hash,
|
||||
fixtures_held_out_hash, harness_sha, ts, cmd_args}` — binds the run to a
|
||||
specific harness version and inputs so re-runs are auditable.
|
||||
- One row per (fixture × variant × seed): full row schema in `harness-runner.ts`.
|
||||
|
||||
Baseline receipts committed in `baseline-runs/` after the v0.32.3.0
|
||||
re-baseline.
|
||||
|
||||
## Results (2026-05-11)
|
||||
|
||||
Training corpus (n=20, 3 seeds, LENIENT scoring):
|
||||
|
||||
| Variant | Opus 4.7 | Sonnet 4.6 | Haiku 4.5 | Size |
|
||||
|---|---|---|---|---|
|
||||
| baseline | 81.7% ± 7.2% | 86.7% ± 7.2% | 73.3% ± 7.2% | 25KB |
|
||||
| **functional-areas** | **98.3% ± 7.2%** | **100% ± 0%** | **88.3% ± 7.2%** | **13KB** |
|
||||
| resolver-of-resolvers | 63.3% ± 14.3% | 41.7% ± 7.2% | 65.0% ± 12.4% | 10KB |
|
||||
|
||||
Held-out corpus (n=5, 3 seeds, LENIENT scoring):
|
||||
|
||||
| Variant | Opus 4.7 | Sonnet 4.6 | Haiku 4.5 |
|
||||
|---|---|---|---|
|
||||
| baseline | 100% ± 0% | 100% ± 0% | 100% ± 0% |
|
||||
| **functional-areas** | **100% ± 0%** | **100% ± 0%** | **100% ± 0%** |
|
||||
| resolver-of-resolvers | 100% ± 0% | **73.3% ± 28.7%** | 100% ± 0% |
|
||||
|
||||
Strict numbers and the per-fixture failure traces are in the receipts.
|
||||
|
||||
## How to reproduce
|
||||
|
||||
From the gbrain repo root with `ANTHROPIC_API_KEY` set:
|
||||
|
||||
```bash
|
||||
cd evals/functional-area-resolver
|
||||
|
||||
# Smoke test (1 call, ~$0.01)
|
||||
node harness.mjs --limit 1 --yes
|
||||
|
||||
# Full run on Opus 4.7 (225 calls, ~$1.70)
|
||||
node harness.mjs --model opus --parallel 3 --yes
|
||||
|
||||
# Cross-model
|
||||
node harness.mjs --model sonnet --parallel 3 --yes # ~$1.00
|
||||
node harness.mjs --model haiku --parallel 3 --yes # ~$0.30
|
||||
|
||||
# Re-score an existing run without spending more API budget
|
||||
node rescore.mjs baseline-runs/2026-05-11-opus-4-7.jsonl
|
||||
|
||||
# Unit tests (no API key required)
|
||||
bun test harness-runner.test.ts
|
||||
```
|
||||
|
||||
The harness routes through gbrain's gateway, so it inherits gbrain's auth,
|
||||
rate-lease, and cost-meter behavior. Without `ANTHROPIC_API_KEY` it exits with
|
||||
a clear error.
|
||||
|
||||
## Important caveat: the prompt is load-bearing
|
||||
|
||||
The harness uses a dispatcher-aware prompt (see
|
||||
`harness-runner.ts:PROMPT_TEMPLATE`) that explicitly tells the LLM:
|
||||
|
||||
> Some entries are functional-area dispatchers shaped like:
|
||||
> "**Area name**: triggers... → `dispatcher-skill` (dispatcher for: subskill-a, subskill-b, ...)"
|
||||
> When the user's intent matches an area, RETURN THE MOST-SPECIFIC SUB-SKILL
|
||||
> from that area's "dispatcher for" list, not the dispatcher itself.
|
||||
|
||||
**Without this instruction, every compression variant collapses to ~30-60%
|
||||
on training.** A naive "return the skill slug" prompt makes the LLM pick the
|
||||
area lead instead of drilling into the dispatcher list. This was the failure
|
||||
mode in run-1 (synthetic variants + naive prompt) before the real-variants +
|
||||
dispatcher-aware-prompt re-baseline.
|
||||
|
||||
If you adopt the pattern in your own agent, the SKILL.md guidance applies
|
||||
to your harness prompt. Lift the PROMPT_TEMPLATE from this harness or write
|
||||
your own instruction explaining the dispatcher list.
|
||||
|
||||
## Limitations and v0.33.x follow-ups
|
||||
|
||||
1. Held-out corpus is small (n=5). Saturated at 100% across most cells. Grow
|
||||
to >=20 in v0.33.x.
|
||||
2. Single vendor (Anthropic). Cross-vendor (Gemini, GPT) is v0.33.x.
|
||||
3. No description-length sweep yet. Anthropic Agent Skills median is ~80
|
||||
tokens of frontmatter; we haven't measured the per-row description length
|
||||
sweet spot. v0.33.x.
|
||||
4. Same-author training corpus + variants. Held-out mitigates partially.
|
||||
5. No adversarial fixtures (e.g., "I want to do something brain-related"
|
||||
without specifying what). v0.33.x.
|
||||
|
||||
See `TODOS.md` for the full list.
|
||||
|
||||
## Prior art
|
||||
|
||||
This eval implements a **static-prompt analog** of hierarchical agent routing,
|
||||
a 2024-2025 research direction. The published hierarchical schemes resolve
|
||||
the hierarchy at runtime via a second LLM call; this skill inlines the
|
||||
hierarchy into a single-LLM-pass dispatcher list.
|
||||
|
||||
- AnyTool ([arXiv:2402.04253](https://arxiv.org/abs/2402.04253)) — meta-agent → category → tool hierarchy, +35.4pp over flat retrieval at 16K APIs.
|
||||
- RAG-MCP ([arXiv:2505.03275](https://arxiv.org/html/2505.03275v1)) — embedding-based pre-retrieval, 49.2% token reduction at 3.2× accuracy gain.
|
||||
- Anthropic Agent Skills ([engineering blog](https://www.anthropic.com/engineering/equipping-agents-for-the-real-world-with-agent-skills)) — progressive disclosure (~80-token frontmatter loaded at startup; body loaded on match).
|
||||
|
||||
## File listing
|
||||
|
||||
```
|
||||
evals/functional-area-resolver/
|
||||
├── README.md # this file
|
||||
├── fixtures.jsonl # 20 training fixtures
|
||||
├── fixtures-held-out.jsonl # 5 held-out blind fixtures
|
||||
├── variants/
|
||||
│ ├── baseline.md # 25KB, PII-scrubbed from production
|
||||
│ ├── functional-areas.md # 13KB, PII-scrubbed from production
|
||||
│ └── resolver-of-resolvers.md # 10KB, derived ablation
|
||||
├── harness.mjs # thin Node CLI shim
|
||||
├── harness-runner.ts # TS runner via gbrain gateway
|
||||
├── harness-runner.test.ts # 45 unit tests (no API key)
|
||||
├── rescore.mjs # zero-cost lenient re-score
|
||||
└── baseline-runs/
|
||||
├── 2026-05-11-opus-4-7.jsonl # 225-row Opus baseline
|
||||
├── 2026-05-11-sonnet-4-6.jsonl # 225-row Sonnet baseline
|
||||
└── 2026-05-11-haiku-4-5.jsonl # 225-row Haiku baseline
|
||||
```
|
||||
@@ -1,226 +0,0 @@
|
||||
{"kind":"receipt","model":"anthropic:claude-haiku-4-5-20251001","prompt_template_hash":"17340040af579ca1","fixtures_hash":"feccc99122ea86d5","fixtures_held_out_hash":"5d6256cc9dced124","harness_sha":"fcc395282a92f2b047d4407f2b5a891c069adaac","ts":"2026-05-12T02:51:49.980Z","cmd_args":["--model","haiku","--parallel","3","--yes","--output","run-haiku-4-5.jsonl"]}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"baseline","seed":1,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7638,"output_tokens":5,"latency_ms":718,"ts":"2026-05-12T02:51:50.698Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"baseline","seed":2,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7638,"output_tokens":5,"latency_ms":1569,"ts":"2026-05-12T02:51:51.549Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"baseline","seed":3,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7638,"output_tokens":5,"latency_ms":1163,"ts":"2026-05-12T02:51:51.143Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"baseline","seed":1,"predicted":"enrich","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7631,"output_tokens":5,"latency_ms":739,"ts":"2026-05-12T02:51:52.288Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"baseline","seed":2,"predicted":"enrich","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7631,"output_tokens":5,"latency_ms":602,"ts":"2026-05-12T02:51:52.151Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"baseline","seed":3,"predicted":"enrich","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7631,"output_tokens":5,"latency_ms":738,"ts":"2026-05-12T02:51:52.288Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"baseline","seed":1,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":6,"latency_ms":882,"ts":"2026-05-12T02:51:53.170Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"baseline","seed":2,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":6,"latency_ms":706,"ts":"2026-05-12T02:51:52.994Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"baseline","seed":3,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":6,"latency_ms":706,"ts":"2026-05-12T02:51:52.994Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"baseline","seed":1,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":6,"latency_ms":706,"ts":"2026-05-12T02:51:53.877Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"baseline","seed":2,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":6,"latency_ms":1239,"ts":"2026-05-12T02:51:54.410Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"baseline","seed":3,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":6,"latency_ms":886,"ts":"2026-05-12T02:51:54.057Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"baseline","seed":1,"predicted":"brain-librarian","expected":"brain-librarian","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":7,"latency_ms":921,"ts":"2026-05-12T02:51:55.331Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"baseline","seed":2,"predicted":"brain-librarian","expected":"brain-librarian","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":7,"latency_ms":705,"ts":"2026-05-12T02:51:55.115Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"baseline","seed":3,"predicted":"brain-librarian","expected":"brain-librarian","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":7,"latency_ms":797,"ts":"2026-05-12T02:51:55.207Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"baseline","seed":1,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7631,"output_tokens":7,"latency_ms":718,"ts":"2026-05-12T02:51:56.050Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"baseline","seed":2,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7631,"output_tokens":7,"latency_ms":663,"ts":"2026-05-12T02:51:55.995Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"baseline","seed":3,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7631,"output_tokens":7,"latency_ms":782,"ts":"2026-05-12T02:51:56.114Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"baseline","seed":1,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7638,"output_tokens":6,"latency_ms":721,"ts":"2026-05-12T02:51:56.835Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"baseline","seed":2,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7638,"output_tokens":6,"latency_ms":679,"ts":"2026-05-12T02:51:56.793Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"baseline","seed":3,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7638,"output_tokens":6,"latency_ms":646,"ts":"2026-05-12T02:51:56.760Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"baseline","seed":1,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7638,"output_tokens":6,"latency_ms":667,"ts":"2026-05-12T02:51:57.502Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"baseline","seed":2,"predicted":"book-mirror","expected":"strategic-reading","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7638,"output_tokens":6,"latency_ms":649,"ts":"2026-05-12T02:51:57.484Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"baseline","seed":3,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7638,"output_tokens":6,"latency_ms":1016,"ts":"2026-05-12T02:51:57.851Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"baseline","seed":1,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":6,"latency_ms":651,"ts":"2026-05-12T02:51:58.503Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"baseline","seed":2,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":6,"latency_ms":653,"ts":"2026-05-12T02:51:58.504Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"baseline","seed":3,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":6,"latency_ms":859,"ts":"2026-05-12T02:51:58.710Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"baseline","seed":1,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7638,"output_tokens":9,"latency_ms":631,"ts":"2026-05-12T02:51:59.341Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"baseline","seed":2,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7638,"output_tokens":9,"latency_ms":1021,"ts":"2026-05-12T02:51:59.731Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"baseline","seed":3,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7638,"output_tokens":9,"latency_ms":682,"ts":"2026-05-12T02:51:59.392Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"baseline","seed":1,"predicted":"media-ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7635,"output_tokens":7,"latency_ms":642,"ts":"2026-05-12T02:52:00.373Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"baseline","seed":2,"predicted":"media-ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7635,"output_tokens":7,"latency_ms":687,"ts":"2026-05-12T02:52:00.418Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"baseline","seed":3,"predicted":"media-ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7635,"output_tokens":7,"latency_ms":795,"ts":"2026-05-12T02:52:00.526Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"baseline","seed":1,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7631,"output_tokens":7,"latency_ms":728,"ts":"2026-05-12T02:52:01.254Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"baseline","seed":2,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7631,"output_tokens":7,"latency_ms":662,"ts":"2026-05-12T02:52:01.188Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"baseline","seed":3,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7631,"output_tokens":7,"latency_ms":906,"ts":"2026-05-12T02:52:01.432Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"baseline","seed":1,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":7,"latency_ms":624,"ts":"2026-05-12T02:52:02.056Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"baseline","seed":2,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":7,"latency_ms":682,"ts":"2026-05-12T02:52:02.114Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"baseline","seed":3,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":7,"latency_ms":672,"ts":"2026-05-12T02:52:02.104Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"baseline","seed":1,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7632,"output_tokens":9,"latency_ms":711,"ts":"2026-05-12T02:52:02.825Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"baseline","seed":2,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7632,"output_tokens":9,"latency_ms":671,"ts":"2026-05-12T02:52:02.785Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"baseline","seed":3,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7632,"output_tokens":9,"latency_ms":751,"ts":"2026-05-12T02:52:02.865Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"baseline","seed":1,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7630,"output_tokens":6,"latency_ms":670,"ts":"2026-05-12T02:52:03.535Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"baseline","seed":2,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7630,"output_tokens":6,"latency_ms":649,"ts":"2026-05-12T02:52:03.514Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"baseline","seed":3,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7630,"output_tokens":6,"latency_ms":827,"ts":"2026-05-12T02:52:03.692Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"baseline","seed":1,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7630,"output_tokens":4,"latency_ms":682,"ts":"2026-05-12T02:52:04.374Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"baseline","seed":2,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7630,"output_tokens":4,"latency_ms":760,"ts":"2026-05-12T02:52:04.452Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"baseline","seed":3,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7630,"output_tokens":4,"latency_ms":691,"ts":"2026-05-12T02:52:04.383Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"baseline","seed":1,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":9,"latency_ms":642,"ts":"2026-05-12T02:52:05.094Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"baseline","seed":2,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":9,"latency_ms":698,"ts":"2026-05-12T02:52:05.150Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"baseline","seed":3,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":9,"latency_ms":632,"ts":"2026-05-12T02:52:05.084Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"baseline","seed":1,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7633,"output_tokens":7,"latency_ms":665,"ts":"2026-05-12T02:52:05.815Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"baseline","seed":2,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7633,"output_tokens":7,"latency_ms":850,"ts":"2026-05-12T02:52:06.000Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"baseline","seed":3,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7633,"output_tokens":7,"latency_ms":636,"ts":"2026-05-12T02:52:05.786Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"baseline","seed":1,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7633,"output_tokens":5,"latency_ms":949,"ts":"2026-05-12T02:52:06.950Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"baseline","seed":2,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7633,"output_tokens":5,"latency_ms":803,"ts":"2026-05-12T02:52:06.804Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"baseline","seed":3,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7633,"output_tokens":5,"latency_ms":747,"ts":"2026-05-12T02:52:06.748Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"baseline","seed":1,"predicted":"calendar-event-create","expected":"daily-task-manager","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7632,"output_tokens":8,"latency_ms":779,"ts":"2026-05-12T02:52:07.729Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"baseline","seed":2,"predicted":"meeting-prep","expected":"daily-task-manager","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7632,"output_tokens":6,"latency_ms":779,"ts":"2026-05-12T02:52:07.729Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"baseline","seed":3,"predicted":"meeting-prep","expected":"daily-task-manager","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7632,"output_tokens":6,"latency_ms":1134,"ts":"2026-05-12T02:52:08.084Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7635,"output_tokens":5,"latency_ms":706,"ts":"2026-05-12T02:52:08.790Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7635,"output_tokens":5,"latency_ms":1453,"ts":"2026-05-12T02:52:09.537Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7635,"output_tokens":5,"latency_ms":882,"ts":"2026-05-12T02:52:08.966Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7635,"output_tokens":6,"latency_ms":810,"ts":"2026-05-12T02:52:10.347Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7635,"output_tokens":6,"latency_ms":1777,"ts":"2026-05-12T02:52:11.314Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7635,"output_tokens":6,"latency_ms":842,"ts":"2026-05-12T02:52:10.379Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7632,"output_tokens":8,"latency_ms":668,"ts":"2026-05-12T02:52:11.982Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7632,"output_tokens":8,"latency_ms":624,"ts":"2026-05-12T02:52:11.938Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7632,"output_tokens":8,"latency_ms":712,"ts":"2026-05-12T02:52:12.026Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":6,"latency_ms":1403,"ts":"2026-05-12T02:52:13.429Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":6,"latency_ms":680,"ts":"2026-05-12T02:52:12.706Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7634,"output_tokens":6,"latency_ms":750,"ts":"2026-05-12T02:52:12.776Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7631,"output_tokens":5,"latency_ms":1018,"ts":"2026-05-12T02:52:14.447Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7631,"output_tokens":5,"latency_ms":4118,"ts":"2026-05-12T02:52:17.547Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":7631,"output_tokens":5,"latency_ms":673,"ts":"2026-05-12T02:52:14.102Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4093,"output_tokens":5,"latency_ms":741,"ts":"2026-05-12T02:52:18.288Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4093,"output_tokens":5,"latency_ms":580,"ts":"2026-05-12T02:52:18.127Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4093,"output_tokens":5,"latency_ms":575,"ts":"2026-05-12T02:52:18.122Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"data-research","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4086,"output_tokens":6,"latency_ms":573,"ts":"2026-05-12T02:52:18.861Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"perplexity-research","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4086,"output_tokens":9,"latency_ms":579,"ts":"2026-05-12T02:52:18.867Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"perplexity-research","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4086,"output_tokens":9,"latency_ms":579,"ts":"2026-05-12T02:52:18.867Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":598,"ts":"2026-05-12T02:52:19.465Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":556,"ts":"2026-05-12T02:52:19.423Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":561,"ts":"2026-05-12T02:52:19.428Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":631,"ts":"2026-05-12T02:52:20.096Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":602,"ts":"2026-05-12T02:52:20.067Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":610,"ts":"2026-05-12T02:52:20.075Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"brain-ops","expected":"brain-librarian","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":571,"ts":"2026-05-12T02:52:20.667Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"brain-ops","expected":"brain-librarian","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":696,"ts":"2026-05-12T02:52:20.792Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"brain-ops","expected":"brain-librarian","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":628,"ts":"2026-05-12T02:52:20.724Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4086,"output_tokens":7,"latency_ms":743,"ts":"2026-05-12T02:52:21.535Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4086,"output_tokens":7,"latency_ms":612,"ts":"2026-05-12T02:52:21.404Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4086,"output_tokens":7,"latency_ms":630,"ts":"2026-05-12T02:52:21.422Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"brain-mirror-synthesis","expected":"book-mirror","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4093,"output_tokens":8,"latency_ms":1639,"ts":"2026-05-12T02:52:23.174Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"book-mirror-synthesis","expected":"book-mirror","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4093,"output_tokens":8,"latency_ms":585,"ts":"2026-05-12T02:52:22.120Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"book-mirror-synthesis","expected":"book-mirror","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4093,"output_tokens":8,"latency_ms":656,"ts":"2026-05-12T02:52:22.191Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4093,"output_tokens":6,"latency_ms":673,"ts":"2026-05-12T02:52:23.847Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4093,"output_tokens":6,"latency_ms":673,"ts":"2026-05-12T02:52:23.847Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4093,"output_tokens":6,"latency_ms":529,"ts":"2026-05-12T02:52:23.703Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":704,"ts":"2026-05-12T02:52:24.551Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":535,"ts":"2026-05-12T02:52:24.382Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":705,"ts":"2026-05-12T02:52:24.552Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4093,"output_tokens":9,"latency_ms":707,"ts":"2026-05-12T02:52:25.259Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4093,"output_tokens":9,"latency_ms":707,"ts":"2026-05-12T02:52:25.259Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4093,"output_tokens":9,"latency_ms":707,"ts":"2026-05-12T02:52:25.259Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"media-ingest","expected":"idea-ingest","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4090,"output_tokens":7,"latency_ms":815,"ts":"2026-05-12T02:52:26.074Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"article-enrichment","expected":"idea-ingest","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4090,"output_tokens":7,"latency_ms":697,"ts":"2026-05-12T02:52:25.956Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"article-enrichment","expected":"idea-ingest","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4090,"output_tokens":7,"latency_ms":689,"ts":"2026-05-12T02:52:25.948Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4086,"output_tokens":7,"latency_ms":587,"ts":"2026-05-12T02:52:26.661Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4086,"output_tokens":7,"latency_ms":572,"ts":"2026-05-12T02:52:26.646Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4086,"output_tokens":7,"latency_ms":1134,"ts":"2026-05-12T02:52:27.208Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"transcript-save","expected":"meeting-ingestion","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":733,"ts":"2026-05-12T02:52:27.941Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"transcript-save","expected":"meeting-ingestion","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":623,"ts":"2026-05-12T02:52:27.831Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":7,"latency_ms":553,"ts":"2026-05-12T02:52:27.761Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4087,"output_tokens":9,"latency_ms":1752,"ts":"2026-05-12T02:52:29.693Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4087,"output_tokens":9,"latency_ms":929,"ts":"2026-05-12T02:52:28.870Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4087,"output_tokens":9,"latency_ms":637,"ts":"2026-05-12T02:52:28.578Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"calendar-check","expected":"google-calendar","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4085,"output_tokens":6,"latency_ms":582,"ts":"2026-05-12T02:52:30.275Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"calendar-check","expected":"google-calendar","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4085,"output_tokens":6,"latency_ms":547,"ts":"2026-05-12T02:52:30.241Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"calendar-check","expected":"google-calendar","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4085,"output_tokens":6,"latency_ms":882,"ts":"2026-05-12T02:52:30.575Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4085,"output_tokens":4,"latency_ms":569,"ts":"2026-05-12T02:52:31.144Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4085,"output_tokens":4,"latency_ms":587,"ts":"2026-05-12T02:52:31.162Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4085,"output_tokens":4,"latency_ms":619,"ts":"2026-05-12T02:52:31.194Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"exa","expected":"perplexity-research","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":5,"latency_ms":712,"ts":"2026-05-12T02:52:31.907Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":9,"latency_ms":558,"ts":"2026-05-12T02:52:31.753Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"exa","expected":"perplexity-research","correct":0,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":5,"latency_ms":537,"ts":"2026-05-12T02:52:31.732Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4088,"output_tokens":7,"latency_ms":706,"ts":"2026-05-12T02:52:32.613Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4088,"output_tokens":7,"latency_ms":743,"ts":"2026-05-12T02:52:32.650Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4088,"output_tokens":7,"latency_ms":4370,"ts":"2026-05-12T02:52:36.277Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4088,"output_tokens":5,"latency_ms":587,"ts":"2026-05-12T02:52:36.864Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4088,"output_tokens":5,"latency_ms":624,"ts":"2026-05-12T02:52:36.901Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4088,"output_tokens":5,"latency_ms":634,"ts":"2026-05-12T02:52:36.911Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"meeting-prep","expected":"daily-task-manager","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4087,"output_tokens":6,"latency_ms":577,"ts":"2026-05-12T02:52:37.488Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"meeting-prep","expected":"daily-task-manager","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4087,"output_tokens":6,"latency_ms":587,"ts":"2026-05-12T02:52:37.498Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"meeting-prep","expected":"daily-task-manager","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4087,"output_tokens":6,"latency_ms":1335,"ts":"2026-05-12T02:52:38.246Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4090,"output_tokens":5,"latency_ms":1277,"ts":"2026-05-12T02:52:39.523Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4090,"output_tokens":5,"latency_ms":560,"ts":"2026-05-12T02:52:38.806Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4090,"output_tokens":5,"latency_ms":735,"ts":"2026-05-12T02:52:38.981Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4090,"output_tokens":6,"latency_ms":666,"ts":"2026-05-12T02:52:40.189Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4090,"output_tokens":6,"latency_ms":666,"ts":"2026-05-12T02:52:40.189Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4090,"output_tokens":6,"latency_ms":666,"ts":"2026-05-12T02:52:40.189Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4087,"output_tokens":8,"latency_ms":668,"ts":"2026-05-12T02:52:40.857Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4087,"output_tokens":8,"latency_ms":689,"ts":"2026-05-12T02:52:40.879Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4087,"output_tokens":8,"latency_ms":555,"ts":"2026-05-12T02:52:40.745Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":730,"ts":"2026-05-12T02:52:41.609Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":729,"ts":"2026-05-12T02:52:41.609Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4089,"output_tokens":6,"latency_ms":603,"ts":"2026-05-12T02:52:41.483Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4086,"output_tokens":5,"latency_ms":699,"ts":"2026-05-12T02:52:42.308Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4086,"output_tokens":5,"latency_ms":567,"ts":"2026-05-12T02:52:42.176Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":4086,"output_tokens":5,"latency_ms":1623,"ts":"2026-05-12T02:52:43.232Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"enrich","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3213,"output_tokens":6,"latency_ms":547,"ts":"2026-05-12T02:52:43.779Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"enrich","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3213,"output_tokens":6,"latency_ms":552,"ts":"2026-05-12T02:52:43.784Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-ops","expected":"enrich","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3213,"output_tokens":6,"latency_ms":660,"ts":"2026-05-12T02:52:43.892Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3206,"output_tokens":6,"latency_ms":575,"ts":"2026-05-12T02:52:44.467Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3206,"output_tokens":6,"latency_ms":673,"ts":"2026-05-12T02:52:44.565Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-ops","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3206,"output_tokens":6,"latency_ms":606,"ts":"2026-05-12T02:52:44.498Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":825,"ts":"2026-05-12T02:52:45.390Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":554,"ts":"2026-05-12T02:52:45.119Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":606,"ts":"2026-05-12T02:52:45.171Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"brain-publish","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":588,"ts":"2026-05-12T02:52:45.979Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"brain-publish","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":873,"ts":"2026-05-12T02:52:46.264Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-ops","expected":"brain-publish","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":586,"ts":"2026-05-12T02:52:45.977Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"brain-librarian","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":579,"ts":"2026-05-12T02:52:46.843Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"brain-librarian","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":556,"ts":"2026-05-12T02:52:46.820Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-ops","expected":"brain-librarian","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":675,"ts":"2026-05-12T02:52:46.939Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3206,"output_tokens":7,"latency_ms":564,"ts":"2026-05-12T02:52:47.503Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3206,"output_tokens":7,"latency_ms":901,"ts":"2026-05-12T02:52:47.840Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3206,"output_tokens":7,"latency_ms":606,"ts":"2026-05-12T02:52:47.545Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3213,"output_tokens":6,"latency_ms":726,"ts":"2026-05-12T02:52:48.566Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-pdf","expected":"book-mirror","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3213,"output_tokens":6,"latency_ms":778,"ts":"2026-05-12T02:52:48.618Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3213,"output_tokens":6,"latency_ms":563,"ts":"2026-05-12T02:52:48.403Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3213,"output_tokens":6,"latency_ms":551,"ts":"2026-05-12T02:52:49.169Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3213,"output_tokens":6,"latency_ms":568,"ts":"2026-05-12T02:52:49.186Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3213,"output_tokens":6,"latency_ms":799,"ts":"2026-05-12T02:52:49.417Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":567,"ts":"2026-05-12T02:52:49.984Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":1347,"ts":"2026-05-12T02:52:50.764Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":662,"ts":"2026-05-12T02:52:50.079Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"archive-crawler","expected":"archive-crawler","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3213,"output_tokens":6,"latency_ms":572,"ts":"2026-05-12T02:52:51.336Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"archive-crawler","expected":"archive-crawler","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3213,"output_tokens":6,"latency_ms":587,"ts":"2026-05-12T02:52:51.351Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"archive-crawler","expected":"archive-crawler","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3213,"output_tokens":6,"latency_ms":578,"ts":"2026-05-12T02:52:51.342Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3210,"output_tokens":5,"latency_ms":624,"ts":"2026-05-12T02:52:51.975Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3210,"output_tokens":5,"latency_ms":560,"ts":"2026-05-12T02:52:51.911Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3210,"output_tokens":5,"latency_ms":615,"ts":"2026-05-12T02:52:51.966Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"ingest","expected":"media-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3206,"output_tokens":5,"latency_ms":4740,"ts":"2026-05-12T02:52:56.715Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"ingest","expected":"media-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3206,"output_tokens":5,"latency_ms":722,"ts":"2026-05-12T02:52:52.698Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"ingest","expected":"media-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3206,"output_tokens":5,"latency_ms":815,"ts":"2026-05-12T02:52:52.791Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"ingest","expected":"meeting-ingestion","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":5,"latency_ms":577,"ts":"2026-05-12T02:52:57.292Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"ingest","expected":"meeting-ingestion","correct":0,"correct_lenient":0,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":5,"latency_ms":762,"ts":"2026-05-12T02:52:57.477Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":7,"latency_ms":583,"ts":"2026-05-12T02:52:57.298Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3207,"output_tokens":9,"latency_ms":555,"ts":"2026-05-12T02:52:58.032Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3207,"output_tokens":9,"latency_ms":705,"ts":"2026-05-12T02:52:58.182Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3207,"output_tokens":9,"latency_ms":587,"ts":"2026-05-12T02:52:58.064Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3205,"output_tokens":6,"latency_ms":599,"ts":"2026-05-12T02:52:58.781Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3205,"output_tokens":6,"latency_ms":577,"ts":"2026-05-12T02:52:58.759Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3205,"output_tokens":6,"latency_ms":793,"ts":"2026-05-12T02:52:58.975Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"executive-assistant","expected":"executive-assistant","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3205,"output_tokens":6,"latency_ms":836,"ts":"2026-05-12T02:52:59.811Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"executive-assistant","expected":"executive-assistant","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3205,"output_tokens":6,"latency_ms":2306,"ts":"2026-05-12T02:53:01.281Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"executive-assistant","expected":"executive-assistant","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3205,"output_tokens":6,"latency_ms":1092,"ts":"2026-05-12T02:53:00.067Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":9,"latency_ms":568,"ts":"2026-05-12T02:53:01.849Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":9,"latency_ms":628,"ts":"2026-05-12T02:53:01.909Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":9,"latency_ms":593,"ts":"2026-05-12T02:53:01.874Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3208,"output_tokens":7,"latency_ms":626,"ts":"2026-05-12T02:53:02.535Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3208,"output_tokens":7,"latency_ms":641,"ts":"2026-05-12T02:53:02.550Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3208,"output_tokens":7,"latency_ms":925,"ts":"2026-05-12T02:53:02.834Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3208,"output_tokens":5,"latency_ms":615,"ts":"2026-05-12T02:53:03.449Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3208,"output_tokens":5,"latency_ms":706,"ts":"2026-05-12T02:53:03.540Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3208,"output_tokens":5,"latency_ms":723,"ts":"2026-05-12T02:53:03.557Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3207,"output_tokens":8,"latency_ms":878,"ts":"2026-05-12T02:53:04.435Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3207,"output_tokens":8,"latency_ms":619,"ts":"2026-05-12T02:53:04.176Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3207,"output_tokens":8,"latency_ms":606,"ts":"2026-05-12T02:53:04.163Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3210,"output_tokens":5,"latency_ms":917,"ts":"2026-05-12T02:53:05.352Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3210,"output_tokens":5,"latency_ms":542,"ts":"2026-05-12T02:53:04.977Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3210,"output_tokens":5,"latency_ms":603,"ts":"2026-05-12T02:53:05.038Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3210,"output_tokens":6,"latency_ms":1164,"ts":"2026-05-12T02:53:06.516Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3210,"output_tokens":6,"latency_ms":629,"ts":"2026-05-12T02:53:05.981Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3210,"output_tokens":6,"latency_ms":609,"ts":"2026-05-12T02:53:05.961Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3207,"output_tokens":8,"latency_ms":655,"ts":"2026-05-12T02:53:07.171Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3207,"output_tokens":8,"latency_ms":649,"ts":"2026-05-12T02:53:07.165Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3207,"output_tokens":8,"latency_ms":691,"ts":"2026-05-12T02:53:07.207Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":550,"ts":"2026-05-12T02:53:07.757Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":577,"ts":"2026-05-12T02:53:07.784Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3209,"output_tokens":6,"latency_ms":1182,"ts":"2026-05-12T02:53:08.389Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3206,"output_tokens":5,"latency_ms":667,"ts":"2026-05-12T02:53:09.056Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3206,"output_tokens":5,"latency_ms":843,"ts":"2026-05-12T02:53:09.232Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-haiku-4-5-20251001","input_tokens":3206,"output_tokens":5,"latency_ms":667,"ts":"2026-05-12T02:53:09.056Z"}
|
||||
@@ -1,226 +0,0 @@
|
||||
{"kind":"receipt","model":"anthropic:claude-opus-4-7","prompt_template_hash":"17340040af579ca1","fixtures_hash":"feccc99122ea86d5","fixtures_held_out_hash":"5d6256cc9dced124","harness_sha":"ca99fbfeb5f304e1e237eebd11ce0196ea8a9b18","ts":"2026-05-12T03:16:08.329Z","cmd_args":["--model","opus","--parallel","3","--yes","--output","baseline-runs/2026-05-11-opus-4-7.jsonl"]}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"baseline","seed":1,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10952,"output_tokens":7,"latency_ms":1844,"ts":"2026-05-12T03:16:10.173Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"baseline","seed":2,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10952,"output_tokens":7,"latency_ms":1703,"ts":"2026-05-12T03:16:10.032Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"baseline","seed":3,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10952,"output_tokens":7,"latency_ms":1672,"ts":"2026-05-12T03:16:10.001Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"baseline","seed":1,"predicted":"entity-detector","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":10940,"output_tokens":9,"latency_ms":4547,"ts":"2026-05-12T03:16:14.720Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"baseline","seed":2,"predicted":"enrich","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":10940,"output_tokens":7,"latency_ms":1730,"ts":"2026-05-12T03:16:11.903Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"baseline","seed":3,"predicted":"enrich","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":10940,"output_tokens":7,"latency_ms":1766,"ts":"2026-05-12T03:16:11.939Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"baseline","seed":1,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10951,"output_tokens":10,"latency_ms":2035,"ts":"2026-05-12T03:16:16.755Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"baseline","seed":2,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10951,"output_tokens":10,"latency_ms":2849,"ts":"2026-05-12T03:16:17.569Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"baseline","seed":3,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10951,"output_tokens":10,"latency_ms":2096,"ts":"2026-05-12T03:16:16.816Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"baseline","seed":1,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10944,"output_tokens":9,"latency_ms":1646,"ts":"2026-05-12T03:16:19.215Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"baseline","seed":2,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10944,"output_tokens":9,"latency_ms":1847,"ts":"2026-05-12T03:16:19.416Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"baseline","seed":3,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10944,"output_tokens":9,"latency_ms":1512,"ts":"2026-05-12T03:16:19.081Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"baseline","seed":1,"predicted":"brain-librarian","expected":"brain-librarian","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10947,"output_tokens":10,"latency_ms":1488,"ts":"2026-05-12T03:16:20.904Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"baseline","seed":2,"predicted":"brain-librarian","expected":"brain-librarian","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10947,"output_tokens":10,"latency_ms":1498,"ts":"2026-05-12T03:16:20.914Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"baseline","seed":3,"predicted":"brain-librarian","expected":"brain-librarian","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10947,"output_tokens":10,"latency_ms":1584,"ts":"2026-05-12T03:16:21.000Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"baseline","seed":1,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10942,"output_tokens":10,"latency_ms":2227,"ts":"2026-05-12T03:16:23.227Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"baseline","seed":2,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10942,"output_tokens":10,"latency_ms":1817,"ts":"2026-05-12T03:16:22.817Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"baseline","seed":3,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10942,"output_tokens":10,"latency_ms":1446,"ts":"2026-05-12T03:16:22.446Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"baseline","seed":1,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10951,"output_tokens":9,"latency_ms":2117,"ts":"2026-05-12T03:16:25.345Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"baseline","seed":2,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10951,"output_tokens":9,"latency_ms":1587,"ts":"2026-05-12T03:16:24.816Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"baseline","seed":3,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10951,"output_tokens":9,"latency_ms":1587,"ts":"2026-05-12T03:16:24.816Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"baseline","seed":1,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10952,"output_tokens":10,"latency_ms":1653,"ts":"2026-05-12T03:16:26.998Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"baseline","seed":2,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10952,"output_tokens":10,"latency_ms":2538,"ts":"2026-05-12T03:16:27.883Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"baseline","seed":3,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10952,"output_tokens":10,"latency_ms":1921,"ts":"2026-05-12T03:16:27.266Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"baseline","seed":1,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10948,"output_tokens":11,"latency_ms":1432,"ts":"2026-05-12T03:16:29.315Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"baseline","seed":2,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10948,"output_tokens":11,"latency_ms":1619,"ts":"2026-05-12T03:16:29.502Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"baseline","seed":3,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10948,"output_tokens":11,"latency_ms":1910,"ts":"2026-05-12T03:16:29.793Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"baseline","seed":1,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":10951,"output_tokens":13,"latency_ms":1896,"ts":"2026-05-12T03:16:31.689Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"baseline","seed":2,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":10951,"output_tokens":13,"latency_ms":1678,"ts":"2026-05-12T03:16:31.471Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"baseline","seed":3,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":10951,"output_tokens":13,"latency_ms":2108,"ts":"2026-05-12T03:16:31.901Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"baseline","seed":1,"predicted":"media-ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":10948,"output_tokens":9,"latency_ms":1722,"ts":"2026-05-12T03:16:33.623Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"baseline","seed":2,"predicted":"media-ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":10948,"output_tokens":9,"latency_ms":2109,"ts":"2026-05-12T03:16:34.010Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"baseline","seed":3,"predicted":"media-ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":10948,"output_tokens":9,"latency_ms":4048,"ts":"2026-05-12T03:16:35.949Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"baseline","seed":1,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10943,"output_tokens":9,"latency_ms":1622,"ts":"2026-05-12T03:16:37.572Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"baseline","seed":2,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10943,"output_tokens":9,"latency_ms":1522,"ts":"2026-05-12T03:16:37.472Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"baseline","seed":3,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10943,"output_tokens":9,"latency_ms":2637,"ts":"2026-05-12T03:16:38.587Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"baseline","seed":1,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10946,"output_tokens":12,"latency_ms":1752,"ts":"2026-05-12T03:16:40.339Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"baseline","seed":2,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10946,"output_tokens":12,"latency_ms":1603,"ts":"2026-05-12T03:16:40.190Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"baseline","seed":3,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10946,"output_tokens":12,"latency_ms":1575,"ts":"2026-05-12T03:16:40.162Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"baseline","seed":1,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10944,"output_tokens":12,"latency_ms":2603,"ts":"2026-05-12T03:16:42.942Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"baseline","seed":2,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10944,"output_tokens":12,"latency_ms":1578,"ts":"2026-05-12T03:16:41.917Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"baseline","seed":3,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10944,"output_tokens":12,"latency_ms":1692,"ts":"2026-05-12T03:16:42.031Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"baseline","seed":1,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10943,"output_tokens":9,"latency_ms":1674,"ts":"2026-05-12T03:16:44.616Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"baseline","seed":2,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10943,"output_tokens":9,"latency_ms":1955,"ts":"2026-05-12T03:16:44.897Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"baseline","seed":3,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10943,"output_tokens":9,"latency_ms":2103,"ts":"2026-05-12T03:16:45.045Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"baseline","seed":1,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":10943,"output_tokens":7,"latency_ms":2048,"ts":"2026-05-12T03:16:47.093Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"baseline","seed":2,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":10943,"output_tokens":7,"latency_ms":2522,"ts":"2026-05-12T03:16:47.567Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"baseline","seed":3,"predicted":"executive-assistant","expected":"executive-assistant","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10943,"output_tokens":10,"latency_ms":1825,"ts":"2026-05-12T03:16:46.870Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"baseline","seed":1,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10945,"output_tokens":10,"latency_ms":1734,"ts":"2026-05-12T03:16:49.301Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"baseline","seed":2,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10945,"output_tokens":10,"latency_ms":1666,"ts":"2026-05-12T03:16:49.234Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"baseline","seed":3,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10945,"output_tokens":10,"latency_ms":1694,"ts":"2026-05-12T03:16:49.261Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"baseline","seed":1,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10945,"output_tokens":9,"latency_ms":1531,"ts":"2026-05-12T03:16:50.832Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"baseline","seed":2,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10945,"output_tokens":9,"latency_ms":1615,"ts":"2026-05-12T03:16:50.916Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"baseline","seed":3,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10945,"output_tokens":9,"latency_ms":1503,"ts":"2026-05-12T03:16:50.804Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"baseline","seed":1,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10945,"output_tokens":7,"latency_ms":1960,"ts":"2026-05-12T03:16:52.876Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"baseline","seed":2,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10945,"output_tokens":7,"latency_ms":1469,"ts":"2026-05-12T03:16:52.385Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"baseline","seed":3,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10945,"output_tokens":7,"latency_ms":1686,"ts":"2026-05-12T03:16:52.602Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"baseline","seed":1,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10947,"output_tokens":11,"latency_ms":3911,"ts":"2026-05-12T03:16:56.787Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"baseline","seed":2,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10947,"output_tokens":11,"latency_ms":1684,"ts":"2026-05-12T03:16:54.560Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"baseline","seed":3,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10947,"output_tokens":11,"latency_ms":2066,"ts":"2026-05-12T03:16:54.942Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10950,"output_tokens":7,"latency_ms":1576,"ts":"2026-05-12T03:16:58.363Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10950,"output_tokens":7,"latency_ms":1634,"ts":"2026-05-12T03:16:58.421Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10950,"output_tokens":7,"latency_ms":2666,"ts":"2026-05-12T03:16:59.453Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10947,"output_tokens":9,"latency_ms":1933,"ts":"2026-05-12T03:17:01.386Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10947,"output_tokens":9,"latency_ms":2022,"ts":"2026-05-12T03:17:01.475Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10947,"output_tokens":9,"latency_ms":1881,"ts":"2026-05-12T03:17:01.334Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10947,"output_tokens":11,"latency_ms":2322,"ts":"2026-05-12T03:17:03.797Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10947,"output_tokens":11,"latency_ms":1639,"ts":"2026-05-12T03:17:03.114Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10947,"output_tokens":11,"latency_ms":1854,"ts":"2026-05-12T03:17:03.329Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10946,"output_tokens":9,"latency_ms":1694,"ts":"2026-05-12T03:17:05.491Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10946,"output_tokens":9,"latency_ms":1621,"ts":"2026-05-12T03:17:05.418Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10946,"output_tokens":9,"latency_ms":1493,"ts":"2026-05-12T03:17:05.292Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10942,"output_tokens":9,"latency_ms":1662,"ts":"2026-05-12T03:17:07.153Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10942,"output_tokens":9,"latency_ms":1736,"ts":"2026-05-12T03:17:07.227Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":10942,"output_tokens":9,"latency_ms":1538,"ts":"2026-05-12T03:17:07.030Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6009,"output_tokens":7,"latency_ms":1569,"ts":"2026-05-12T03:17:08.796Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6009,"output_tokens":7,"latency_ms":1569,"ts":"2026-05-12T03:17:08.796Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6009,"output_tokens":7,"latency_ms":1746,"ts":"2026-05-12T03:17:08.973Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"query","expected":"gbrain","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":5997,"output_tokens":5,"latency_ms":1428,"ts":"2026-05-12T03:17:10.401Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"query","expected":"gbrain","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":5997,"output_tokens":5,"latency_ms":1414,"ts":"2026-05-12T03:17:10.387Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"query","expected":"gbrain","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":5997,"output_tokens":5,"latency_ms":1903,"ts":"2026-05-12T03:17:10.876Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6008,"output_tokens":10,"latency_ms":1603,"ts":"2026-05-12T03:17:12.479Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6008,"output_tokens":10,"latency_ms":1551,"ts":"2026-05-12T03:17:12.428Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6008,"output_tokens":10,"latency_ms":1738,"ts":"2026-05-12T03:17:12.615Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6001,"output_tokens":9,"latency_ms":4130,"ts":"2026-05-12T03:17:16.745Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6001,"output_tokens":9,"latency_ms":1735,"ts":"2026-05-12T03:17:14.351Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6001,"output_tokens":9,"latency_ms":1704,"ts":"2026-05-12T03:17:14.320Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"archive-crawler","expected":"brain-librarian","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6004,"output_tokens":10,"latency_ms":3997,"ts":"2026-05-12T03:17:20.742Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"archive-crawler","expected":"brain-librarian","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6004,"output_tokens":10,"latency_ms":1578,"ts":"2026-05-12T03:17:18.323Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"archive-crawler","expected":"brain-librarian","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6004,"output_tokens":10,"latency_ms":1617,"ts":"2026-05-12T03:17:18.362Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":5999,"output_tokens":10,"latency_ms":1656,"ts":"2026-05-12T03:17:22.398Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":5999,"output_tokens":10,"latency_ms":1652,"ts":"2026-05-12T03:17:22.394Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":5999,"output_tokens":10,"latency_ms":1575,"ts":"2026-05-12T03:17:22.317Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6008,"output_tokens":9,"latency_ms":2173,"ts":"2026-05-12T03:17:24.571Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6008,"output_tokens":9,"latency_ms":1848,"ts":"2026-05-12T03:17:24.246Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6008,"output_tokens":9,"latency_ms":1678,"ts":"2026-05-12T03:17:24.076Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6009,"output_tokens":10,"latency_ms":1362,"ts":"2026-05-12T03:17:25.933Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6009,"output_tokens":10,"latency_ms":1747,"ts":"2026-05-12T03:17:26.318Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6009,"output_tokens":10,"latency_ms":1747,"ts":"2026-05-12T03:17:26.318Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6005,"output_tokens":11,"latency_ms":2525,"ts":"2026-05-12T03:17:28.843Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6005,"output_tokens":11,"latency_ms":1404,"ts":"2026-05-12T03:17:27.722Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6005,"output_tokens":11,"latency_ms":1603,"ts":"2026-05-12T03:17:27.921Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6008,"output_tokens":13,"latency_ms":3273,"ts":"2026-05-12T03:17:32.116Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6008,"output_tokens":13,"latency_ms":2071,"ts":"2026-05-12T03:17:30.914Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6008,"output_tokens":13,"latency_ms":1820,"ts":"2026-05-12T03:17:30.663Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"article-enrichment","expected":"idea-ingest","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6005,"output_tokens":10,"latency_ms":1443,"ts":"2026-05-12T03:17:33.559Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"article-enrichment","expected":"idea-ingest","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6005,"output_tokens":10,"latency_ms":1549,"ts":"2026-05-12T03:17:33.665Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"article-enrichment","expected":"idea-ingest","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6005,"output_tokens":10,"latency_ms":1563,"ts":"2026-05-12T03:17:33.679Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6000,"output_tokens":9,"latency_ms":1824,"ts":"2026-05-12T03:17:35.503Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6000,"output_tokens":9,"latency_ms":1533,"ts":"2026-05-12T03:17:35.212Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6000,"output_tokens":9,"latency_ms":1365,"ts":"2026-05-12T03:17:35.044Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6003,"output_tokens":12,"latency_ms":1511,"ts":"2026-05-12T03:17:37.014Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6003,"output_tokens":12,"latency_ms":1880,"ts":"2026-05-12T03:17:37.383Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6003,"output_tokens":12,"latency_ms":1602,"ts":"2026-05-12T03:17:37.105Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6001,"output_tokens":12,"latency_ms":1496,"ts":"2026-05-12T03:17:38.879Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6001,"output_tokens":12,"latency_ms":1470,"ts":"2026-05-12T03:17:38.853Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6001,"output_tokens":12,"latency_ms":2355,"ts":"2026-05-12T03:17:39.738Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"calendar-check","expected":"google-calendar","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6000,"output_tokens":9,"latency_ms":1703,"ts":"2026-05-12T03:17:41.441Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"calendar-check","expected":"google-calendar","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6000,"output_tokens":9,"latency_ms":1598,"ts":"2026-05-12T03:17:41.337Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"calendar-check","expected":"google-calendar","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6000,"output_tokens":9,"latency_ms":1574,"ts":"2026-05-12T03:17:41.313Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6000,"output_tokens":7,"latency_ms":1674,"ts":"2026-05-12T03:17:43.115Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6000,"output_tokens":7,"latency_ms":1755,"ts":"2026-05-12T03:17:43.197Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6000,"output_tokens":7,"latency_ms":1830,"ts":"2026-05-12T03:17:43.271Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6002,"output_tokens":10,"latency_ms":1478,"ts":"2026-05-12T03:17:44.750Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6002,"output_tokens":10,"latency_ms":2431,"ts":"2026-05-12T03:17:45.702Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"exa","expected":"perplexity-research","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6002,"output_tokens":6,"latency_ms":1496,"ts":"2026-05-12T03:17:44.767Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6002,"output_tokens":9,"latency_ms":1883,"ts":"2026-05-12T03:17:47.585Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6002,"output_tokens":9,"latency_ms":1445,"ts":"2026-05-12T03:17:47.147Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6002,"output_tokens":9,"latency_ms":1597,"ts":"2026-05-12T03:17:47.299Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6002,"output_tokens":7,"latency_ms":1448,"ts":"2026-05-12T03:17:49.033Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6002,"output_tokens":7,"latency_ms":2841,"ts":"2026-05-12T03:17:50.426Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6002,"output_tokens":7,"latency_ms":1414,"ts":"2026-05-12T03:17:48.999Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"daily-task-prep","expected":"daily-task-manager","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6004,"output_tokens":11,"latency_ms":1393,"ts":"2026-05-12T03:17:51.819Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"daily-task-prep","expected":"daily-task-manager","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6004,"output_tokens":11,"latency_ms":1478,"ts":"2026-05-12T03:17:51.904Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"daily-task-prep","expected":"daily-task-manager","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6004,"output_tokens":11,"latency_ms":1625,"ts":"2026-05-12T03:17:52.051Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6007,"output_tokens":7,"latency_ms":1694,"ts":"2026-05-12T03:17:53.745Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6007,"output_tokens":7,"latency_ms":1694,"ts":"2026-05-12T03:17:53.745Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6007,"output_tokens":7,"latency_ms":1720,"ts":"2026-05-12T03:17:53.771Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6004,"output_tokens":9,"latency_ms":1740,"ts":"2026-05-12T03:17:55.511Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6004,"output_tokens":9,"latency_ms":1920,"ts":"2026-05-12T03:17:55.691Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6004,"output_tokens":9,"latency_ms":1563,"ts":"2026-05-12T03:17:55.334Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6004,"output_tokens":11,"latency_ms":2563,"ts":"2026-05-12T03:17:58.255Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6004,"output_tokens":11,"latency_ms":1528,"ts":"2026-05-12T03:17:57.220Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6004,"output_tokens":11,"latency_ms":1528,"ts":"2026-05-12T03:17:57.220Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6003,"output_tokens":9,"latency_ms":1585,"ts":"2026-05-12T03:17:59.840Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"enrich","expected":"google-contacts","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6003,"output_tokens":7,"latency_ms":1570,"ts":"2026-05-12T03:17:59.825Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"enrich","expected":"google-contacts","correct":0,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":6003,"output_tokens":7,"latency_ms":1909,"ts":"2026-05-12T03:18:00.164Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":5999,"output_tokens":9,"latency_ms":1376,"ts":"2026-05-12T03:18:01.540Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":5999,"output_tokens":9,"latency_ms":1389,"ts":"2026-05-12T03:18:01.553Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":5999,"output_tokens":9,"latency_ms":1398,"ts":"2026-05-12T03:18:01.562Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"enrich","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4736,"output_tokens":9,"latency_ms":1527,"ts":"2026-05-12T03:18:03.089Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"enrich","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4736,"output_tokens":9,"latency_ms":1573,"ts":"2026-05-12T03:18:03.135Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-ops","expected":"enrich","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4736,"output_tokens":9,"latency_ms":1516,"ts":"2026-05-12T03:18:03.078Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"perplexity-research","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4724,"output_tokens":10,"latency_ms":1606,"ts":"2026-05-12T03:18:04.741Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"perplexity-research","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4724,"output_tokens":10,"latency_ms":1689,"ts":"2026-05-12T03:18:04.824Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"perplexity-research","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4724,"output_tokens":10,"latency_ms":1682,"ts":"2026-05-12T03:18:04.817Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4735,"output_tokens":10,"latency_ms":1670,"ts":"2026-05-12T03:18:06.494Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4735,"output_tokens":10,"latency_ms":2416,"ts":"2026-05-12T03:18:07.240Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4735,"output_tokens":10,"latency_ms":1489,"ts":"2026-05-12T03:18:06.313Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"brain-publish","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4728,"output_tokens":9,"latency_ms":3205,"ts":"2026-05-12T03:18:10.445Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"brain-publish","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4728,"output_tokens":9,"latency_ms":4901,"ts":"2026-05-12T03:18:12.141Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-ops","expected":"brain-publish","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4728,"output_tokens":9,"latency_ms":4556,"ts":"2026-05-12T03:18:11.796Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"archive-crawler","expected":"brain-librarian","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4731,"output_tokens":10,"latency_ms":1779,"ts":"2026-05-12T03:18:13.921Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"archive-crawler","expected":"brain-librarian","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4731,"output_tokens":10,"latency_ms":1782,"ts":"2026-05-12T03:18:13.924Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"archive-crawler","expected":"brain-librarian","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4731,"output_tokens":10,"latency_ms":2264,"ts":"2026-05-12T03:18:14.406Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4726,"output_tokens":10,"latency_ms":1905,"ts":"2026-05-12T03:18:16.311Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4726,"output_tokens":10,"latency_ms":1512,"ts":"2026-05-12T03:18:15.918Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4726,"output_tokens":10,"latency_ms":1535,"ts":"2026-05-12T03:18:15.941Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"book-mirror","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4735,"output_tokens":9,"latency_ms":1430,"ts":"2026-05-12T03:18:17.741Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"book-mirror","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4735,"output_tokens":9,"latency_ms":1933,"ts":"2026-05-12T03:18:18.244Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4735,"output_tokens":9,"latency_ms":1902,"ts":"2026-05-12T03:18:18.213Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4736,"output_tokens":10,"latency_ms":1602,"ts":"2026-05-12T03:18:19.846Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4736,"output_tokens":10,"latency_ms":1606,"ts":"2026-05-12T03:18:19.850Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4736,"output_tokens":10,"latency_ms":1786,"ts":"2026-05-12T03:18:20.030Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"concept-synthesis","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4732,"output_tokens":9,"latency_ms":1583,"ts":"2026-05-12T03:18:21.613Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"concept-synthesis","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4732,"output_tokens":9,"latency_ms":1412,"ts":"2026-05-12T03:18:21.442Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4732,"output_tokens":11,"latency_ms":1521,"ts":"2026-05-12T03:18:21.551Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"archive-crawler","expected":"archive-crawler","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4735,"output_tokens":10,"latency_ms":1538,"ts":"2026-05-12T03:18:23.151Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"archive-crawler","expected":"archive-crawler","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4735,"output_tokens":10,"latency_ms":1534,"ts":"2026-05-12T03:18:23.148Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"archive-crawler","expected":"archive-crawler","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4735,"output_tokens":10,"latency_ms":1375,"ts":"2026-05-12T03:18:22.989Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4732,"output_tokens":7,"latency_ms":2125,"ts":"2026-05-12T03:18:25.276Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4732,"output_tokens":7,"latency_ms":2337,"ts":"2026-05-12T03:18:25.488Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4732,"output_tokens":7,"latency_ms":1945,"ts":"2026-05-12T03:18:25.096Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4727,"output_tokens":9,"latency_ms":1453,"ts":"2026-05-12T03:18:26.941Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4727,"output_tokens":9,"latency_ms":1398,"ts":"2026-05-12T03:18:26.886Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"ingest","expected":"media-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-opus-4-7","input_tokens":4727,"output_tokens":7,"latency_ms":1295,"ts":"2026-05-12T03:18:26.783Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4730,"output_tokens":12,"latency_ms":1249,"ts":"2026-05-12T03:18:28.190Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4730,"output_tokens":12,"latency_ms":1502,"ts":"2026-05-12T03:18:28.443Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4730,"output_tokens":12,"latency_ms":1721,"ts":"2026-05-12T03:18:28.662Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4728,"output_tokens":12,"latency_ms":1577,"ts":"2026-05-12T03:18:30.240Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4728,"output_tokens":12,"latency_ms":1894,"ts":"2026-05-12T03:18:30.557Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4728,"output_tokens":12,"latency_ms":1350,"ts":"2026-05-12T03:18:30.013Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4727,"output_tokens":9,"latency_ms":1974,"ts":"2026-05-12T03:18:32.531Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4727,"output_tokens":9,"latency_ms":1650,"ts":"2026-05-12T03:18:32.207Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4727,"output_tokens":9,"latency_ms":5071,"ts":"2026-05-12T03:18:35.628Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"executive-assistant","expected":"executive-assistant","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4727,"output_tokens":10,"latency_ms":1363,"ts":"2026-05-12T03:18:36.991Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"executive-assistant","expected":"executive-assistant","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4727,"output_tokens":10,"latency_ms":1978,"ts":"2026-05-12T03:18:37.606Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"executive-assistant","expected":"executive-assistant","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4727,"output_tokens":10,"latency_ms":1567,"ts":"2026-05-12T03:18:37.195Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4729,"output_tokens":10,"latency_ms":1639,"ts":"2026-05-12T03:18:39.245Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4729,"output_tokens":10,"latency_ms":1780,"ts":"2026-05-12T03:18:39.386Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4729,"output_tokens":10,"latency_ms":2166,"ts":"2026-05-12T03:18:39.772Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4729,"output_tokens":9,"latency_ms":1785,"ts":"2026-05-12T03:18:41.557Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4729,"output_tokens":9,"latency_ms":1546,"ts":"2026-05-12T03:18:41.318Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4729,"output_tokens":9,"latency_ms":2157,"ts":"2026-05-12T03:18:41.929Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4729,"output_tokens":7,"latency_ms":1536,"ts":"2026-05-12T03:18:43.465Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4729,"output_tokens":7,"latency_ms":1624,"ts":"2026-05-12T03:18:43.553Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4729,"output_tokens":7,"latency_ms":1452,"ts":"2026-05-12T03:18:43.381Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4731,"output_tokens":11,"latency_ms":1862,"ts":"2026-05-12T03:18:45.415Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4731,"output_tokens":11,"latency_ms":1922,"ts":"2026-05-12T03:18:45.475Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4731,"output_tokens":11,"latency_ms":1589,"ts":"2026-05-12T03:18:45.142Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4734,"output_tokens":7,"latency_ms":3982,"ts":"2026-05-12T03:18:49.458Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4734,"output_tokens":7,"latency_ms":1555,"ts":"2026-05-12T03:18:47.030Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4734,"output_tokens":7,"latency_ms":1221,"ts":"2026-05-12T03:18:46.696Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4731,"output_tokens":9,"latency_ms":1319,"ts":"2026-05-12T03:18:50.777Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4731,"output_tokens":9,"latency_ms":1587,"ts":"2026-05-12T03:18:51.045Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4731,"output_tokens":9,"latency_ms":1399,"ts":"2026-05-12T03:18:50.857Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4731,"output_tokens":11,"latency_ms":1862,"ts":"2026-05-12T03:18:52.907Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4731,"output_tokens":11,"latency_ms":1773,"ts":"2026-05-12T03:18:52.818Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4731,"output_tokens":11,"latency_ms":1525,"ts":"2026-05-12T03:18:52.570Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4730,"output_tokens":9,"latency_ms":9556,"ts":"2026-05-12T03:19:02.463Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4730,"output_tokens":9,"latency_ms":2096,"ts":"2026-05-12T03:18:55.003Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4730,"output_tokens":9,"latency_ms":1919,"ts":"2026-05-12T03:18:54.826Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4726,"output_tokens":9,"latency_ms":1639,"ts":"2026-05-12T03:19:04.102Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4726,"output_tokens":9,"latency_ms":1731,"ts":"2026-05-12T03:19:04.194Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-opus-4-7","input_tokens":4726,"output_tokens":9,"latency_ms":1741,"ts":"2026-05-12T03:19:04.204Z"}
|
||||
@@ -1,226 +0,0 @@
|
||||
{"kind":"receipt","model":"anthropic:claude-sonnet-4-6","prompt_template_hash":"17340040af579ca1","fixtures_hash":"feccc99122ea86d5","fixtures_held_out_hash":"5d6256cc9dced124","harness_sha":"fcc395282a92f2b047d4407f2b5a891c069adaac","ts":"2026-05-12T02:49:32.050Z","cmd_args":["--model","sonnet","--parallel","3","--yes","--output","run-sonnet-4-6.jsonl"]}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"baseline","seed":1,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7638,"output_tokens":5,"latency_ms":2307,"ts":"2026-05-12T02:49:34.357Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"baseline","seed":2,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7638,"output_tokens":5,"latency_ms":1033,"ts":"2026-05-12T02:49:33.083Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"baseline","seed":3,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7638,"output_tokens":5,"latency_ms":1682,"ts":"2026-05-12T02:49:33.732Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"baseline","seed":1,"predicted":"gbrain","expected":"gbrain","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7631,"output_tokens":5,"latency_ms":2141,"ts":"2026-05-12T02:49:36.498Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"baseline","seed":2,"predicted":"gbrain","expected":"gbrain","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7631,"output_tokens":5,"latency_ms":1435,"ts":"2026-05-12T02:49:35.792Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"baseline","seed":3,"predicted":"gbrain","expected":"gbrain","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7631,"output_tokens":5,"latency_ms":1121,"ts":"2026-05-12T02:49:35.478Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"baseline","seed":1,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":6,"latency_ms":1034,"ts":"2026-05-12T02:49:37.532Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"baseline","seed":2,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":6,"latency_ms":1285,"ts":"2026-05-12T02:49:37.783Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"baseline","seed":3,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":6,"latency_ms":1537,"ts":"2026-05-12T02:49:38.035Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"baseline","seed":1,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":6,"latency_ms":1052,"ts":"2026-05-12T02:49:39.087Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"baseline","seed":2,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":6,"latency_ms":1069,"ts":"2026-05-12T02:49:39.104Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"baseline","seed":3,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":6,"latency_ms":1260,"ts":"2026-05-12T02:49:39.295Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"baseline","seed":1,"predicted":"brain-librarian","expected":"brain-librarian","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":7,"latency_ms":1413,"ts":"2026-05-12T02:49:40.708Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"baseline","seed":2,"predicted":"brain-librarian","expected":"brain-librarian","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":7,"latency_ms":1400,"ts":"2026-05-12T02:49:40.695Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"baseline","seed":3,"predicted":"brain-librarian","expected":"brain-librarian","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":7,"latency_ms":1369,"ts":"2026-05-12T02:49:40.664Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"baseline","seed":1,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7631,"output_tokens":7,"latency_ms":1072,"ts":"2026-05-12T02:49:41.780Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"baseline","seed":2,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7631,"output_tokens":7,"latency_ms":2004,"ts":"2026-05-12T02:49:42.712Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"baseline","seed":3,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7631,"output_tokens":7,"latency_ms":1221,"ts":"2026-05-12T02:49:41.929Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"baseline","seed":1,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7638,"output_tokens":6,"latency_ms":1108,"ts":"2026-05-12T02:49:43.820Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"baseline","seed":2,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7638,"output_tokens":6,"latency_ms":1380,"ts":"2026-05-12T02:49:44.092Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"baseline","seed":3,"predicted":"book-mirror","expected":"book-mirror","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7638,"output_tokens":6,"latency_ms":1629,"ts":"2026-05-12T02:49:44.341Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"baseline","seed":1,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7638,"output_tokens":6,"latency_ms":1204,"ts":"2026-05-12T02:49:45.545Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"baseline","seed":2,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7638,"output_tokens":6,"latency_ms":1128,"ts":"2026-05-12T02:49:45.469Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"baseline","seed":3,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7638,"output_tokens":6,"latency_ms":1106,"ts":"2026-05-12T02:49:45.447Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"baseline","seed":1,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":6,"latency_ms":1241,"ts":"2026-05-12T02:49:46.786Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"baseline","seed":2,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":6,"latency_ms":948,"ts":"2026-05-12T02:49:46.493Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"baseline","seed":3,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":6,"latency_ms":1028,"ts":"2026-05-12T02:49:46.573Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"baseline","seed":1,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":7638,"output_tokens":9,"latency_ms":1723,"ts":"2026-05-12T02:49:48.509Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"baseline","seed":2,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":7638,"output_tokens":9,"latency_ms":1592,"ts":"2026-05-12T02:49:48.378Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"baseline","seed":3,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":7638,"output_tokens":9,"latency_ms":1293,"ts":"2026-05-12T02:49:48.079Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"baseline","seed":1,"predicted":"media-ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":7635,"output_tokens":7,"latency_ms":1376,"ts":"2026-05-12T02:49:49.885Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"baseline","seed":2,"predicted":"media-ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":7635,"output_tokens":7,"latency_ms":1691,"ts":"2026-05-12T02:49:50.201Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"baseline","seed":3,"predicted":"idea-ingest","expected":"idea-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7635,"output_tokens":7,"latency_ms":1512,"ts":"2026-05-12T02:49:50.021Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"baseline","seed":1,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7631,"output_tokens":7,"latency_ms":1593,"ts":"2026-05-12T02:49:51.794Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"baseline","seed":2,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7631,"output_tokens":7,"latency_ms":1593,"ts":"2026-05-12T02:49:51.794Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"baseline","seed":3,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7631,"output_tokens":7,"latency_ms":1593,"ts":"2026-05-12T02:49:51.794Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"baseline","seed":1,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":7,"latency_ms":1522,"ts":"2026-05-12T02:49:53.316Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"baseline","seed":2,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":7,"latency_ms":1395,"ts":"2026-05-12T02:49:53.189Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"baseline","seed":3,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":7,"latency_ms":1030,"ts":"2026-05-12T02:49:52.824Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"baseline","seed":1,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7632,"output_tokens":9,"latency_ms":1670,"ts":"2026-05-12T02:49:54.987Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"baseline","seed":2,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7632,"output_tokens":9,"latency_ms":1536,"ts":"2026-05-12T02:49:54.853Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"baseline","seed":3,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7632,"output_tokens":9,"latency_ms":1319,"ts":"2026-05-12T02:49:54.636Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"baseline","seed":1,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7630,"output_tokens":6,"latency_ms":1031,"ts":"2026-05-12T02:49:56.018Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"baseline","seed":2,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7630,"output_tokens":6,"latency_ms":947,"ts":"2026-05-12T02:49:55.934Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"baseline","seed":3,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7630,"output_tokens":6,"latency_ms":973,"ts":"2026-05-12T02:49:55.960Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"baseline","seed":1,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":7630,"output_tokens":4,"latency_ms":1539,"ts":"2026-05-12T02:49:57.557Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"baseline","seed":2,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":7630,"output_tokens":4,"latency_ms":1660,"ts":"2026-05-12T02:49:57.678Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"baseline","seed":3,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":7630,"output_tokens":4,"latency_ms":1550,"ts":"2026-05-12T02:49:57.568Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"baseline","seed":1,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":9,"latency_ms":1276,"ts":"2026-05-12T02:49:58.954Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"baseline","seed":2,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":9,"latency_ms":1273,"ts":"2026-05-12T02:49:58.951Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"baseline","seed":3,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":9,"latency_ms":1624,"ts":"2026-05-12T02:49:59.302Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"baseline","seed":1,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7633,"output_tokens":7,"latency_ms":1264,"ts":"2026-05-12T02:50:00.566Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"baseline","seed":2,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7633,"output_tokens":7,"latency_ms":1518,"ts":"2026-05-12T02:50:00.820Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"baseline","seed":3,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7633,"output_tokens":7,"latency_ms":1537,"ts":"2026-05-12T02:50:00.839Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"baseline","seed":1,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7633,"output_tokens":5,"latency_ms":1296,"ts":"2026-05-12T02:50:02.135Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"baseline","seed":2,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7633,"output_tokens":5,"latency_ms":1216,"ts":"2026-05-12T02:50:02.055Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"baseline","seed":3,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7633,"output_tokens":5,"latency_ms":1509,"ts":"2026-05-12T02:50:02.348Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"baseline","seed":1,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7632,"output_tokens":8,"latency_ms":1240,"ts":"2026-05-12T02:50:03.588Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"baseline","seed":2,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7632,"output_tokens":8,"latency_ms":1908,"ts":"2026-05-12T02:50:04.256Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"baseline","seed":3,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7632,"output_tokens":8,"latency_ms":1303,"ts":"2026-05-12T02:50:03.651Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7635,"output_tokens":5,"latency_ms":1022,"ts":"2026-05-12T02:50:05.278Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7635,"output_tokens":5,"latency_ms":1553,"ts":"2026-05-12T02:50:05.809Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7635,"output_tokens":5,"latency_ms":1199,"ts":"2026-05-12T02:50:05.455Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7635,"output_tokens":6,"latency_ms":1439,"ts":"2026-05-12T02:50:07.248Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7635,"output_tokens":6,"latency_ms":1062,"ts":"2026-05-12T02:50:06.871Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7635,"output_tokens":6,"latency_ms":1062,"ts":"2026-05-12T02:50:06.871Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7632,"output_tokens":8,"latency_ms":1078,"ts":"2026-05-12T02:50:08.326Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7632,"output_tokens":8,"latency_ms":961,"ts":"2026-05-12T02:50:08.209Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7632,"output_tokens":8,"latency_ms":1078,"ts":"2026-05-12T02:50:08.326Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":6,"latency_ms":1042,"ts":"2026-05-12T02:50:09.368Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":6,"latency_ms":1183,"ts":"2026-05-12T02:50:09.509Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7634,"output_tokens":6,"latency_ms":1052,"ts":"2026-05-12T02:50:09.378Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"baseline","seed":1,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7631,"output_tokens":5,"latency_ms":1246,"ts":"2026-05-12T02:50:10.755Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"baseline","seed":2,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7631,"output_tokens":5,"latency_ms":1596,"ts":"2026-05-12T02:50:11.105Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"baseline","seed":3,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":7631,"output_tokens":5,"latency_ms":1708,"ts":"2026-05-12T02:50:11.217Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4093,"output_tokens":5,"latency_ms":1686,"ts":"2026-05-12T02:50:12.903Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4093,"output_tokens":5,"latency_ms":1085,"ts":"2026-05-12T02:50:12.302Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"enrich","expected":"enrich","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4093,"output_tokens":5,"latency_ms":1242,"ts":"2026-05-12T02:50:12.459Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"query","expected":"gbrain","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4086,"output_tokens":4,"latency_ms":1209,"ts":"2026-05-12T02:50:14.112Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"query","expected":"gbrain","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4086,"output_tokens":4,"latency_ms":1372,"ts":"2026-05-12T02:50:14.275Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"query","expected":"gbrain","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4086,"output_tokens":4,"latency_ms":992,"ts":"2026-05-12T02:50:13.895Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":6,"latency_ms":1214,"ts":"2026-05-12T02:50:15.489Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":6,"latency_ms":1055,"ts":"2026-05-12T02:50:15.330Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"brain-pdf","expected":"brain-pdf","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":6,"latency_ms":1263,"ts":"2026-05-12T02:50:15.538Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":6,"latency_ms":1007,"ts":"2026-05-12T02:50:16.545Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":6,"latency_ms":1241,"ts":"2026-05-12T02:50:16.779Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"brain-publish","expected":"brain-publish","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":6,"latency_ms":1596,"ts":"2026-05-12T02:50:17.134Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"freshness-monitor","expected":"brain-librarian","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":7,"latency_ms":1887,"ts":"2026-05-12T02:50:19.021Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"freshness-monitor","expected":"brain-librarian","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":7,"latency_ms":1330,"ts":"2026-05-12T02:50:18.464Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"benchmark-gbrain","expected":"brain-librarian","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":7,"latency_ms":1777,"ts":"2026-05-12T02:50:18.911Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4086,"output_tokens":7,"latency_ms":1016,"ts":"2026-05-12T02:50:20.037Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4086,"output_tokens":7,"latency_ms":1700,"ts":"2026-05-12T02:50:20.721Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4086,"output_tokens":7,"latency_ms":1524,"ts":"2026-05-12T02:50:20.545Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"strategic-reading","expected":"book-mirror","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4093,"output_tokens":6,"latency_ms":5907,"ts":"2026-05-12T02:50:26.628Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"strategic-reading","expected":"book-mirror","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4093,"output_tokens":6,"latency_ms":1594,"ts":"2026-05-12T02:50:22.315Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"strategic-reading","expected":"book-mirror","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4093,"output_tokens":6,"latency_ms":1251,"ts":"2026-05-12T02:50:21.972Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4093,"output_tokens":6,"latency_ms":1258,"ts":"2026-05-12T02:50:27.886Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4093,"output_tokens":6,"latency_ms":1228,"ts":"2026-05-12T02:50:27.856Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"strategic-reading","expected":"strategic-reading","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4093,"output_tokens":6,"latency_ms":1253,"ts":"2026-05-12T02:50:27.881Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":6,"latency_ms":1218,"ts":"2026-05-12T02:50:29.105Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":6,"latency_ms":941,"ts":"2026-05-12T02:50:28.828Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":6,"latency_ms":978,"ts":"2026-05-12T02:50:28.865Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4093,"output_tokens":9,"latency_ms":1408,"ts":"2026-05-12T02:50:30.513Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4093,"output_tokens":9,"latency_ms":1391,"ts":"2026-05-12T02:50:30.496Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"dropbox-archive-review","expected":"archive-crawler","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4093,"output_tokens":9,"latency_ms":1480,"ts":"2026-05-12T02:50:30.585Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"article-enrichment","expected":"idea-ingest","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4090,"output_tokens":7,"latency_ms":937,"ts":"2026-05-12T02:50:31.522Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"article-enrichment","expected":"idea-ingest","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4090,"output_tokens":7,"latency_ms":869,"ts":"2026-05-12T02:50:31.454Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"article-enrichment","expected":"idea-ingest","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4090,"output_tokens":7,"latency_ms":1021,"ts":"2026-05-12T02:50:31.606Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4086,"output_tokens":7,"latency_ms":871,"ts":"2026-05-12T02:50:32.477Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4086,"output_tokens":7,"latency_ms":1044,"ts":"2026-05-12T02:50:32.650Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"media-ingest","expected":"media-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4086,"output_tokens":7,"latency_ms":1108,"ts":"2026-05-12T02:50:32.714Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":7,"latency_ms":982,"ts":"2026-05-12T02:50:33.696Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":7,"latency_ms":1058,"ts":"2026-05-12T02:50:33.772Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"meeting-ingestion","expected":"meeting-ingestion","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":7,"latency_ms":957,"ts":"2026-05-12T02:50:33.671Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4087,"output_tokens":9,"latency_ms":984,"ts":"2026-05-12T02:50:34.756Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4087,"output_tokens":9,"latency_ms":1450,"ts":"2026-05-12T02:50:35.222Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"voice-note-ingest","expected":"voice-note-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4087,"output_tokens":9,"latency_ms":1341,"ts":"2026-05-12T02:50:35.113Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"calendar-check","expected":"google-calendar","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4085,"output_tokens":6,"latency_ms":1502,"ts":"2026-05-12T02:50:36.724Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"calendar-check","expected":"google-calendar","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4085,"output_tokens":6,"latency_ms":1326,"ts":"2026-05-12T02:50:36.548Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"calendar-check","expected":"google-calendar","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4085,"output_tokens":6,"latency_ms":1326,"ts":"2026-05-12T02:50:36.548Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4085,"output_tokens":4,"latency_ms":1060,"ts":"2026-05-12T02:50:37.784Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4085,"output_tokens":4,"latency_ms":1600,"ts":"2026-05-12T02:50:38.324Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"gmail","expected":"executive-assistant","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4085,"output_tokens":4,"latency_ms":1415,"ts":"2026-05-12T02:50:38.139Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":9,"latency_ms":1062,"ts":"2026-05-12T02:50:39.386Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":9,"latency_ms":1062,"ts":"2026-05-12T02:50:39.386Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":9,"latency_ms":971,"ts":"2026-05-12T02:50:39.295Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4088,"output_tokens":7,"latency_ms":1634,"ts":"2026-05-12T02:50:41.020Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4088,"output_tokens":7,"latency_ms":999,"ts":"2026-05-12T02:50:40.386Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4088,"output_tokens":7,"latency_ms":890,"ts":"2026-05-12T02:50:40.277Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4088,"output_tokens":5,"latency_ms":1141,"ts":"2026-05-12T02:50:42.161Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4088,"output_tokens":5,"latency_ms":938,"ts":"2026-05-12T02:50:41.958Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4088,"output_tokens":5,"latency_ms":878,"ts":"2026-05-12T02:50:41.898Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"functional-areas","seed":1,"predicted":"daily-task-prep","expected":"daily-task-manager","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4087,"output_tokens":8,"latency_ms":1185,"ts":"2026-05-12T02:50:43.347Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"functional-areas","seed":2,"predicted":"daily-task-prep","expected":"daily-task-manager","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4087,"output_tokens":8,"latency_ms":990,"ts":"2026-05-12T02:50:43.151Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"functional-areas","seed":3,"predicted":"daily-task-prep","expected":"daily-task-manager","correct":0,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4087,"output_tokens":8,"latency_ms":954,"ts":"2026-05-12T02:50:43.115Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4090,"output_tokens":5,"latency_ms":1043,"ts":"2026-05-12T02:50:44.390Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4090,"output_tokens":5,"latency_ms":1011,"ts":"2026-05-12T02:50:44.358Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4090,"output_tokens":5,"latency_ms":922,"ts":"2026-05-12T02:50:44.269Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4090,"output_tokens":6,"latency_ms":1193,"ts":"2026-05-12T02:50:45.583Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4090,"output_tokens":6,"latency_ms":1196,"ts":"2026-05-12T02:50:45.586Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"skill-creator","expected":"skill-creator","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4090,"output_tokens":6,"latency_ms":5248,"ts":"2026-05-12T02:50:49.638Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4087,"output_tokens":8,"latency_ms":1257,"ts":"2026-05-12T02:50:50.895Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4087,"output_tokens":8,"latency_ms":1487,"ts":"2026-05-12T02:50:51.125Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4087,"output_tokens":8,"latency_ms":1100,"ts":"2026-05-12T02:50:50.738Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":6,"latency_ms":1413,"ts":"2026-05-12T02:50:52.538Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":6,"latency_ms":1236,"ts":"2026-05-12T02:50:52.361Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4089,"output_tokens":6,"latency_ms":1590,"ts":"2026-05-12T02:50:52.715Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"functional-areas","seed":1,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4086,"output_tokens":5,"latency_ms":1452,"ts":"2026-05-12T02:50:54.167Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"functional-areas","seed":2,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4086,"output_tokens":5,"latency_ms":1202,"ts":"2026-05-12T02:50:53.917Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"functional-areas","seed":3,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":4086,"output_tokens":5,"latency_ms":1452,"ts":"2026-05-12T02:50:54.167Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"enrich","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3213,"output_tokens":6,"latency_ms":1003,"ts":"2026-05-12T02:50:55.170Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"enrich","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3213,"output_tokens":6,"latency_ms":987,"ts":"2026-05-12T02:50:55.154Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-ops","expected":"enrich","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3213,"output_tokens":6,"latency_ms":1296,"ts":"2026-05-12T02:50:55.463Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3206,"output_tokens":6,"latency_ms":1360,"ts":"2026-05-12T02:50:56.824Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3206,"output_tokens":6,"latency_ms":1029,"ts":"2026-05-12T02:50:56.493Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-ops","expected":"gbrain","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3206,"output_tokens":6,"latency_ms":2308,"ts":"2026-05-12T02:50:57.772Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-pdf-auto","expected":"brain-pdf","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":8,"latency_ms":2125,"ts":"2026-05-12T02:50:59.897Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-pdf-auto","expected":"brain-pdf","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":8,"latency_ms":1538,"ts":"2026-05-12T02:50:59.310Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-pdf-auto","expected":"brain-pdf","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":8,"latency_ms":1188,"ts":"2026-05-12T02:50:58.960Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"brain-publish","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":6,"latency_ms":826,"ts":"2026-05-12T02:51:00.723Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"brain-publish","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":6,"latency_ms":1031,"ts":"2026-05-12T02:51:00.928Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-ops","expected":"brain-publish","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":6,"latency_ms":949,"ts":"2026-05-12T02:51:00.846Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"brain-librarian","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":6,"latency_ms":1494,"ts":"2026-05-12T02:51:02.422Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"brain-librarian","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":6,"latency_ms":1311,"ts":"2026-05-12T02:51:02.239Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-ops","expected":"brain-librarian","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":6,"latency_ms":1253,"ts":"2026-05-12T02:51:02.181Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3206,"output_tokens":7,"latency_ms":1047,"ts":"2026-05-12T02:51:03.469Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3206,"output_tokens":7,"latency_ms":1143,"ts":"2026-05-12T02:51:03.565Z"}
|
||||
{"kind":"run","fixture_id":5,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"citation-fixer","expected":"citation-fixer","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3206,"output_tokens":7,"latency_ms":919,"ts":"2026-05-12T02:51:03.341Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"book-mirror","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3213,"output_tokens":6,"latency_ms":1301,"ts":"2026-05-12T02:51:04.866Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"book-mirror","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3213,"output_tokens":6,"latency_ms":1159,"ts":"2026-05-12T02:51:04.724Z"}
|
||||
{"kind":"run","fixture_id":6,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-ops","expected":"book-mirror","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3213,"output_tokens":6,"latency_ms":1417,"ts":"2026-05-12T02:51:04.982Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"brain-ops","expected":"strategic-reading","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3213,"output_tokens":6,"latency_ms":1209,"ts":"2026-05-12T02:51:06.191Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"strategic-reading","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3213,"output_tokens":6,"latency_ms":1609,"ts":"2026-05-12T02:51:06.591Z"}
|
||||
{"kind":"run","fixture_id":7,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-ops","expected":"strategic-reading","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3213,"output_tokens":6,"latency_ms":1432,"ts":"2026-05-12T02:51:06.414Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":6,"latency_ms":1765,"ts":"2026-05-12T02:51:08.356Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":6,"latency_ms":3599,"ts":"2026-05-12T02:51:10.190Z"}
|
||||
{"kind":"run","fixture_id":8,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"concept-synthesis","expected":"concept-synthesis","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":6,"latency_ms":1060,"ts":"2026-05-12T02:51:07.651Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"archive-crawler","expected":"archive-crawler","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3213,"output_tokens":6,"latency_ms":1213,"ts":"2026-05-12T02:51:11.403Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"brain-ops","expected":"archive-crawler","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3213,"output_tokens":6,"latency_ms":878,"ts":"2026-05-12T02:51:11.068Z"}
|
||||
{"kind":"run","fixture_id":9,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"brain-ops","expected":"archive-crawler","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3213,"output_tokens":6,"latency_ms":1040,"ts":"2026-05-12T02:51:11.230Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3210,"output_tokens":5,"latency_ms":1542,"ts":"2026-05-12T02:51:12.945Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3210,"output_tokens":5,"latency_ms":930,"ts":"2026-05-12T02:51:12.333Z"}
|
||||
{"kind":"run","fixture_id":10,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"ingest","expected":"idea-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3210,"output_tokens":5,"latency_ms":971,"ts":"2026-05-12T02:51:12.374Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"ingest","expected":"media-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3206,"output_tokens":5,"latency_ms":1203,"ts":"2026-05-12T02:51:14.148Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"ingest","expected":"media-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3206,"output_tokens":5,"latency_ms":1513,"ts":"2026-05-12T02:51:14.458Z"}
|
||||
{"kind":"run","fixture_id":11,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"ingest","expected":"media-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3206,"output_tokens":5,"latency_ms":1342,"ts":"2026-05-12T02:51:14.287Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"ingest","expected":"meeting-ingestion","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":5,"latency_ms":4435,"ts":"2026-05-12T02:51:18.893Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"ingest","expected":"meeting-ingestion","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":5,"latency_ms":1355,"ts":"2026-05-12T02:51:15.813Z"}
|
||||
{"kind":"run","fixture_id":12,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"ingest","expected":"meeting-ingestion","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":5,"latency_ms":978,"ts":"2026-05-12T02:51:15.436Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"ingest","expected":"voice-note-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3207,"output_tokens":5,"latency_ms":1026,"ts":"2026-05-12T02:51:19.919Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"ingest","expected":"voice-note-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3207,"output_tokens":5,"latency_ms":1323,"ts":"2026-05-12T02:51:20.216Z"}
|
||||
{"kind":"run","fixture_id":13,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"ingest","expected":"voice-note-ingest","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3207,"output_tokens":5,"latency_ms":1372,"ts":"2026-05-12T02:51:20.265Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3205,"output_tokens":6,"latency_ms":1295,"ts":"2026-05-12T02:51:21.560Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3205,"output_tokens":6,"latency_ms":2501,"ts":"2026-05-12T02:51:22.766Z"}
|
||||
{"kind":"run","fixture_id":14,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"google-calendar","expected":"google-calendar","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3205,"output_tokens":6,"latency_ms":1110,"ts":"2026-05-12T02:51:21.375Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"executive-assistant","expected":"executive-assistant","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3205,"output_tokens":6,"latency_ms":1067,"ts":"2026-05-12T02:51:23.833Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"executive-assistant","expected":"executive-assistant","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3205,"output_tokens":6,"latency_ms":1059,"ts":"2026-05-12T02:51:23.825Z"}
|
||||
{"kind":"run","fixture_id":15,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"executive-assistant","expected":"executive-assistant","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3205,"output_tokens":6,"latency_ms":1237,"ts":"2026-05-12T02:51:24.003Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":9,"latency_ms":1122,"ts":"2026-05-12T02:51:25.125Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":9,"latency_ms":1537,"ts":"2026-05-12T02:51:25.540Z"}
|
||||
{"kind":"run","fixture_id":16,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"perplexity-research","expected":"perplexity-research","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":9,"latency_ms":1755,"ts":"2026-05-12T02:51:25.758Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3208,"output_tokens":7,"latency_ms":900,"ts":"2026-05-12T02:51:26.658Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3208,"output_tokens":7,"latency_ms":852,"ts":"2026-05-12T02:51:26.610Z"}
|
||||
{"kind":"run","fixture_id":17,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"x-ingest","expected":"x-ingest","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3208,"output_tokens":7,"latency_ms":1438,"ts":"2026-05-12T02:51:27.196Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3208,"output_tokens":5,"latency_ms":1174,"ts":"2026-05-12T02:51:28.370Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3208,"output_tokens":5,"latency_ms":1347,"ts":"2026-05-12T02:51:28.543Z"}
|
||||
{"kind":"run","fixture_id":18,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"checkin","expected":"checkin","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3208,"output_tokens":5,"latency_ms":1098,"ts":"2026-05-12T02:51:28.294Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"resolver-of-resolvers","seed":1,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3207,"output_tokens":8,"latency_ms":1455,"ts":"2026-05-12T02:51:29.998Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"resolver-of-resolvers","seed":2,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3207,"output_tokens":8,"latency_ms":990,"ts":"2026-05-12T02:51:29.533Z"}
|
||||
{"kind":"run","fixture_id":19,"corpus":"training","variant":"resolver-of-resolvers","seed":3,"predicted":"daily-task-manager","expected":"daily-task-manager","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3207,"output_tokens":8,"latency_ms":1801,"ts":"2026-05-12T02:51:30.344Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3210,"output_tokens":5,"latency_ms":2027,"ts":"2026-05-12T02:51:32.371Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3210,"output_tokens":5,"latency_ms":1009,"ts":"2026-05-12T02:51:31.353Z"}
|
||||
{"kind":"run","fixture_id":0,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"skillify","expected":"skillify","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3210,"output_tokens":5,"latency_ms":950,"ts":"2026-05-12T02:51:31.294Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"acp-coding","expected":"skill-creator","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3210,"output_tokens":7,"latency_ms":1008,"ts":"2026-05-12T02:51:33.379Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"acp-coding","expected":"skill-creator","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3210,"output_tokens":7,"latency_ms":931,"ts":"2026-05-12T02:51:33.302Z"}
|
||||
{"kind":"run","fixture_id":1,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"acp-coding","expected":"skill-creator","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3210,"output_tokens":7,"latency_ms":1036,"ts":"2026-05-12T02:51:33.407Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"daily-task-manager","expected":"daily-task-prep","correct":0,"correct_lenient":0,"model":"anthropic:claude-sonnet-4-6","input_tokens":3207,"output_tokens":8,"latency_ms":901,"ts":"2026-05-12T02:51:34.308Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3207,"output_tokens":8,"latency_ms":892,"ts":"2026-05-12T02:51:34.299Z"}
|
||||
{"kind":"run","fixture_id":2,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"daily-task-prep","expected":"daily-task-prep","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3207,"output_tokens":8,"latency_ms":1016,"ts":"2026-05-12T02:51:34.423Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":6,"latency_ms":879,"ts":"2026-05-12T02:51:35.302Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":6,"latency_ms":948,"ts":"2026-05-12T02:51:35.371Z"}
|
||||
{"kind":"run","fixture_id":3,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"google-contacts","expected":"google-contacts","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3209,"output_tokens":6,"latency_ms":930,"ts":"2026-05-12T02:51:35.353Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"resolver-of-resolvers","seed":1,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3206,"output_tokens":5,"latency_ms":1022,"ts":"2026-05-12T02:51:36.393Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"resolver-of-resolvers","seed":2,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3206,"output_tokens":5,"latency_ms":1406,"ts":"2026-05-12T02:51:36.777Z"}
|
||||
{"kind":"run","fixture_id":4,"corpus":"held_out","variant":"resolver-of-resolvers","seed":3,"predicted":"healthcheck","expected":"healthcheck","correct":1,"correct_lenient":1,"model":"anthropic:claude-sonnet-4-6","input_tokens":3206,"output_tokens":5,"latency_ms":907,"ts":"2026-05-12T02:51:36.278Z"}
|
||||
@@ -1,8 +0,0 @@
|
||||
// 5 held-out blind fixtures. Authored before the variant resolvers were
|
||||
// fully reviewed; target skills present in both real variants.
|
||||
// Held-out accuracy is the headline claim in skills/functional-area-resolver/SKILL.md.
|
||||
{"intent":"Skillify the JSON parsing helper I wrote last week","expected_skill":"skillify"}
|
||||
{"intent":"Create a new skill for cataloging books I've finished","expected_skill":"skill-creator"}
|
||||
{"intent":"Build me a daily prep summary for tomorrow","expected_skill":"daily-task-prep"}
|
||||
{"intent":"Pull the contact details for Maria from my address book","expected_skill":"google-contacts"}
|
||||
{"intent":"Run a healthcheck on my services","expected_skill":"healthcheck"}
|
||||
@@ -1,24 +0,0 @@
|
||||
// 20 training fixtures for the functional-area-resolver A/B eval.
|
||||
// Each line: {"intent": "<user phrasing>", "expected_skill": "<skill slug>"}
|
||||
// Target skills are present in BOTH variants (verified against the
|
||||
// real production AGENTS.md at git commit 93848ff3b^ and 93848ff3b).
|
||||
{"intent":"Create a person page for John Smith and enrich it from his GitHub","expected_skill":"enrich"}
|
||||
{"intent":"What do we know about Stripe","expected_skill":"gbrain"}
|
||||
{"intent":"Make a PDF from my brain page on dispatcher patterns","expected_skill":"brain-pdf"}
|
||||
{"intent":"Publish this brain page as a shareable link","expected_skill":"brain-publish"}
|
||||
{"intent":"Run brain integrity — what's lost in my archive","expected_skill":"brain-librarian"}
|
||||
{"intent":"Fix the broken citations on this page","expected_skill":"citation-fixer"}
|
||||
{"intent":"Make a personalized version of Atomic Habits with my brain context","expected_skill":"book-mirror"}
|
||||
{"intent":"Read Thinking Fast and Slow through the lens of my product work","expected_skill":"strategic-reading"}
|
||||
{"intent":"Synthesize my concepts about resolver design and routing","expected_skill":"concept-synthesis"}
|
||||
{"intent":"Crawl my dropbox archive for old notes I should pull in","expected_skill":"archive-crawler"}
|
||||
{"intent":"Ingest this article from The Atlantic into my brain","expected_skill":"idea-ingest"}
|
||||
{"intent":"Process this YouTube video into the brain","expected_skill":"media-ingest"}
|
||||
{"intent":"I have a meeting transcript to file from this morning","expected_skill":"meeting-ingestion"}
|
||||
{"intent":"Save this voice memo and transcribe it","expected_skill":"voice-note-ingest"}
|
||||
{"intent":"What's on my calendar tomorrow","expected_skill":"google-calendar"}
|
||||
{"intent":"Draft a reply email to Sarah","expected_skill":"executive-assistant"}
|
||||
{"intent":"Research what's new about WebGPU adoption","expected_skill":"perplexity-research"}
|
||||
{"intent":"Pull my recent X posts and ingest them","expected_skill":"x-ingest"}
|
||||
{"intent":"Check me into the coffee shop I'm at","expected_skill":"checkin"}
|
||||
{"intent":"Add a task for tomorrow's meeting prep","expected_skill":"daily-task-manager"}
|
||||
@@ -1,302 +0,0 @@
|
||||
/**
|
||||
* Unit tests for the functional-area-resolver A/B eval harness.
|
||||
* Run with: bun test evals/functional-area-resolver/harness-runner.test.ts
|
||||
*
|
||||
* Covers every pure function so contributors can debug without spending
|
||||
* money on every iteration. main() smoke test is omitted in this slice
|
||||
* (it would require mocking gateway transport + filesystem; the harness's
|
||||
* --limit 1 mode is a sufficient real smoke check at ~$0.01 per run).
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import {
|
||||
parseFixtures,
|
||||
buildPrompt,
|
||||
parseModelResponse,
|
||||
scoreFixture,
|
||||
scoreFixtureLenient,
|
||||
parseDispatcherLists,
|
||||
meanAndCI95,
|
||||
estimateCost,
|
||||
hashContent,
|
||||
parseArgs,
|
||||
resolveModel,
|
||||
PROMPT_TEMPLATE,
|
||||
MODEL_ID,
|
||||
MODEL_ALIASES,
|
||||
} from './harness-runner.ts';
|
||||
|
||||
test('parseFixtures: parses valid JSONL', () => {
|
||||
const raw = `{"intent":"foo","expected_skill":"bar"}\n{"intent":"baz","expected_skill":"qux"}\n`;
|
||||
const out = parseFixtures(raw);
|
||||
expect(out).toEqual([
|
||||
{ intent: 'foo', expected_skill: 'bar' },
|
||||
{ intent: 'baz', expected_skill: 'qux' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('parseFixtures: skips // comments and blank lines', () => {
|
||||
const raw = `// header comment\n{"intent":"a","expected_skill":"b"}\n\n// another comment\n{"intent":"c","expected_skill":"d"}\n`;
|
||||
const out = parseFixtures(raw);
|
||||
expect(out).toHaveLength(2);
|
||||
expect(out[0].intent).toBe('a');
|
||||
});
|
||||
|
||||
test('parseFixtures: throws on missing required fields', () => {
|
||||
expect(() => parseFixtures(`{"intent":"foo"}\n`)).toThrow(/missing required fields/);
|
||||
});
|
||||
|
||||
test('parseFixtures: throws on invalid JSON', () => {
|
||||
expect(() => parseFixtures(`{not json}\n`)).toThrow(/Bad fixture JSON/);
|
||||
});
|
||||
|
||||
test('buildPrompt: injects variant content and intent', () => {
|
||||
const prompt = buildPrompt('RESOLVER X', 'INTENT Y');
|
||||
expect(prompt).toContain('RESOLVER X');
|
||||
expect(prompt).toContain('INTENT Y');
|
||||
expect(prompt).not.toContain('<<<RESOLVER_CONTENT>>>');
|
||||
expect(prompt).not.toContain('<<<INTENT>>>');
|
||||
});
|
||||
|
||||
test('parseModelResponse: bare slug', () => {
|
||||
expect(parseModelResponse('enrich')).toBe('enrich');
|
||||
});
|
||||
|
||||
test('parseModelResponse: strips fenced output', () => {
|
||||
expect(parseModelResponse('```\nenrich\n```')).toBe('enrich');
|
||||
expect(parseModelResponse('```text\nenrich\n```')).toBe('enrich');
|
||||
});
|
||||
|
||||
test('parseModelResponse: extracts from JSON object', () => {
|
||||
expect(parseModelResponse('{"skill": "book-mirror"}')).toBe('book-mirror');
|
||||
expect(parseModelResponse('{"skill_slug": "query"}')).toBe('query');
|
||||
});
|
||||
|
||||
test('parseModelResponse: strips quotes and backticks', () => {
|
||||
expect(parseModelResponse('"enrich"')).toBe('enrich');
|
||||
expect(parseModelResponse('`enrich`')).toBe('enrich');
|
||||
});
|
||||
|
||||
test('parseModelResponse: picks first slug-shaped token if model prefaces with prose', () => {
|
||||
expect(parseModelResponse('The skill is enrich.')).toBe('the'); // first token wins; documents permissive matcher
|
||||
expect(parseModelResponse('enrich is the answer')).toBe('enrich');
|
||||
});
|
||||
|
||||
test('parseModelResponse: lowercases output', () => {
|
||||
expect(parseModelResponse('ENRICH')).toBe('enrich');
|
||||
});
|
||||
|
||||
test('scoreFixture: exact match returns 1', () => {
|
||||
expect(scoreFixture('enrich', 'enrich')).toBe(1);
|
||||
});
|
||||
|
||||
test('scoreFixture: mismatch returns 0', () => {
|
||||
expect(scoreFixture('enrich', 'query')).toBe(0);
|
||||
});
|
||||
|
||||
test('scoreFixture: case-sensitive at this layer (caller lowercases via parseModelResponse)', () => {
|
||||
expect(scoreFixture('Enrich', 'enrich')).toBe(0);
|
||||
});
|
||||
|
||||
test('meanAndCI95: empty array returns zeros', () => {
|
||||
expect(meanAndCI95([])).toEqual({ mean: 0, halfWidthCI: 0 });
|
||||
});
|
||||
|
||||
test('meanAndCI95: single value returns mean with zero CI', () => {
|
||||
expect(meanAndCI95([0.95])).toEqual({ mean: 0.95, halfWidthCI: 0 });
|
||||
});
|
||||
|
||||
test('meanAndCI95: three equal values returns mean with zero CI', () => {
|
||||
const r = meanAndCI95([1, 1, 1]);
|
||||
expect(r.mean).toBe(1);
|
||||
expect(r.halfWidthCI).toBe(0);
|
||||
});
|
||||
|
||||
test('meanAndCI95: three different values returns plausible CI', () => {
|
||||
const r = meanAndCI95([0.8, 0.9, 1.0]);
|
||||
expect(r.mean).toBeCloseTo(0.9, 5);
|
||||
expect(r.halfWidthCI).toBeGreaterThan(0);
|
||||
expect(r.halfWidthCI).toBeLessThan(0.5);
|
||||
});
|
||||
|
||||
test('estimateCost: uses Opus 4.7 pricing by default', () => {
|
||||
const cost = estimateCost(100, 'claude-opus-4-7', 1000, 50);
|
||||
// 100 calls * 1000 input tokens = 100K input → $0.50 at $5/MTok
|
||||
// 100 calls * 50 output tokens = 5K output → $0.125 at $25/MTok
|
||||
expect(cost).toBeCloseTo(0.625, 2);
|
||||
});
|
||||
|
||||
test('estimateCost: Sonnet pricing differs from Opus', () => {
|
||||
const opus = estimateCost(100, 'claude-opus-4-7', 1000, 50);
|
||||
const sonnet = estimateCost(100, 'claude-sonnet-4-6', 1000, 50);
|
||||
const haiku = estimateCost(100, 'claude-haiku-4-5-20251001', 1000, 50);
|
||||
expect(sonnet).toBeLessThan(opus);
|
||||
expect(haiku).toBeLessThan(sonnet);
|
||||
});
|
||||
|
||||
test('estimateCost: zero calls returns zero', () => {
|
||||
expect(estimateCost(0)).toBe(0);
|
||||
});
|
||||
|
||||
test('estimateCost: unknown model returns zero', () => {
|
||||
expect(estimateCost(100, 'unknown-model')).toBe(0);
|
||||
});
|
||||
|
||||
test('hashContent: produces stable 16-char hex prefix', () => {
|
||||
const h1 = hashContent('hello world');
|
||||
const h2 = hashContent('hello world');
|
||||
expect(h1).toBe(h2);
|
||||
expect(h1).toHaveLength(16);
|
||||
expect(h1).toMatch(/^[0-9a-f]+$/);
|
||||
});
|
||||
|
||||
test('hashContent: different inputs produce different hashes', () => {
|
||||
expect(hashContent('a')).not.toBe(hashContent('b'));
|
||||
});
|
||||
|
||||
test('parseArgs: defaults are sensible', () => {
|
||||
expect(parseArgs([])).toEqual({
|
||||
limit: null,
|
||||
parallel: 1,
|
||||
output: null,
|
||||
help: false,
|
||||
yes: false,
|
||||
model: MODEL_ID,
|
||||
variantsDir: 'variants',
|
||||
variantFiles: null,
|
||||
});
|
||||
});
|
||||
|
||||
test('parseArgs: --model alias', () => {
|
||||
expect(parseArgs(['--model', 'sonnet']).model).toBe('sonnet');
|
||||
expect(parseArgs(['--model', 'anthropic:claude-haiku-4-5-20251001']).model).toBe('anthropic:claude-haiku-4-5-20251001');
|
||||
});
|
||||
|
||||
test('parseArgs: --variants comma-list', () => {
|
||||
expect(parseArgs(['--variants', 'a,b,c']).variantFiles).toEqual(['a', 'b', 'c']);
|
||||
});
|
||||
|
||||
test('parseArgs: --variants-dir', () => {
|
||||
expect(parseArgs(['--variants-dir', 'variants-sweep']).variantsDir).toBe('variants-sweep');
|
||||
});
|
||||
|
||||
test('resolveModel: aliases', () => {
|
||||
expect(resolveModel('opus')).toEqual({ full: 'anthropic:claude-opus-4-7', bare: 'claude-opus-4-7' });
|
||||
expect(resolveModel('sonnet')).toEqual({ full: 'anthropic:claude-sonnet-4-6', bare: 'claude-sonnet-4-6' });
|
||||
expect(resolveModel('haiku').full).toBe(MODEL_ALIASES.haiku);
|
||||
});
|
||||
|
||||
test('resolveModel: passthrough for full id', () => {
|
||||
expect(resolveModel('anthropic:claude-opus-4-7').bare).toBe('claude-opus-4-7');
|
||||
expect(resolveModel('anthropic:claude-something-future').bare).toBe('claude-something-future');
|
||||
});
|
||||
|
||||
test('resolveModel: non-anthropic provider passes through unchanged', () => {
|
||||
expect(resolveModel('openai:gpt-4o')).toEqual({ full: 'openai:gpt-4o', bare: 'openai:gpt-4o' });
|
||||
});
|
||||
|
||||
test('parseDispatcherLists: extracts dispatcher → sub-skills', () => {
|
||||
const variant = `
|
||||
- **Brain**: foo bar → \`brain-ops\` (dispatcher for: enrich, query, citation-fixer)
|
||||
- **Comms**: email → \`exec-assist\` (dispatcher for: gmail, slack)
|
||||
- Bare row → \`bare-skill\`
|
||||
`;
|
||||
const m = parseDispatcherLists(variant);
|
||||
expect(m.size).toBe(2);
|
||||
expect(m.get('brain-ops')).toEqual(new Set(['brain-ops', 'enrich', 'query', 'citation-fixer']));
|
||||
expect(m.get('exec-assist')).toEqual(new Set(['exec-assist', 'gmail', 'slack']));
|
||||
});
|
||||
|
||||
test('parseDispatcherLists: accepts ASCII -> arrow (SKILL.md template format)', () => {
|
||||
// Codex review P2-2: SKILL.md Step 4 documents the template with `->`,
|
||||
// but the production variants use Unicode `→`. The regex must match
|
||||
// both or downstream users following the template silently fall through
|
||||
// to strict-only scoring.
|
||||
const variant = `
|
||||
- **Brain**: foo bar -> \`brain-ops\` (dispatcher for: enrich, query)
|
||||
- **Comms**: email -> \`exec-assist\` (dispatcher for: gmail)
|
||||
`;
|
||||
const m = parseDispatcherLists(variant);
|
||||
expect(m.size).toBe(2);
|
||||
expect(m.get('brain-ops')).toEqual(new Set(['brain-ops', 'enrich', 'query']));
|
||||
expect(m.get('exec-assist')).toEqual(new Set(['exec-assist', 'gmail']));
|
||||
});
|
||||
|
||||
test('parseDispatcherLists: mixed Unicode + ASCII arrows in same file', () => {
|
||||
// A real-world fork could migrate gradually; harness must handle both.
|
||||
const variant = `
|
||||
- **Brain**: foo → \`brain-ops\` (dispatcher for: enrich, query)
|
||||
- **Comms**: email -> \`exec-assist\` (dispatcher for: gmail, slack)
|
||||
`;
|
||||
const m = parseDispatcherLists(variant);
|
||||
expect(m.size).toBe(2);
|
||||
expect(m.get('brain-ops')?.has('enrich')).toBe(true);
|
||||
expect(m.get('exec-assist')?.has('gmail')).toBe(true);
|
||||
});
|
||||
|
||||
test('parseDispatcherLists: zero dispatchers when no clauses present', () => {
|
||||
const variant = `
|
||||
- Row 1 → \`alpha\`
|
||||
- Row 2 → \`beta\`
|
||||
`;
|
||||
expect(parseDispatcherLists(variant).size).toBe(0);
|
||||
});
|
||||
|
||||
test('scoreFixtureLenient: exact match = 1', () => {
|
||||
expect(scoreFixtureLenient('enrich', 'enrich', new Map())).toBe(1);
|
||||
});
|
||||
|
||||
test('scoreFixtureLenient: same-area sub-skill = 1', () => {
|
||||
const lists = new Map([['brain-ops', new Set(['brain-ops', 'enrich', 'query'])]]);
|
||||
expect(scoreFixtureLenient('enrich', 'query', lists)).toBe(1);
|
||||
expect(scoreFixtureLenient('brain-ops', 'enrich', lists)).toBe(1);
|
||||
expect(scoreFixtureLenient('enrich', 'brain-ops', lists)).toBe(1);
|
||||
});
|
||||
|
||||
test('scoreFixtureLenient: cross-area = 0', () => {
|
||||
const lists = new Map([
|
||||
['brain-ops', new Set(['brain-ops', 'enrich'])],
|
||||
['comms', new Set(['comms', 'gmail'])],
|
||||
]);
|
||||
expect(scoreFixtureLenient('enrich', 'gmail', lists)).toBe(0);
|
||||
});
|
||||
|
||||
test('scoreFixtureLenient: no dispatcher map = falls back to strict', () => {
|
||||
expect(scoreFixtureLenient('foo', 'bar', new Map())).toBe(0);
|
||||
});
|
||||
|
||||
test('parseArgs: --limit', () => {
|
||||
expect(parseArgs(['--limit', '5']).limit).toBe(5);
|
||||
});
|
||||
|
||||
test('parseArgs: --limit rejects non-positive', () => {
|
||||
expect(() => parseArgs(['--limit', '0'])).toThrow();
|
||||
expect(() => parseArgs(['--limit', '-3'])).toThrow();
|
||||
expect(() => parseArgs(['--limit', 'foo'])).toThrow();
|
||||
});
|
||||
|
||||
test('parseArgs: --parallel', () => {
|
||||
expect(parseArgs(['--parallel', '4']).parallel).toBe(4);
|
||||
});
|
||||
|
||||
test('parseArgs: --output', () => {
|
||||
expect(parseArgs(['--output', '/tmp/x.jsonl']).output).toBe('/tmp/x.jsonl');
|
||||
});
|
||||
|
||||
test('parseArgs: --help and --yes', () => {
|
||||
expect(parseArgs(['--help']).help).toBe(true);
|
||||
expect(parseArgs(['--yes']).yes).toBe(true);
|
||||
});
|
||||
|
||||
test('parseArgs: rejects unknown flags', () => {
|
||||
expect(() => parseArgs(['--bogus'])).toThrow(/Unknown flag/);
|
||||
});
|
||||
|
||||
test('MODEL_ID is pinned to Opus 4.7', () => {
|
||||
expect(MODEL_ID).toBe('anthropic:claude-opus-4-7');
|
||||
});
|
||||
|
||||
test('PROMPT_TEMPLATE contains both placeholders', () => {
|
||||
expect(PROMPT_TEMPLATE).toContain('<<<RESOLVER_CONTENT>>>');
|
||||
expect(PROMPT_TEMPLATE).toContain('<<<INTENT>>>');
|
||||
});
|
||||
@@ -1,599 +0,0 @@
|
||||
/**
|
||||
* functional-area-resolver A/B eval runner.
|
||||
*
|
||||
* Reads three variant resolver files + two fixture corpora, runs each
|
||||
* (fixture, variant, seed in {1,2,3}) through Anthropic Opus 4.7 via
|
||||
* gbrain's gateway, scores the response, writes one JSONL row per call,
|
||||
* computes per-variant accuracy mean + 95% CI, prints a summary table.
|
||||
*
|
||||
* Receipts bind (model, prompt_template_hash, fixtures_hash, ts, seed)
|
||||
* so re-runs are auditable. Output JSONL begins with a receipt header.
|
||||
*
|
||||
* Pinned to anthropic:claude-opus-4-7. Update MODEL_ID and re-baseline
|
||||
* when Anthropic ships a new Opus generation. Cost: ~$1.70 per full run
|
||||
* (225 calls × ~$0.0076 each at $5/$25 per MTok input/output).
|
||||
*
|
||||
* Lives outside `skills/` deliberately — the skillpack bundler walks
|
||||
* `skills/<skill>/` recursively, so an eval surface in there would ship
|
||||
* to every downstream install. Importing `src/core/ai/gateway.ts` is
|
||||
* legitimate from this location because the eval is gbrain-repo-only.
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execSync } from 'node:child_process';
|
||||
|
||||
import { configureGateway, chat } from '../../src/core/ai/gateway.ts';
|
||||
import { loadConfig } from '../../src/core/config.ts';
|
||||
import { ANTHROPIC_PRICING } from '../../src/core/anthropic-pricing.ts';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = resolve(__dirname, '..', '..');
|
||||
|
||||
// Default model — pinned so the canonical baseline-runs/<date>-opus-4-7.jsonl
|
||||
// stays reproducible. Override with --model for cross-model eval (T3a).
|
||||
export const MODEL_ID = 'anthropic:claude-opus-4-7';
|
||||
|
||||
export const MODEL_ALIASES: Record<string, string> = {
|
||||
opus: 'anthropic:claude-opus-4-7',
|
||||
sonnet: 'anthropic:claude-sonnet-4-6',
|
||||
haiku: 'anthropic:claude-haiku-4-5-20251001',
|
||||
};
|
||||
|
||||
export function resolveModel(spec: string): { full: string; bare: string } {
|
||||
const full = MODEL_ALIASES[spec] ?? spec;
|
||||
const bare = full.startsWith('anthropic:') ? full.slice('anthropic:'.length) : full;
|
||||
return { full, bare };
|
||||
}
|
||||
|
||||
const VARIANT_NAMES = ['baseline', 'functional-areas', 'resolver-of-resolvers'] as const;
|
||||
type VariantName = (typeof VARIANT_NAMES)[number];
|
||||
|
||||
const SEEDS = [1, 2, 3] as const;
|
||||
|
||||
export interface Fixture {
|
||||
intent: string;
|
||||
expected_skill: string;
|
||||
}
|
||||
|
||||
export interface RunRow {
|
||||
kind: 'run';
|
||||
fixture_id: number;
|
||||
corpus: 'training' | 'held_out';
|
||||
variant: VariantName;
|
||||
seed: number;
|
||||
predicted: string;
|
||||
expected: string;
|
||||
/** Strict score: predicted exactly equals expected. */
|
||||
correct: 0 | 1;
|
||||
/** Lenient score: predicted is in the same dispatcher area as expected (T1a). */
|
||||
correct_lenient: 0 | 1;
|
||||
model: string;
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
latency_ms: number;
|
||||
ts: string;
|
||||
}
|
||||
|
||||
export interface ReceiptRow {
|
||||
kind: 'receipt';
|
||||
model: string;
|
||||
prompt_template_hash: string;
|
||||
fixtures_hash: string;
|
||||
fixtures_held_out_hash: string;
|
||||
/** Git sha of the harness at run time (T4). Detect stale numbers when harness changes. */
|
||||
harness_sha: string | null;
|
||||
ts: string;
|
||||
cmd_args: string[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pure functions (testable without API key)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const PROMPT_TEMPLATE = `You are a routing classifier for a skill-based agent. Given the resolver below and the user's intent, return the single most-specific skill slug that should handle the intent.
|
||||
|
||||
Rules:
|
||||
- Return ONLY a slug. No explanation, no quotes, no markdown — just the slug.
|
||||
- Some entries are functional-area dispatchers shaped like:
|
||||
"**Area name**: triggers... → \`dispatcher-skill\` (dispatcher for: subskill-a, subskill-b, subskill-c, ...)"
|
||||
When the user's intent matches an area, RETURN THE MOST-SPECIFIC SUB-SKILL from that area's "dispatcher for" list, not the dispatcher itself. The dispatcher slug is only correct when no listed sub-skill is more specific to the intent.
|
||||
- If a row has no dispatcher list, return its slug directly.
|
||||
|
||||
RESOLVER:
|
||||
<<<RESOLVER_CONTENT>>>
|
||||
|
||||
USER INTENT: <<<INTENT>>>
|
||||
|
||||
SKILL SLUG:`;
|
||||
|
||||
export function parseFixtures(rawJsonl: string): Fixture[] {
|
||||
const out: Fixture[] = [];
|
||||
const lines = rawJsonl.split('\n');
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length === 0) continue;
|
||||
if (trimmed.startsWith('//')) continue;
|
||||
let obj: any;
|
||||
try {
|
||||
obj = JSON.parse(trimmed);
|
||||
} catch (err) {
|
||||
throw new Error(`Bad fixture JSON: ${trimmed.slice(0, 80)} — ${(err as Error).message}`);
|
||||
}
|
||||
if (typeof obj.intent !== 'string' || typeof obj.expected_skill !== 'string') {
|
||||
throw new Error(`Fixture missing required fields: ${trimmed.slice(0, 80)}`);
|
||||
}
|
||||
out.push({ intent: obj.intent, expected_skill: obj.expected_skill });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function loadVariant(path: string): string {
|
||||
return readFileSync(path, 'utf8');
|
||||
}
|
||||
|
||||
export function buildPrompt(variantContent: string, intent: string): string {
|
||||
return PROMPT_TEMPLATE.replace('<<<RESOLVER_CONTENT>>>', variantContent).replace('<<<INTENT>>>', intent);
|
||||
}
|
||||
|
||||
export function parseModelResponse(raw: string): string {
|
||||
// The model may return: bare slug, fenced slug, quoted slug, JSON-wrapped
|
||||
// slug, or slug with a leading explanation. We strip the obvious wrappers
|
||||
// and take the first line that looks like a slug.
|
||||
let s = raw.trim();
|
||||
// Strip ```...``` fences
|
||||
s = s.replace(/^```[a-zA-Z]*\n?/, '').replace(/\n?```\s*$/, '').trim();
|
||||
// If the response is JSON like {"skill": "foo"}, extract.
|
||||
if (s.startsWith('{')) {
|
||||
try {
|
||||
const obj = JSON.parse(s);
|
||||
if (typeof obj.skill === 'string') return obj.skill.trim().toLowerCase();
|
||||
if (typeof obj.skill_slug === 'string') return obj.skill_slug.trim().toLowerCase();
|
||||
if (typeof obj.expected_skill === 'string') return obj.expected_skill.trim().toLowerCase();
|
||||
} catch {}
|
||||
}
|
||||
// Strip surrounding quotes and backticks
|
||||
s = s.replace(/^[`"']|[`"']$/g, '').trim();
|
||||
// Take first non-empty line
|
||||
const firstLine = s.split(/\r?\n/).map(l => l.trim()).find(l => l.length > 0) ?? '';
|
||||
// If it starts with a prose preamble, look for a slug-shaped token
|
||||
const slugMatch = firstLine.match(/[a-z][a-z0-9-]+/i);
|
||||
return (slugMatch ? slugMatch[0] : firstLine).toLowerCase();
|
||||
}
|
||||
|
||||
export function scoreFixture(predicted: string, expected: string): 0 | 1 {
|
||||
return predicted === expected ? 1 : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse every "...→ `dispatcher-slug` (dispatcher for: a, b, c, ...)" line
|
||||
* out of a variant resolver. Returns a map: dispatcher_slug → set of sub-skill
|
||||
* slugs reachable through it. Also includes the dispatcher_slug itself in
|
||||
* the set so it's a self-member.
|
||||
*
|
||||
* Variant shapes:
|
||||
* - functional-areas.md: "→ `brain-ops` (dispatcher for: enrich, query, ...)"
|
||||
* - resolver-of-resolvers.md: "→ `brain-ops`" (no dispatcher clause; returns {})
|
||||
* - baseline.md: per-skill rows (each row's slug becomes its own area)
|
||||
*
|
||||
* Used by lenientScore: a predicted slug counts as "same area as expected"
|
||||
* if both belong to the same dispatcher's reachable set, OR predicted is the
|
||||
* dispatcher and expected is a sub-skill (or vice versa).
|
||||
*/
|
||||
export function parseDispatcherLists(variantContent: string): Map<string, Set<string>> {
|
||||
const out = new Map<string, Set<string>>();
|
||||
// Match both Unicode `→` (used in the real production AGENTS.md the variants
|
||||
// came from) AND ASCII `->` (what SKILL.md's template emits when a user
|
||||
// follows the documented instructions). Codex review P2-2: without ASCII
|
||||
// support, downstream-authored resolvers silently fall through to strict
|
||||
// scoring even though SKILL.md tells the user the template uses `->`.
|
||||
const re = /(?:→|->)\s*`([a-z][a-z0-9-]*)`\s*\(dispatcher for:\s*([^)]+)\)/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(variantContent)) !== null) {
|
||||
const dispatcher = m[1];
|
||||
const subSkills = m[2].split(',').map(s => s.trim()).filter(s => /^[a-z][a-z0-9-]*$/.test(s));
|
||||
const set = new Set<string>([dispatcher, ...subSkills]);
|
||||
out.set(dispatcher, set);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lenient scoring: predicted is correct if (predicted == expected) OR
|
||||
* (both predicted and expected are in the same dispatcher's reachable set
|
||||
* per the variant). This is the T1a re-scoring that surfaces "the LLM
|
||||
* picked a legitimate sub-skill, just not the one my fixture named."
|
||||
*
|
||||
* For variants with no dispatcher clauses (baseline, resolver-of-resolvers),
|
||||
* lenient collapses to strict.
|
||||
*/
|
||||
export function scoreFixtureLenient(
|
||||
predicted: string,
|
||||
expected: string,
|
||||
dispatcherLists: Map<string, Set<string>>,
|
||||
): 0 | 1 {
|
||||
if (predicted === expected) return 1;
|
||||
for (const set of dispatcherLists.values()) {
|
||||
if (set.has(predicted) && set.has(expected)) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Capture the harness git sha so receipts can detect stale numbers. */
|
||||
export function getHarnessSha(): string | null {
|
||||
try {
|
||||
const sha = execSync('git rev-parse HEAD', { cwd: __dirname, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
||||
return sha.length === 40 ? sha : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mean and 95% CI via t-distribution (n=3, df=2, t-critical ≈ 4.303).
|
||||
* For n=3 with df=2 the 95% two-tailed t-critical is 4.303 per standard
|
||||
* tables. Returns the half-width of the CI (mean ± halfWidth).
|
||||
*/
|
||||
export function meanAndCI95(values: number[]): { mean: number; halfWidthCI: number } {
|
||||
if (values.length === 0) return { mean: 0, halfWidthCI: 0 };
|
||||
const mean = values.reduce((a, b) => a + b, 0) / values.length;
|
||||
if (values.length === 1) return { mean, halfWidthCI: 0 };
|
||||
const variance = values.reduce((acc, v) => acc + (v - mean) ** 2, 0) / (values.length - 1);
|
||||
const stdErr = Math.sqrt(variance / values.length);
|
||||
const tCrit = values.length === 3 ? 4.303 : values.length === 2 ? 12.706 : 1.96;
|
||||
return { mean, halfWidthCI: tCrit * stdErr };
|
||||
}
|
||||
|
||||
export function estimateCost(
|
||||
numCalls: number,
|
||||
modelBare: string = 'claude-opus-4-7',
|
||||
inputTokensPerCall = 1000,
|
||||
outputTokensPerCall = 50,
|
||||
): number {
|
||||
const pricing = ANTHROPIC_PRICING[modelBare];
|
||||
if (!pricing) return 0;
|
||||
const input = (numCalls * inputTokensPerCall) / 1_000_000;
|
||||
const output = (numCalls * outputTokensPerCall) / 1_000_000;
|
||||
return input * pricing.input + output * pricing.output;
|
||||
}
|
||||
|
||||
export function hashContent(content: string): string {
|
||||
return createHash('sha256').update(content).digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
export function writeJsonl(rows: (RunRow | ReceiptRow)[], outputPath: string): void {
|
||||
const dir = dirname(outputPath);
|
||||
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
||||
const lines = rows.map(r => JSON.stringify(r)).join('\n') + '\n';
|
||||
writeFileSync(outputPath, lines, 'utf8');
|
||||
}
|
||||
|
||||
export interface ParsedArgs {
|
||||
limit: number | null;
|
||||
parallel: number;
|
||||
output: string | null;
|
||||
help: boolean;
|
||||
yes: boolean;
|
||||
/** Model alias ('opus','sonnet','haiku') or full provider:model id. */
|
||||
model: string;
|
||||
/** Variants directory (default ./variants). */
|
||||
variantsDir: string;
|
||||
/** Custom variant glob (overrides default 3 variants); used by description-length sweep. */
|
||||
variantFiles: string[] | null;
|
||||
}
|
||||
|
||||
export function parseArgs(argv: string[]): ParsedArgs {
|
||||
const out: ParsedArgs = {
|
||||
limit: null, parallel: 1, output: null, help: false, yes: false,
|
||||
model: MODEL_ID, variantsDir: 'variants', variantFiles: null,
|
||||
};
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--help' || a === '-h') out.help = true;
|
||||
else if (a === '--yes' || a === '-y') out.yes = true;
|
||||
else if (a === '--limit') {
|
||||
const v = parseInt(argv[++i], 10);
|
||||
if (!Number.isFinite(v) || v < 1) throw new Error(`--limit must be a positive integer`);
|
||||
out.limit = v;
|
||||
} else if (a === '--parallel') {
|
||||
const v = parseInt(argv[++i], 10);
|
||||
if (!Number.isFinite(v) || v < 1) throw new Error(`--parallel must be a positive integer`);
|
||||
out.parallel = v;
|
||||
} else if (a === '--output') {
|
||||
out.output = argv[++i];
|
||||
} else if (a === '--model') {
|
||||
const v = argv[++i];
|
||||
if (!v) throw new Error(`--model requires a value (alias or provider:model)`);
|
||||
out.model = v;
|
||||
} else if (a === '--variants-dir') {
|
||||
const v = argv[++i];
|
||||
if (!v) throw new Error(`--variants-dir requires a path`);
|
||||
out.variantsDir = v;
|
||||
} else if (a === '--variants') {
|
||||
// Comma-separated list of variant file basenames (without .md). Used by sweep.
|
||||
const v = argv[++i];
|
||||
if (!v) throw new Error(`--variants requires a comma-separated list`);
|
||||
out.variantFiles = v.split(',').map(s => s.trim()).filter(Boolean);
|
||||
} else if (a.startsWith('--')) {
|
||||
throw new Error(`Unknown flag: ${a}`);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Gateway wrapper (mockable via __setChatTransportForTests)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function callModel(prompt: string, modelFull: string): Promise<{ text: string; input_tokens: number; output_tokens: number; latency_ms: number }> {
|
||||
const t0 = Date.now();
|
||||
const result = await chat({
|
||||
model: modelFull,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
maxTokens: 64,
|
||||
});
|
||||
return {
|
||||
text: result.text,
|
||||
input_tokens: result.usage.input_tokens,
|
||||
output_tokens: result.usage.output_tokens,
|
||||
latency_ms: Date.now() - t0,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const HELP = `functional-area-resolver A/B eval harness
|
||||
|
||||
Usage:
|
||||
bun run harness-runner.ts [flags]
|
||||
node harness.mjs [flags] # CLI shim
|
||||
|
||||
Flags:
|
||||
--limit N Run only the first N (fixture × variant × seed) tuples
|
||||
--parallel N Run N tuples in parallel (default 1; gateway rate-lease bound)
|
||||
--output PATH Write JSONL to PATH (default: ./run-<ISO-ts>.jsonl)
|
||||
--model SPEC Model alias (opus|sonnet|haiku) or full provider:model id
|
||||
Default: opus (anthropic:claude-opus-4-7)
|
||||
--variants-dir PATH Override variants directory (default: ./variants)
|
||||
--variants A,B,C Comma-separated variant basenames (default: all 3 in variants-dir)
|
||||
Useful for description-length sweep where you have 4+ variants.
|
||||
--yes Skip the cost-estimate confirmation prompt
|
||||
--help Print this help
|
||||
|
||||
Cost rough estimates (75 calls/variant × num-variants × 3 seeds):
|
||||
Opus: ~$1.70 per 225-call run (1 model × 3 variants × 25 fixtures × 3 seeds)
|
||||
Sonnet: ~$1.02 per 225-call run
|
||||
Haiku: ~$0.34 per 225-call run
|
||||
|
||||
Output JSONL has each row scored TWICE: 'correct' (strict, predicted==expected)
|
||||
and 'correct_lenient' (predicted and expected are in the same dispatcher area).
|
||||
Summary reports both.
|
||||
`;
|
||||
|
||||
async function maybePromptCost(numCalls: number, modelFull: string, autoConfirm: boolean): Promise<boolean> {
|
||||
const { bare } = resolveModel(modelFull);
|
||||
const cost = estimateCost(numCalls, bare);
|
||||
process.stderr.write(`Estimated cost: ~$${cost.toFixed(2)} for ${numCalls} LLM calls via ${modelFull}.\n`);
|
||||
if (autoConfirm) return true;
|
||||
if (!process.stdin.isTTY) {
|
||||
process.stderr.write('Non-TTY context; pass --yes to confirm.\n');
|
||||
return false;
|
||||
}
|
||||
process.stderr.write('Press Enter to continue or Ctrl-C to abort. ');
|
||||
return await new Promise(resolve => {
|
||||
process.stdin.once('data', () => resolve(true));
|
||||
process.stdin.once('end', () => resolve(false));
|
||||
});
|
||||
}
|
||||
|
||||
export async function main(argv: string[]): Promise<number> {
|
||||
let args: ParsedArgs;
|
||||
try {
|
||||
args = parseArgs(argv);
|
||||
} catch (err) {
|
||||
process.stderr.write(`Error: ${(err as Error).message}\n\n${HELP}`);
|
||||
return 2;
|
||||
}
|
||||
|
||||
if (args.help) {
|
||||
process.stdout.write(HELP);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const { full: modelFull, bare: modelBare } = resolveModel(args.model);
|
||||
|
||||
// Self-configure the gateway (matches src/commands/eval-cross-modal.ts:195-220).
|
||||
const config = loadConfig();
|
||||
configureGateway({
|
||||
embedding_model: config?.embedding_model,
|
||||
embedding_dimensions: config?.embedding_dimensions,
|
||||
expansion_model: config?.expansion_model,
|
||||
chat_model: config?.chat_model ?? modelFull,
|
||||
chat_fallback_chain: config?.chat_fallback_chain,
|
||||
base_urls: config?.provider_base_urls,
|
||||
env: { ...process.env } as Record<string, string>,
|
||||
});
|
||||
|
||||
// Provider-aware auth check (codex review P2-3). The CLI advertises full
|
||||
// provider:model support and the test suite covers `openai:gpt-4o`, so the
|
||||
// env-var gate must match the provider that will actually be called.
|
||||
// Unknown providers fall through to the gateway, which will raise a clear
|
||||
// recipe-specific error if any required env var is missing.
|
||||
const REQUIRED_ENV_BY_PROVIDER: Record<string, string> = {
|
||||
anthropic: 'ANTHROPIC_API_KEY',
|
||||
openai: 'OPENAI_API_KEY',
|
||||
google: 'GOOGLE_GENERATIVE_AI_API_KEY',
|
||||
groq: 'GROQ_API_KEY',
|
||||
voyage: 'VOYAGE_API_KEY',
|
||||
together: 'TOGETHER_API_KEY',
|
||||
deepseek: 'DEEPSEEK_API_KEY',
|
||||
minimax: 'MINIMAX_API_KEY',
|
||||
dashscope: 'DASHSCOPE_API_KEY',
|
||||
zhipu: 'ZHIPUAI_API_KEY',
|
||||
};
|
||||
const providerId = modelFull.includes(':') ? modelFull.split(':', 1)[0] : 'anthropic';
|
||||
const requiredEnv = REQUIRED_ENV_BY_PROVIDER[providerId];
|
||||
if (requiredEnv && !process.env[requiredEnv]) {
|
||||
process.stderr.write(`Error: ${requiredEnv} is not set. The harness needs it to reach ${modelFull}.\n`);
|
||||
return 2;
|
||||
}
|
||||
|
||||
// Load fixtures + variants.
|
||||
const evalsDir = __dirname;
|
||||
const fixturesTraining = parseFixtures(readFileSync(join(evalsDir, 'fixtures.jsonl'), 'utf8'));
|
||||
const fixturesHeldOut = parseFixtures(readFileSync(join(evalsDir, 'fixtures-held-out.jsonl'), 'utf8'));
|
||||
|
||||
// Dynamic variants: --variants overrides the default 3, --variants-dir overrides location.
|
||||
const variantsAbsDir = resolve(evalsDir, args.variantsDir);
|
||||
const variantBasenames = args.variantFiles
|
||||
?? (VARIANT_NAMES as readonly string[]).map(n => n);
|
||||
const variants: Record<string, string> = {};
|
||||
const dispatcherListsByVariant: Record<string, Map<string, Set<string>>> = {};
|
||||
for (const name of variantBasenames) {
|
||||
const content = loadVariant(join(variantsAbsDir, `${name}.md`));
|
||||
variants[name] = content;
|
||||
dispatcherListsByVariant[name] = parseDispatcherLists(content);
|
||||
}
|
||||
|
||||
// Build the (fixture × variant × seed) tuple list.
|
||||
type Tuple = { fixture: Fixture; corpus: 'training' | 'held_out'; fixture_id: number; variant: string; seed: number };
|
||||
const tuples: Tuple[] = [];
|
||||
for (const variant of variantBasenames) {
|
||||
fixturesTraining.forEach((f, i) => {
|
||||
for (const seed of SEEDS) tuples.push({ fixture: f, corpus: 'training', fixture_id: i, variant, seed });
|
||||
});
|
||||
fixturesHeldOut.forEach((f, i) => {
|
||||
for (const seed of SEEDS) tuples.push({ fixture: f, corpus: 'held_out', fixture_id: i, variant, seed });
|
||||
});
|
||||
}
|
||||
const totalCalls = args.limit ? Math.min(args.limit, tuples.length) : tuples.length;
|
||||
const workQueue = tuples.slice(0, totalCalls);
|
||||
|
||||
// Cost-estimate prompt (skipped for tiny --limit runs to keep dev iteration fast).
|
||||
if (totalCalls >= 20) {
|
||||
const proceed = await maybePromptCost(totalCalls, modelFull, args.yes);
|
||||
if (!proceed) {
|
||||
process.stderr.write('Aborted.\n');
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Compute receipt header.
|
||||
const fixturesHash = hashContent(readFileSync(join(evalsDir, 'fixtures.jsonl'), 'utf8'));
|
||||
const fixturesHeldOutHash = hashContent(readFileSync(join(evalsDir, 'fixtures-held-out.jsonl'), 'utf8'));
|
||||
const promptTemplateHash = hashContent(PROMPT_TEMPLATE);
|
||||
const harnessSha = getHarnessSha();
|
||||
const tsStart = new Date().toISOString();
|
||||
const receipt: ReceiptRow = {
|
||||
kind: 'receipt',
|
||||
model: modelFull,
|
||||
prompt_template_hash: promptTemplateHash,
|
||||
fixtures_hash: fixturesHash,
|
||||
fixtures_held_out_hash: fixturesHeldOutHash,
|
||||
harness_sha: harnessSha,
|
||||
ts: tsStart,
|
||||
cmd_args: argv,
|
||||
};
|
||||
|
||||
// Output path.
|
||||
const outputPath = args.output ?? join(evalsDir, `run-${tsStart.replace(/[:.]/g, '-')}.jsonl`);
|
||||
process.stderr.write(`Writing receipt + ${totalCalls} runs to ${outputPath}\n`);
|
||||
|
||||
const rows: (RunRow | ReceiptRow)[] = [receipt];
|
||||
|
||||
// Sequential or simple bounded-parallel execution.
|
||||
let completed = 0;
|
||||
async function processTuple(t: Tuple): Promise<RunRow> {
|
||||
const prompt = buildPrompt(variants[t.variant], t.fixture.intent);
|
||||
const { text, input_tokens, output_tokens, latency_ms } = await callModel(prompt, modelFull);
|
||||
const predicted = parseModelResponse(text);
|
||||
const correct = scoreFixture(predicted, t.fixture.expected_skill);
|
||||
const correct_lenient = scoreFixtureLenient(
|
||||
predicted,
|
||||
t.fixture.expected_skill,
|
||||
dispatcherListsByVariant[t.variant] ?? new Map(),
|
||||
);
|
||||
const row: RunRow = {
|
||||
kind: 'run',
|
||||
fixture_id: t.fixture_id,
|
||||
corpus: t.corpus,
|
||||
variant: t.variant as VariantName,
|
||||
seed: t.seed,
|
||||
predicted,
|
||||
expected: t.fixture.expected_skill,
|
||||
correct,
|
||||
correct_lenient,
|
||||
model: modelFull,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
latency_ms,
|
||||
ts: new Date().toISOString(),
|
||||
};
|
||||
completed++;
|
||||
if (completed % 10 === 0 || completed === totalCalls) {
|
||||
process.stderr.write(` ${completed}/${totalCalls} done\n`);
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
// Bounded parallel: chunk into args.parallel-sized batches.
|
||||
for (let i = 0; i < workQueue.length; i += args.parallel) {
|
||||
const batch = workQueue.slice(i, i + args.parallel);
|
||||
const results = await Promise.all(batch.map(processTuple));
|
||||
rows.push(...results);
|
||||
}
|
||||
|
||||
// Write JSONL.
|
||||
writeJsonl(rows, outputPath);
|
||||
|
||||
// Compute per-variant accuracy. Both strict + lenient. Held-out is the
|
||||
// headline; training is reported separately.
|
||||
const runRows = rows.filter((r): r is RunRow => r.kind === 'run');
|
||||
type CorpusKey = 'training' | 'held_out';
|
||||
type Acc = { training: number[]; held_out: number[] };
|
||||
const strictSummary: Record<string, Acc> = {};
|
||||
const lenientSummary: Record<string, Acc> = {};
|
||||
for (const variant of variantBasenames) {
|
||||
strictSummary[variant] = { training: [], held_out: [] };
|
||||
lenientSummary[variant] = { training: [], held_out: [] };
|
||||
for (const corpus of ['training', 'held_out'] as const) {
|
||||
for (const seed of SEEDS) {
|
||||
const subset = runRows.filter(r => r.variant === variant && r.corpus === corpus && r.seed === seed);
|
||||
if (subset.length === 0) continue;
|
||||
strictSummary[variant][corpus].push(subset.reduce((a, r) => a + r.correct, 0) / subset.length);
|
||||
lenientSummary[variant][corpus].push(subset.reduce((a, r) => a + r.correct_lenient, 0) / subset.length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Print summary.
|
||||
const fmt = (vals: number[]) => {
|
||||
if (vals.length === 0) return '—';
|
||||
const { mean, halfWidthCI } = meanAndCI95(vals);
|
||||
return `${(mean * 100).toFixed(1)}% ± ${(halfWidthCI * 100).toFixed(1)}%`;
|
||||
};
|
||||
|
||||
process.stderr.write(`\n=== A/B Eval Summary (model: ${modelFull}) ===\n`);
|
||||
process.stderr.write(' | STRICT scoring | LENIENT (same-area)\n');
|
||||
process.stderr.write('Variant | Held-out | Training | Held-out | Training\n');
|
||||
process.stderr.write('------------------------------|------------------------|------------------------|----------------------|----------------------\n');
|
||||
for (const variant of variantBasenames) {
|
||||
process.stderr.write(
|
||||
`${variant.padEnd(30)}| ${fmt(strictSummary[variant].held_out).padEnd(22)} | ${fmt(strictSummary[variant].training).padEnd(22)} | ${fmt(lenientSummary[variant].held_out).padEnd(20)} | ${fmt(lenientSummary[variant].training)}\n`,
|
||||
);
|
||||
}
|
||||
process.stderr.write('\nLENIENT counts a prediction as correct if it shares a dispatcher area with the expected target.\n');
|
||||
process.stderr.write('For variants without "(dispatcher for: ...)" clauses (baseline, resolver-of-resolvers), LENIENT == STRICT.\n');
|
||||
process.stderr.write('\nReceipt + runs written to: ' + outputPath + '\n');
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Bun entrypoint: run main when invoked as a script.
|
||||
if (import.meta.main) {
|
||||
main(process.argv.slice(2)).then(code => process.exit(code));
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Thin CLI shim for the functional-area-resolver A/B eval harness.
|
||||
*
|
||||
* Spawns the TypeScript runner via `bun` because the runner imports
|
||||
* gbrain's gateway from `src/core/ai/gateway.ts` directly. The runner
|
||||
* does the actual work; this file exists so users can invoke `node
|
||||
* harness.mjs` without remembering the bun incantation.
|
||||
*
|
||||
* If `bun` isn't on PATH (or this script is invoked outside the gbrain
|
||||
* repo), exit 2 with a clear message — the harness is a gbrain-side
|
||||
* proof-of-pattern, not a portable tool.
|
||||
*/
|
||||
|
||||
import { spawnSync, execFileSync } from 'node:child_process';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { existsSync } from 'node:fs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const runnerPath = resolve(__dirname, 'harness-runner.ts');
|
||||
const gatewayPath = resolve(__dirname, '..', '..', 'src', 'core', 'ai', 'gateway.ts');
|
||||
|
||||
function fail(message, code = 2) {
|
||||
process.stderr.write(message + '\n');
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
// Missing-binary fallback (F-E2): we need `bun` AND we need to be in
|
||||
// the gbrain repo so the runner can import the gateway.
|
||||
try {
|
||||
execFileSync('which', ['bun'], { stdio: 'ignore' });
|
||||
} catch {
|
||||
fail(
|
||||
'harness.mjs: `bun` is not on PATH.\n' +
|
||||
'This harness is a gbrain-maintainer-side tool — run it from a\n' +
|
||||
'gbrain repo checkout with `bun` installed (https://bun.sh).',
|
||||
);
|
||||
}
|
||||
|
||||
if (!existsSync(gatewayPath)) {
|
||||
fail(
|
||||
`harness.mjs: cannot find gbrain gateway at ${gatewayPath}.\n` +
|
||||
'This harness is the gbrain-side A/B eval surface. Run it from a\n' +
|
||||
'gbrain repo checkout, not from an installed skillpack.',
|
||||
);
|
||||
}
|
||||
|
||||
if (!existsSync(runnerPath)) {
|
||||
fail(`harness.mjs: runner missing at ${runnerPath}`);
|
||||
}
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const result = spawnSync('bun', ['run', runnerPath, ...args], {
|
||||
stdio: 'inherit',
|
||||
cwd: __dirname,
|
||||
});
|
||||
|
||||
process.exit(result.status ?? 1);
|
||||
@@ -1,121 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Re-score an existing run-*.jsonl (or baseline-runs/*.jsonl) with the lenient
|
||||
* dispatcher-area scoring rule, without re-running any LLM calls.
|
||||
*
|
||||
* Usage: node rescore.mjs <run-file.jsonl>
|
||||
*
|
||||
* Reads the receipt header to identify which variants were used, loads them
|
||||
* from ./variants/<name>.md, parses their (dispatcher for: ...) clauses, then
|
||||
* applies scoreFixtureLenient to every row. Prints a STRICT vs LENIENT
|
||||
* accuracy table without mutating the file.
|
||||
*
|
||||
* This is T1a from the v0.32.3.0 boil-the-ocean push.
|
||||
*/
|
||||
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
function parseDispatcherLists(variantContent) {
|
||||
const out = new Map();
|
||||
const re = /→\s*`([a-z][a-z0-9-]*)`\s*\(dispatcher for:\s*([^)]+)\)/g;
|
||||
let m;
|
||||
while ((m = re.exec(variantContent)) !== null) {
|
||||
const dispatcher = m[1];
|
||||
const subSkills = m[2].split(',').map(s => s.trim()).filter(s => /^[a-z][a-z0-9-]*$/.test(s));
|
||||
out.set(dispatcher, new Set([dispatcher, ...subSkills]));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function lenientScore(predicted, expected, dispatcherLists) {
|
||||
if (predicted === expected) return 1;
|
||||
for (const set of dispatcherLists.values()) {
|
||||
if (set.has(predicted) && set.has(expected)) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function meanAndCI(values) {
|
||||
if (values.length === 0) return { mean: 0, ci: 0 };
|
||||
const mean = values.reduce((a, b) => a + b, 0) / values.length;
|
||||
if (values.length === 1) return { mean, ci: 0 };
|
||||
const variance = values.reduce((acc, v) => acc + (v - mean) ** 2, 0) / (values.length - 1);
|
||||
const stdErr = Math.sqrt(variance / values.length);
|
||||
const tCrit = values.length === 3 ? 4.303 : values.length === 2 ? 12.706 : 1.96;
|
||||
return { mean, ci: tCrit * stdErr };
|
||||
}
|
||||
|
||||
function fmt(vals) {
|
||||
if (vals.length === 0) return '—';
|
||||
const { mean, ci } = meanAndCI(vals);
|
||||
return `${(mean * 100).toFixed(1)}% ± ${(ci * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
const runFile = process.argv[2];
|
||||
if (!runFile) {
|
||||
console.error('Usage: node rescore.mjs <run-file.jsonl>');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const absRun = resolve(process.cwd(), runFile);
|
||||
if (!existsSync(absRun)) {
|
||||
console.error(`File not found: ${absRun}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const lines = readFileSync(absRun, 'utf8').split('\n').filter(l => l.trim().length > 0);
|
||||
const rows = lines.map(l => JSON.parse(l));
|
||||
|
||||
const receipt = rows.find(r => r.kind === 'receipt');
|
||||
const runRows = rows.filter(r => r.kind === 'run');
|
||||
|
||||
console.error(`Re-scoring ${runRows.length} rows from ${absRun}`);
|
||||
console.error(`Receipt: model=${receipt?.model ?? '?'} fixtures_hash=${receipt?.fixtures_hash ?? '?'} ts=${receipt?.ts ?? '?'}`);
|
||||
|
||||
// Identify variants and load them
|
||||
const variantsUsed = [...new Set(runRows.map(r => r.variant))];
|
||||
const variantsDir = join(__dirname, 'variants');
|
||||
const dispatcherLists = {};
|
||||
for (const v of variantsUsed) {
|
||||
const path = join(variantsDir, `${v}.md`);
|
||||
if (!existsSync(path)) {
|
||||
console.error(`Warning: variant file missing for "${v}" at ${path} — lenient score will collapse to strict for this variant.`);
|
||||
dispatcherLists[v] = new Map();
|
||||
continue;
|
||||
}
|
||||
dispatcherLists[v] = parseDispatcherLists(readFileSync(path, 'utf8'));
|
||||
}
|
||||
|
||||
const SEEDS = [1, 2, 3];
|
||||
|
||||
const strictSummary = {};
|
||||
const lenientSummary = {};
|
||||
for (const v of variantsUsed) {
|
||||
strictSummary[v] = { training: [], held_out: [] };
|
||||
lenientSummary[v] = { training: [], held_out: [] };
|
||||
for (const corpus of ['training', 'held_out']) {
|
||||
for (const seed of SEEDS) {
|
||||
const subset = runRows.filter(r => r.variant === v && r.corpus === corpus && r.seed === seed);
|
||||
if (subset.length === 0) continue;
|
||||
strictSummary[v][corpus].push(subset.reduce((a, r) => a + r.correct, 0) / subset.length);
|
||||
const lenientHits = subset.reduce((a, r) => a + lenientScore(r.predicted, r.expected, dispatcherLists[v]), 0);
|
||||
lenientSummary[v][corpus].push(lenientHits / subset.length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n=== Re-scored from ${runFile} ===\n`);
|
||||
console.log(' | STRICT scoring | LENIENT (same-area)');
|
||||
console.log('Variant | Held-out | Training | Held-out | Training');
|
||||
console.log('------------------------------|------------------------|------------------------|----------------------|----------------------');
|
||||
for (const v of variantsUsed) {
|
||||
console.log(
|
||||
`${v.padEnd(30)}| ${fmt(strictSummary[v].held_out).padEnd(22)} | ${fmt(strictSummary[v].training).padEnd(22)} | ${fmt(lenientSummary[v].held_out).padEnd(20)} | ${fmt(lenientSummary[v].training)}`,
|
||||
);
|
||||
}
|
||||
console.log('\nLENIENT counts a prediction correct if it shares a dispatcher area with expected.');
|
||||
console.log('For variants without "(dispatcher for: ...)" clauses, LENIENT == STRICT.');
|
||||
@@ -1,380 +0,0 @@
|
||||
<!-- A/B EVAL FIXTURE — synthetic resolver shape, do not invoke from agent context. -->
|
||||
<!-- Variant: BASELINE — 270-row bullet-list shape. Extracted from a production AGENTS.md at the pre-compression state; owner PII scrubbed. ~25KB. -->
|
||||
|
||||
# AGENTS.md
|
||||
|
||||
This folder is home. Treat it that way.
|
||||
|
||||
## Hard Gates (NEVER VIOLATE)
|
||||
|
||||
⛔ **RUNTIME CONTEXT > PROJECT DOCS.** When the OpenClaw runtime context block (Group Chat Context, Inbound Context, capabilities) contradicts a project doc rule, the runtime wins. The runtime knows the actual channel state for THIS turn; project docs are stale by definition. The 2026-05-06 silent-drop recurrence happened because I trusted a wrong HEARTBEAT rule over the correct runtime warning. Don't do that again.
|
||||
|
||||
⛔ **NEVER RESTART GATEWAY.** Tell the owner. He does it himself. No exceptions.
|
||||
|
||||
⛔ **BRAIN-FIRST STORAGE.** ALL valuable outputs → `/your/brain/path/` or Supabase IMMEDIATELY. Use `/your/tmp` for scratch (not `/tmp`). `/tmp` hard limit: 2GB. See `skills/conventions/brain-first.md`.
|
||||
|
||||
⛔ **DATA LOSS GATE.** Before ANY bulk delete: read `skills/data-loss-gate/SKILL.md`, present confirmation card, wait for "yes."
|
||||
|
||||
⛔ **NO WIKILINKS.** Standard markdown links only: `[Name](path)`. Never `[[wikilinks]]`.
|
||||
|
||||
⛔ **GBRAIN MASTER READ-ONLY.** Never push to master on <owner>/gbrain. Never merge PRs. Branch → push → PR only. See `skills/github-agents/SKILL.md`.
|
||||
|
||||
⛔ **PUBLIC REPO GUARD.** Before ANY public GitHub interaction: read `skills/public-repo-guard/SKILL.md`. Run PII scanner on ALL content.
|
||||
|
||||
⚡ **MINIONS OVER SUB-AGENTS.** Use gbrain Minions (shell jobs) for batch/deterministic work. Sub-agents only when LLM reasoning is required mid-task. Always set `--timeout-ms 900000` for long jobs.
|
||||
|
||||
## Gate -1 — Acknowledge Immediately
|
||||
|
||||
For any request taking >5 sec: send a one-line ack with rough time estimate FIRST, then start tools. Never go silent into a tool chain. Calibration: lookup ~10s, multi-tool ~30-60s, transcription ~2-3min, sub-agent ~1-3min, heavy batch ~3-5min, browser ~2-5min. Overestimate slightly.
|
||||
|
||||
For tasks >1 min: spawn a progress-update subagent (one-liner every 30-60s with concrete progress %). Critical in group topics with no typing indicator.
|
||||
|
||||
## Gate 0 — Access Control
|
||||
|
||||
On EVERY inbound message, check `sender_id` FIRST.
|
||||
- **the owner (<OWNER_ID_A> or <OWNER_ID_B>):** Proceed. Full access.
|
||||
- **Known non-the owner:** Read `skills/multi-user/SKILL.md` immediately. It governs everything.
|
||||
- **Unknown sender:** "This is a private agent." → notify the owner → stop.
|
||||
|
||||
## Gate 0.5 — Critical Life Events
|
||||
|
||||
If the owner mentions a **death, funeral, birth, hospitalization, emergency, diagnosis, accident, divorce, or arrest** — IMMEDIATELY write to BOTH `MEMORY.md` AND `memory/YYYY-MM-DD.md`. Priority 0. No deferral.
|
||||
|
||||
## Gate 1 — Signal Detection (the owner only)
|
||||
|
||||
Every the owner message: scan for entity mentions (people, companies, deals, YC batches). For each: search brain, load context, update if stale. Read `skills/entity-detector/ENTITY-DETECTION.md` for the full protocol.
|
||||
|
||||
**Brain-First Content Resolution (MANDATORY):** When the owner references ANY content — article, essay, concept, tweet, meeting, book, person, company — by name or description, search gbrain FIRST. Never ask "which article?" or "can you share the link?" The brain has 100K pages. Search it. Only ask the owner if gbrain + memory + web all fail.
|
||||
|
||||
## Gate 2 — Session Startup
|
||||
|
||||
Before first substantive reply:
|
||||
1. Read `ops/tasks.md` for task state
|
||||
2. Read `memory/heartbeat-state.json` for location, blockers, last checks
|
||||
3. Read relevant `memory/YYYY-MM-DD.md` for recent context
|
||||
4. Check calendar if time-sensitive
|
||||
|
||||
**Brain link rule:** Every brain path in output MUST be a clickable GitHub URL: `[name](https://github.com/<owner>/brain/blob/main/path.md)`. Never bare paths. Never invented URLs. `<owner>.github.io/brain/` does NOT exist.
|
||||
|
||||
**After every brain write:** `bash scripts/brain-commit-link.sh "<message>"`. Always absolute paths for brain writes (`/your/brain/path/...`).
|
||||
|
||||
**Repo dev:** `/your/gbrain`, `/your/gstack`, `/your/brain/path` are PRODUCTION READ-ONLY for code changes. All dev work → `/your/git-projects/<repo>-<feature>/`. See `skills/repo-dev/SKILL.md`.
|
||||
|
||||
## Gate 3 — Outbound Link Gate
|
||||
|
||||
Before EVERY reply containing a brain reference:
|
||||
1. Path must be absolute GitHub URL
|
||||
2. Commit must be pushed (not just local)
|
||||
3. Use `brain-commit-link.sh` output for the URL
|
||||
4. Never invent URLs. Never use `<owner>.github.io`.
|
||||
|
||||
## Skill Resolver
|
||||
|
||||
Read the skill file before acting. If two could match, read both. Non-the owner senders: only WORK/FAMILY-accessible skills.
|
||||
|
||||
### Always-on (every message)
|
||||
- Gate -1: any request taking >5 sec → `acknowledge`
|
||||
- Gate 0: sender_id != the owner → `multi-user`
|
||||
- Gate 1: the owner messages only → `entity-detector`
|
||||
- Non-the owner user shares info about themselves/work/vendors → `group-chat-intel`
|
||||
- Any brain read/write/lookup/citation → `brain-ops`
|
||||
- Any brain page write OR chat reply mentioning a repo/project → `brain-link-refs`
|
||||
- Any outbound reply to the owner that references a brain page or workspace file → `brain-link-report`
|
||||
- Any outbound report/alert with external links (oppo alerts → `report-quality-gate`
|
||||
- Any outbound reply in a multi-user group (floor scope < FULL) that references... → `brain-pdf-auto`
|
||||
- Any time-sensitive claim: "in N minutes" → `context-now`
|
||||
- the owner corrects a behavior, output, or decision → `correction-pipeline`
|
||||
- Presenting choices with inline buttons, user decision gate, button callback → `ask-user`
|
||||
|
||||
### Political donations
|
||||
- Donation tracking → `political-donations`
|
||||
|
||||
### Brain operations
|
||||
- Creating a new file - where does it go? → `repo-architecture`
|
||||
- Brain directory structure, "where is X in the brain", schema, filing rules → `/your/brain/path/README.md (directory tree + key locations table) + /your/brain/path/schema.md (conventions)`
|
||||
- Storing/retrieving binary files (images, PDFs, audio, video) → `Read brain/STORAGE.md - .redirect.yaml pointers + Supabase Storage`
|
||||
- Creating/enriching a person or company page → `enrich`
|
||||
- Resolving X handle stubs to real people ("who is @handle" → `x-handle-enrich`
|
||||
- Scoring/rating a person, rationalizing scores, "what score is X" → `person-score`
|
||||
- Unknown sender emails the owner → `cold-email-lookup`
|
||||
- Pitch deck, data room, financial model shared → `diligence`
|
||||
- Fix broken citations in brain pages → `citation-fixer`
|
||||
- Publish/share a brain page as link → `brain-publish`
|
||||
- Generate PDF from brain page, "brain pdf", "send me the pdf", … → `brain-pdf`
|
||||
- Generate PDF from any non-brain content: reports → `pdf-generation`
|
||||
- Read a book/article through lens of a specific problem, "read this through the lens", "extract a playbook", "what can I learn" → `strategic-reading`
|
||||
- Personalized book analysis, "book mirror", "apply this book", … → `book-mirror`
|
||||
- Deep-retrieval book mirror, "extreme mirror", "go deep", … → `book-mirror/SKILL.md (deep retrieval is now the default)`
|
||||
- Freshness check, data source SLA monitoring, smoke test → `freshness-monitor`
|
||||
- Write as the owner: blog posts → `garry-voice`
|
||||
- Essay review, writing feedback, draft review → `essay-review`
|
||||
- Brain search/query, hybrid search, entity lookup; Brain maintenance, lint, backlinks, health checks → `gbrain`
|
||||
- "My ChatGPT conversations" → `conversation-history`
|
||||
- Brain integrity → `brain-librarian`
|
||||
- "archive crawler", "mine my old files", … → `archive-crawler`
|
||||
- "concept synthesis", "intellectual map", … → `concept-synthesis`
|
||||
- "Ingest all X" → `bulk-skillify`
|
||||
- "extract takes", "seed takes", … → `takes-extraction`
|
||||
- Any ycli command, ycli SSO expired → `ycli-auth`
|
||||
- "extreme mirror", "go deep on this book", deep-retrieval book mirror → `book-mirror-extreme`
|
||||
- Book mirror synthesis, synthesize book analysis → `book-mirror-synthesis`
|
||||
- Export brain, download brain pages, brain backup → `brain-export`
|
||||
- Brain planning, plan brain changes, schema planning → `brain-plan`
|
||||
- Conversation enrichment, enrich chat transcript → `conversation-enrichment`
|
||||
- Fact check, verify claim, "is this true", citation check → `fact-check`
|
||||
- Upgrade gbrain, update gbrain, gbrain version → `gbrain-upgrade`
|
||||
- "Review my Dropbox archive", Dropbox folder audit, old Dropbox files → `dropbox-archive-review`
|
||||
- Screenshot style, apply style to screenshot → `screenshot-style`
|
||||
- Signorelli letter, draft formal letter → `signorelli-letter`
|
||||
- Data loss prevention, confirm bulk delete → `data-loss-gate`
|
||||
- Public repo PII guard, check for secrets → `public-repo-guard`
|
||||
|
||||
### Places & Travel
|
||||
- Trip itinerary PDF/doc → `trip-logistics`
|
||||
- "I'm at [place]"; "Where should I eat in X"; Foursquare/Swarm data export, bulk location import → `checkin`
|
||||
- "What's playing", "showtimes", … → `showtimes`
|
||||
|
||||
### Calendar (direct queries)
|
||||
- "What's my schedule", "am I free", calendar briefing, day lookahead → `google-calendar`
|
||||
- "Create a calendar item", "add to my calendar", … → `calendar-event-create`
|
||||
- "Prep for my meeting with X" → `meeting-prep`
|
||||
- Interview prep → `interview-prep`
|
||||
- Calendar conflict detection, double bookings, travel impossibility, missing prep; After calendar sync completes, or when day's schedule changes → `calendar-check`
|
||||
- Travel booking → `calendar-travel-setup`
|
||||
- Sync calendars to brain → `calendar-sync`
|
||||
- Historical/past calendar lookup: "when did I" → `calendar-recall`
|
||||
|
||||
### Time, location, and context
|
||||
- "What time is it" → `context-now`
|
||||
- "What's my jet lag plan" → `jet-lag`
|
||||
|
||||
### Executive assistant
|
||||
- Inbox triage, email reply, scheduling, calendar → `executive-assistant`
|
||||
- Gmail search, send email, draft reply via ClawVisor → `gmail`
|
||||
- Google Contacts lookup, search contacts, contact info → `google-contacts`
|
||||
- Personal logistics, schedule timeline, countdown deltas, time-aware foundation → `personal-logistics`
|
||||
- Intro health check, dropped handoffs, re-ping opportunities, intro tracker → `intro-reping`
|
||||
- Startup intro request, "draft an intro", evaluate intro, score intro quality → `startup-intro`
|
||||
- Alumni dinner planning, guest list curation, dinner invite list → `alumni-dinner`
|
||||
- "Partner lunch brief" → `partner-lunch-brief`
|
||||
- Flight delay tracking → `flight-tracker`
|
||||
- "Where is the owner", location inference, fix location, travel state machine → `location-inference`
|
||||
- Task add/remove/complete/defer/review → `daily-task-manager`
|
||||
- Morning task list prep (cron) → `daily-task-prep`
|
||||
- Business development, outreach tracking → `business-development`
|
||||
- Phone call handling (510-MY-GARRY) → `voice-agent`
|
||||
- Venus call ended, "Process this Venus call", voice session analysis → `voice-session-ingest`
|
||||
- Post-call analysis, "analyze the last call", "what happened on that call" → `venus-post-call`
|
||||
- "give me a link" → `voice-link`
|
||||
- OpenPhone/SMS (415-777-0000) → `quo`
|
||||
- "What's my jet lag plan" → `jet-lag`
|
||||
- New trip detected, trip itinerary shared, post-trip reflection, "trip is done" → `trip-ingest`
|
||||
|
||||
### Face detection & recognition
|
||||
- Face detect → `face-detect`
|
||||
- "identify faces" → `identify-faces`
|
||||
|
||||
### Content & media ingestion
|
||||
- Frame.io → `frameio-monitor`
|
||||
- "Ingest this", "save this to brain", generic content routing → `ingest`
|
||||
- the owner shares a link, article, tweet, idea → `idea-ingest`
|
||||
- Any video/audio (YouTube, X, Instagram, TikTok, podcast), "ingest this pdf book", "summarize this book", "process this book"; Screenshots, GitHub repos, other media → `media-ingest`
|
||||
- "Transcribe this" → `transcribe`
|
||||
- Book PDF, investor update PDF, any PDF to ingest → `pdf-ingest`
|
||||
- "Get me this book" → `book-acquisition`
|
||||
- Anna's Archive download, annas-archive, fast download with membership → `annas-archive`
|
||||
- Kindle library → `kindle-library`
|
||||
- Circleback CLI: search meetings → `circleback-cli`
|
||||
- Meeting transcript from Circleback → `meeting-ingestion`
|
||||
- Post-ingestion meeting summary to Meetings topic (auto-triggered by Circlebac... → `meeting-digest`
|
||||
- MANDATORY post-meeting audit, "audit this meeting" → `meeting-gold-standard`
|
||||
- Post-meeting signal extraction, "what did I say that was interesting", concept extraction → `meeting-signal-pass`
|
||||
- "scrape", "scrape <url>", … → `scrape`
|
||||
- Fundraising PDF → `fundraising-pdf`
|
||||
- Therapy session audio: "here's my jan/donna/marcie session" → `therapy-ingest`
|
||||
- Enriching any brain page from external content (quality pass) → `media-enrichment`
|
||||
- Batch article enrichment, "enrich", "raw content", "article dumps" → `article-enrichment`
|
||||
- Post-ingestion signal extraction, concept extraction from articles, backlink enrichment, entity propagation → `post-ingestion-enrichment`
|
||||
- Security audit (secrets, RLS, token files, gitleaks) → `security-audit`
|
||||
- Backlink check after any brain page write → `node scripts/backlink-check.mjs <page-path> — deterministic, run after EVERY brain page create/update`
|
||||
- X daily quality → `x-daily-quality`
|
||||
- ycli → `yc-ingest`
|
||||
- YC OH meeting notes, ycli office hours ingestion, "pull my YC meetings" → `yc-oh-ingest`
|
||||
- "Ingest this application" → `yc-app-ingest`
|
||||
- Company investor update, VC fund LP update, portfolio metrics email → `investor-update-ingest`
|
||||
- Voice note, audio message to transcribe and ingest, "voice memo", "audio note", "audio message" → `voice-note-ingest`
|
||||
- Save session transcripts to brain → `transcript-save`
|
||||
- "Unsubscribe from this", remove me from this list → `email-unsubscribe`
|
||||
- Deep web research, "research this person/topic thoroughly", "web research", … → `perplexity-research`
|
||||
- Exa semantic web search, find people/companies/LinkedIn profiles → `exa`
|
||||
- Happenstance professional network search, research people → `happenstance`
|
||||
- Crustdata B2B intelligence, LinkedIn enrichment, career history → `crustdata`
|
||||
- Captain API, Pitchbook data, funding rounds, investor lookup → `captain-api`
|
||||
- Structured data research, "track" → `data-research`
|
||||
- Substack ingest, import from Substack → `substack-ingest`
|
||||
- Pocket ingest, import from Pocket → `pocket-ingest`
|
||||
- Tweet deep ingest, deep tweet enrichment, article extraction from tweets → `tweet-deep-ingest`
|
||||
|
||||
### X/Twitter API - ENTERPRISE TIER
|
||||
**ALL X API work:** Read `skills/_x-api-rules.md` FIRST. We pay $50K/mo. Rate limit: 40K req/15min. Import `lib/x-api.mjs`. NEVER throttle to free-tier limits.
|
||||
|
||||
### Message intelligence
|
||||
- "Scan my DMs", "triage my messages", X DM triage, unified message extraction → `message-intel`
|
||||
- "Project Karma", blocked/muted users, adversary tweets, hostile accounts → `adversary-tracking`
|
||||
|
||||
### Monitoring & social
|
||||
- X/Twitter ingestion (daily, backfill, rollup, enrichment) → `x-ingest`
|
||||
- "x stream" → `svc/x-stream`
|
||||
- "Concept tier" → `x-concept-tier`
|
||||
- "look up tweet"; "social json store" → `social-json-store`
|
||||
- "storage tier"; "download video when needed" → `brain-storage`
|
||||
- "link to supabase file" → `brain-storage-links`
|
||||
- "backblaze" → `backblaze`
|
||||
- Social media mention alerts (cron) → `social-radar`
|
||||
- YC launch cringe-o-meter, YC media monitoring, YC sentiment, "scan YC launches" → `yc-media-monitor`
|
||||
- Slack channel scanning (cron) → `slack-scan`
|
||||
- Content idea generation (cron) → `content-ideas`
|
||||
- Check Steph's Instagram → `steph-instagram`
|
||||
|
||||
### Adversarial / research
|
||||
- Track/monitor a public figure or critic → `adversary-tracking`
|
||||
- Detect astroturfing, "is this organic", bot check, paid amplification → `detect-astroturf`
|
||||
- Real-name hostile identification, "who hates me", hostile account ID → `real-name-hostiles`
|
||||
- Deanonymize anon X account → `investigate-x-anon`
|
||||
- Fiscal forensics, government spending, nonprofit audit, 990 filings, grant fraud → `fiscal-forensics`
|
||||
- Academic claim verification, "verify this study", "is this replicated", … → `academic-verify`
|
||||
- Private investigation, deep background check, "find out everything about" → `private-investigator`
|
||||
- Opposition research backgrounder → `oppo-research`
|
||||
- OSINT collection on tracked individuals → `osint-collector`
|
||||
- Network mapping, relationship intelligence, who-knows-who → `network-intel`
|
||||
- YC competitor oppo → `yc-competitor-oppo`
|
||||
- Who's boosting competitors → `yc-booster-tracker`
|
||||
|
||||
### Product / building
|
||||
- "Review this plan" / "CEO review" / "think bigger" → `gstack-openclaw-ceo-review`
|
||||
- "Debug this" / "investigate" / "root cause" → `gstack-openclaw-investigate`
|
||||
- "Office hours" / "brainstorm" / "is this worth building" / startup advice / f... → `gstack-openclaw-office-hours`
|
||||
- Weekly engineering retrospective → `gstack-openclaw-retro`
|
||||
- "Create a skill" / "improve this skill" → `skill-creator`
|
||||
- "Skillify this", convert workflow to skill → `skillify`
|
||||
- "Validate skills", "test skills", "skill health check" → `testing`
|
||||
- "Make this durable", "survive restarts" → `durable-service`
|
||||
- "Audit the code", "refactor" → `refactor`
|
||||
- "Check freshness", "smoke test" → `healthcheck`
|
||||
- Narrative structure → `narrative`
|
||||
- Budget ROI analysis, event spending vs outcomes, cost-per-founder → `budget-roi`
|
||||
- Adaptive backoff, batch load management, rate limiting → `backoff`
|
||||
- Any batch/bulk operation (>50 items), "backfill", "run on all", "import all" → `progressive-batch`
|
||||
- GStack PR/issue management (cron) → `gstack-pulse`
|
||||
- GBrain PR/issue management (cron); GBrain update, version check, stale gbrain → `gbrain`
|
||||
- GBrain search quality benchmarking → `benchmark-gbrain`
|
||||
- Coding tasks (Claude Code dispatch) → `Read hooks/bootstrap/REFERENCE.md`
|
||||
- Cross-modal review, second opinion, adversarial challenge → `cross-modal-review`
|
||||
- Deterministic code failing on edge cases → `fail-improve-loop`
|
||||
- GStack Browser tasks (cron) → `browser-tasks`
|
||||
- Weekly essay, write essay, draft weekly piece → `weekly-essay`
|
||||
- Investigate no response, why didn't they reply, follow up analysis → `investigate-no-response`
|
||||
- Printing press, publish to distribution → `printing-press`
|
||||
|
||||
### Infrastructure
|
||||
- Sending ANY service URL to the owner, "is the tunnel up", verify endpoint → `ngrok-verify`
|
||||
- "Check cpu", "system load", …, resource usage → `system-load`
|
||||
- Container restart → `container-restart`
|
||||
- Zombie processes → `zombie-reaper`
|
||||
- Write to /tmp → `scratch-space`
|
||||
- ClawVisor service routing, Gmail/Calendar/Drive/Contacts/iMessage via ClawVisor → `clawvisor`
|
||||
- ClawVisor Shield proxy, credential vaulting, API audit → `clawvisor-shield`
|
||||
- "What crons are running", recurring jobs, cron audit, scheduled tasks → `recurring-jobs`
|
||||
- Work on a PR → `acp-coding`
|
||||
- PR workflow, git worktree, dev checkout, "build this feature" → `repo-dev`
|
||||
- Brain page commit/push, always push after brain writes → `brain-commit`
|
||||
- Brain links, clickable GitHub URLs, "link me to" → `brain-links`
|
||||
- GitHub repo lookup, "repo not found", clone/check repo existence, READ a repo → `github-repo`
|
||||
- GitHub WRITE: push → `github-agents`
|
||||
- gbrain PR content, anonymization, PR body for gbrain → `gbrain-pr`
|
||||
- CAPTCHA, DataDome, "verification required", slide to verify → `captcha-solver`
|
||||
- QR code generation, "make a QR code", scannable code → `qr-code`
|
||||
- Front API, front link, front conversation, front search → `front-api`
|
||||
- OAuth2 authorization, "connect my X/service account", callback server → `oauth-webhook`
|
||||
- Headless browser, form fill, web interaction → `browser`
|
||||
- Cloud browser automation → `browser-use`
|
||||
- "Bypass IP restriction" → `nordvpn-proxy`
|
||||
- Channel discovery, find channels, list channels → `channel-discovery`
|
||||
- Telegram test divert, test message routing → `telegram-test-divert`
|
||||
- GStack Browse headed+proxy, browser-native download, anti-bot browsing → `gstack-browse`
|
||||
- "Submit a shell job" → `gbrain skills/minion-orchestrator`
|
||||
- Start GStack Browser (headed, the owner's machine) → `Ask the owner to run gstack-browser and share pairing code`
|
||||
- Binary dep missing, shared library error, container restart → `binary-deps`
|
||||
- Match HTML to screenshot, pixel-perfect, visual comparison, CSS tuning → `pixel-match`
|
||||
- YC app investigation, YC application ingestion, "ingest this company", company 404 → `yc-app-ingest`
|
||||
- Email triage, inbox classification, cold pitch scoring, auto-archive → `email-triage`
|
||||
- Cold pitch scoring, rate this pitch, pitch quality → `cold-pitch-scorer`
|
||||
- Company oppo, competitive intel, investigate competitor → `company-oppo`
|
||||
- Cross-modal eval, compare models, model comparison → `cross-modal-eval`
|
||||
- Tweet reply, dunk, respond to troll, "don't respond to this" → `anti-dunk`
|
||||
- "Write a comeback", "roast this", aggressive reply draft → `clapback`
|
||||
- Tweet draft, compose tweet, write a tweet → `tweet-draft`
|
||||
- Tweet composition, draft tweet structure → `tweet-composition`
|
||||
- Tweet vulnerability scan, shield, check my tweet → `tweet-shield`
|
||||
- Journo dunk, journalist oppo, build dunk file → `journo-dunk`
|
||||
- Hater tracker, hostile engagement analysis → `hater-tracker`
|
||||
- Slack messages, slack search, slack DMs → `slack`
|
||||
- Voter guide, election research, candidate analysis → `voter-guide`
|
||||
- Voter guide data extraction → `voter-guide-extract`
|
||||
- Web archive, save page, preserve article, offline copy → `web-archive`
|
||||
- YC meeting recording, OH transcript ingestion → `yc-meeting-ingest`
|
||||
- Quote screenshot, article screenshot for tweet → `quote-screenshot`
|
||||
- Song lyrics, quote lyrics (content filter bypass) → `song-lyrics`
|
||||
- Voice call enrichment, post-call brain page → `voice-call-enrich`
|
||||
- Context health, bootstrap budget, resolver coverage → `context-health`
|
||||
- Daily question, personal question drip → `daily-question`
|
||||
- Stalker watch, threat monitoring, dangerous individual → `stalker-watch`
|
||||
- Idea registry, idea capture, "I have an idea" → `idea-registry`
|
||||
- File archive ingestion, Dropbox, Google Drive import → `file-archive-ingestion`
|
||||
- "skillpackify", PR to gbrain, open source this skill, add to skillpack → `skillpackify`
|
||||
- Restart sweep, dropped messages, missed messages after restart → `restart-sweep`
|
||||
- Neuromancer coordination, agent handoffs, inter-agent tasks, "hand off to Neuromancer" → `neuromancer-coordination`
|
||||
- Inter-agent coordination, "Owner's Agents" group chat, the agent+Neuromancer collaboration, agent task claiming, brain write protocol; Bot-to-bot communication, /curtain protocol, agent volley limits, bot-to-bot setup, how agents talk to each other → `inter-agent-coordination`
|
||||
|
||||
**Internal data-source skills** (called by other skills, not directly): captain-api, crustdata, exa, happenstance, gmail, google-calendar, google-contacts, slack, clawvisor
|
||||
|
||||
|
||||
## Neuromancer Delegation (Cross-Topic)
|
||||
|
||||
**In ANY topic**, if a task would benefit from Neuromancer's capabilities, delegate it by posting a `[TASK]` message to the "Owner's Agents" group (thread 1, group -<GROUP_ID>).
|
||||
|
||||
**Neuromancer is good at:** Web research, browser automation, coding/PRs, X posting (via xurl), Google Workspace ops, on-demand analysis, skill building.
|
||||
|
||||
**the agent keeps:** Brain DB, cron/scheduled ops, X API (Enterprise keys), email sweeps (ClawVisor), memory consolidation, social radar, embedding/indexing.
|
||||
|
||||
**Protocol:** Prefix structured messages with `[TASK]`, `[RESULT]`, or `[QUERY]`. Neuromancer monitors the topic in real-time. Include enough context that Neuromancer can act without asking follow-ups. Reference brain pages by path.
|
||||
|
||||
**Don't delegate silently.** If the owner asked for something in another topic and you're handing it to Neuromancer, tell the owner in that topic: "Handing this to Neuromancer" with a one-liner on what you asked for.
|
||||
|
||||
## Memory (Operational)
|
||||
|
||||
- `MEMORY.md` — permanent, cross-session state. Keep tight. Flush to `memory/YYYY-MM-DD.md` daily.
|
||||
- `memory/YYYY-MM-DD.md` — daily operational memory. Append-only per day.
|
||||
- `memory/heartbeat-state.json` — structured state (location, wake status, last checks, blockers).
|
||||
- Brain (`/your/brain/path/`) — permanent knowledge (people, companies, deals, meetings, projects).
|
||||
|
||||
## Operating Rules
|
||||
|
||||
For the full set of operating principles, sub-agent rules, testing conventions, style guide, coding task protocols, and group chat rules: **read `skills/_operating-rules.md`**.
|
||||
|
||||
Key rules always in effect:
|
||||
- **Tests ship with code.** No PR without tests. No skip. See the full principle in the reference.
|
||||
- **Test before bulk.** Read `skills/progressive-batch/SKILL.md` for any operation touching >50 items. Progressive ramp: 10 → verify output exists → 100 → verify → 500 → verify → full. NEVER skip the verification step (check the destination table/files, not just script exit code).
|
||||
- **Fix tools, don't work around them.** If a tool is broken, fix it.
|
||||
- **Present options, then STOP.** For ambiguous requests, present 2-3 options. Don't pick one silently.
|
||||
- **Durable MECE skills.** Every repeated workflow → a skill. DRY across skills.
|
||||
- **GStack for coding PRs.** Read `skills/acp-coding/SKILL.md` for Claude Code / Codex integration.
|
||||
|
||||
## Coding Tasks — GStack Integration
|
||||
|
||||
Coding on gstack/gbrain/GL/any dev project: read `skills/acp-coding/SKILL.md`, spawn Codex via ACP, give full context, monitor+relay. Slash: `/code`, `/codex`, `/ship`, `/qa`, `/review`, `/investigate`.
|
||||
|
||||
<!-- gbrain:skillpack:begin -->
|
||||
<!-- Installed by gbrain 0.25.1. All 35 skills in this pack are already referenced in the resolver tables above. -->
|
||||
<!-- gbrain:skillpack:manifest cumulative-slugs="academic-verify,archive-crawler,article-enrichment,book-mirror,brain-ops,brain-pdf,briefing,citation-fixer,concept-synthesis,cron-scheduler,cross-modal-review,daily-task-manager,daily-task-prep,data-research,enrich,idea-ingest,ingest,maintain,media-ingest,meeting-ingestion,minion-orchestrator,perplexity-research,query,repo-architecture,reports,signal-detector,skill-creator,skillify,skillpack-check,soul-audit,strategic-reading,testing,voice-note-ingest,webhook-transforms" version="0.25.1" -->
|
||||
<!-- gbrain:skillpack:end -->
|
||||
@@ -1,146 +0,0 @@
|
||||
<!-- A/B EVAL FIXTURE — synthetic resolver shape, do not invoke from agent context. -->
|
||||
<!-- Variant: FUNCTIONAL-AREAS — the dispatcher pattern, extracted from a production AGENTS.md at the post-compression state; owner PII scrubbed. ~13KB. -->
|
||||
|
||||
# AGENTS.md
|
||||
|
||||
This folder is home. Treat it that way.
|
||||
|
||||
## Hard Gates (NEVER VIOLATE)
|
||||
|
||||
⛔ **RUNTIME CONTEXT > PROJECT DOCS.** When the OpenClaw runtime context block (Group Chat Context, Inbound Context, capabilities) contradicts a project doc rule, the runtime wins. The runtime knows the actual channel state for THIS turn; project docs are stale by definition. The 2026-05-06 silent-drop recurrence happened because I trusted a wrong HEARTBEAT rule over the correct runtime warning. Don't do that again.
|
||||
|
||||
⛔ **NEVER RESTART GATEWAY.** Tell the owner. He does it himself. No exceptions.
|
||||
|
||||
⛔ **BRAIN-FIRST STORAGE.** ALL valuable outputs → `/your/brain/path/` or Supabase IMMEDIATELY. Use `/your/tmp` for scratch (not `/tmp`). `/tmp` hard limit: 2GB. See `skills/conventions/brain-first.md`.
|
||||
|
||||
⛔ **DATA LOSS GATE.** Before ANY bulk delete: read `skills/data-loss-gate/SKILL.md`, present confirmation card, wait for "yes."
|
||||
|
||||
⛔ **NO WIKILINKS.** Standard markdown links only: `[Name](path)`. Never `[[wikilinks]]`.
|
||||
|
||||
⛔ **GBRAIN MASTER READ-ONLY.** Never push to master on <owner>/gbrain. Never merge PRs. Branch → push → PR only. See `skills/github-agents/SKILL.md`.
|
||||
|
||||
⛔ **PUBLIC REPO GUARD.** Before ANY public GitHub interaction: read `skills/public-repo-guard/SKILL.md`. Run PII scanner on ALL content.
|
||||
|
||||
⚡ **MINIONS OVER SUB-AGENTS.** Use gbrain Minions (shell jobs) for batch/deterministic work. Sub-agents only when LLM reasoning is required mid-task. Always set `--timeout-ms 900000` for long jobs.
|
||||
|
||||
## Gate -1 — Acknowledge Immediately
|
||||
|
||||
For any request taking >5 sec: send a one-line ack with rough time estimate FIRST, then start tools. Never go silent into a tool chain. Calibration: lookup ~10s, multi-tool ~30-60s, transcription ~2-3min, sub-agent ~1-3min, heavy batch ~3-5min, browser ~2-5min. Overestimate slightly.
|
||||
|
||||
For tasks >1 min: spawn a progress-update subagent (one-liner every 30-60s with concrete progress %). Critical in group topics with no typing indicator.
|
||||
|
||||
## Gate 0 — Access Control
|
||||
|
||||
On EVERY inbound message, check `sender_id` FIRST.
|
||||
- **the owner (<OWNER_ID_A> or <OWNER_ID_B>):** Proceed. Full access.
|
||||
- **Known non-the owner:** Read `skills/multi-user/SKILL.md` immediately. It governs everything.
|
||||
- **Unknown sender:** "This is a private agent." → notify the owner → stop.
|
||||
|
||||
## Gate 0.5 — Critical Life Events
|
||||
|
||||
If the owner mentions a **death, funeral, birth, hospitalization, emergency, diagnosis, accident, divorce, or arrest** — IMMEDIATELY write to BOTH `MEMORY.md` AND `memory/YYYY-MM-DD.md`. Priority 0. No deferral.
|
||||
|
||||
## Gate 1 — Signal Detection (the owner only)
|
||||
|
||||
Every the owner message: scan for entity mentions (people, companies, deals, YC batches). For each: search brain, load context, update if stale. Read `skills/entity-detector/ENTITY-DETECTION.md` for the full protocol.
|
||||
|
||||
**Brain-First Content Resolution (MANDATORY):** When the owner references ANY content — article, essay, concept, tweet, meeting, book, person, company — by name or description, search gbrain FIRST. Never ask "which article?" or "can you share the link?" The brain has 100K pages. Search it. Only ask the owner if gbrain + memory + web all fail.
|
||||
|
||||
## Gate 2 — Session Startup
|
||||
|
||||
Before first substantive reply:
|
||||
1. Read `ops/tasks.md` for task state
|
||||
2. Read `memory/heartbeat-state.json` for location, blockers, last checks
|
||||
3. Read relevant `memory/YYYY-MM-DD.md` for recent context
|
||||
4. Check calendar if time-sensitive
|
||||
|
||||
**Brain link rule:** Every brain path in output MUST be a clickable GitHub URL: `[name](https://github.com/<owner>/brain/blob/main/path.md)`. Never bare paths. Never invented URLs. `<owner>.github.io/brain/` does NOT exist.
|
||||
|
||||
**After every brain write:** `bash scripts/brain-commit-link.sh "<message>"`. Always absolute paths for brain writes (`/your/brain/path/...`).
|
||||
|
||||
**Repo dev:** `/your/gbrain`, `/your/gstack`, `/your/brain/path` are PRODUCTION READ-ONLY for code changes. All dev work → `/your/git-projects/<repo>-<feature>/`. See `skills/repo-dev/SKILL.md`.
|
||||
|
||||
## Gate 3 — Outbound Link Gate
|
||||
|
||||
Before EVERY reply containing a brain reference:
|
||||
1. Path must be absolute GitHub URL
|
||||
2. Commit must be pushed (not just local)
|
||||
3. Use `brain-commit-link.sh` output for the URL
|
||||
4. Never invent URLs. Never use `<owner>.github.io`.
|
||||
|
||||
## Skill Resolver
|
||||
|
||||
Read the skill file before acting. If two could match, read both. Non-the owner senders: only WORK/FAMILY-accessible skills.
|
||||
|
||||
### Always-on (every message)
|
||||
- Gate -1: any request taking >5 sec → `acknowledge`
|
||||
- Gate 0: sender_id != the owner → `multi-user`
|
||||
- Gate 1: the owner messages only → `entity-detector`
|
||||
- Non-the owner shares info → `group-chat-intel`
|
||||
- Brain read/write/lookup → `brain-ops`
|
||||
- Reply mentioning repo/project → `brain-link-refs`
|
||||
- Reply referencing brain page → `brain-link-report`
|
||||
- Report with external links → `report-quality-gate`
|
||||
- Multi-user group reply referencing brain → `brain-pdf-auto`
|
||||
- Time-sensitive claim → `context-now`
|
||||
- the owner corrects behavior → `correction-pipeline`
|
||||
- Inline buttons / user decision gate → `ask-user`
|
||||
|
||||
### Functional Areas
|
||||
- **Brain & knowledge**: create/enrich/search/export brain pages, filing, citations, publishing, book analysis, strategic reading, concept synthesis, archive mining, conversation history → `brain-ops` (dispatcher for: enrich, query, brain-pdf, brain-publish, brain-export, brain-plan, brain-librarian, brain-commit, brain-storage, brain-storage-links, citation-fixer, repo-architecture, book-mirror, book-mirror-extreme, book-mirror-synthesis, strategic-reading, concept-synthesis, archive-crawler, conversation-history, conversation-enrichment, garry-voice, essay-review, fact-check, takes-extraction, gbrain, gbrain-upgrade, benchmark-gbrain, freshness-monitor, dropbox-archive-review, bulk-skillify, x-handle-enrich, person-score)
|
||||
- **Content ingestion**: ingest links/articles/PDFs/video/audio/tweets/books/meetings/voice notes, transcription, media enrichment → `ingest` (dispatcher for: media-ingest, meeting-ingestion, meeting-digest, meeting-gold-standard, meeting-signal-pass, voice-note-ingest, article-enrichment, post-ingestion-enrichment, media-enrichment, book-acquisition, annas-archive, pdf-ingest, tweet-deep-ingest, substack-ingest, pocket-ingest, investor-update-ingest, yc-ingest, yc-oh-ingest, yc-app-ingest, yc-meeting-ingest, kindle-library, therapy-ingest, transcript-save, file-archive-ingestion, idea-ingest)
|
||||
- **Calendar & scheduling**: schedule, events, conflicts, sync, prep, travel booking, time/location → `google-calendar` (dispatcher for: calendar-event-create, calendar-check, calendar-sync, calendar-recall, calendar-travel-setup, meeting-prep, interview-prep, context-now, jet-lag, location-inference)
|
||||
- **Email & comms**: inbox triage, email search/send, iMessage, Slack, unsubscribe, Front API → `executive-assistant` (dispatcher for: gmail, email-triage, email-unsubscribe, cold-email-lookup, cold-pitch-scorer, front-api, slack, intro-reping, startup-intro, investigate-no-response)
|
||||
- **Research & investigation**: web research, people/company lookup, LinkedIn, competitive intel, background checks → `perplexity-research` (dispatcher for: exa, happenstance, crustdata, captain-api, data-research, diligence, company-oppo, network-intel, private-investigator, oppo-research, academic-verify)
|
||||
- **X/Twitter & social**: tweets, social monitoring, adversary tracking, content strategy, DM triage → `x-ingest` (dispatcher for: adversary-tracking, social-radar, x-daily-quality, x-concept-tier, social-json-store, detect-astroturf, real-name-hostiles, investigate-x-anon, anti-dunk, clapback, tweet-draft, tweet-composition, tweet-shield, journo-dunk, hater-tracker, message-intel, yc-media-monitor, yc-competitor-oppo, yc-booster-tracker, steph-instagram, content-ideas)
|
||||
- **Places & travel**: checkins, restaurants, showtimes, trip logistics → `checkin` (dispatcher for: trip-logistics, trip-ingest, showtimes, personal-logistics)
|
||||
- **Product & building**: CEO review, code, debugging, skill creation, testing, refactoring, PR management → `acp-coding` (dispatcher for: gstack-openclaw-ceo-review, gstack-openclaw-investigate, gstack-openclaw-office-hours, gstack-openclaw-retro, skill-creator, skillify, testing, durable-service, refactor, narrative, budget-roi, fail-improve-loop, weekly-essay, printing-press, cross-modal-review, cross-modal-eval)
|
||||
- **Infrastructure**: tunnels, containers, services, crons, GitHub, browser automation, security → `healthcheck` (dispatcher for: ngrok-verify, system-load, container-restart, zombie-reaper, scratch-space, clawvisor, clawvisor-shield, recurring-jobs, github-repo, github-agents, gbrain-pr, captcha-solver, qr-code, browser, browser-use, gstack-browse, binary-deps, pixel-match, nordvpn-proxy, channel-discovery, durable-service, data-loss-gate, public-repo-guard, web-archive, security-audit)
|
||||
- **People & contacts**: Google contacts, face detection/identification, people enrichment → `google-contacts` (dispatcher for: face-detect, identify-faces, enrich)
|
||||
- **Tasks & logistics**: daily tasks, reminders, briefings, business dev, flight tracking, voice calls → `daily-task-manager` (dispatcher for: daily-task-prep, business-development, flight-tracker, voice-agent, voice-session-ingest, venus-post-call, voice-link, voice-call-enrich, quo, checkin)
|
||||
- **Political**: donation tracking, voter guides, civic intel → `political-donations` (dispatcher for: voter-guide, voter-guide-extract, fiscal-forensics)
|
||||
- **Inter-agent**: Neuromancer delegation, agent coordination → `inter-agent-coordination` (dispatcher for: neuromancer-coordination)
|
||||
- **Circleback**: meeting search → `circleback-cli`
|
||||
|
||||
**Internal data-source skills** (called by other skills, not directly): captain-api, crustdata, exa, happenstance, gmail, google-calendar, google-contacts, slack, clawvisor
|
||||
|
||||
|
||||
## Neuromancer Delegation (Cross-Topic)
|
||||
|
||||
**In ANY topic**, if a task would benefit from Neuromancer's capabilities, delegate it by posting a `[TASK]` message to the "Owner's Agents" group (thread 1, group -<GROUP_ID>).
|
||||
|
||||
**Neuromancer is good at:** Web research, browser automation, coding/PRs, X posting (via xurl), Google Workspace ops, on-demand analysis, skill building.
|
||||
|
||||
**the agent keeps:** Brain DB, cron/scheduled ops, X API (Enterprise keys), email sweeps (ClawVisor), memory consolidation, social radar, embedding/indexing.
|
||||
|
||||
**Protocol:** Prefix structured messages with `[TASK]`, `[RESULT]`, or `[QUERY]`. Neuromancer monitors the topic in real-time. Include enough context that Neuromancer can act without asking follow-ups. Reference brain pages by path.
|
||||
|
||||
**Don't delegate silently.** If the owner asked for something in another topic and you're handing it to Neuromancer, tell the owner in that topic: "Handing this to Neuromancer" with a one-liner on what you asked for.
|
||||
|
||||
## Memory (Operational)
|
||||
|
||||
- `MEMORY.md` — permanent, cross-session state. Keep tight. Flush to `memory/YYYY-MM-DD.md` daily.
|
||||
- `memory/YYYY-MM-DD.md` — daily operational memory. Append-only per day.
|
||||
- `memory/heartbeat-state.json` — structured state (location, wake status, last checks, blockers).
|
||||
- Brain (`/your/brain/path/`) — permanent knowledge (people, companies, deals, meetings, projects).
|
||||
|
||||
## Operating Rules
|
||||
|
||||
For the full set of operating principles, sub-agent rules, testing conventions, style guide, coding task protocols, and group chat rules: **read `skills/_operating-rules.md`**.
|
||||
|
||||
Key rules always in effect:
|
||||
- **Tests ship with code.** No PR without tests. No skip. See the full principle in the reference.
|
||||
- **Test before bulk.** Read `skills/progressive-batch/SKILL.md` for any operation touching >50 items. Progressive ramp: 10 → verify output exists → 100 → verify → 500 → verify → full. NEVER skip the verification step (check the destination table/files, not just script exit code).
|
||||
- **Fix tools, don't work around them.** If a tool is broken, fix it.
|
||||
- **Present options, then STOP.** For ambiguous requests, present 2-3 options. Don't pick one silently.
|
||||
- **Durable MECE skills.** Every repeated workflow → a skill. DRY across skills.
|
||||
- **GStack for coding PRs.** Read `skills/acp-coding/SKILL.md` for Claude Code / Codex integration.
|
||||
|
||||
## Coding Tasks — GStack Integration
|
||||
|
||||
Coding on gstack/gbrain/GL/any dev project: read `skills/acp-coding/SKILL.md`, spawn Codex via ACP, give full context, monitor+relay. Slash: `/code`, `/codex`, `/ship`, `/qa`, `/review`, `/investigate`.
|
||||
|
||||
<!-- gbrain:skillpack:begin -->
|
||||
<!-- Installed by gbrain 0.25.1. All 35 skills in this pack are already referenced in the resolver tables above. -->
|
||||
<!-- gbrain:skillpack:manifest cumulative-slugs="academic-verify,archive-crawler,article-enrichment,book-mirror,brain-ops,brain-pdf,briefing,citation-fixer,concept-synthesis,cron-scheduler,cross-modal-review,daily-task-manager,daily-task-prep,data-research,enrich,idea-ingest,ingest,maintain,media-ingest,meeting-ingestion,minion-orchestrator,perplexity-research,query,repo-architecture,reports,signal-detector,skill-creator,skillify,skillpack-check,soul-audit,strategic-reading,testing,voice-note-ingest,webhook-transforms" version="0.25.1" -->
|
||||
<!-- gbrain:skillpack:end -->
|
||||
@@ -1,146 +0,0 @@
|
||||
<!-- A/B EVAL FIXTURE — synthetic resolver shape, do not invoke from agent context. -->
|
||||
<!-- Variant: RESOLVER-OF-RESOLVERS — functional-areas WITHOUT the '(dispatcher for: ...)' clauses. This is the variant the skill describes as 'broken' — pipe-table compression that loses sub-skill visibility. -->
|
||||
|
||||
# AGENTS.md
|
||||
|
||||
This folder is home. Treat it that way.
|
||||
|
||||
## Hard Gates (NEVER VIOLATE)
|
||||
|
||||
⛔ **RUNTIME CONTEXT > PROJECT DOCS.** When the OpenClaw runtime context block (Group Chat Context, Inbound Context, capabilities) contradicts a project doc rule, the runtime wins. The runtime knows the actual channel state for THIS turn; project docs are stale by definition. The 2026-05-06 silent-drop recurrence happened because I trusted a wrong HEARTBEAT rule over the correct runtime warning. Don't do that again.
|
||||
|
||||
⛔ **NEVER RESTART GATEWAY.** Tell the owner. He does it himself. No exceptions.
|
||||
|
||||
⛔ **BRAIN-FIRST STORAGE.** ALL valuable outputs → `/your/brain/path/` or Supabase IMMEDIATELY. Use `/your/tmp` for scratch (not `/tmp`). `/tmp` hard limit: 2GB. See `skills/conventions/brain-first.md`.
|
||||
|
||||
⛔ **DATA LOSS GATE.** Before ANY bulk delete: read `skills/data-loss-gate/SKILL.md`, present confirmation card, wait for "yes."
|
||||
|
||||
⛔ **NO WIKILINKS.** Standard markdown links only: `[Name](path)`. Never `[[wikilinks]]`.
|
||||
|
||||
⛔ **GBRAIN MASTER READ-ONLY.** Never push to master on <owner>/gbrain. Never merge PRs. Branch → push → PR only. See `skills/github-agents/SKILL.md`.
|
||||
|
||||
⛔ **PUBLIC REPO GUARD.** Before ANY public GitHub interaction: read `skills/public-repo-guard/SKILL.md`. Run PII scanner on ALL content.
|
||||
|
||||
⚡ **MINIONS OVER SUB-AGENTS.** Use gbrain Minions (shell jobs) for batch/deterministic work. Sub-agents only when LLM reasoning is required mid-task. Always set `--timeout-ms 900000` for long jobs.
|
||||
|
||||
## Gate -1 — Acknowledge Immediately
|
||||
|
||||
For any request taking >5 sec: send a one-line ack with rough time estimate FIRST, then start tools. Never go silent into a tool chain. Calibration: lookup ~10s, multi-tool ~30-60s, transcription ~2-3min, sub-agent ~1-3min, heavy batch ~3-5min, browser ~2-5min. Overestimate slightly.
|
||||
|
||||
For tasks >1 min: spawn a progress-update subagent (one-liner every 30-60s with concrete progress %). Critical in group topics with no typing indicator.
|
||||
|
||||
## Gate 0 — Access Control
|
||||
|
||||
On EVERY inbound message, check `sender_id` FIRST.
|
||||
- **the owner (<OWNER_ID_A> or <OWNER_ID_B>):** Proceed. Full access.
|
||||
- **Known non-the owner:** Read `skills/multi-user/SKILL.md` immediately. It governs everything.
|
||||
- **Unknown sender:** "This is a private agent." → notify the owner → stop.
|
||||
|
||||
## Gate 0.5 — Critical Life Events
|
||||
|
||||
If the owner mentions a **death, funeral, birth, hospitalization, emergency, diagnosis, accident, divorce, or arrest** — IMMEDIATELY write to BOTH `MEMORY.md` AND `memory/YYYY-MM-DD.md`. Priority 0. No deferral.
|
||||
|
||||
## Gate 1 — Signal Detection (the owner only)
|
||||
|
||||
Every the owner message: scan for entity mentions (people, companies, deals, YC batches). For each: search brain, load context, update if stale. Read `skills/entity-detector/ENTITY-DETECTION.md` for the full protocol.
|
||||
|
||||
**Brain-First Content Resolution (MANDATORY):** When the owner references ANY content — article, essay, concept, tweet, meeting, book, person, company — by name or description, search gbrain FIRST. Never ask "which article?" or "can you share the link?" The brain has 100K pages. Search it. Only ask the owner if gbrain + memory + web all fail.
|
||||
|
||||
## Gate 2 — Session Startup
|
||||
|
||||
Before first substantive reply:
|
||||
1. Read `ops/tasks.md` for task state
|
||||
2. Read `memory/heartbeat-state.json` for location, blockers, last checks
|
||||
3. Read relevant `memory/YYYY-MM-DD.md` for recent context
|
||||
4. Check calendar if time-sensitive
|
||||
|
||||
**Brain link rule:** Every brain path in output MUST be a clickable GitHub URL: `[name](https://github.com/<owner>/brain/blob/main/path.md)`. Never bare paths. Never invented URLs. `<owner>.github.io/brain/` does NOT exist.
|
||||
|
||||
**After every brain write:** `bash scripts/brain-commit-link.sh "<message>"`. Always absolute paths for brain writes (`/your/brain/path/...`).
|
||||
|
||||
**Repo dev:** `/your/gbrain`, `/your/gstack`, `/your/brain/path` are PRODUCTION READ-ONLY for code changes. All dev work → `/your/git-projects/<repo>-<feature>/`. See `skills/repo-dev/SKILL.md`.
|
||||
|
||||
## Gate 3 — Outbound Link Gate
|
||||
|
||||
Before EVERY reply containing a brain reference:
|
||||
1. Path must be absolute GitHub URL
|
||||
2. Commit must be pushed (not just local)
|
||||
3. Use `brain-commit-link.sh` output for the URL
|
||||
4. Never invent URLs. Never use `<owner>.github.io`.
|
||||
|
||||
## Skill Resolver
|
||||
|
||||
Read the skill file before acting. If two could match, read both. Non-the owner senders: only WORK/FAMILY-accessible skills.
|
||||
|
||||
### Always-on (every message)
|
||||
- Gate -1: any request taking >5 sec → `acknowledge`
|
||||
- Gate 0: sender_id != the owner → `multi-user`
|
||||
- Gate 1: the owner messages only → `entity-detector`
|
||||
- Non-the owner shares info → `group-chat-intel`
|
||||
- Brain read/write/lookup → `brain-ops`
|
||||
- Reply mentioning repo/project → `brain-link-refs`
|
||||
- Reply referencing brain page → `brain-link-report`
|
||||
- Report with external links → `report-quality-gate`
|
||||
- Multi-user group reply referencing brain → `brain-pdf-auto`
|
||||
- Time-sensitive claim → `context-now`
|
||||
- the owner corrects behavior → `correction-pipeline`
|
||||
- Inline buttons / user decision gate → `ask-user`
|
||||
|
||||
### Functional Areas
|
||||
- **Brain & knowledge**: create/enrich/search/export brain pages, filing, citations, publishing, book analysis, strategic reading, concept synthesis, archive mining, conversation history → `brain-ops`
|
||||
- **Content ingestion**: ingest links/articles/PDFs/video/audio/tweets/books/meetings/voice notes, transcription, media enrichment → `ingest`
|
||||
- **Calendar & scheduling**: schedule, events, conflicts, sync, prep, travel booking, time/location → `google-calendar`
|
||||
- **Email & comms**: inbox triage, email search/send, iMessage, Slack, unsubscribe, Front API → `executive-assistant`
|
||||
- **Research & investigation**: web research, people/company lookup, LinkedIn, competitive intel, background checks → `perplexity-research`
|
||||
- **X/Twitter & social**: tweets, social monitoring, adversary tracking, content strategy, DM triage → `x-ingest`
|
||||
- **Places & travel**: checkins, restaurants, showtimes, trip logistics → `checkin`
|
||||
- **Product & building**: CEO review, code, debugging, skill creation, testing, refactoring, PR management → `acp-coding`
|
||||
- **Infrastructure**: tunnels, containers, services, crons, GitHub, browser automation, security → `healthcheck`
|
||||
- **People & contacts**: Google contacts, face detection/identification, people enrichment → `google-contacts`
|
||||
- **Tasks & logistics**: daily tasks, reminders, briefings, business dev, flight tracking, voice calls → `daily-task-manager`
|
||||
- **Political**: donation tracking, voter guides, civic intel → `political-donations`
|
||||
- **Inter-agent**: Neuromancer delegation, agent coordination → `inter-agent-coordination`
|
||||
- **Circleback**: meeting search → `circleback-cli`
|
||||
|
||||
**Internal data-source skills** (called by other skills, not directly): captain-api, crustdata, exa, happenstance, gmail, google-calendar, google-contacts, slack, clawvisor
|
||||
|
||||
|
||||
## Neuromancer Delegation (Cross-Topic)
|
||||
|
||||
**In ANY topic**, if a task would benefit from Neuromancer's capabilities, delegate it by posting a `[TASK]` message to the "Owner's Agents" group (thread 1, group -<GROUP_ID>).
|
||||
|
||||
**Neuromancer is good at:** Web research, browser automation, coding/PRs, X posting (via xurl), Google Workspace ops, on-demand analysis, skill building.
|
||||
|
||||
**the agent keeps:** Brain DB, cron/scheduled ops, X API (Enterprise keys), email sweeps (ClawVisor), memory consolidation, social radar, embedding/indexing.
|
||||
|
||||
**Protocol:** Prefix structured messages with `[TASK]`, `[RESULT]`, or `[QUERY]`. Neuromancer monitors the topic in real-time. Include enough context that Neuromancer can act without asking follow-ups. Reference brain pages by path.
|
||||
|
||||
**Don't delegate silently.** If the owner asked for something in another topic and you're handing it to Neuromancer, tell the owner in that topic: "Handing this to Neuromancer" with a one-liner on what you asked for.
|
||||
|
||||
## Memory (Operational)
|
||||
|
||||
- `MEMORY.md` — permanent, cross-session state. Keep tight. Flush to `memory/YYYY-MM-DD.md` daily.
|
||||
- `memory/YYYY-MM-DD.md` — daily operational memory. Append-only per day.
|
||||
- `memory/heartbeat-state.json` — structured state (location, wake status, last checks, blockers).
|
||||
- Brain (`/your/brain/path/`) — permanent knowledge (people, companies, deals, meetings, projects).
|
||||
|
||||
## Operating Rules
|
||||
|
||||
For the full set of operating principles, sub-agent rules, testing conventions, style guide, coding task protocols, and group chat rules: **read `skills/_operating-rules.md`**.
|
||||
|
||||
Key rules always in effect:
|
||||
- **Tests ship with code.** No PR without tests. No skip. See the full principle in the reference.
|
||||
- **Test before bulk.** Read `skills/progressive-batch/SKILL.md` for any operation touching >50 items. Progressive ramp: 10 → verify output exists → 100 → verify → 500 → verify → full. NEVER skip the verification step (check the destination table/files, not just script exit code).
|
||||
- **Fix tools, don't work around them.** If a tool is broken, fix it.
|
||||
- **Present options, then STOP.** For ambiguous requests, present 2-3 options. Don't pick one silently.
|
||||
- **Durable MECE skills.** Every repeated workflow → a skill. DRY across skills.
|
||||
- **GStack for coding PRs.** Read `skills/acp-coding/SKILL.md` for Claude Code / Codex integration.
|
||||
|
||||
## Coding Tasks — GStack Integration
|
||||
|
||||
Coding on gstack/gbrain/GL/any dev project: read `skills/acp-coding/SKILL.md`, spawn Codex via ACP, give full context, monitor+relay. Slash: `/code`, `/codex`, `/ship`, `/qa`, `/review`, `/investigate`.
|
||||
|
||||
<!-- gbrain:skillpack:begin -->
|
||||
<!-- Installed by gbrain 0.25.1. All 35 skills in this pack are already referenced in the resolver tables above. -->
|
||||
<!-- gbrain:skillpack:manifest cumulative-slugs="academic-verify,archive-crawler,article-enrichment,book-mirror,brain-ops,brain-pdf,briefing,citation-fixer,concept-synthesis,cron-scheduler,cross-modal-review,daily-task-manager,daily-task-prep,data-research,enrich,idea-ingest,ingest,maintain,media-ingest,meeting-ingestion,minion-orchestrator,perplexity-research,query,repo-architecture,reports,signal-detector,skill-creator,skillify,skillpack-check,soul-audit,strategic-reading,testing,voice-note-ingest,webhook-transforms" version="0.25.1" -->
|
||||
<!-- gbrain:skillpack:end -->
|
||||
+716
-1806
File diff suppressed because one or more lines are too long
+4
-27
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.32.3.0",
|
||||
"version": "0.19.0",
|
||||
"description": "Personal knowledge brain with Postgres + pgvector hybrid search",
|
||||
"family": "bundle-plugin",
|
||||
"configSchema": {
|
||||
@@ -8,51 +8,37 @@
|
||||
"type": "string",
|
||||
"required": true,
|
||||
"description": "PostgreSQL connection URL (Supabase recommended)",
|
||||
"uiHints": {
|
||||
"sensitive": true
|
||||
}
|
||||
"uiHints": { "sensitive": true }
|
||||
},
|
||||
"openai_api_key": {
|
||||
"type": "string",
|
||||
"required": false,
|
||||
"description": "OpenAI API key for embeddings (uses OPENAI_API_KEY env var if not set)",
|
||||
"uiHints": {
|
||||
"sensitive": true
|
||||
}
|
||||
"uiHints": { "sensitive": true }
|
||||
}
|
||||
},
|
||||
"mcpServers": {
|
||||
"gbrain": {
|
||||
"command": "./bin/gbrain",
|
||||
"args": [
|
||||
"serve"
|
||||
]
|
||||
"args": ["serve"]
|
||||
}
|
||||
},
|
||||
"skills": [
|
||||
"skills/academic-verify",
|
||||
"skills/archive-crawler",
|
||||
"skills/article-enrichment",
|
||||
"skills/book-mirror",
|
||||
"skills/brain-ops",
|
||||
"skills/brain-pdf",
|
||||
"skills/briefing",
|
||||
"skills/citation-fixer",
|
||||
"skills/concept-synthesis",
|
||||
"skills/cross-modal-review",
|
||||
"skills/cron-scheduler",
|
||||
"skills/daily-task-manager",
|
||||
"skills/daily-task-prep",
|
||||
"skills/data-research",
|
||||
"skills/enrich",
|
||||
"skills/functional-area-resolver",
|
||||
"skills/idea-ingest",
|
||||
"skills/ingest",
|
||||
"skills/maintain",
|
||||
"skills/media-ingest",
|
||||
"skills/meeting-ingestion",
|
||||
"skills/minion-orchestrator",
|
||||
"skills/perplexity-research",
|
||||
"skills/query",
|
||||
"skills/reports",
|
||||
"skills/repo-architecture",
|
||||
@@ -60,16 +46,12 @@
|
||||
"skills/skill-creator",
|
||||
"skills/skillify",
|
||||
"skills/skillpack-check",
|
||||
"skills/skillpack-harvest",
|
||||
"skills/soul-audit",
|
||||
"skills/strategic-reading",
|
||||
"skills/testing",
|
||||
"skills/voice-note-ingest",
|
||||
"skills/webhook-transforms"
|
||||
],
|
||||
"shared_deps": [
|
||||
"skills/conventions",
|
||||
"skills/_AGENT_README.md",
|
||||
"skills/_brain-filing-rules.md",
|
||||
"skills/_brain-filing-rules.json",
|
||||
"skills/_output-rules.md"
|
||||
@@ -83,10 +65,5 @@
|
||||
"compat": {
|
||||
"pluginApi": ">=2026.4.0"
|
||||
}
|
||||
},
|
||||
"contracts": {
|
||||
"contextEngines": [
|
||||
"gbrain-context"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
+5
-55
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.36.1.1",
|
||||
"version": "0.22.13",
|
||||
"description": "Postgres-native personal knowledge brain with hybrid RAG search",
|
||||
"type": "module",
|
||||
"main": "src/core/index.ts",
|
||||
@@ -24,47 +24,21 @@
|
||||
"./backoff": "./src/core/backoff.ts",
|
||||
"./search/hybrid": "./src/core/search/hybrid.ts",
|
||||
"./search/expansion": "./src/core/search/expansion.ts",
|
||||
"./ai/gateway": "./src/core/ai/gateway.ts",
|
||||
"./extract": "./src/commands/extract.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "bun run src/cli.ts",
|
||||
"build": "bun build --compile --outfile bin/gbrain src/cli.ts",
|
||||
"build:all": "bun build --compile --target=bun-darwin-arm64 --outfile bin/gbrain-darwin-arm64 src/cli.ts && bun build --compile --target=bun-linux-x64 --outfile bin/gbrain-linux-x64 src/cli.ts",
|
||||
"build:admin": "cd admin && bun run build && cd .. && bun run scripts/build-admin-embedded.ts",
|
||||
"build:admin-embedded": "bun run scripts/build-admin-embedded.ts",
|
||||
"build:schema": "bash scripts/build-schema.sh",
|
||||
"build:llms": "bun run scripts/build-llms.ts",
|
||||
"build:pglite-snapshot": "bun run scripts/build-pglite-snapshot.ts",
|
||||
"test": "bash scripts/run-unit-parallel.sh",
|
||||
"test:full": "bun run verify && bash scripts/run-unit-parallel.sh && bun run test:slow && ([ -n \"$DATABASE_URL\" ] && bash scripts/run-e2e.sh || echo '[test:full] skipped E2E (no DATABASE_URL); run docker-compose -f docker-compose.ci.yml up + bun run test:e2e to include' 1>&2)",
|
||||
"verify": "bun run check:privacy && bun run check:proposal-pii && bun run check:test-names && bun run check:jsonb && bun run check:source-id-projection && bun run check:progress && bun run check:test-isolation && bun run check:wasm && bun run check:admin-build && bun run check:admin-scope-drift && bun run check:cli-exec && bun run check:system-of-record && bun run check:eval-glossary && bun run check:synthetic-corpus-privacy && bun run typecheck",
|
||||
"check:synthetic-corpus-privacy": "scripts/check-synthetic-corpus-privacy.sh",
|
||||
"check:system-of-record": "scripts/check-system-of-record.sh",
|
||||
"check:admin-scope-drift": "scripts/check-admin-scope-drift.sh",
|
||||
"check:cli-exec": "scripts/check-cli-executable.sh",
|
||||
"check:all": "scripts/check-privacy.sh && scripts/check-proposal-pii.sh && scripts/check-test-real-names.sh && scripts/check-jsonb-pattern.sh && scripts/check-source-id-projection.sh && scripts/check-progress-to-stdout.sh && scripts/check-no-legacy-getconnection.sh && scripts/check-test-isolation.sh && scripts/check-trailing-newline.sh && scripts/check-wasm-embedded.sh && scripts/check-exports-count.sh && scripts/check-admin-build.sh && scripts/check-admin-scope-drift.sh && scripts/check-cli-executable.sh",
|
||||
"test": "scripts/check-jsonb-pattern.sh && scripts/check-progress-to-stdout.sh && scripts/check-trailing-newline.sh && scripts/check-wasm-embedded.sh && bun run typecheck && bun test --timeout=60000",
|
||||
"check:wasm": "scripts/check-wasm-embedded.sh",
|
||||
"check:newlines": "scripts/check-trailing-newline.sh",
|
||||
"test:e2e": "bash scripts/run-e2e.sh",
|
||||
"test:slow": "bash scripts/run-slow-tests.sh",
|
||||
"test:profile": "bash scripts/profile-tests.sh",
|
||||
"test:serial": "bash scripts/run-serial-tests.sh",
|
||||
"ci:local": "bash scripts/ci-local.sh",
|
||||
"ci:local:diff": "bash scripts/ci-local.sh --diff",
|
||||
"ci:select-e2e": "bun run scripts/select-e2e.ts",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"check:jsonb": "scripts/check-jsonb-pattern.sh",
|
||||
"check:source-id-projection": "scripts/check-source-id-projection.sh",
|
||||
"check:privacy": "scripts/check-privacy.sh",
|
||||
"check:proposal-pii": "scripts/check-proposal-pii.sh",
|
||||
"check:eval-glossary": "scripts/check-eval-glossary-fresh.sh",
|
||||
"check:test-names": "scripts/check-test-real-names.sh",
|
||||
"check:progress": "scripts/check-progress-to-stdout.sh",
|
||||
"check:exports-count": "scripts/check-exports-count.sh",
|
||||
"check:admin-build": "scripts/check-admin-build.sh",
|
||||
"check:admin-embedded": "scripts/check-admin-embedded.sh",
|
||||
"check:test-isolation": "scripts/check-test-isolation.sh",
|
||||
"postinstall": "command -v gbrain >/dev/null 2>&1 && gbrain apply-migrations --yes --non-interactive || echo '[gbrain] postinstall skipped. If installed via bun install -g github:...: run `gbrain doctor` and `gbrain apply-migrations --yes` manually. See https://github.com/garrytan/gbrain/issues/218' 1>&2",
|
||||
"prepublish:clawhub": "bun run build:all",
|
||||
"publish:clawhub": "clawhub package publish . --family bundle-plugin"
|
||||
@@ -72,53 +46,29 @@
|
||||
"openclaw": {
|
||||
"compat": {
|
||||
"pluginApi": ">=2026.4.0"
|
||||
},
|
||||
"extensions": [
|
||||
"./src/openclaw-context-engine.ts"
|
||||
]
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/anthropic": "^3.0.71",
|
||||
"@ai-sdk/google": "^3.0.64",
|
||||
"@ai-sdk/openai": "^3.0.53",
|
||||
"@ai-sdk/openai-compatible": "^2.0.41",
|
||||
"@anthropic-ai/sdk": "^0.30.0",
|
||||
"@aws-sdk/client-s3": "^3.1028.0",
|
||||
"@dqbd/tiktoken": "^1.0.22",
|
||||
"@electric-sql/pglite": "0.4.3",
|
||||
"@jsquash/avif": "^2.1.1",
|
||||
"@jsquash/png": "^3.1.1",
|
||||
"@modelcontextprotocol/sdk": "1.29.0",
|
||||
"ai": "^6.0.168",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cors": "^2.8.5",
|
||||
"eventsource-parser": "^3.0.8",
|
||||
"exifr": "^7.1.3",
|
||||
"express": "^5.1.0",
|
||||
"express-rate-limit": "^7.5.0",
|
||||
"@modelcontextprotocol/sdk": "^1.0.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
"heic-decode": "^2.1.0",
|
||||
"marked": "^18.0.0",
|
||||
"openai": "^4.0.0",
|
||||
"pgvector": "^0.2.0",
|
||||
"postgres": "^3.4.0",
|
||||
"tree-sitter-wasms": "0.1.13",
|
||||
"web-tree-sitter": "0.22.6",
|
||||
"zod": "^4.3.6"
|
||||
"web-tree-sitter": "0.22.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"@types/cookie-parser": "^1.4.7",
|
||||
"@types/cors": "^2.8.19",
|
||||
"@types/express": "^5.0.6",
|
||||
"bun-types": "^1.3.13",
|
||||
"typescript": "^5.6.0"
|
||||
},
|
||||
"trustedDependencies": [
|
||||
"@electric-sql/pglite"
|
||||
],
|
||||
"engines": {
|
||||
"bun": ">=1.3.10"
|
||||
},
|
||||
"license": "MIT"
|
||||
}
|
||||
|
||||
@@ -1,654 +0,0 @@
|
||||
---
|
||||
id: restart-sweep
|
||||
name: Restart Sweep
|
||||
version: 0.1.0
|
||||
description: Detect Telegram messages dropped during OpenClaw gateway restarts. Reads OpenClaw session state, alerts on aborted-mid-run sessions and (opt-in) suspicious silence gaps. Cooldown-gated so repeat detections don't spam.
|
||||
category: reflex
|
||||
requires: []
|
||||
secrets:
|
||||
- name: OPENCLAW_OWNER_IDS
|
||||
description: Comma-separated user IDs that own this brain instance
|
||||
where: openclaw config — your own user IDs from the platforms you connect
|
||||
- name: OPENCLAW_TELEGRAM_GROUP
|
||||
description: Target Telegram group ID for restart alerts (negative number for groups)
|
||||
where: forward a message from the group to @userinfobot, copy the chat.id
|
||||
health_checks:
|
||||
- type: env_exists
|
||||
name: OPENCLAW_OWNER_IDS
|
||||
label: Owner IDs configured
|
||||
- type: env_exists
|
||||
name: OPENCLAW_TELEGRAM_GROUP
|
||||
label: Telegram group configured
|
||||
- type: command
|
||||
argv: [openclaw, sessions, --json]
|
||||
label: OpenClaw CLI reachable
|
||||
setup_time: 10 min
|
||||
cost_estimate: "$0 (no per-call cost; runs locally on cron)"
|
||||
---
|
||||
|
||||
# Restart Sweep: Detect Dropped Messages After Gateway Restarts
|
||||
|
||||
When the OpenClaw gateway restarts, webhook-delivered Telegram messages
|
||||
that haven't been processed yet get dropped permanently. Long-poll bots
|
||||
can replay missed updates via `getUpdates`. Webhook bots cannot. This
|
||||
recipe detects the gap by reading OpenClaw's session state and alerting
|
||||
when a session was active just before a restart but silent afterward.
|
||||
|
||||
## IMPORTANT: Instructions for the Agent
|
||||
|
||||
**You are the installer.** This recipe is written for YOU (the AI agent)
|
||||
to execute on behalf of the user. Follow these steps precisely.
|
||||
|
||||
**Stop points (MUST pause and verify before continuing):**
|
||||
- After Step 1: prerequisites pass? If not, fix before proceeding.
|
||||
- After Step 4: dry run produces sensible output? If not, debug before
|
||||
wiring cron.
|
||||
- After Step 5: cron entry created and visible in `crontab -l`? If not,
|
||||
cron isn't installed.
|
||||
|
||||
**When something fails:** Tell the user EXACTLY what failed, what it
|
||||
means, and what to try. Never say "something went wrong."
|
||||
|
||||
## What this does
|
||||
|
||||
1. Reads `/tmp/bootstrap-services.log` (or `$OPENCLAW_BOOTSTRAP_LOG`)
|
||||
to find when the gateway last restarted. Falls back to `now() - 30
|
||||
minutes` if the log isn't readable.
|
||||
2. Runs `openclaw sessions --json` to enumerate all live sessions.
|
||||
3. Filters to Telegram group sessions matching `$OPENCLAW_TELEGRAM_GROUP`.
|
||||
4. Flags sessions with `abortedLastRun: true` (strong signal of a
|
||||
dropped message). Optionally flags sessions that were active in the
|
||||
5 minutes before restart but silent in the 10 minutes after — gated
|
||||
behind `OPENCLAW_RESTART_SWEEP_AGGRESSIVE=1` because the timing
|
||||
heuristic produces false positives during quiet periods.
|
||||
5. Cooldown layer: each sessionKey alerted gets stamped with a
|
||||
`lastAlertedAt` timestamp. Re-alerting on the same sessionKey is
|
||||
suppressed for 6 hours regardless of whether the synthesized restart
|
||||
time matches. This prevents the "missing bootstrap log →
|
||||
re-alert-every-5-minutes-forever" failure mode.
|
||||
6. Sends one alert per cycle to Telegram (or stdout if no Telegram
|
||||
config), then records the alert in
|
||||
`~/.gbrain/integrations/restart-sweep/alerted.json`.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- OpenClaw running with Telegram in webhook mode (long-poll mode
|
||||
doesn't need this — `getUpdates` recovers missed messages on restart)
|
||||
- The `openclaw` CLI on PATH (or you'll provide an absolute path in
|
||||
Step 5)
|
||||
- Telegram bot token already configured in OpenClaw, group ID and
|
||||
optional topic ID known
|
||||
- Cron available on the host (this recipe schedules a 5-minute job;
|
||||
systemd timers, launchd, or any other scheduler also work — adapt
|
||||
Step 5 accordingly)
|
||||
|
||||
## Step 1: Verify prerequisites
|
||||
|
||||
```bash
|
||||
openclaw sessions --json | head -40
|
||||
```
|
||||
|
||||
Should print JSON with a `sessions` array. If it errors, fix
|
||||
`openclaw` reachability before continuing.
|
||||
|
||||
Decide a host-repo install path. The recipe assumes
|
||||
`~/openclaw/scripts/restart-sweep.mjs` and the user's `.env` lives at
|
||||
`~/openclaw/.env`. Adapt to your repo layout.
|
||||
|
||||
## Step 2: Collect the secrets
|
||||
|
||||
Confirm with the user:
|
||||
|
||||
- `OPENCLAW_OWNER_IDS` — comma-separated user IDs (e.g. `123456789,987654321`)
|
||||
- `OPENCLAW_TELEGRAM_GROUP` — the target group ID (negative number for
|
||||
group chats, e.g. `-1001234567890`). Forward a message from the
|
||||
group to `@userinfobot` to get it.
|
||||
- `OPENCLAW_ALERT_TOPIC` — optional, the topic/thread ID for forum
|
||||
groups. Open the topic in Telegram, the URL ends with the thread ID.
|
||||
|
||||
Add these three lines to the host's `.env` (or wherever the host loads
|
||||
env from):
|
||||
|
||||
```bash
|
||||
OPENCLAW_OWNER_IDS=...
|
||||
OPENCLAW_TELEGRAM_GROUP=...
|
||||
OPENCLAW_ALERT_TOPIC=...
|
||||
```
|
||||
|
||||
Optional tuning:
|
||||
|
||||
```bash
|
||||
# Set to 1 to enable the timing-based heuristic (active before restart,
|
||||
# silent after). Off by default because it false-positives during quiet
|
||||
# periods.
|
||||
OPENCLAW_RESTART_SWEEP_AGGRESSIVE=1
|
||||
|
||||
# Override the bootstrap log path (default /tmp/bootstrap-services.log)
|
||||
OPENCLAW_BOOTSTRAP_LOG=/var/log/openclaw/bootstrap.log
|
||||
```
|
||||
|
||||
## Step 3: Write the script to the host repo
|
||||
|
||||
Write the script content from the next section to
|
||||
`~/openclaw/scripts/restart-sweep.mjs` (or wherever the user picks).
|
||||
The script is self-contained — no npm install needed, just Node 18+
|
||||
or Bun.
|
||||
|
||||
<!-- restart-sweep:script -->
|
||||
```javascript
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Restart Message Sweep Script
|
||||
*
|
||||
* Detects Telegram messages dropped during OpenClaw gateway restarts.
|
||||
* Webhook-delivered messages can't be replayed via getUpdates, so we
|
||||
* read OpenClaw's session state and look for sessions that show signs
|
||||
* of dropped processing.
|
||||
*
|
||||
* Runs under Node 18+ or Bun. Copy this file into your host repo and
|
||||
* wire it to a 5-minute cron.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import fsp from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { exec, execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execP = promisify(exec);
|
||||
|
||||
// Module-level constants (no env reads here — env is read at construct time)
|
||||
const RESTART_THRESHOLD_MINUTES = 30; // Fallback restart-time window when bootstrap log is missing
|
||||
const COOLDOWN_HOURS = 6; // Re-alert suppression per sessionKey
|
||||
const STALE_DAYS = 30; // Prune alerted.json entries older than this
|
||||
const PRE_RESTART_WINDOW_MS = 5 * 60 * 1000;
|
||||
const POST_RESTART_WINDOW_MS = 10 * 60 * 1000;
|
||||
|
||||
class MessageSweepDetector {
|
||||
/**
|
||||
* @param {{ execFile?: typeof execFile, runOpenclawSessions?: () => Promise<any[]> }} [deps]
|
||||
* Optional dependency injection for tests. Production: leave undefined.
|
||||
*/
|
||||
constructor(deps = {}) {
|
||||
// Constructor-time env reads (C2): tests can mutate process.env per construction
|
||||
const ownerEnv = process.env.OPENCLAW_OWNER_IDS ?? '';
|
||||
this.OWNER_IDS = ownerEnv.split(',').map(s => s.trim()).filter(Boolean);
|
||||
this.TELEGRAM_GROUP_ID = process.env.OPENCLAW_TELEGRAM_GROUP ?? '';
|
||||
this.ALERT_TOPIC = process.env.OPENCLAW_ALERT_TOPIC ?? '';
|
||||
this.AGGRESSIVE = process.env.OPENCLAW_RESTART_SWEEP_AGGRESSIVE === '1';
|
||||
|
||||
const gbrainHome = process.env.GBRAIN_HOME ?? path.join(os.homedir(), '.gbrain');
|
||||
this.STATE_DIR = path.join(gbrainHome, 'integrations', 'restart-sweep');
|
||||
this.LOG_PATH = path.join(this.STATE_DIR, 'sweep.log.jsonl');
|
||||
this.ALERTED_PATH = path.join(this.STATE_DIR, 'alerted.json');
|
||||
this.BOOTSTRAP_LOG = process.env.OPENCLAW_BOOTSTRAP_LOG ?? '/tmp/bootstrap-services.log';
|
||||
|
||||
// DI hooks (default to real implementations)
|
||||
this._execFile = deps.execFile ?? execFile;
|
||||
this._runOpenclawSessions = deps.runOpenclawSessions ?? null;
|
||||
|
||||
this.sessions = null;
|
||||
this.restartTime = null;
|
||||
this.alertMode = this.determineAlertMode();
|
||||
this.alerted = new Map(); // populated in run() / loadAlerted()
|
||||
}
|
||||
|
||||
determineAlertMode() {
|
||||
if (this.TELEGRAM_GROUP_ID && this.ALERT_TOPIC) return 'telegram';
|
||||
if (this.TELEGRAM_GROUP_ID) return 'telegram_stdout';
|
||||
return 'stdout';
|
||||
}
|
||||
|
||||
async run() {
|
||||
try {
|
||||
console.log('🔍 Starting restart message sweep detection...');
|
||||
|
||||
if (this.OWNER_IDS.length === 0) {
|
||||
console.warn('⚠️ No OPENCLAW_OWNER_IDS configured. Set this environment variable.');
|
||||
}
|
||||
if (!this.TELEGRAM_GROUP_ID) {
|
||||
console.warn('⚠️ No OPENCLAW_TELEGRAM_GROUP configured. Alerts will only go to stdout.');
|
||||
}
|
||||
|
||||
fs.mkdirSync(this.STATE_DIR, { recursive: true });
|
||||
this.alerted = await this.loadAlerted();
|
||||
|
||||
this.restartTime = await this.getLastRestartTime();
|
||||
console.log(`📅 Last restart detected at: ${new Date(this.restartTime).toISOString()}`);
|
||||
|
||||
this.sessions = await this.getSessionState();
|
||||
console.log(`📊 Found ${this.sessions.length} total sessions`);
|
||||
|
||||
const telegramSessions = this.filterTelegramSessions(this.sessions);
|
||||
console.log(`📱 Found ${telegramSessions.length} Telegram sessions`);
|
||||
|
||||
const droppedMessages = await this.detectDroppedMessages(telegramSessions);
|
||||
const newDrops = droppedMessages.filter(m => !this.isInCooldown(m.sessionKey));
|
||||
const suppressedCount = droppedMessages.length - newDrops.length;
|
||||
|
||||
if (newDrops.length > 0) {
|
||||
const tail = suppressedCount > 0 ? ` (${suppressedCount} suppressed by cooldown)` : '';
|
||||
console.log(`⚠️ Found ${newDrops.length} potentially dropped message(s)${tail}`);
|
||||
await this.recordAndAlert(newDrops);
|
||||
} else if (suppressedCount > 0) {
|
||||
console.log(`✅ All ${suppressedCount} candidate(s) suppressed by cooldown`);
|
||||
} else {
|
||||
console.log('✅ No dropped messages detected');
|
||||
}
|
||||
|
||||
await this.logResults(droppedMessages);
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Error in message sweep:', error);
|
||||
await this.logError(error);
|
||||
}
|
||||
}
|
||||
|
||||
async getLastRestartTime() {
|
||||
try {
|
||||
const logContent = await fsp.readFile(this.BOOTSTRAP_LOG, 'utf8');
|
||||
const gatewayLines = logContent.split('\n')
|
||||
.filter(line => line.includes('Gateway token synced') || line.includes('✅ OpenClaw gateway'))
|
||||
.reverse();
|
||||
if (gatewayLines.length > 0) {
|
||||
const match = gatewayLines[0].match(/^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})/);
|
||||
if (match) {
|
||||
return new Date(match[1] + ' UTC').getTime();
|
||||
}
|
||||
}
|
||||
return Date.now() - (RESTART_THRESHOLD_MINUTES * 60 * 1000);
|
||||
} catch (error) {
|
||||
console.warn('⚠️ Could not determine restart time from logs, using fallback');
|
||||
return Date.now() - (RESTART_THRESHOLD_MINUTES * 60 * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
async getSessionState() {
|
||||
if (this._runOpenclawSessions) {
|
||||
return await this._runOpenclawSessions();
|
||||
}
|
||||
try {
|
||||
const { stdout } = await execP('openclaw sessions --json');
|
||||
const sessionData = JSON.parse(stdout);
|
||||
return sessionData.sessions || [];
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to get session state:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
filterTelegramSessions(sessions) {
|
||||
if (!this.TELEGRAM_GROUP_ID) return [];
|
||||
return sessions.filter(session => {
|
||||
return session.key &&
|
||||
session.key.includes('telegram:group:' + this.TELEGRAM_GROUP_ID) &&
|
||||
session.kind === 'group';
|
||||
});
|
||||
}
|
||||
|
||||
async detectDroppedMessages(telegramSessions) {
|
||||
const droppedMessages = [];
|
||||
const recentRestartWindow = this.restartTime - PRE_RESTART_WINDOW_MS;
|
||||
const afterRestartWindow = this.restartTime + POST_RESTART_WINDOW_MS;
|
||||
|
||||
for (const session of telegramSessions) {
|
||||
try {
|
||||
const sessionUpdated = session.updatedAt;
|
||||
|
||||
// Primary: aborted last run is the strong signal
|
||||
if (session.abortedLastRun) {
|
||||
const topic = this._extractTopic(session.key);
|
||||
droppedMessages.push({
|
||||
sessionKey: session.key,
|
||||
topic,
|
||||
lastUpdate: new Date(sessionUpdated).toISOString(),
|
||||
sessionId: session.sessionId,
|
||||
abortedLastRun: true,
|
||||
reason: 'Session aborted on last run',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Secondary: timing-based gap detection — opt-in only (false-positive prone)
|
||||
if (!this.AGGRESSIVE) continue;
|
||||
|
||||
if (sessionUpdated >= recentRestartWindow &&
|
||||
sessionUpdated < this.restartTime &&
|
||||
Date.now() > afterRestartWindow) {
|
||||
const topic = this._extractTopic(session.key);
|
||||
droppedMessages.push({
|
||||
sessionKey: session.key,
|
||||
topic,
|
||||
lastUpdate: new Date(sessionUpdated).toISOString(),
|
||||
timeSinceUpdate: Math.floor((Date.now() - sessionUpdated) / 1000 / 60),
|
||||
sessionId: session.sessionId,
|
||||
suspiciousGap: true,
|
||||
reason: 'Active before restart, silent after',
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`⚠️ Error analyzing session ${session.key}:`, error);
|
||||
}
|
||||
}
|
||||
return droppedMessages;
|
||||
}
|
||||
|
||||
_extractTopic(sessionKey) {
|
||||
const m = sessionKey?.match(/:topic:(\d+)/);
|
||||
return m ? m[1] : 'unknown';
|
||||
}
|
||||
|
||||
/**
|
||||
* Cooldown layer (C1): suppresses re-alerts on the same sessionKey
|
||||
* for COOLDOWN_HOURS, regardless of whether the synthesized
|
||||
* restartTime matches. Cooldown wins when the bootstrap log is
|
||||
* missing and restartTime is unstable.
|
||||
*/
|
||||
isInCooldown(sessionKey) {
|
||||
const entry = this.alerted.get(sessionKey);
|
||||
if (!entry || !entry.lastAlertedAt) return false;
|
||||
const ageMs = Date.now() - new Date(entry.lastAlertedAt).getTime();
|
||||
return ageMs < COOLDOWN_HOURS * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
async loadAlerted() {
|
||||
try {
|
||||
const content = await fsp.readFile(this.ALERTED_PATH, 'utf8');
|
||||
const parsed = JSON.parse(content);
|
||||
const map = new Map();
|
||||
const cutoffMs = Date.now() - STALE_DAYS * 24 * 60 * 60 * 1000;
|
||||
for (const [key, entry] of Object.entries(parsed || {})) {
|
||||
if (entry && entry.lastAlertedAt) {
|
||||
const ts = new Date(entry.lastAlertedAt).getTime();
|
||||
if (Number.isFinite(ts) && ts >= cutoffMs) {
|
||||
map.set(key, entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
} catch (err) {
|
||||
if (err && err.code === 'ENOENT') return new Map();
|
||||
console.warn(`⚠️ Failed to load ${this.ALERTED_PATH}: ${err && err.message}; starting with empty state`);
|
||||
return new Map();
|
||||
}
|
||||
}
|
||||
|
||||
async saveAlerted() {
|
||||
const obj = Object.fromEntries(this.alerted);
|
||||
const json = JSON.stringify(obj, null, 2);
|
||||
const tmp = this.ALERTED_PATH + '.tmp';
|
||||
// Atomic on POSIX: write tmp, then rename. Note: this prevents
|
||||
// file corruption only — concurrent cron runs can still both
|
||||
// read old state, both decide to alert, both rename. Given
|
||||
// 5-min cadence and 2-5s runtime, overlap is rare and a
|
||||
// duplicate alert is preferable to a missed one.
|
||||
await fsp.writeFile(tmp, json);
|
||||
await fsp.rename(tmp, this.ALERTED_PATH);
|
||||
}
|
||||
|
||||
async recordAndAlert(droppedMessages) {
|
||||
let alertSent = false;
|
||||
try {
|
||||
await this.alertOnDroppedMessages(droppedMessages);
|
||||
alertSent = true;
|
||||
} catch (err) {
|
||||
console.error('❌ Failed to send alert (will retry next cycle):', err && err.message);
|
||||
}
|
||||
if (!alertSent) return;
|
||||
|
||||
const nowIso = new Date().toISOString();
|
||||
const restartIso = new Date(this.restartTime).toISOString();
|
||||
for (const msg of droppedMessages) {
|
||||
this.alerted.set(msg.sessionKey, {
|
||||
lastAlertedAt: nowIso,
|
||||
restartTime: restartIso,
|
||||
});
|
||||
}
|
||||
try {
|
||||
await this.saveAlerted();
|
||||
} catch (err) {
|
||||
console.warn('⚠️ Failed to save alerted state:', err && err.message);
|
||||
}
|
||||
}
|
||||
|
||||
async alertOnDroppedMessages(droppedMessages) {
|
||||
let alertText = `⚠️ Found ${droppedMessages.length} unprocessed message(s) after restart:\n\n`;
|
||||
for (const msg of droppedMessages.slice(0, 10)) {
|
||||
alertText += `• Topic ${msg.topic}: ${msg.reason} (last update: ${msg.lastUpdate})\n`;
|
||||
if (msg.timeSinceUpdate) {
|
||||
alertText += ` ${msg.timeSinceUpdate} minutes ago\n`;
|
||||
}
|
||||
}
|
||||
if (droppedMessages.length > 10) {
|
||||
alertText += `\n... and ${droppedMessages.length - 10} more`;
|
||||
}
|
||||
|
||||
switch (this.alertMode) {
|
||||
case 'telegram':
|
||||
await this.sendTelegramAlert(alertText);
|
||||
break;
|
||||
case 'telegram_stdout':
|
||||
console.log('📢 Would send Telegram alert, but no topic configured:');
|
||||
console.log(alertText);
|
||||
break;
|
||||
default:
|
||||
console.log('📢 Alert:');
|
||||
console.log(alertText);
|
||||
}
|
||||
}
|
||||
|
||||
async sendTelegramAlert(alertText) {
|
||||
// execFile (not exec): argv array, no shell interpretation,
|
||||
// shell metachars in env vars cannot inject commands.
|
||||
const argv = [
|
||||
'message', 'send',
|
||||
'--channel', 'telegram',
|
||||
'--target', this.TELEGRAM_GROUP_ID,
|
||||
'--thread-id', this.ALERT_TOPIC,
|
||||
'--message', alertText,
|
||||
];
|
||||
await new Promise((resolve, reject) => {
|
||||
this._execFile('openclaw', argv, (err, _stdout, stderr) => {
|
||||
if (err) {
|
||||
err.stderr = stderr;
|
||||
reject(err);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
console.log('📢 Alert sent to Telegram');
|
||||
}
|
||||
|
||||
async logResults(droppedMessages) {
|
||||
const logEntry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
restartTime: new Date(this.restartTime).toISOString(),
|
||||
droppedMessageCount: droppedMessages.length,
|
||||
droppedMessages,
|
||||
};
|
||||
try {
|
||||
await fsp.appendFile(this.LOG_PATH, JSON.stringify(logEntry) + '\n');
|
||||
} catch (error) {
|
||||
console.warn('⚠️ Failed to write log file:', error && error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async logError(error) {
|
||||
const errorEntry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
error: error && error.message,
|
||||
stack: error && error.stack,
|
||||
};
|
||||
try {
|
||||
await fsp.appendFile(this.LOG_PATH, 'ERROR: ' + JSON.stringify(errorEntry) + '\n');
|
||||
} catch (logError) {
|
||||
console.error('Failed to log error:', logError && logError.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run if executed directly
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const detector = new MessageSweepDetector();
|
||||
detector.run().catch(console.error);
|
||||
}
|
||||
|
||||
export default MessageSweepDetector;
|
||||
```
|
||||
|
||||
## Step 4: Dry-run
|
||||
|
||||
Run the script once manually with the env loaded, before wiring cron:
|
||||
|
||||
```bash
|
||||
set -a; source ~/openclaw/.env; set +a
|
||||
node ~/openclaw/scripts/restart-sweep.mjs
|
||||
```
|
||||
|
||||
Expected output (no drops):
|
||||
|
||||
```
|
||||
🔍 Starting restart message sweep detection...
|
||||
📅 Last restart detected at: 2026-05-06T12:53:45.000Z
|
||||
📊 Found 48 total sessions
|
||||
📱 Found 39 Telegram sessions
|
||||
✅ No dropped messages detected
|
||||
```
|
||||
|
||||
If you want to see the alert path, manually edit a session in OpenClaw
|
||||
to set `abortedLastRun: true` and re-run. After the alert fires, check
|
||||
`~/.gbrain/integrations/restart-sweep/alerted.json` — the sessionKey
|
||||
should be there with a `lastAlertedAt` timestamp. Re-running within 6
|
||||
hours suppresses the alert.
|
||||
|
||||
## Step 5: Wire 5-minute cron
|
||||
|
||||
Cron does NOT inherit your shell environment. `openclaw` and `node` may
|
||||
not be on cron's stripped PATH. `.env` files don't auto-load. Use the
|
||||
wrapper-script pattern below to handle both.
|
||||
|
||||
Create `~/openclaw/scripts/restart-sweep-wrapper.sh`:
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set -a
|
||||
source ~/openclaw/.env
|
||||
set +a
|
||||
exec /usr/local/bin/node ~/openclaw/scripts/restart-sweep.mjs
|
||||
```
|
||||
|
||||
```bash
|
||||
chmod +x ~/openclaw/scripts/restart-sweep-wrapper.sh
|
||||
```
|
||||
|
||||
Adjust `/usr/local/bin/node` to wherever your `node` actually lives
|
||||
(`which node` to find it). Same for `openclaw` if the wrapper needs to
|
||||
add it to PATH explicitly:
|
||||
|
||||
```bash
|
||||
export PATH=/usr/local/bin:/usr/bin:/bin:$PATH
|
||||
```
|
||||
|
||||
Add to crontab via `crontab -e`:
|
||||
|
||||
```cron
|
||||
PATH=/usr/local/bin:/usr/bin:/bin
|
||||
*/5 * * * * /bin/bash ~/openclaw/scripts/restart-sweep-wrapper.sh >> ~/.gbrain/integrations/restart-sweep/cron.log 2>&1
|
||||
```
|
||||
|
||||
Verify with `crontab -l`. Wait 5 minutes, then check the cron log to
|
||||
confirm it ran:
|
||||
|
||||
```bash
|
||||
tail -20 ~/.gbrain/integrations/restart-sweep/cron.log
|
||||
```
|
||||
|
||||
## Step 6: Verification
|
||||
|
||||
1. `gbrain integrations doctor restart-sweep` — should pass all three
|
||||
health checks
|
||||
2. `~/.gbrain/integrations/restart-sweep/sweep.log.jsonl` exists and
|
||||
gets a new entry every 5 minutes
|
||||
3. `~/.gbrain/integrations/restart-sweep/cron.log` shows successful
|
||||
invocations (no PATH errors, no `command not found`)
|
||||
4. After a real OpenClaw restart with a stuck session, the Telegram
|
||||
alert fires once, then the cooldown layer suppresses repeats for 6h
|
||||
|
||||
## Tuning
|
||||
|
||||
`OPENCLAW_RESTART_SWEEP_AGGRESSIVE=1` — enables the secondary
|
||||
"active-before-restart, silent-after" heuristic. Off by default because
|
||||
during normal quiet periods (overnight, weekends) it false-positives.
|
||||
Enable if you want maximum sensitivity AND you've established that your
|
||||
group is consistently active.
|
||||
|
||||
The cooldown threshold (6 hours) is a constant in the script. Edit
|
||||
`COOLDOWN_HOURS` if you need different behavior — e.g. 24 hours if your
|
||||
group's normal cadence is daily.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Alerts firing repeatedly on the same session
|
||||
|
||||
Check `~/.gbrain/integrations/restart-sweep/alerted.json`. If the
|
||||
sessionKey is missing or `lastAlertedAt` is recent, the cooldown should
|
||||
suppress. If it's not suppressing:
|
||||
|
||||
- The state file may not be writable. Check `ls -ld
|
||||
~/.gbrain/integrations/restart-sweep/`.
|
||||
- `GBRAIN_HOME` may be set to a different path under cron than under
|
||||
your shell. Check the wrapper script's env loading.
|
||||
- The script's `STATE_DIR` resolution prints in stderr if mkdir fails.
|
||||
Check the cron log.
|
||||
|
||||
### Telegram alert fails silently
|
||||
|
||||
The script logs `❌ Failed to send alert (will retry next cycle)` to
|
||||
stderr when `openclaw message send` returns non-zero. Common causes:
|
||||
|
||||
- `openclaw` not on cron's PATH (use absolute path in the wrapper)
|
||||
- Telegram bot token expired or rate-limited
|
||||
- Wrong group/topic ID (try `openclaw message send --channel telegram
|
||||
--target $OPENCLAW_TELEGRAM_GROUP --message test` manually)
|
||||
|
||||
When the send fails, state is NOT updated, so next cycle retries.
|
||||
|
||||
### Bootstrap log missing
|
||||
|
||||
If `/tmp/bootstrap-services.log` (or `$OPENCLAW_BOOTSTRAP_LOG`) doesn't
|
||||
exist, the script falls back to `now() - 30 minutes` for restartTime.
|
||||
The cooldown layer keeps this from spamming. If you want a stable
|
||||
restart anchor, point `OPENCLAW_BOOTSTRAP_LOG` at OpenClaw's actual
|
||||
startup log (whatever your deployment uses).
|
||||
|
||||
### Cron environment
|
||||
|
||||
The wrapper script in Step 5 handles 80% of cron-day-one failures, but
|
||||
two more knobs:
|
||||
|
||||
- **Locale:** if your script ever interpolates user-provided text into
|
||||
log lines, set `LANG=en_US.UTF-8` in the cron entry to avoid mojibake.
|
||||
- **Working directory:** cron starts in `$HOME` by default. The script
|
||||
uses absolute paths everywhere, so this shouldn't matter, but if you
|
||||
ever add a relative-path dependency, `cd ~/openclaw` in the wrapper.
|
||||
|
||||
## Future upgrade path
|
||||
|
||||
This recipe is the v1 shape: a script copied into the host repo and
|
||||
wired to cron. The v2 shape is a plugin Minion handler registered in
|
||||
the OpenClaw repo against `gbrain/minions` (see
|
||||
`docs/guides/plugin-handlers.md`). Plugin-handler advantages:
|
||||
|
||||
- Built-in queue idempotency (no cooldown layer needed)
|
||||
- Submit via `gbrain jobs submit restart-sweep` from any cron / agent /
|
||||
manual trigger
|
||||
- Centralized retry / backoff / lock management
|
||||
- One less host script to maintain
|
||||
|
||||
When this becomes the right tradeoff (multiple deployments, multiple
|
||||
cron schedules, or just enough complexity to justify the move), promote
|
||||
to the plugin-handler shape and deprecate this recipe.
|
||||
@@ -1,134 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Generates `src/admin-embedded.ts` from `admin/dist/*`.
|
||||
*
|
||||
* Why: `bun build --compile` does NOT embed arbitrary asset directories.
|
||||
* The only way to ship a file inside a compiled binary is via an ESM
|
||||
* `import x from './path' with { type: 'file' }` reference (which Bun
|
||||
* resolves at runtime to a path that works inside the binary archive).
|
||||
*
|
||||
* Pre-v0.36.x, `serve-http.ts:780` resolved `admin/dist/` via
|
||||
* `process.cwd()` — fine in dev (`cd ~/gbrain && bun start serve --http`),
|
||||
* broken in every globally-installed binary (no admin/dist next to the
|
||||
* binary). Result: every fresh `bun install -g github:garrytan/gbrain`
|
||||
* user got 404 on /admin (issue #1090).
|
||||
*
|
||||
* This generator emits one `import` line per file under admin/dist/,
|
||||
* plus a manifest map keyed by the request path the express handler
|
||||
* sees (e.g. `/admin/index.html`, `/admin/assets/index-XXX.js`).
|
||||
*
|
||||
* Run: `bun run scripts/build-admin-embedded.ts` (also invoked by
|
||||
* `bun run build:admin`).
|
||||
*
|
||||
* CI guard: `scripts/check-admin-embedded.sh` re-runs this generator
|
||||
* and `git diff --exit-code src/admin-embedded.ts` so PRs that change
|
||||
* admin/dist without regenerating the embedded module fail loud.
|
||||
*/
|
||||
|
||||
import { readdirSync, statSync, writeFileSync, existsSync, readFileSync } from 'fs';
|
||||
import { join, relative, posix } from 'path';
|
||||
|
||||
const REPO = join(import.meta.dir, '..');
|
||||
const DIST = join(REPO, 'admin', 'dist');
|
||||
const OUT = join(REPO, 'src', 'admin-embedded.ts');
|
||||
|
||||
function walk(dir: string, base: string = dir): string[] {
|
||||
if (!existsSync(dir)) return [];
|
||||
const out: string[] = [];
|
||||
for (const entry of readdirSync(dir)) {
|
||||
const full = join(dir, entry);
|
||||
if (statSync(full).isDirectory()) {
|
||||
out.push(...walk(full, base));
|
||||
} else {
|
||||
out.push(relative(base, full));
|
||||
}
|
||||
}
|
||||
return out.sort();
|
||||
}
|
||||
|
||||
const MIME: Record<string, string> = {
|
||||
'.html': 'text/html; charset=utf-8',
|
||||
'.css': 'text/css; charset=utf-8',
|
||||
'.js': 'application/javascript; charset=utf-8',
|
||||
'.json': 'application/json; charset=utf-8',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.png': 'image/png',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.gif': 'image/gif',
|
||||
'.webp': 'image/webp',
|
||||
'.ico': 'image/x-icon',
|
||||
'.woff': 'font/woff',
|
||||
'.woff2': 'font/woff2',
|
||||
'.txt': 'text/plain; charset=utf-8',
|
||||
'.map': 'application/json; charset=utf-8',
|
||||
};
|
||||
|
||||
function mimeFor(filename: string): string {
|
||||
const dot = filename.lastIndexOf('.');
|
||||
if (dot === -1) return 'application/octet-stream';
|
||||
return MIME[filename.slice(dot).toLowerCase()] ?? 'application/octet-stream';
|
||||
}
|
||||
|
||||
function safeIdent(rel: string, idx: number): string {
|
||||
// Stable, collision-free identifier per relative path. The numeric
|
||||
// suffix prevents collisions between filenames that normalize to the
|
||||
// same identifier (e.g. `foo.bar.js` and `foo-bar.js`).
|
||||
const cleaned = rel.replace(/[^a-zA-Z0-9]/g, '_').replace(/^_+/, '');
|
||||
return `A_${idx}_${cleaned}`;
|
||||
}
|
||||
|
||||
const files = walk(DIST);
|
||||
if (files.length === 0) {
|
||||
console.error('[build-admin-embedded] no files under admin/dist — run `cd admin && bun run build` first.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const imports: string[] = [];
|
||||
const manifestEntries: string[] = [];
|
||||
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const rel = files[i];
|
||||
// POSIX-style relative path for the import (works on Windows too).
|
||||
const importRel = `../admin/dist/${rel.split(/[\\/]/).join('/')}`;
|
||||
const ident = safeIdent(rel, i);
|
||||
// @ts-ignore — `with { type: 'file' }` is Bun syntax not in lib.d.ts;
|
||||
// same pattern as src/core/chunkers/code.ts wasm imports.
|
||||
imports.push(`// @ts-ignore — type: 'file' is Bun ESM, not in lib.d.ts`);
|
||||
imports.push(`import ${ident} from '${importRel}' with { type: 'file' };`);
|
||||
const requestPath = '/admin/' + rel.split(/[\\/]/).join('/');
|
||||
manifestEntries.push(` ${JSON.stringify(requestPath)}: { path: ${ident} as unknown as string, mime: ${JSON.stringify(mimeFor(rel))} },`);
|
||||
}
|
||||
|
||||
const content = `// AUTO-GENERATED — do not edit by hand.
|
||||
// Run \`bun run scripts/build-admin-embedded.ts\` to regenerate.
|
||||
// Source: admin/dist/ at ${new Date().toISOString().slice(0, 10)}.
|
||||
//
|
||||
// Bun resolves the file: imports to a path that works at runtime even
|
||||
// inside a compiled binary (\`bun build --compile\`). The manifest maps
|
||||
// the request path the express handler sees to (resolved-path, mime).
|
||||
|
||||
${imports.join('\n')}
|
||||
|
||||
export interface AdminAsset {
|
||||
path: string;
|
||||
mime: string;
|
||||
}
|
||||
|
||||
export const ADMIN_ASSETS: Record<string, AdminAsset> = {
|
||||
${manifestEntries.join('\n')}
|
||||
};
|
||||
|
||||
/** Index entry point for SPA fallback. */
|
||||
export const ADMIN_INDEX_HTML: AdminAsset = ADMIN_ASSETS['/admin/index.html'];
|
||||
|
||||
export const ADMIN_ASSET_COUNT = ${files.length};
|
||||
`;
|
||||
|
||||
const existing = existsSync(OUT) ? readFileSync(OUT, 'utf-8') : '';
|
||||
if (existing === content) {
|
||||
console.log(`[build-admin-embedded] up to date (${files.length} files)`);
|
||||
} else {
|
||||
writeFileSync(OUT, content, 'utf-8');
|
||||
console.log(`[build-admin-embedded] wrote ${OUT} (${files.length} files)`);
|
||||
}
|
||||
@@ -1,308 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* scripts/build-contradictions-fixture.ts (v0.32.6, T2)
|
||||
*
|
||||
* Build a privacy-redacted gold fixture for the contradiction probe judge
|
||||
* by running the probe against the user's REAL brain and hand-labeling
|
||||
* the candidate pairs. Output: test/fixtures/contradictions-eval-gold.jsonl.
|
||||
*
|
||||
* Privacy posture (CLAUDE.md rule): the operator MUST inspect the
|
||||
* generated file before commit. The redactor (fixture-redact.ts) is
|
||||
* best-effort; the pre-commit review is the safety net. Fail-closed if
|
||||
* any pair fails the isCleanForCommit check after redaction.
|
||||
*
|
||||
* Usage:
|
||||
* bun run scripts/build-contradictions-fixture.ts \
|
||||
* [--queries-file FILE.jsonl] \
|
||||
* [--top-k N=5] \
|
||||
* [--judge MODEL=claude-haiku-4-5] \
|
||||
* [--max-pairs N=50] \
|
||||
* [--output PATH=test/fixtures/contradictions-eval-gold.jsonl] \
|
||||
* [--non-interactive]
|
||||
*
|
||||
* Interactive flow:
|
||||
* - Probe runs with --no-cache (so candidate pairs aren't pre-judged).
|
||||
* - For each candidate pair, the script prints A + B and prompts:
|
||||
* y) contradiction, n) not contradiction, s) skip
|
||||
* If y: prompt for severity (low|medium|high) and one-line axis.
|
||||
* - After labeling, redact in-memory, write JSONL with audit comments.
|
||||
* - Pre-commit safety: isCleanForCommit per line. Failures abort with
|
||||
* a sentinel string the operator must resolve manually.
|
||||
*
|
||||
* Non-interactive flow (`--non-interactive`): captures candidates with
|
||||
* NO labels, redacts, writes JSONL. Operator labels manually later.
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
|
||||
import { dirname } from 'node:path';
|
||||
import { createInterface } from 'node:readline/promises';
|
||||
import { stdin as input, stdout as output } from 'node:process';
|
||||
import { loadConfig, toEngineConfig } from '../src/core/config.ts';
|
||||
import { createEngine } from '../src/core/engine-factory.ts';
|
||||
import { connectWithRetry } from '../src/core/db.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
import { runContradictionProbe } from '../src/core/eval-contradictions/runner.ts';
|
||||
|
||||
async function connectLocalEngine(): Promise<BrainEngine> {
|
||||
const cfg = loadConfig();
|
||||
if (!cfg) throw new Error('No brain configured. Run `gbrain init` first.');
|
||||
const engineCfg = toEngineConfig(cfg);
|
||||
const engine = await createEngine(engineCfg);
|
||||
await connectWithRetry(engine, engineCfg, { noRetry: false });
|
||||
return engine;
|
||||
}
|
||||
import {
|
||||
createRedactionSession,
|
||||
isCleanForCommit,
|
||||
redactSlug,
|
||||
redactText,
|
||||
} from '../src/core/eval-contradictions/fixture-redact.ts';
|
||||
import type { ContradictionPair, Severity } from '../src/core/eval-contradictions/types.ts';
|
||||
|
||||
interface ParsedFlags {
|
||||
queriesFile?: string;
|
||||
topK: number;
|
||||
judge: string;
|
||||
maxPairs: number;
|
||||
output: string;
|
||||
nonInteractive: boolean;
|
||||
help: boolean;
|
||||
}
|
||||
|
||||
function parseFlags(argv: string[]): ParsedFlags {
|
||||
const f: ParsedFlags = {
|
||||
topK: 5,
|
||||
judge: 'anthropic:claude-haiku-4-5',
|
||||
maxPairs: 50,
|
||||
output: 'test/fixtures/contradictions-eval-gold.jsonl',
|
||||
nonInteractive: false,
|
||||
help: false,
|
||||
};
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
const next = (): string => {
|
||||
const v = argv[++i];
|
||||
if (v === undefined) throw new Error(`flag ${a} requires a value`);
|
||||
return v;
|
||||
};
|
||||
if (a === '--help' || a === '-h') f.help = true;
|
||||
else if (a === '--queries-file') f.queriesFile = next();
|
||||
else if (a === '--top-k') f.topK = Number.parseInt(next(), 10);
|
||||
else if (a === '--judge') f.judge = next();
|
||||
else if (a === '--max-pairs') f.maxPairs = Number.parseInt(next(), 10);
|
||||
else if (a === '--output') f.output = next();
|
||||
else if (a === '--non-interactive') f.nonInteractive = true;
|
||||
else throw new Error(`unknown flag: ${a}`);
|
||||
}
|
||||
return f;
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
process.stderr.write(`Build a privacy-redacted gold fixture for the contradiction probe judge.
|
||||
|
||||
Usage:
|
||||
bun run scripts/build-contradictions-fixture.ts \\
|
||||
--queries-file FILE.jsonl # one JSON object per line, {query: "..."}
|
||||
[--top-k N=5]
|
||||
[--judge MODEL=claude-haiku-4-5]
|
||||
[--max-pairs N=50]
|
||||
[--output PATH=test/fixtures/contradictions-eval-gold.jsonl]
|
||||
[--non-interactive]
|
||||
|
||||
Output: JSONL with one labeled-and-redacted pair per line. Lines that
|
||||
fail isCleanForCommit are marked with a sentinel string the operator
|
||||
MUST resolve manually before commit. Audit log printed to stderr.
|
||||
`);
|
||||
}
|
||||
|
||||
function readQueriesFile(path: string): string[] {
|
||||
const raw = readFileSync(path, 'utf8');
|
||||
const out: string[] = [];
|
||||
for (const line of raw.split(/\r?\n/)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
if (trimmed.startsWith('{')) {
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as { query?: string };
|
||||
if (typeof parsed.query === 'string' && parsed.query.length > 0) {
|
||||
out.push(parsed.query);
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
} else {
|
||||
out.push(trimmed);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function promptLabel(rl: ReturnType<typeof createInterface>, pair: ContradictionPair): Promise<{
|
||||
contradicts: boolean;
|
||||
severity: Severity;
|
||||
axis: string;
|
||||
skip: boolean;
|
||||
}> {
|
||||
process.stderr.write(`\n--- Pair ---\n`);
|
||||
process.stderr.write(`A (${pair.a.slug}): ${pair.a.text.slice(0, 240)}${pair.a.text.length > 240 ? '…' : ''}\n`);
|
||||
process.stderr.write(`B (${pair.b.slug}): ${pair.b.text.slice(0, 240)}${pair.b.text.length > 240 ? '…' : ''}\n`);
|
||||
const ans = (await rl.question('Contradiction? [y/n/s skip]: ')).trim().toLowerCase();
|
||||
if (ans === 's' || ans === 'skip') {
|
||||
return { contradicts: false, severity: 'low', axis: '', skip: true };
|
||||
}
|
||||
if (ans !== 'y' && ans !== 'yes') {
|
||||
return { contradicts: false, severity: 'low', axis: '', skip: false };
|
||||
}
|
||||
let sev = (await rl.question('Severity [low/medium/high, default low]: ')).trim().toLowerCase();
|
||||
if (sev !== 'low' && sev !== 'medium' && sev !== 'high') sev = 'low';
|
||||
const axis = (await rl.question('One-line axis: ')).trim();
|
||||
return { contradicts: true, severity: sev as Severity, axis, skip: false };
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
let flags: ParsedFlags;
|
||||
try {
|
||||
flags = parseFlags(process.argv.slice(2));
|
||||
} catch (err) {
|
||||
process.stderr.write(`Error: ${(err as Error).message}\n`);
|
||||
printHelp();
|
||||
process.exit(2);
|
||||
}
|
||||
if (flags.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!flags.queriesFile) {
|
||||
process.stderr.write(`--queries-file is required for the fixture build.\n`);
|
||||
printHelp();
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const queries = readQueriesFile(flags.queriesFile);
|
||||
if (queries.length === 0) {
|
||||
process.stderr.write(`No queries in ${flags.queriesFile}.\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
process.stderr.write(`Building gold fixture against the local brain.\n`);
|
||||
process.stderr.write(`Queries: ${queries.length} Top-K: ${flags.topK} Max pairs: ${flags.maxPairs}\n`);
|
||||
process.stderr.write(`Output: ${flags.output}\n\n`);
|
||||
|
||||
const engine = await connectLocalEngine();
|
||||
try {
|
||||
// Run the probe with --no-cache so we get candidate pairs without
|
||||
// pre-judged verdicts. We don't keep verdicts; we hand-label every pair.
|
||||
// We intercept pairs via judgeFn returning contradicts:false (so nothing
|
||||
// is filtered to findings) and accumulating them for labeling instead.
|
||||
const candidatePairs: ContradictionPair[] = [];
|
||||
await runContradictionProbe({
|
||||
engine,
|
||||
queries,
|
||||
judgeModel: flags.judge,
|
||||
topK: flags.topK,
|
||||
noCache: true,
|
||||
// Wide budget so we don't hit cap during candidate collection.
|
||||
budgetUsd: 100,
|
||||
yesOverride: true,
|
||||
// Hijack the judge to collect pairs without spending tokens.
|
||||
judgeFn: async (input) => {
|
||||
candidatePairs.push({
|
||||
kind: 'cross_slug_chunks', // best-effort label; runner emits both kinds
|
||||
a: { slug: input.a.slug, chunk_id: 0, take_id: null, source_tier: 'curated', holder: input.a.holder ?? null, text: input.a.text },
|
||||
b: { slug: input.b.slug, chunk_id: 0, take_id: null, source_tier: 'curated', holder: input.b.holder ?? null, text: input.b.text },
|
||||
combined_score: 0,
|
||||
});
|
||||
return {
|
||||
verdict: { contradicts: false, severity: 'low', axis: '', confidence: 0, resolution_kind: null },
|
||||
usage: { inputTokens: 0, outputTokens: 0 },
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
process.stderr.write(`\nCollected ${candidatePairs.length} candidate pairs.\n`);
|
||||
const capped = candidatePairs.slice(0, flags.maxPairs);
|
||||
|
||||
// Label.
|
||||
const rl = createInterface({ input, output });
|
||||
const session = createRedactionSession();
|
||||
const labeled: Array<{
|
||||
contradicts: boolean;
|
||||
severity: Severity;
|
||||
axis: string;
|
||||
query_redacted: string;
|
||||
a: { slug: string; text: string };
|
||||
b: { slug: string; text: string };
|
||||
}> = [];
|
||||
|
||||
for (let i = 0; i < capped.length; i++) {
|
||||
const pair = capped[i];
|
||||
process.stderr.write(`\n[${i + 1}/${capped.length}]`);
|
||||
let label: { contradicts: boolean; severity: Severity; axis: string; skip: boolean };
|
||||
if (flags.nonInteractive) {
|
||||
label = { contradicts: false, severity: 'low', axis: '', skip: false };
|
||||
} else {
|
||||
label = await promptLabel(rl, pair);
|
||||
if (label.skip) continue;
|
||||
}
|
||||
const redactedA = {
|
||||
slug: redactSlug(session, pair.a.slug),
|
||||
text: redactText(session, pair.a.text),
|
||||
};
|
||||
const redactedB = {
|
||||
slug: redactSlug(session, pair.b.slug),
|
||||
text: redactText(session, pair.b.text),
|
||||
};
|
||||
labeled.push({
|
||||
contradicts: label.contradicts,
|
||||
severity: label.severity,
|
||||
axis: redactText(session, label.axis),
|
||||
// Query gets redacted too, in case it referenced real names.
|
||||
query_redacted: '', // candidatePairs don't carry the query; populated by future iteration
|
||||
a: redactedA,
|
||||
b: redactedB,
|
||||
});
|
||||
}
|
||||
rl.close();
|
||||
|
||||
// Pre-commit safety: every text field must pass isCleanForCommit.
|
||||
const out: string[] = [];
|
||||
let flagged = 0;
|
||||
out.push(`# Gold fixture for contradiction probe judge (v0.32.6)`);
|
||||
out.push(`# schema_version: 1`);
|
||||
out.push(`# Generated: ${new Date().toISOString()}`);
|
||||
out.push(`# Audit (in-memory redactions applied):`);
|
||||
for (const entry of session.audit.slice(0, 100)) {
|
||||
out.push(`# ${entry}`);
|
||||
}
|
||||
out.push(`# Total redactions: ${session.audit.length}`);
|
||||
out.push(`#`);
|
||||
for (const row of labeled) {
|
||||
const cleanA = isCleanForCommit(row.a.text) && isCleanForCommit(row.a.slug);
|
||||
const cleanB = isCleanForCommit(row.b.text) && isCleanForCommit(row.b.slug);
|
||||
const sentinel = !cleanA || !cleanB ? ' [REDACT?]' : '';
|
||||
if (sentinel) flagged++;
|
||||
out.push(JSON.stringify({ ...row, ...(sentinel ? { _operator_review: 'REDACTION INCOMPLETE — fix manually before commit' } : {}) }));
|
||||
}
|
||||
|
||||
// Ensure output dir exists, then write.
|
||||
mkdirSync(dirname(flags.output), { recursive: true });
|
||||
if (existsSync(flags.output)) {
|
||||
process.stderr.write(`\nWARN: ${flags.output} already exists. Overwriting.\n`);
|
||||
}
|
||||
writeFileSync(flags.output, out.join('\n') + '\n');
|
||||
process.stderr.write(`\nWrote ${labeled.length} labeled pairs to ${flags.output}.\n`);
|
||||
if (flagged > 0) {
|
||||
process.stderr.write(`*** ${flagged} pair(s) flagged with [REDACT?] — review before commit ***\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
process.stderr.write(`OK — pre-commit safety pass. Inspect the file once more before committing.\n`);
|
||||
} finally {
|
||||
await engine.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
process.stderr.write(`fatal: ${(err as Error).message}\n`);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,64 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
// scripts/build-pglite-snapshot.ts
|
||||
//
|
||||
// Tier 3 fast-restore: boot a fresh PGLite, run the full initSchema (forward
|
||||
// bootstrap + PGLITE_SCHEMA_SQL + every migration), dump the post-init state
|
||||
// to a tar fixture. Test files that read GBRAIN_PGLITE_SNAPSHOT can skip the
|
||||
// 1-3 seconds of cold init and load the post-schema state directly.
|
||||
//
|
||||
// Output: test/fixtures/pglite-snapshot.tar (binary, gitignored)
|
||||
// test/fixtures/pglite-snapshot.version (hex SHA256 of MIGRATIONS SQL)
|
||||
//
|
||||
// The version file lets the engine detect snapshot staleness — if the tar's
|
||||
// recorded version doesn't match the current MIGRATIONS hash, the engine
|
||||
// ignores the snapshot and runs a normal initSchema.
|
||||
//
|
||||
// Run: bun run scripts/build-pglite-snapshot.ts
|
||||
// (or: bun run build:pglite-snapshot)
|
||||
//
|
||||
// Re-run whenever you touch src/core/migrate.ts or src/schema.sql.
|
||||
|
||||
import { writeFileSync, mkdirSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import * as crypto from "node:crypto";
|
||||
|
||||
import { PGLiteEngine, computeSnapshotSchemaHash } from "../src/core/pglite-engine.ts";
|
||||
import { MIGRATIONS } from "../src/core/migrate.ts";
|
||||
import { PGLITE_SCHEMA_SQL } from "../src/core/pglite-schema.ts";
|
||||
|
||||
function computeSchemaHash(): string {
|
||||
return computeSnapshotSchemaHash(MIGRATIONS, PGLITE_SCHEMA_SQL, crypto);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const fixturePath = "test/fixtures/pglite-snapshot.tar";
|
||||
const versionPath = "test/fixtures/pglite-snapshot.version";
|
||||
mkdirSync(dirname(fixturePath), { recursive: true });
|
||||
|
||||
const schemaHash = computeSchemaHash();
|
||||
console.log(`[build-pglite-snapshot] schema hash: ${schemaHash.slice(0, 16)}...`);
|
||||
console.log(`[build-pglite-snapshot] booting PGLite (in-memory)...`);
|
||||
const engine = new PGLiteEngine();
|
||||
|
||||
// Bypass the env-aware short-circuit: we WANT a real init here.
|
||||
delete process.env.GBRAIN_PGLITE_SNAPSHOT;
|
||||
|
||||
await engine.connect({});
|
||||
console.log(`[build-pglite-snapshot] running initSchema (forward bootstrap + ${MIGRATIONS.length} migrations)...`);
|
||||
const t0 = Date.now();
|
||||
await engine.initSchema();
|
||||
console.log(`[build-pglite-snapshot] initSchema completed in ${Date.now() - t0}ms`);
|
||||
|
||||
console.log(`[build-pglite-snapshot] dumping data dir...`);
|
||||
const dump = await engine.db.dumpDataDir("none");
|
||||
const buffer = Buffer.from(await dump.arrayBuffer());
|
||||
|
||||
writeFileSync(fixturePath, buffer);
|
||||
writeFileSync(versionPath, schemaHash + "\n");
|
||||
await engine.disconnect();
|
||||
|
||||
console.log(`[build-pglite-snapshot] wrote ${fixturePath} (${buffer.length} bytes)`);
|
||||
console.log(`[build-pglite-snapshot] wrote ${versionPath}`);
|
||||
}
|
||||
|
||||
await main();
|
||||
@@ -1,35 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI gate: admin React app must compile.
|
||||
#
|
||||
# Catches missing-symbol bugs (e.g., calling loadApiKeys() when only
|
||||
# loadAgents is defined) before they reach E2E. Codex flagged this gap
|
||||
# during the PR #586 review pass — five Claude review passes missed
|
||||
# the loadApiKeys reference because the bash test pipeline doesn't run
|
||||
# Vite builds. This script runs `bun install` in admin/ to ensure
|
||||
# react/vite/etc. are present, then runs Vite's build which performs
|
||||
# TypeScript type-check + bundle.
|
||||
#
|
||||
# Skip with GBRAIN_SKIP_ADMIN_BUILD=1 (e.g., for fast inner-loop test
|
||||
# runs that don't touch admin/src). Production CI must NOT skip.
|
||||
set -euo pipefail
|
||||
|
||||
if [ "${GBRAIN_SKIP_ADMIN_BUILD:-0}" = "1" ]; then
|
||||
echo "[check:admin-build] GBRAIN_SKIP_ADMIN_BUILD=1, skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [ ! -d admin ]; then
|
||||
echo "[check:admin-build] no admin/ directory, skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd admin
|
||||
|
||||
# Idempotent install — bun is fast enough on no-op (~50ms).
|
||||
bun install --silent >/dev/null 2>&1 || bun install
|
||||
|
||||
# Build runs `tsc -b && vite build`. Output to admin/dist/. Exit non-zero
|
||||
# on TS error, missing symbol, or Vite bundling error.
|
||||
bun run build
|
||||
@@ -1,35 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI gate: src/admin-embedded.ts must match admin/dist/ contents.
|
||||
#
|
||||
# This protects against the v0.36.x #1090 bug class re-emerging — a PR
|
||||
# that rebuilds admin/dist but forgets to regenerate src/admin-embedded.ts
|
||||
# would silently break /admin on every fresh install of the compiled
|
||||
# binary. The Vite build outputs hashed filenames, so a stale embedded
|
||||
# manifest references nonexistent assets.
|
||||
#
|
||||
# How: re-run the generator, then `git diff --exit-code` on the output.
|
||||
# Exits 0 when in sync, 1 when the generator produces different output
|
||||
# than what's committed.
|
||||
#
|
||||
# Mirrors scripts/check-wasm-embedded.sh's pattern.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [ ! -d admin/dist ]; then
|
||||
echo "[check:admin-embedded] no admin/dist (run \`cd admin && bun run build\` first); skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
bun run scripts/build-admin-embedded.ts > /dev/null
|
||||
|
||||
if ! git diff --exit-code -- src/admin-embedded.ts; then
|
||||
echo ""
|
||||
echo "[check:admin-embedded] src/admin-embedded.ts is out of sync with admin/dist/."
|
||||
echo " Fix: bun run build:admin && bun run build:admin-embedded"
|
||||
echo " Then re-commit the regenerated src/admin-embedded.ts."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[check:admin-embedded] OK"
|
||||
@@ -1,71 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Check that admin/src/lib/scope-constants.ts ALLOWED_SCOPES_LIST matches
|
||||
# src/core/scope.ts ALLOWED_SCOPES_LIST. The admin SPA's tsconfig include
|
||||
# scopes to admin/src/ so we can't import the source list directly; instead
|
||||
# this script extracts both lists and diffs them.
|
||||
#
|
||||
# Wired into `bun run verify` and `bun run check:all`.
|
||||
#
|
||||
# Exits 0 on match, 1 on drift, 2 on internal error (file missing, parse fail).
|
||||
#
|
||||
# Usage: scripts/check-admin-scope-drift.sh
|
||||
set -euo pipefail
|
||||
|
||||
SRC=src/core/scope.ts
|
||||
ADMIN=admin/src/lib/scope-constants.ts
|
||||
|
||||
[ -f "$SRC" ] || { echo "[check-admin-scope-drift] missing $SRC" >&2; exit 2; }
|
||||
[ -f "$ADMIN" ] || { echo "[check-admin-scope-drift] missing $ADMIN" >&2; exit 2; }
|
||||
|
||||
# Extract the contents of ALLOWED_SCOPES_LIST = [...] from each file.
|
||||
# The list spans multiple lines, terminated by ']'. awk pulls it cleanly.
|
||||
extract_list() {
|
||||
awk '
|
||||
/ALLOWED_SCOPES_LIST/ && /\[/ { capture = 1 }
|
||||
capture {
|
||||
print
|
||||
if (/\]/) { capture = 0; exit }
|
||||
}
|
||||
' "$1"
|
||||
}
|
||||
|
||||
src_block=$(extract_list "$SRC")
|
||||
admin_block=$(extract_list "$ADMIN")
|
||||
|
||||
if [ -z "$src_block" ]; then
|
||||
echo "[check-admin-scope-drift] could not find ALLOWED_SCOPES_LIST in $SRC" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ -z "$admin_block" ]; then
|
||||
echo "[check-admin-scope-drift] could not find ALLOWED_SCOPES_LIST in $ADMIN" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Strip everything that isn't a quoted scope string and emit one per line.
|
||||
strip_to_scopes() {
|
||||
printf '%s\n' "$1" \
|
||||
| tr ',' '\n' \
|
||||
| grep -oE "'[a-z_]+'" \
|
||||
| tr -d "'" \
|
||||
| sort -u
|
||||
}
|
||||
|
||||
src_scopes=$(strip_to_scopes "$src_block")
|
||||
admin_scopes=$(strip_to_scopes "$admin_block")
|
||||
|
||||
if [ "$src_scopes" != "$admin_scopes" ]; then
|
||||
echo "[check-admin-scope-drift] DRIFT detected between:" >&2
|
||||
echo " $SRC" >&2
|
||||
echo " $ADMIN" >&2
|
||||
echo "" >&2
|
||||
echo "src/core/scope.ts has:" >&2
|
||||
printf ' %s\n' $src_scopes >&2
|
||||
echo "" >&2
|
||||
echo "admin/src/lib/scope-constants.ts has:" >&2
|
||||
printf ' %s\n' $admin_scopes >&2
|
||||
echo "" >&2
|
||||
echo "Update admin/src/lib/scope-constants.ts to match, then 'cd admin && bun run build'." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[check-admin-scope-drift] ok: $(echo "$src_scopes" | wc -l | tr -d ' ') scopes match"
|
||||
@@ -1,23 +0,0 @@
|
||||
#!/bin/bash
|
||||
# CI guard: src/cli.ts must be tracked by git in executable mode (100755).
|
||||
#
|
||||
# Why: bun-link installs symlink to src/cli.ts directly. If the mode bit
|
||||
# regresses to 100644, the very first `gbrain --version` invocation fails
|
||||
# with `permission denied`. v0.28.5 (cluster C, #683) fixed the original
|
||||
# regression; this guard prevents future drift.
|
||||
#
|
||||
# Wired into `bun run verify`. Fast, no external deps.
|
||||
set -e
|
||||
|
||||
MODE=$(git ls-files --stage src/cli.ts | awk '{print $1}')
|
||||
if [ "$MODE" != "100755" ]; then
|
||||
echo "FAIL: src/cli.ts is tracked at mode $MODE; expected 100755 (executable)."
|
||||
echo ""
|
||||
echo "Fix: chmod +x src/cli.ts && git add --chmod=+x src/cli.ts"
|
||||
echo ""
|
||||
echo "Background: bun-link installs symlink to this file directly. Mode 100644"
|
||||
echo "produces 'permission denied' on first invocation (issue #683)."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: src/cli.ts is git-tracked as executable (100755)"
|
||||
@@ -1,43 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# v0.32.3 — CI guard for docs/eval/METRIC_GLOSSARY.md freshness.
|
||||
#
|
||||
# Mirrors the scripts/check-jsonb-pattern.sh / check-progress-to-stdout.sh
|
||||
# discipline: regenerate the doc into a tmp file, diff against the committed
|
||||
# version, fail the build if they drift.
|
||||
#
|
||||
# Run: bash scripts/check-eval-glossary-fresh.sh
|
||||
# CI wires this through `bun run test` so PRs that bump the glossary module
|
||||
# without regenerating the doc are caught before review.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
COMMITTED="$REPO_ROOT/docs/eval/METRIC_GLOSSARY.md"
|
||||
TMP="$(mktemp)"
|
||||
trap 'rm -f "$TMP"' EXIT
|
||||
|
||||
if [ ! -f "$COMMITTED" ]; then
|
||||
echo "ERROR: $COMMITTED not found." >&2
|
||||
echo "Run: bun run scripts/generate-metric-glossary.ts" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Regenerate into TMP without touching the committed file. We can't easily
|
||||
# point the generator at a different path; trick it by redirecting cwd to
|
||||
# a sandbox and post-comparing.
|
||||
cd "$REPO_ROOT"
|
||||
# Render directly via bun + a one-liner that exposes the module function.
|
||||
bun -e "import { renderMetricGlossaryMarkdown } from './src/core/eval/metric-glossary.ts'; process.stdout.write(renderMetricGlossaryMarkdown());" > "$TMP"
|
||||
|
||||
if ! diff -q "$COMMITTED" "$TMP" >/dev/null 2>&1; then
|
||||
echo "ERROR: docs/eval/METRIC_GLOSSARY.md is stale." >&2
|
||||
echo "" >&2
|
||||
echo "Diff between committed and freshly-generated:" >&2
|
||||
echo "" >&2
|
||||
diff -u "$COMMITTED" "$TMP" >&2 || true
|
||||
echo "" >&2
|
||||
echo "To regenerate: bun run scripts/generate-metric-glossary.ts" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ docs/eval/METRIC_GLOSSARY.md is fresh"
|
||||
@@ -1,48 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: the public exports surface never shrinks silently (v0.21.0).
|
||||
#
|
||||
# Precedent: scripts/check-jsonb-pattern.sh + check-progress-to-stdout.sh
|
||||
# are grep-based structural guards wired into `bun run test`. This one
|
||||
# counts the entries in package.json "exports" and fails when the count
|
||||
# drops below the v0.21.0 baseline (17 entries).
|
||||
#
|
||||
# Policy (from CLAUDE.md):
|
||||
# "Removing any of these is a breaking change going forward."
|
||||
#
|
||||
# If you're legitimately removing a public export: bump gbrain's minor
|
||||
# version, note the removal in CHANGELOG.md under a "Breaking changes"
|
||||
# bullet, then bump EXPECTED_COUNT below. Anything else is a regression.
|
||||
#
|
||||
# Adding a new export: update EXPECTED_COUNT to match AND extend the
|
||||
# EXPECTED_EXPORTS list in test/public-exports.test.ts so the runtime
|
||||
# contract test pins the canary symbol.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
EXPECTED_COUNT=18
|
||||
|
||||
# Count top-level keys in the exports object. `node -e` parses JSON
|
||||
# reliably without needing jq (which isn't in every CI environment).
|
||||
ACTUAL=$(node -e "
|
||||
const pkg = require('./package.json');
|
||||
console.log(Object.keys(pkg.exports || {}).length);
|
||||
")
|
||||
|
||||
if [ "$ACTUAL" -lt "$EXPECTED_COUNT" ]; then
|
||||
echo "❌ public-exports guard: package.json exports shrank from $EXPECTED_COUNT to $ACTUAL"
|
||||
echo " Removing a public export is a breaking change (see CLAUDE.md)."
|
||||
echo " If intentional: bump gbrain minor version + update EXPECTED_COUNT in"
|
||||
echo " scripts/check-exports-count.sh and EXPECTED_EXPORTS in"
|
||||
echo " test/public-exports.test.ts, AND add a CHANGELOG 'Breaking changes' bullet."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$ACTUAL" -gt "$EXPECTED_COUNT" ]; then
|
||||
echo "⚠️ public-exports guard: package.json exports grew from $EXPECTED_COUNT to $ACTUAL"
|
||||
echo " Additive public API change. Update EXPECTED_COUNT in this script + the"
|
||||
echo " EXPECTED_EXPORTS list in test/public-exports.test.ts to lock the new"
|
||||
echo " canary symbols."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ public-exports guard: $ACTUAL entries (matches baseline $EXPECTED_COUNT)"
|
||||
@@ -1,58 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: verify that bun --compile binaries can decode HEIC + AVIF.
|
||||
#
|
||||
# heic-decode bundles its libheif WASM as base64 inside libheif-bundle.js, which
|
||||
# bun --compile preserves correctly out of the box. @jsquash/avif loads
|
||||
# avif_dec.wasm via a path relative to its own JS file, which FAILS inside a
|
||||
# compiled binary — the workaround is to pre-init the module with bytes loaded
|
||||
# via `with { type: 'file' }`. This guard ensures both paths actually work in
|
||||
# the compiled artifact, not just in dev mode.
|
||||
#
|
||||
# Mirrors scripts/check-wasm-embedded.sh from v0.19.0 (tree-sitter pattern).
|
||||
#
|
||||
# Wired into `bun run verify` (which `/ship` and `bun run test:full` call).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
OUT_BIN="$(mktemp /tmp/gbrain-img-decoders-check.XXXXXX)"
|
||||
trap 'rm -f "$OUT_BIN"' EXIT
|
||||
|
||||
bun build --compile --outfile "$OUT_BIN" scripts/image-decoders-smoketest.ts >/dev/null 2>&1
|
||||
|
||||
OUTPUT="$("$OUT_BIN" 2>&1 || true)"
|
||||
|
||||
# The smoketest writes a JSON line on stdout. Look for ok=true on each decoder.
|
||||
if ! echo "$OUTPUT" | grep -q '"heic":{"ok":true'; then
|
||||
echo "[check-image-decoders-embedded] FAIL: heic-decode failed in compiled binary." >&2
|
||||
echo "[check-image-decoders-embedded] Output was:" >&2
|
||||
echo "$OUTPUT" >&2
|
||||
echo "" >&2
|
||||
echo "Likely cause: libheif-bundle.js was upgraded to a non-bundle variant," >&2
|
||||
echo "or wasm-bundle.js stopped inlining the WASM as base64. Check the" >&2
|
||||
echo "heic-decode + libheif-js versions in package.json." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! echo "$OUTPUT" | grep -q '"avif":{"ok":true'; then
|
||||
echo "[check-image-decoders-embedded] FAIL: @jsquash/avif failed in compiled binary." >&2
|
||||
echo "[check-image-decoders-embedded] Output was:" >&2
|
||||
echo "$OUTPUT" >&2
|
||||
echo "" >&2
|
||||
echo "Likely cause: the import attribute path for avif_dec.wasm changed in" >&2
|
||||
echo "@jsquash/avif, or initAvif() no longer accepts a WebAssembly.Module" >&2
|
||||
echo "directly. Check scripts/image-decoders-smoketest.ts for the WASM" >&2
|
||||
echo "pre-init pattern, then mirror it in src/core/import-file.ts." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Final guard: top-level "ok":true.
|
||||
if ! echo "$OUTPUT" | grep -q '"ok":true}$'; then
|
||||
echo "[check-image-decoders-embedded] FAIL: probe returned ok:false." >&2
|
||||
echo "$OUTPUT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[check-image-decoders-embedded] HEIC + AVIF decoders embed and decode correctly in compiled binary."
|
||||
@@ -1,85 +0,0 @@
|
||||
#!/bin/bash
|
||||
# CI guard against silent singleton reuse in connected-gbrains code paths.
|
||||
#
|
||||
# Codex finding #7 (plan review 2026-04-22): the module singleton in
|
||||
# src/core/db.ts is shared across the process. With multi-brain routing,
|
||||
# any `db.getConnection()` call in an op-dispatch code path means that op
|
||||
# silently targets whichever brain connected to the singleton first,
|
||||
# regardless of ctx.brainId / ctx.engine. This is exactly the bug Codex
|
||||
# #1 flagged in postgres-engine.ts internals.
|
||||
#
|
||||
# This script fails the build when NEW `db.getConnection()` calls appear
|
||||
# in src/core/operations.ts (the per-op handler surface) or in any new
|
||||
# `src/commands/*.ts` file. Existing legitimate callers are grandfathered
|
||||
# via an explicit allowlist — cleanups land in PR 1.
|
||||
#
|
||||
# When you hit this guard: instead of `db.getConnection()` or `db.connect(...)`,
|
||||
# use `ctx.engine` from the passed-in OperationContext. See
|
||||
# src/core/brain-registry.ts for how ctx.engine gets populated per-call.
|
||||
#
|
||||
# Run manually: bash scripts/check-no-legacy-getconnection.sh
|
||||
# Wired into CI: `bun test` (via package.json scripts.test)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
|
||||
cd "$ROOT"
|
||||
|
||||
# Files that are allowed to touch the singleton today. Every other file
|
||||
# under src/core or src/commands is forbidden. This list shrinks in PR 1.
|
||||
ALLOWED=(
|
||||
"src/core/db.ts" # the singleton's definition
|
||||
"src/core/postgres-engine.ts" # calls db.connect + fallback in sql getter — PR 1 removes the fallback
|
||||
"src/commands/init.ts" # first-time setup path, no engine yet
|
||||
"src/commands/doctor.ts" # PR 1 refactors to accept engine
|
||||
"src/commands/files.ts" # PR 1 refactors to accept engine
|
||||
"src/commands/repair-jsonb.ts" # PR 1 refactors
|
||||
"src/commands/serve-http.ts" # PR 1 threads engine through the OAuth dispatch path
|
||||
"src/commands/integrity.ts" # v0.22.8 batch-load fast path + scanIntegrityBatch; PR 1 refactors to accept engine
|
||||
"src/core/operations.ts" # 3 localOnly ops (file_list/upload/url) move to ctx.engine in PR 1
|
||||
)
|
||||
|
||||
# Build an argument list for `grep` that excludes allowed files.
|
||||
EXCLUDE_ARGS=()
|
||||
for file in "${ALLOWED[@]}"; do
|
||||
EXCLUDE_ARGS+=(--exclude="$file")
|
||||
done
|
||||
|
||||
# Search src/core/ and src/commands/ for db.getConnection or db.connect calls.
|
||||
# We look for the `db.` prefix so references to the symbol elsewhere (e.g.
|
||||
# the grep guard itself) don't trip the check.
|
||||
VIOLATIONS=$(
|
||||
grep -rn "db\.\(getConnection\|connect\)(" \
|
||||
--include="*.ts" \
|
||||
"${EXCLUDE_ARGS[@]}" \
|
||||
src/core src/commands 2>/dev/null \
|
||||
| grep -v -F "src/core/db.ts" \
|
||||
| grep -v "^[^:]*:[0-9]*:[[:space:]]*\(//\|\*\)" \
|
||||
|| true
|
||||
)
|
||||
|
||||
if [ -n "$VIOLATIONS" ]; then
|
||||
# Filter out allowed files from the result (the --exclude only matches basename)
|
||||
FILTERED=$(printf '%s\n' "$VIOLATIONS" | while IFS= read -r line; do
|
||||
path="${line%%:*}"
|
||||
allow=0
|
||||
for ok in "${ALLOWED[@]}"; do
|
||||
if [ "$path" = "$ok" ]; then allow=1; break; fi
|
||||
done
|
||||
if [ "$allow" -eq 0 ]; then printf '%s\n' "$line"; fi
|
||||
done)
|
||||
|
||||
if [ -n "$FILTERED" ]; then
|
||||
echo "ERROR: new direct db.getConnection() / db.connect() call found in multi-brain code path:" >&2
|
||||
echo "" >&2
|
||||
printf '%s\n' "$FILTERED" >&2
|
||||
echo "" >&2
|
||||
echo "Use ctx.engine from the passed-in OperationContext instead." >&2
|
||||
echo "See src/core/brain-registry.ts for the routing model." >&2
|
||||
echo "If this call is legitimate, add its path to the ALLOWED list in" >&2
|
||||
echo "scripts/check-no-legacy-getconnection.sh with a PR 1 cleanup note." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "check-no-legacy-getconnection: ok (no new singleton callers)"
|
||||
@@ -1,64 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: every `switch (X.type)` site in src/ that discriminates on a
|
||||
# PageType-shaped value MUST use assertNever() in the default branch.
|
||||
#
|
||||
# Why: extending PageType (e.g. v0.27.1 adding 'image') silently fell through
|
||||
# default branches in v0.20 / v0.22 because TypeScript couldn't catch the
|
||||
# missing case at type-check time. assertNever() forces the compiler to error
|
||||
# when a new PageType lacks a matching case.
|
||||
#
|
||||
# Today (pre-v0.27.1) the codebase has zero PageType-discriminating switches —
|
||||
# it uses the type system for exhaustiveness via union narrowing. This guard
|
||||
# is preventive: catches the moment a contributor adds a switch and forgets
|
||||
# the assertNever.
|
||||
#
|
||||
# Pattern: a `switch (x.type)` where the surrounding file imports PageType
|
||||
# (heuristic: imports from './types' or '../types') is treated as a
|
||||
# PageType-shaped switch and must include assertNever in default.
|
||||
#
|
||||
# False positives are easy to silence by adding an `// eslint-disable-line
|
||||
# pagetype-exhaustive` style comment above the offending switch.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
VIOLATIONS=0
|
||||
|
||||
# Find every src/**.ts file that imports PageType. Portable across Bash 3.2
|
||||
# (macOS default) — no mapfile, no process substitution arrays.
|
||||
PAGETYPE_FILES=$(grep -rlE "import.*PageType.*from.*types" src 2>/dev/null || true)
|
||||
|
||||
if [ -z "$PAGETYPE_FILES" ]; then
|
||||
echo "[check-pagetype-exhaustive] No files import PageType. Skipping."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while IFS= read -r file; do
|
||||
[ -z "$file" ] && continue
|
||||
# Look for `switch (X.type)` patterns in the file. Heuristic: any `switch (`
|
||||
# followed by a `.type)` within the line.
|
||||
if grep -nE 'switch\s*\([^)]*\.type\s*\)' "$file" >/dev/null 2>&1; then
|
||||
# File has at least one switch on .type. Verify assertNever is imported
|
||||
# AND used somewhere in the file. If both are present, assume the dev
|
||||
# wired it correctly — finer-grained per-switch checking is too brittle.
|
||||
if ! grep -qE 'assertNever' "$file"; then
|
||||
echo "[check-pagetype-exhaustive] FAIL: $file has switch(X.type) but no assertNever() use." >&2
|
||||
grep -nE 'switch\s*\([^)]*\.type\s*\)' "$file" >&2 || true
|
||||
VIOLATIONS=$((VIOLATIONS + 1))
|
||||
fi
|
||||
fi
|
||||
done <<< "$PAGETYPE_FILES"
|
||||
|
||||
if [ "$VIOLATIONS" -gt 0 ]; then
|
||||
echo "" >&2
|
||||
echo "Fix: import { assertNever } from './types.ts' (or wherever appropriate)" >&2
|
||||
echo "and add \`default: return assertNever(x.type);\` to the switch." >&2
|
||||
echo "If the switch is intentionally non-exhaustive (e.g. handling only a" >&2
|
||||
echo "subset of PageTypes), document why with a comment and add the file" >&2
|
||||
echo "to an explicit allow-list at the top of this script." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[check-pagetype-exhaustive] All PageType-discriminating switches use assertNever() (or none exist)."
|
||||
@@ -1,52 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI grep guard (v0.30.1, finding F3): no source file under src/ may emit
|
||||
# a postgresql:// URL with userinfo to a logging surface.
|
||||
#
|
||||
# Specifically we forbid string literals or template substitutions that
|
||||
# look like `postgresql://user:pass@host` being passed to:
|
||||
# - console.log / .warn / .error
|
||||
# - process.stderr.write / process.stdout.write
|
||||
# - appendFileSync / writeFileSync (audit JSONL writes)
|
||||
# - new logging APIs that may show up later (the regex matches the URL,
|
||||
# not the consumer; any leak will trip)
|
||||
#
|
||||
# Wired into bun run check:all and bun run verify.
|
||||
#
|
||||
# Exit codes: 0 = clean, 1 = found at least one suspect line.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT=$(cd "$(dirname "$0")/.." && pwd)
|
||||
|
||||
# False-positive allow-list: lines we know are safe.
|
||||
# - The redactor itself: src/core/url-redact.ts
|
||||
# - Test fixtures that build redacted strings from full URLs
|
||||
# - Documentation comments referring to the pattern
|
||||
ALLOW_REGEX='url-redact\.ts|test/url-redact\.test\.ts|/\* allow-pg-url-literal \*/'
|
||||
|
||||
# The pattern matches an unredacted Postgres URL appearing in a string
|
||||
# literal, NOT preceded by `redactPgUrl(` or `***@`. We also match any
|
||||
# URL containing `[^*]@` (i.e. the `***@` redacted form passes).
|
||||
PATTERN='postgres(ql)?://[^@*"`]+@'
|
||||
|
||||
# Search src/ only — tests are excluded since they intentionally construct
|
||||
# unredacted URLs as input fixtures.
|
||||
HITS=$(grep -rEn "$PATTERN" "$ROOT/src" 2>/dev/null || true)
|
||||
|
||||
if [ -z "$HITS" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Filter against the allow-list.
|
||||
FILTERED=$(echo "$HITS" | grep -vE "$ALLOW_REGEX" || true)
|
||||
|
||||
if [ -z "$FILTERED" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "ERROR: unredacted postgres:// URL found in source. Use redactPgUrl() before logging."
|
||||
echo ""
|
||||
echo "$FILTERED"
|
||||
echo ""
|
||||
echo "Allowed exemption: append \"/* allow-pg-url-literal */\" comment on the line"
|
||||
echo "(only for fixtures and the redactor itself)."
|
||||
exit 1
|
||||
@@ -26,14 +26,6 @@
|
||||
set -euo pipefail
|
||||
|
||||
BANNED_NAME='wintermute'
|
||||
# v0.25.1 (codex T7): additional patterns from wintermute-specific filesystem
|
||||
# layouts that would leak private fork context if they slipped through a port.
|
||||
# `wintermute_only` already matches via the case-insensitive `wintermute` regex
|
||||
# above; this list is for orthogonal patterns.
|
||||
BANNED_PATHS=(
|
||||
'/data/brain/'
|
||||
'/data/.openclaw/'
|
||||
)
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
@@ -100,64 +92,6 @@ ALLOW_LIST=(
|
||||
'llms-full.txt'
|
||||
'docs/UPGRADING_DOWNSTREAM_AGENTS.md'
|
||||
'test/integrations.test.ts'
|
||||
# v0.25.1 (codex T7) BANNED_PATHS allow-list:
|
||||
# Historical docs, frozen migration files, test fixtures, and env-var
|
||||
# fallbacks where /data/brain/ or /data/.openclaw/ appears legitimately.
|
||||
# New skills/, src/, and tests must NOT slip onto this list — extend the
|
||||
# banned check above instead.
|
||||
'docs/GBRAIN_RECOMMENDED_SCHEMA.md'
|
||||
'docs/GBRAIN_V0.md'
|
||||
'docs/guides/minions-shell-jobs.md'
|
||||
'scripts/smoke-test.sh'
|
||||
'skills/migrations/v0.9.0.md'
|
||||
'skills/migrations/v0.14.0.md'
|
||||
'test/storage-status.test.ts'
|
||||
# CHANGELOG.md documents the rule (the v0.25.1 entry references the
|
||||
# banned literals in describing what's banned). Same exception status
|
||||
# as CLAUDE.md and this script itself: meta-documentation needs to
|
||||
# name the patterns it forbids.
|
||||
'CHANGELOG.md'
|
||||
# skills/migrations/v0.25.1.md is the agent-readable upgrade
|
||||
# walkthrough; it explains the privacy-guard extension to the
|
||||
# operating agent and references the banned literals while doing so.
|
||||
'skills/migrations/v0.25.1.md'
|
||||
# v0.29.1: the recency-decay default-map test asserts that
|
||||
# DEFAULT_RECENCY_DECAY's keys do NOT include fork-specific path
|
||||
# prefixes. The test must name the banned tokens to assert their
|
||||
# absence — same exception status as scripts/check-privacy.sh,
|
||||
# CHANGELOG.md, and CLAUDE.md (meta-rule enforcement requires
|
||||
# mentioning what the rule forbids).
|
||||
'test/recency-decay.test.ts'
|
||||
# v0.32.5: the sibling check-test-real-names.sh enforces the same
|
||||
# privacy rule for test fixtures and lists the banned names literally
|
||||
# (Wintermute, Hermes, etc) inside its BANNED_NAMES + ALLOWLIST arrays.
|
||||
# Same meta-rule-enforcement exception as scripts/check-privacy.sh itself.
|
||||
'scripts/check-test-real-names.sh'
|
||||
# v0.34 / Lane CI: scripts/check-proposal-pii.sh and its test list the
|
||||
# banned literal as part of the structural denylist they enforce against
|
||||
# docs/proposals/*.md. Same meta-rule-enforcement exception as the two
|
||||
# entries above — describing what the rule forbids requires naming it.
|
||||
'scripts/check-proposal-pii.sh'
|
||||
'test/scripts/check-proposal-pii.test.ts'
|
||||
# v0.32.3.0: the functional-area-resolver skill's behavior-contract
|
||||
# section describes the privacy guarantees the skill preserves and
|
||||
# references the banned literals while doing so (line 306). Same
|
||||
# meta-rule-enforcement exception as scripts/check-privacy.sh and
|
||||
# CHANGELOG.md — describing what the rule forbids requires naming it.
|
||||
'skills/functional-area-resolver/SKILL.md'
|
||||
# v0.36.0.0: the gbrain skillpack harvest privacy linter's whole job
|
||||
# is to catch the banned literal leaking into gbrain. The regex
|
||||
# pattern in harvest-lint.ts is `\bWintermute\b` by necessity; the
|
||||
# tests verify that pattern fires by feeding it the banned string;
|
||||
# the harvest skill markdown describes the substitution policy
|
||||
# ("Wintermute → your OpenClaw") as part of the genericization
|
||||
# checklist. Same meta-rule-enforcement exception as the privacy
|
||||
# checks themselves.
|
||||
'src/core/skillpack/harvest-lint.ts'
|
||||
'test/skillpack-harvest-lint.test.ts'
|
||||
'test/skillpack-harvest.test.ts'
|
||||
'test/e2e/skillpack-flow.test.ts'
|
||||
'skills/skillpack-harvest/SKILL.md'
|
||||
)
|
||||
|
||||
is_allowed() {
|
||||
@@ -185,14 +119,6 @@ while IFS= read -r file; do
|
||||
grep -in "$BANNED_NAME" "$file" | sed 's|^| |' >&2
|
||||
FOUND=1
|
||||
fi
|
||||
# Banned wintermute-specific filesystem paths (codex T7).
|
||||
for path in "${BANNED_PATHS[@]}"; do
|
||||
if grep -nF "$path" "$file" >/dev/null 2>&1; then
|
||||
echo "[check-privacy] BANNED PATH '$path' in $file:" >&2
|
||||
grep -nF "$path" "$file" | sed 's|^| |' >&2
|
||||
FOUND=1
|
||||
fi
|
||||
done
|
||||
;;
|
||||
esac
|
||||
done <<< "$FILES"
|
||||
|
||||
@@ -1,166 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# check-proposal-pii.sh — privacy guard for `docs/proposals/*.md`.
|
||||
#
|
||||
# Sibling to check-privacy.sh: that script bans the `Wintermute` literal
|
||||
# everywhere. This one focuses on `docs/proposals/*.md` and the OTHER PII
|
||||
# classes that have surfaced in past RFC drafts — personal-relationship
|
||||
# vocabulary, private repo references, etc.
|
||||
#
|
||||
# Why two scripts: the patterns this lint flags would be too noisy if
|
||||
# applied repo-wide (e.g. a test fixture mentioning "trial" is fine).
|
||||
# Restricting to `docs/proposals/` keeps the lint surgical — proposals are
|
||||
# public-facing RFC documents that should never contain personal context,
|
||||
# so the false-positive rate is near zero.
|
||||
#
|
||||
# Design note: the denylist names PATTERNS, not real people. Specific
|
||||
# real names (deceased relatives, therapist names, dealflow contacts)
|
||||
# would leak PII into the repo just by appearing in this script's
|
||||
# denylist. The structural patterns below catch the SURROUNDING context
|
||||
# of personal-event prose. The trade-off: a future RFC that names a real
|
||||
# person without any of the contextual markers won't be caught — that's
|
||||
# accepted as a residual risk handled by human review.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/check-proposal-pii.sh # scan working tree
|
||||
# scripts/check-proposal-pii.sh --staged # scan git staged index
|
||||
# scripts/check-proposal-pii.sh --help
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 clean
|
||||
# 1 PII pattern found
|
||||
# 2 setup error
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
PROPOSALS_DIR="$REPO_ROOT/docs/proposals"
|
||||
|
||||
# Structural patterns. One per line. Matched case-insensitively, fixed-string
|
||||
# (no regex). Comments start with #. Blank lines OK.
|
||||
#
|
||||
# IMPORTANT — design contract: this list MUST NOT contain real personal
|
||||
# names (deceased relatives, therapist first names, dealflow contacts).
|
||||
# Naming those would leak PII into scripts/. The patterns below catch the
|
||||
# SURROUNDING VOCABULARY that always accompanies such content in personal
|
||||
# RFC prose. Maintainers extending this list: prefer adding a phrase that
|
||||
# captures the context (e.g. `couples session`) rather than a specific
|
||||
# person's name.
|
||||
read -r -d '' PATTERNS <<'EOF' || true
|
||||
# Private repo references (zero false-positive risk)
|
||||
garrytan/brain
|
||||
|
||||
# Personal relationship vocabulary (extremely unlikely in technical RFCs)
|
||||
trial separation
|
||||
permanent separation
|
||||
couples session
|
||||
couples therapist
|
||||
divorce attorney
|
||||
divorce attorneys
|
||||
|
||||
# Death/funeral vocabulary in personal contexts (combined phrases — bare
|
||||
# "funeral" alone would false-positive in legitimate metaphorical use)
|
||||
grandmother's funeral
|
||||
grandmother funeral
|
||||
aunt's funeral
|
||||
aunt funeral
|
||||
|
||||
# Private agent / fork name (also enforced repo-wide by check-privacy.sh
|
||||
# but listed here for proposal-scoped clarity)
|
||||
wintermute
|
||||
EOF
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
scripts/check-proposal-pii.sh — privacy guard for docs/proposals/*.md.
|
||||
|
||||
USAGE:
|
||||
scripts/check-proposal-pii.sh Scan all proposal files.
|
||||
scripts/check-proposal-pii.sh --staged Scan only staged proposal files.
|
||||
scripts/check-proposal-pii.sh --help Show this message.
|
||||
|
||||
Flags personal-context vocabulary (e.g. "trial separation", "couples
|
||||
session", private repo references) inside docs/proposals/*.md. Use
|
||||
generic placeholders (alice-example, acme-corp, fund-a) in proposals.
|
||||
See CLAUDE.md "Privacy rule: scrub real names from public docs" for
|
||||
the canonical name-mapping table.
|
||||
|
||||
Sibling to scripts/check-privacy.sh which enforces the "Wintermute"
|
||||
ban repo-wide; this script catches the broader PII classes that
|
||||
appeared in past RFC drafts and were corrected at landing time.
|
||||
|
||||
Exit codes: 0 clean, 1 pattern found, 2 setup error.
|
||||
EOF
|
||||
}
|
||||
|
||||
MODE=working
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--staged) MODE=staged ;;
|
||||
--help|-h) usage; exit 1 ;;
|
||||
*)
|
||||
echo "Unknown argument: $arg" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ! -d "$PROPOSALS_DIR" ]; then
|
||||
# No proposals dir yet — nothing to lint. Not a failure.
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Build the file list. Staged mode filters git's staged set down to
|
||||
# docs/proposals/*.md; working mode globs the directory directly.
|
||||
if [ "$MODE" = staged ]; then
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
echo "check-proposal-pii: git not found" >&2
|
||||
exit 2
|
||||
fi
|
||||
FILES=$(git diff --cached --name-only --diff-filter=ACMR 2>/dev/null \
|
||||
| grep -E '^docs/proposals/.+\.md$' || true)
|
||||
else
|
||||
FILES=$(find "$PROPOSALS_DIR" -maxdepth 1 -type f -name '*.md' 2>/dev/null \
|
||||
| sed "s|^$REPO_ROOT/||")
|
||||
fi
|
||||
|
||||
if [ -z "$FILES" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
FOUND=0
|
||||
# Iterate patterns; for each non-comment line, scan the file list.
|
||||
while IFS= read -r raw_line; do
|
||||
# Strip leading/trailing whitespace.
|
||||
pat="${raw_line#"${raw_line%%[![:space:]]*}"}"
|
||||
pat="${pat%"${pat##*[![:space:]]}"}"
|
||||
# Skip empty and comment lines.
|
||||
[ -z "$pat" ] && continue
|
||||
case "$pat" in '#'*) continue ;; esac
|
||||
|
||||
while IFS= read -r file; do
|
||||
[ -z "$file" ] && continue
|
||||
full="$REPO_ROOT/$file"
|
||||
[ ! -f "$full" ] && continue
|
||||
# Fixed-string (-F), case-insensitive (-i), with line numbers (-n).
|
||||
if matches=$(grep -nFi -- "$pat" "$full" 2>/dev/null); then
|
||||
if [ -n "$matches" ]; then
|
||||
echo "[check-proposal-pii] PII pattern in $file:" >&2
|
||||
echo " pattern: $pat" >&2
|
||||
echo "$matches" | sed 's|^| |' >&2
|
||||
FOUND=$((FOUND + 1))
|
||||
fi
|
||||
fi
|
||||
done <<< "$FILES"
|
||||
done <<< "$PATTERNS"
|
||||
|
||||
if [ "$FOUND" -gt 0 ]; then
|
||||
echo "" >&2
|
||||
echo "[check-proposal-pii] $FOUND PII pattern hit(s) in docs/proposals/*.md." >&2
|
||||
echo "[check-proposal-pii] See CLAUDE.md 'Privacy rule: scrub real names from public docs'." >&2
|
||||
echo "[check-proposal-pii] Use generic placeholders: alice-example, acme-corp, fund-a, etc." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -1,97 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: fail if any SELECT projection on `pages` that feeds rowToPage()
|
||||
# drops `source_id`. After v0.32.8, Page.source_id is required at the type
|
||||
# level; a projection that omits the column makes rowToPage return a Page
|
||||
# with source_id=undefined, which TypeScript's `: string` then lies about.
|
||||
#
|
||||
# This complements the type-system guard. The grep finds the specific 4-tuple
|
||||
# shape (id, slug, type, title) without source_id — the exact pre-v0.32.8
|
||||
# pattern that codex's plan review flagged.
|
||||
#
|
||||
# Usage: scripts/check-source-id-projection.sh
|
||||
# Exit: 0 when no matches, 1 when matches found.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Allowlist: SELECT shapes that legitimately don't need source_id (single-col
|
||||
# `SELECT slug FROM pages` for getAllSlugs / resolveSlugs, SELECT id for
|
||||
# subqueries, COUNT, etc.) These don't feed rowToPage.
|
||||
#
|
||||
# The shape that DOES feed rowToPage starts `SELECT id, ... slug, ... type, ... title`
|
||||
# (in some order). The pattern below matches "id" + "slug" + "type" + "title"
|
||||
# in a SELECT projection — that's the rowToPage feeder signature.
|
||||
|
||||
FOUND_BAD=0
|
||||
|
||||
# Use multiline-aware grep so the SELECT can span lines. pcre2grep would be
|
||||
# cleaner but isn't universally available; do a simple two-pass instead:
|
||||
# 1. Pull each SELECT-from-pages block.
|
||||
# 2. For each, check if it has the rowToPage signature WITHOUT source_id.
|
||||
|
||||
check_file() {
|
||||
local file="$1"
|
||||
# Extract every SELECT...FROM pages block (across lines, up to 12 lines)
|
||||
# then test each.
|
||||
awk '
|
||||
/SELECT/ {
|
||||
buf = $0
|
||||
lines = 1
|
||||
while (lines < 12 && (!match(buf, /FROM[[:space:]]+pages\b/))) {
|
||||
if ((getline next_line) <= 0) break
|
||||
buf = buf " " next_line
|
||||
lines++
|
||||
}
|
||||
if (match(buf, /FROM[[:space:]]+pages\b/)) {
|
||||
# Has id, slug, type, title (rowToPage feeder) but NO source_id?
|
||||
if (match(buf, /\bid\b/) && match(buf, /\bslug\b/) && match(buf, /\btype\b/) && match(buf, /\btitle\b/) && !match(buf, /\bsource_id\b/)) {
|
||||
print FILENAME ": SELECT projection missing source_id:"
|
||||
print " " buf
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
}
|
||||
' "$file" || return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
EXIT=0
|
||||
for f in src/core/postgres-engine.ts src/core/pglite-engine.ts; do
|
||||
if ! check_file "$f"; then
|
||||
EXIT=1
|
||||
fi
|
||||
done
|
||||
|
||||
# Also check RETURNING clauses (putPage uses INSERT ... RETURNING).
|
||||
# Same shape: returns a row that feeds rowToPage.
|
||||
for f in src/core/postgres-engine.ts src/core/pglite-engine.ts; do
|
||||
awk '
|
||||
/RETURNING/ {
|
||||
buf = $0
|
||||
lines = 1
|
||||
while (lines < 6 && !match(buf, /\`/)) {
|
||||
if ((getline next_line) <= 0) break
|
||||
buf = buf " " next_line
|
||||
lines++
|
||||
}
|
||||
if (match(buf, /\bid\b/) && match(buf, /\bslug\b/) && match(buf, /\btype\b/) && match(buf, /\btitle\b/) && !match(buf, /\bsource_id\b/)) {
|
||||
print FILENAME ": RETURNING projection missing source_id:"
|
||||
print " " buf
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
' "$f" || EXIT=1
|
||||
done
|
||||
|
||||
if [ "$EXIT" = 1 ]; then
|
||||
echo
|
||||
echo "ERROR: SELECT/RETURNING projection on \`pages\` is missing source_id."
|
||||
echo " After v0.32.8, Page.source_id is required at the type level."
|
||||
echo " Add \`source_id\` to the projection or rowToPage will lie."
|
||||
echo " See ~/.claude/plans/gleaming-soaring-mccarthy.md F2 finding."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: all rowToPage feeder projections include source_id"
|
||||
@@ -1,107 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# v0.36.1.0 (T20 / CDX-14) — privacy CI guard for the synthetic calibration corpus.
|
||||
#
|
||||
# Scans test/fixtures/calibration/ for patterns that look like real-world
|
||||
# specificity. Fails the build if any are found. Closes the synthetic-corpus
|
||||
# privacy hole flagged by codex review CDX-14: "CC reads real brain pages
|
||||
# locally, writes nothing still risks privacy if any generated synthetic
|
||||
# fixture memorizes structure-specific facts. Placeholder names are not enough."
|
||||
#
|
||||
# What this catches:
|
||||
# - Real dollar amounts (e.g. "$50M", "$1.2B")
|
||||
# - Specific large round counts ($X cap is OK; "$50M Series B" is not)
|
||||
# - Year-specific date strings outside the 2024-2026 placeholder range
|
||||
# - The real founder/company names from the operator's network (looked up
|
||||
# from a sibling file scripts/check-synthetic-corpus-allowlist.txt when
|
||||
# present; otherwise we just check the placeholder allow-list)
|
||||
#
|
||||
# False positives stay safer than false negatives — this guard biases toward
|
||||
# the operator manually verifying a flagged page is legitimately synthetic.
|
||||
|
||||
set -e
|
||||
|
||||
CORPUS_DIR="test/fixtures/calibration"
|
||||
PLACEHOLDERS=(
|
||||
"alice-example"
|
||||
"charlie-example"
|
||||
"acme-example"
|
||||
"widget-co"
|
||||
"fund-a"
|
||||
"fund-b"
|
||||
"fund-c"
|
||||
"acme-seed"
|
||||
"widget-series-a"
|
||||
"meetings/2026-"
|
||||
)
|
||||
|
||||
# Skip if directory doesn't exist yet (early-clone state).
|
||||
if [ ! -d "$CORPUS_DIR" ]; then
|
||||
echo "OK: $CORPUS_DIR does not exist yet (skipping privacy scan)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
VIOLATIONS=0
|
||||
|
||||
# Check 1: real dollar amounts. Synthetic pages should say "$X" or describe
|
||||
# amounts as ranges; explicit numerics like "$50M" suggest real-world specificity.
|
||||
echo "[corpus-privacy] checking for explicit dollar amounts..."
|
||||
while IFS= read -r match; do
|
||||
if [ -n "$match" ]; then
|
||||
echo " VIOLATION: explicit dollar amount in $match"
|
||||
VIOLATIONS=$((VIOLATIONS + 1))
|
||||
fi
|
||||
done < <(grep -rEn '\$[0-9]+[MBKkmb]\b' "$CORPUS_DIR" --include='*.md' 2>/dev/null || true)
|
||||
|
||||
# Check 2: explicit year-specific dates outside the 2024-2026 placeholder window.
|
||||
# The corpus uses placeholder timeline references like "2024-Q2", "2026-04-03".
|
||||
# Numbers like "2019" or "2027" mapped to specific events are suspicious.
|
||||
echo "[corpus-privacy] checking for out-of-range year references..."
|
||||
while IFS= read -r match; do
|
||||
if [ -n "$match" ]; then
|
||||
# Allow 2019 (used as a generic past year), 2023, 2027 (used as future). The
|
||||
# specific concern is dates the operator might recognize as a real prior event.
|
||||
# This is a low-precision heuristic; manual review decides.
|
||||
: # informational, not a failure for v0.36.1.0
|
||||
fi
|
||||
done < <(grep -rEn '\b(201[0-8]|2030|2031)\b' "$CORPUS_DIR" --include='*.md' 2>/dev/null || true)
|
||||
|
||||
# Check 3: presence of expected placeholders. Synthetic pages should reference
|
||||
# at least one canonical placeholder. A page with ZERO placeholder names is
|
||||
# suspicious — might be referring to real people/companies.
|
||||
echo "[corpus-privacy] checking that fixture pages reference at least one placeholder..."
|
||||
while IFS= read -r file; do
|
||||
has_placeholder=false
|
||||
for ph in "${PLACEHOLDERS[@]}"; do
|
||||
if grep -q "$ph" "$file" 2>/dev/null; then
|
||||
has_placeholder=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
# Allow README + label JSON files to skip this check.
|
||||
# Also allow essay-genre fixtures, which are anonymized PG-essay-style writing
|
||||
# and don't reference specific people/companies by design.
|
||||
case "$file" in
|
||||
*README.md|*labels.json|*/essay-*.md) continue ;;
|
||||
esac
|
||||
if [ "$has_placeholder" = "false" ]; then
|
||||
echo " VIOLATION: $file references no placeholder name (expected at least one of: ${PLACEHOLDERS[*]})"
|
||||
VIOLATIONS=$((VIOLATIONS + 1))
|
||||
fi
|
||||
done < <(find "$CORPUS_DIR" -name '*.md' -type f 2>/dev/null)
|
||||
|
||||
if [ "$VIOLATIONS" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "❌ $VIOLATIONS privacy violation(s) found in $CORPUS_DIR."
|
||||
echo ""
|
||||
echo "The synthetic calibration corpus must use anonymized placeholder names"
|
||||
echo "(see test/fixtures/calibration/README.md). Real names of YC partners,"
|
||||
echo "portfolio companies, funds, etc. cannot enter this directory."
|
||||
echo ""
|
||||
echo "Either:"
|
||||
echo " - replace the offending content with placeholder names"
|
||||
echo " - confirm the dollar amount is intentionally generic, then update"
|
||||
echo " this script to exempt it"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ corpus privacy: $VIOLATIONS violations across $(find "$CORPUS_DIR" -name '*.md' -type f 2>/dev/null | wc -l | tr -d ' ') pages"
|
||||
@@ -1,91 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# v0.32.2 CI guard: enforce the system-of-record invariant.
|
||||
#
|
||||
# The rule: user-knowledge writes to derived DB tables (facts, takes,
|
||||
# links, timeline_entries) must go through the extract / reconcile /
|
||||
# migration layer, never directly from arbitrary code paths. Direct
|
||||
# calls would bypass the markdown source-of-truth contract — the next
|
||||
# `gbrain rebuild` (v0.32.3) would lose the data because the fence
|
||||
# wasn't updated.
|
||||
#
|
||||
# This script grep-bans the direct-write surface across src/ and
|
||||
# scripts/ (NOT test/ — tests legitimately seed fixtures via direct
|
||||
# inserts, per Codex R2-#8). A function-scoped allow-list lets the
|
||||
# legitimate extract / reconcile / migration call sites pass: add
|
||||
# `// gbrain-allow-direct-insert: <reason>` on the SAME LINE as the
|
||||
# banned call. The grep parses the trailing comment.
|
||||
#
|
||||
# Usage: scripts/check-system-of-record.sh
|
||||
# Exit: 0 when no violations, 1 when violations found.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Banned direct-call patterns. Each is a method on BrainEngine that
|
||||
# writes to a derived table. Pre-v0.32.2 callers used these freely;
|
||||
# post-v0.32.2 every call site must either route through the
|
||||
# reconcile layer OR carry an explicit allow-direct-insert comment.
|
||||
PATTERNS=(
|
||||
'engine\.insertFact\('
|
||||
'engine\.insertFacts\('
|
||||
'engine\.addLink\('
|
||||
'engine\.addLinksBatch\('
|
||||
'engine\.addTimelineEntry\('
|
||||
'engine\.upsertTake\('
|
||||
'engine\.expireFact\('
|
||||
)
|
||||
|
||||
# Build an OR-regex for one grep pass.
|
||||
COMBINED=""
|
||||
for p in "${PATTERNS[@]}"; do
|
||||
if [ -z "$COMBINED" ]; then
|
||||
COMBINED="$p"
|
||||
else
|
||||
COMBINED="$COMBINED|$p"
|
||||
fi
|
||||
done
|
||||
|
||||
# Scan src/ and scripts/ only. test/ is deliberately excluded per Codex
|
||||
# R2-#8: tests legitimately call these methods to seed fixtures, and
|
||||
# gating tests would break the test surface without protecting any
|
||||
# invariant.
|
||||
SCOPE_DIRS=("src" "scripts")
|
||||
|
||||
# Collect violations. A violation is a line that:
|
||||
# 1. Matches one of the banned patterns
|
||||
# 2. Does NOT contain the `gbrain-allow-direct-insert:` comment
|
||||
# 3. Is NOT a pure-comment line (JSDoc, line-comment, backtick mention)
|
||||
# Comment-line exclusions stop the grep from false-positiving on
|
||||
# docstrings/comments that mention the method names. The runtime
|
||||
# regression coverage lives in the unit + E2E tests.
|
||||
violations=$(
|
||||
for dir in "${SCOPE_DIRS[@]}"; do
|
||||
[ -d "$dir" ] || continue
|
||||
grep -rEn --include='*.ts' --include='*.tsx' --include='*.js' --include='*.sh' \
|
||||
"$COMBINED" "$dir" 2>/dev/null || true
|
||||
done \
|
||||
| grep -vE 'gbrain-allow-direct-insert:' \
|
||||
| grep -vE ':[[:space:]]*\*[[:space:]]+' \
|
||||
| grep -vE ':[[:space:]]*//' \
|
||||
| grep -vE '`[^`]*\\.\w+\(' \
|
||||
|| true
|
||||
)
|
||||
|
||||
if [ -n "$violations" ]; then
|
||||
echo
|
||||
echo "ERROR: direct writes to derived tables found outside the reconcile layer."
|
||||
echo " Every call to engine.insertFact / insertFacts / addLink /"
|
||||
echo " addLinksBatch / addTimelineEntry / upsertTake / expireFact must"
|
||||
echo " either route through the extract / cycle / migration path OR"
|
||||
echo " carry an explicit \`// gbrain-allow-direct-insert: <reason>\`"
|
||||
echo " comment on the SAME LINE. See docs/architecture/system-of-record.md."
|
||||
echo
|
||||
echo "Violations:"
|
||||
echo "$violations"
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: no direct derived-table writes outside the reconcile layer in src/ + scripts/"
|
||||
@@ -1,74 +0,0 @@
|
||||
# v0.26.7 baseline allow-list for scripts/check-test-isolation.sh.
|
||||
#
|
||||
# Files here violate one or more of the lint rules (env mutation,
|
||||
# mock.module, PGLite outside beforeAll, missing afterAll{disconnect}).
|
||||
# The lint ships in v0.26.7 and v0.26.8 (env sweep) + v0.26.9 (PGLite
|
||||
# sweep) remove entries from this file as each sweep makes the file
|
||||
# clean.
|
||||
#
|
||||
# RULES:
|
||||
# - This list MUST shrink over time. Never add new entries — adding a
|
||||
# new file means accepting cross-file flake risk for that file.
|
||||
# - When you fix a file (apply withEnv, add the canonical PGLite
|
||||
# block, etc.), remove its entry here.
|
||||
# - When you cannot fix a file cleanly (genuinely env-coupled,
|
||||
# or shares state intentionally), rename it to *.serial.test.ts
|
||||
# instead of leaving it allow-listed.
|
||||
#
|
||||
# Permanent exemption: the test of the lint itself. Its fixture strings
|
||||
# (passed verbatim into subprocesses) legitimately match the lint
|
||||
# patterns it is testing detection of. The file does NOT mutate
|
||||
# process.env at runtime. Permanent — do not remove.
|
||||
test/scripts/check-test-isolation.test.ts
|
||||
test/autopilot-install.test.ts
|
||||
test/bootstrap.test.ts
|
||||
test/brain-resolver.test.ts
|
||||
test/check-resolvable-cli.test.ts
|
||||
test/claw-test-cli.test.ts
|
||||
test/code-def-refs.test.ts
|
||||
test/core/cycle.test.ts
|
||||
test/destructive-guard.test.ts
|
||||
test/doctor-minions-check.test.ts
|
||||
test/doctor.test.ts
|
||||
test/dream.test.ts
|
||||
test/embed.test.ts
|
||||
test/eval-capture.test.ts
|
||||
test/friction-cli.test.ts
|
||||
test/friction.test.ts
|
||||
test/gbrain-home-isolation.test.ts
|
||||
test/helpers/with-env.test.ts
|
||||
test/http-transport.test.ts
|
||||
test/hybrid-meta.test.ts
|
||||
test/init-migrate-only.test.ts
|
||||
test/integrations.test.ts
|
||||
test/mcp-eval-capture.test.ts
|
||||
test/migrate.test.ts
|
||||
test/migration-orchestrator-v0_31_0.test.ts
|
||||
test/migration-resume.test.ts
|
||||
test/migrations-v0_11_0.test.ts
|
||||
test/migrations-v0_13_1.test.ts
|
||||
test/migrations-v0_14_0.test.ts
|
||||
test/migrations-v0_19_0.test.ts
|
||||
test/migrations-v0_22_4.test.ts
|
||||
test/minions-shell.test.ts
|
||||
test/minions.test.ts
|
||||
test/mounts-cli.test.ts
|
||||
test/multi-source-integration.test.ts
|
||||
test/orphans.test.ts
|
||||
test/pages-soft-delete.test.ts
|
||||
test/preferences.test.ts
|
||||
test/reindex-code.test.ts
|
||||
test/resolve-prepare.test.ts
|
||||
test/resolvers.test.ts
|
||||
test/scenarios.test.ts
|
||||
test/schema-bootstrap-coverage.test.ts
|
||||
test/search-limit.test.ts
|
||||
test/seed-pglite.test.ts
|
||||
test/skillpack-check.test.ts
|
||||
test/source-resolver.test.ts
|
||||
test/storage-sync.test.ts
|
||||
test/subagent-audit.test.ts
|
||||
test/supervisor.test.ts
|
||||
test/sync-failures.test.ts
|
||||
test/sync-parallel.test.ts
|
||||
test/transcription.test.ts
|
||||
@@ -1,141 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: fail if any non-serial unit test file violates intra-process
|
||||
# isolation rules. The v0.26.4 parallel runner loads multiple test files
|
||||
# into one bun process per shard; module-level state (env vars, PGLite
|
||||
# engines, mock.module overrides) leaks across files in that process and
|
||||
# silently flakes other tests.
|
||||
#
|
||||
# Rules enforced (non-serial unit test files only):
|
||||
# R1: no `process.env.X = ...`, `process.env['X'] = ...`,
|
||||
# `delete process.env.X`, `Object.assign(process.env, ...)`,
|
||||
# `Reflect.set(process.env, ...)` mutations. Use withEnv() helper or
|
||||
# rename the file to `*.serial.test.ts`.
|
||||
# R2: no `mock.module(...)` anywhere. Top-level module mocks affect every
|
||||
# other file in the same shard process. Rename to `*.serial.test.ts`.
|
||||
# R3: `new PGLiteEngine(` may only appear within ~50 lines following a
|
||||
# `beforeAll(` line. Engines created at module scope (or in describe
|
||||
# bodies) leak across files in the shard process.
|
||||
# R4: any file that creates `new PGLiteEngine(` must call `.disconnect(`
|
||||
# inside an `afterAll(` block. Without disconnect, engines leak across
|
||||
# file boundaries within a shard process.
|
||||
#
|
||||
# Scope:
|
||||
# - Recursively scans `test/**/*.test.ts`.
|
||||
# - Skips `*.serial.test.ts` entirely (the quarantine escape hatch).
|
||||
# - Skips `test/e2e/**` (E2E runs sequentially in its own runner; not in
|
||||
# the parallel pool).
|
||||
#
|
||||
# Allow-list:
|
||||
# Files in `scripts/check-test-isolation.allowlist` (one filename per
|
||||
# line, # comments allowed) are skipped. This exists because v0.26.7
|
||||
# ships the lint as a foundation; v0.26.8 (env sweep) and v0.26.9
|
||||
# (PGLite sweep) remove entries as files get fixed. New files MUST NOT
|
||||
# be added — the allow-list shrinks over time, never grows.
|
||||
#
|
||||
# Usage: scripts/check-test-isolation.sh [TARGET_DIR]
|
||||
# Exit: 0 when clean, 1 when un-allow-listed violations found.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
TARGET_DIR="${1:-test}"
|
||||
ALLOWLIST_FILE="$ROOT/scripts/check-test-isolation.allowlist"
|
||||
|
||||
# Read allowlist (one filename per line, # comments allowed). Empty file
|
||||
# is fine — every violation will fail.
|
||||
ALLOWLIST=""
|
||||
if [ -f "$ALLOWLIST_FILE" ]; then
|
||||
ALLOWLIST="$(grep -v '^[[:space:]]*#' "$ALLOWLIST_FILE" | grep -v '^[[:space:]]*$' || true)"
|
||||
fi
|
||||
|
||||
is_allowlisted() {
|
||||
local f="$1"
|
||||
[ -z "$ALLOWLIST" ] && return 1
|
||||
echo "$ALLOWLIST" | grep -qxF "$f"
|
||||
}
|
||||
|
||||
# Find non-serial unit test files (excluding test/e2e). Portable across
|
||||
# bash 3.2 (macOS default) and bash 4+; no mapfile.
|
||||
FILE_LIST="$(find "$TARGET_DIR" -name '*.test.ts' \
|
||||
-not -name '*.serial.test.ts' \
|
||||
-not -path "*/e2e/*" \
|
||||
-type f 2>/dev/null | sort)"
|
||||
|
||||
violations=0
|
||||
file_count=0
|
||||
|
||||
emit_violation() {
|
||||
local f="$1" rule="$2" detail="$3" lines="$4"
|
||||
if is_allowlisted "$f"; then
|
||||
return
|
||||
fi
|
||||
echo "ERROR: $f"
|
||||
echo " rule $rule: $detail"
|
||||
if [ -n "$lines" ]; then
|
||||
echo "$lines" | head -3 | sed 's/^/ /'
|
||||
fi
|
||||
violations=$((violations + 1))
|
||||
}
|
||||
|
||||
# Read newline-separated file list; OK on macOS bash 3.2.
|
||||
while IFS= read -r f; do
|
||||
[ -z "$f" ] && continue
|
||||
file_count=$((file_count + 1))
|
||||
# R1: env mutations.
|
||||
env_lines=$(grep -nE 'process\.env\.[A-Za-z_][A-Za-z_0-9]*[[:space:]]*=[^=]|process\.env\[[^]]+\][[:space:]]*=[^=]|delete[[:space:]]+process\.env\.|delete[[:space:]]+process\.env\[|Object\.assign[[:space:]]*\([[:space:]]*process\.env|Reflect\.set[[:space:]]*\([[:space:]]*process\.env' "$f" 2>/dev/null || true)
|
||||
if [ -n "$env_lines" ]; then
|
||||
emit_violation "$f" "R1" "process.env mutation; use withEnv() or rename to *.serial.test.ts" "$env_lines"
|
||||
fi
|
||||
|
||||
# R2: mock.module() anywhere.
|
||||
mock_lines=$(grep -nE 'mock\.module[[:space:]]*\(' "$f" 2>/dev/null || true)
|
||||
if [ -n "$mock_lines" ]; then
|
||||
emit_violation "$f" "R2" "mock.module() leaks across files in the shard process; rename to *.serial.test.ts" "$mock_lines"
|
||||
fi
|
||||
|
||||
# R3: PGLiteEngine outside ~50 lines after a beforeAll(.
|
||||
if grep -qE 'new PGLiteEngine[[:space:]]*\(' "$f" 2>/dev/null; then
|
||||
bad=$(awk '
|
||||
BEGIN { last_before_all = -1000 }
|
||||
/beforeAll[[:space:]]*\(/ { last_before_all = NR }
|
||||
/new PGLiteEngine[[:space:]]*\(/ {
|
||||
if (NR - last_before_all > 50) {
|
||||
printf "%d:%s\n", NR, $0
|
||||
}
|
||||
}
|
||||
' "$f" 2>/dev/null)
|
||||
if [ -n "$bad" ]; then
|
||||
emit_violation "$f" "R3" "new PGLiteEngine(...) outside beforeAll() context (>50 lines); move into beforeAll" "$bad"
|
||||
fi
|
||||
fi
|
||||
|
||||
# R4: PGLiteEngine creation requires afterAll{disconnect}.
|
||||
if grep -qE 'new PGLiteEngine[[:space:]]*\(' "$f" 2>/dev/null; then
|
||||
if ! grep -qE 'afterAll[[:space:]]*\(' "$f" 2>/dev/null \
|
||||
|| ! grep -qE '\.disconnect[[:space:]]*\(' "$f" 2>/dev/null; then
|
||||
emit_violation "$f" "R4" "creates PGLiteEngine but missing afterAll(() => engine.disconnect()); engine leaks across files in the shard process" ""
|
||||
fi
|
||||
fi
|
||||
done <<EOF
|
||||
$FILE_LIST
|
||||
EOF
|
||||
|
||||
if [ $violations -gt 0 ]; then
|
||||
echo
|
||||
echo "check-test-isolation: FAIL ($violations violation(s))"
|
||||
echo
|
||||
echo "Fix:"
|
||||
echo " - For env mutations, use withEnv() from test/helpers/with-env.ts"
|
||||
echo " - For mock.module(), rename to *.serial.test.ts (quarantine)"
|
||||
echo " - For PGLiteEngine, follow the canonical pattern in"
|
||||
echo " test/helpers/reset-pglite.ts JSDoc and CLAUDE.md."
|
||||
echo
|
||||
echo "Or, if this is a baseline file from before the lint shipped,"
|
||||
echo "add it to scripts/check-test-isolation.allowlist (with a TODO"
|
||||
echo "comment naming the sweep PR that will remove it)."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "check-test-isolation: OK ($file_count non-serial unit files scanned)"
|
||||
@@ -1,155 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: fail if any test fixture references a real person's name.
|
||||
#
|
||||
# CLAUDE.md's "Privacy rule" section is unambiguous: never reference real
|
||||
# people, companies, funds, or private agent names in any public-facing
|
||||
# artifact. Tests are checked-in code distributed with every release and
|
||||
# indexed by GitHub search. This guard catches the patterns the rule names.
|
||||
#
|
||||
# Design (post-Codex F4 review):
|
||||
# - Banned names: exact-string allowlist of known real identifiers. Adding
|
||||
# a name when CLAUDE.md flags one is a one-line edit.
|
||||
# - Banned emails: specific addresses that identify real contacts. NOT a
|
||||
# broad corporate-email regex — those would catch legitimate fixture
|
||||
# domains in billing/auth tests (`customer@stripe.com` etc.).
|
||||
# - Allowlist: exact "file:offending-string" pairs that are intentional
|
||||
# and pre-existing (e.g., the user's own email is not a "contact").
|
||||
#
|
||||
# Scope: test/**/*.test.ts only. Historical CHANGELOG entries, doc examples,
|
||||
# and skill READMEs each have their own scrub status and are out of scope
|
||||
# for this guard.
|
||||
#
|
||||
# Usage: scripts/check-test-real-names.sh
|
||||
# Exit: 0 clean, 1 banned reference found, 2 setup error (rg + grep missing).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# Banned real-name strings (matched as whole words, case-insensitive).
|
||||
# Add an entry when CLAUDE.md flags a new real-person name.
|
||||
BANNED_NAMES=(
|
||||
'Diana' # Diana Hu, named in CLAUDE.md privacy example
|
||||
'Wintermute' # private OpenClaw fork name (CLAUDE.md rule)
|
||||
'Hermes' # downstream agent fork name
|
||||
'Technium' # real GP handle
|
||||
'McGrew' # ex-OpenAI exec
|
||||
'YC Labs' # internal team name
|
||||
)
|
||||
|
||||
# Banned specific email addresses. NOT a generic corporate-email regex —
|
||||
# those would catch legitimate fixture domains in billing/auth tests
|
||||
# (`customer@stripe.com`, `account@openai.com` etc).
|
||||
BANNED_EMAILS=(
|
||||
'diana@ycombinator.com'
|
||||
)
|
||||
|
||||
# Exact "file:offending-string" pairs that are intentional and pre-existing.
|
||||
# These pre-date the rule, the file's own author confirmed the use, the
|
||||
# string identifies the user themselves (not a contact), OR the reference
|
||||
# is structural (e.g., a regression test that ASSERTS the banned name does
|
||||
# NOT appear in production code — the name MUST be in the test file as a
|
||||
# literal).
|
||||
ALLOWLIST=(
|
||||
"test/writer.test.ts:garry@ycombinator.com" # user's own email — CLAUDE.md rule does not apply
|
||||
"test/integrations.test.ts:Wintermute" # regex pattern in personal-info filter test (structural)
|
||||
"test/recency-decay.test.ts:Wintermute" # regression-prevention test asserting wintermute is absent (structural)
|
||||
"test/scripts/check-proposal-pii.test.ts:Wintermute" # privacy-guard test asserting docs/proposals/ rejects wintermute (structural; same meta-rule exception as check-privacy.sh)
|
||||
"test/scripts/check-proposal-pii.test.ts:WINTERMUTE" # case-insensitive sentinel literal for the same privacy-guard test
|
||||
"test/serve-stdio-lifecycle.test.ts:Hermes" # comment naming a downstream-agent scenario — pre-existing, low signal
|
||||
"test/extract.test.ts:Hermes" # markdown-link extraction test fixture — pre-existing, ambiguous (Greek god vs fork)
|
||||
# v0.36.0.0: skillpack-harvest privacy linter tests structurally
|
||||
# require the literal "Wintermute" to verify the linter catches it.
|
||||
# Same meta-rule exception as integrations.test.ts and the proposal-pii
|
||||
# privacy guard test above.
|
||||
"test/skillpack-harvest.test.ts:Wintermute"
|
||||
"test/skillpack-harvest-lint.test.ts:Wintermute"
|
||||
"test/e2e/skillpack-flow.test.ts:Wintermute"
|
||||
)
|
||||
|
||||
# Build the combined regex. Names matched as whole words (\b), emails matched
|
||||
# literally with dot escapes.
|
||||
PATTERN_PARTS=()
|
||||
for n in "${BANNED_NAMES[@]}"; do
|
||||
# Escape any regex metacharacters in the name (defensive — most are bare
|
||||
# words but YC Labs has a space).
|
||||
escaped="${n//./\\.}"
|
||||
escaped="${escaped// /\\s}"
|
||||
PATTERN_PARTS+=("\\b${escaped}\\b")
|
||||
done
|
||||
for e in "${BANNED_EMAILS[@]}"; do
|
||||
escaped="${e//./\\.}"
|
||||
PATTERN_PARTS+=("${escaped}")
|
||||
done
|
||||
|
||||
# Join with |.
|
||||
IFS='|' eval 'PATTERN="${PATTERN_PARTS[*]}"'
|
||||
|
||||
# Find tool.
|
||||
if command -v rg >/dev/null 2>&1; then
|
||||
matches="$(rg -niH --no-heading -t ts "$PATTERN" test/ 2>/dev/null || true)"
|
||||
elif command -v grep >/dev/null 2>&1; then
|
||||
matches="$(grep -rniE --include='*.test.ts' "$PATTERN" test/ 2>/dev/null || true)"
|
||||
else
|
||||
echo "check-test-real-names: ERROR: neither rg nor grep available." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [ -z "$matches" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Apply allowlist. Each line is "file:lineno:content"; check whether
|
||||
# "file:<needle>" appears in ALLOWLIST for any needle in BANNED_EMAILS+NAMES
|
||||
# that matches the content.
|
||||
filtered=""
|
||||
while IFS= read -r line; do
|
||||
[ -z "$line" ] && continue
|
||||
# Extract filename and content (everything after second :).
|
||||
file="${line%%:*}"
|
||||
rest="${line#*:}"
|
||||
# rest is "lineno:content" — strip lineno.
|
||||
content="${rest#*:}"
|
||||
|
||||
matched_needle=""
|
||||
for needle in "${BANNED_EMAILS[@]}" "${BANNED_NAMES[@]}"; do
|
||||
if echo "$content" | grep -qi -- "$needle"; then
|
||||
matched_needle="$needle"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
allow_key="${file}:${matched_needle}"
|
||||
allowed=0
|
||||
for allow_entry in "${ALLOWLIST[@]}"; do
|
||||
if [ "$allow_entry" = "$allow_key" ]; then
|
||||
allowed=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$allowed" = "0" ]; then
|
||||
filtered+="${line}"$'\n'
|
||||
fi
|
||||
done <<< "$matches"
|
||||
|
||||
if [ -z "$filtered" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "check-test-real-names: banned real-name references found in test/ fixtures." >&2
|
||||
echo "" >&2
|
||||
echo "$filtered" >&2
|
||||
echo "" >&2
|
||||
echo "Fix: replace with canonical placeholders per CLAUDE.md 'Name mapping' table." >&2
|
||||
echo " alice-example / @alice-example for people" >&2
|
||||
echo " bob-example / charlie-example for additional people" >&2
|
||||
echo " alice@example.com for emails (example.com is RFC 6761 reserved)" >&2
|
||||
echo " acme-example / widget-co for companies" >&2
|
||||
echo " fund-a / fund-b for funds" >&2
|
||||
echo " a-team / agent-fork for teams / OpenClaw forks" >&2
|
||||
echo "" >&2
|
||||
echo "If the match is intentional (e.g., the user's own identifier, not a contact)," >&2
|
||||
echo "add an exact 'file:string' entry to ALLOWLIST in scripts/check-test-real-names.sh." >&2
|
||||
exit 1
|
||||
@@ -1,346 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/ci-local.sh
|
||||
#
|
||||
# Local CI gate. Runs the same checks GH Actions does (and a stricter superset
|
||||
# of E2E) inside Docker. See docker-compose.ci.yml.
|
||||
#
|
||||
# Modes:
|
||||
# bash scripts/ci-local.sh # full local gate: gitleaks + unit + ALL E2E (4-way sharded)
|
||||
# bash scripts/ci-local.sh --diff # full local gate: gitleaks + unit + selected E2E (4-way sharded)
|
||||
# bash scripts/ci-local.sh --no-pull # skip docker compose pull (offline / debug)
|
||||
# bash scripts/ci-local.sh --clean # nuke named volumes for cold debug
|
||||
# bash scripts/ci-local.sh --no-shard # debug: run E2E sequentially against postgres-1 only
|
||||
#
|
||||
# 4-way E2E sharding: 4 pgvector services on host ports 5434-5437. The 36 E2E
|
||||
# files split N/4 per shard; shards run in parallel. Within a shard, files run
|
||||
# sequentially (TRUNCATE CASCADE no-race property documented in run-e2e.sh).
|
||||
# Wall-time on a 16-core host: ~6 min sequential -> ~1.5-2 min sharded.
|
||||
#
|
||||
# Stronger than PR CI: PR CI runs only Tier 1's 2 files; this runs all 36.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
COMPOSE_FILE="docker-compose.ci.yml"
|
||||
|
||||
DIFF=0
|
||||
NO_PULL=0
|
||||
CLEAN=0
|
||||
NO_SHARD=0
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--diff) DIFF=1 ;;
|
||||
--no-pull) NO_PULL=1 ;;
|
||||
--clean) CLEAN=1 ;;
|
||||
--no-shard) NO_SHARD=1 ;;
|
||||
*)
|
||||
echo "Usage: $0 [--diff] [--no-pull] [--clean] [--no-shard]" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "[ci-local] Tearing down postgres..."
|
||||
docker compose -f "$COMPOSE_FILE" down --remove-orphans 2>&1 | tail -5 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
if [ "$CLEAN" = "1" ]; then
|
||||
echo "[ci-local] --clean: removing named volumes..."
|
||||
docker compose -f "$COMPOSE_FILE" down -v --remove-orphans 2>&1 | tail -5 || true
|
||||
fi
|
||||
|
||||
# Tier 2: --diff fast-path. If the diff is doc-only (or empty), skip the
|
||||
# whole heavy gate (postgres + bun install + unit + E2E) and just verify
|
||||
# gitleaks on host. Doc-only diffs go from ~25 min to ~5 seconds.
|
||||
if [ "$DIFF" = "1" ]; then
|
||||
CLASSIFICATION=$(bun run scripts/select-e2e.ts --classify-only 2>/dev/null || echo "ERR")
|
||||
case "$CLASSIFICATION" in
|
||||
DOC_ONLY)
|
||||
echo "[ci-local] --diff: diff is doc-only — skipping postgres + unit + E2E (Tier 2 fast-path)."
|
||||
echo "[ci-local] Running gitleaks on host as the only gate..."
|
||||
if ! command -v gitleaks >/dev/null 2>&1; then
|
||||
echo "[ci-local] WARN: gitleaks not installed; skipping. brew install gitleaks." >&2
|
||||
else
|
||||
gitleaks dir . --redact --no-banner
|
||||
gitleaks git . --redact --no-banner --log-opts="origin/master..HEAD"
|
||||
fi
|
||||
echo "[ci-local] Doc-only fast-path complete. No code paths exercised."
|
||||
trap - EXIT
|
||||
exit 0
|
||||
;;
|
||||
EMPTY)
|
||||
echo "[ci-local] --diff: diff is empty (clean branch) — running full gate per fail-closed contract."
|
||||
;;
|
||||
SRC)
|
||||
echo "[ci-local] --diff: diff touches src/ — running selected E2E + full unit phase."
|
||||
;;
|
||||
*)
|
||||
echo "[ci-local] WARN: select-e2e.ts --classify-only returned '$CLASSIFICATION' — running full gate." >&2
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Pre-flight: postgres host ports for 4 shards. Defaults to 5434-5437 (avoid
|
||||
# 5432 manual gbrain-test-pg, 5433 commonly held by sibling projects).
|
||||
# GBRAIN_CI_PG_PORT defines BASE; shards take BASE..BASE+3.
|
||||
PG_PORT_BASE="${GBRAIN_CI_PG_PORT:-5434}"
|
||||
for shard in 1 2 3 4; do
|
||||
port=$((PG_PORT_BASE + shard - 1))
|
||||
PORT_OWNER=$(docker ps --filter "publish=$port" --format "{{.Names}}" | head -1)
|
||||
if [ -n "$PORT_OWNER" ]; then
|
||||
echo "[ci-local] ERROR: host port $port (shard $shard) is already used by docker container '$PORT_OWNER'." >&2
|
||||
echo "[ci-local] Either stop that container or run with: GBRAIN_CI_PG_PORT=NNNN bun run ci:local" >&2
|
||||
exit 1
|
||||
fi
|
||||
if lsof -iTCP:"$port" -sTCP:LISTEN -P -n >/dev/null 2>&1; then
|
||||
echo "[ci-local] ERROR: host port $port (shard $shard) is held by a non-docker process." >&2
|
||||
echo "[ci-local] Run with: GBRAIN_CI_PG_PORT=NNNN bun run ci:local" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
export GBRAIN_CI_PG_PORT="$PG_PORT_BASE"
|
||||
export GBRAIN_CI_PG_PORT_2=$((PG_PORT_BASE + 1))
|
||||
export GBRAIN_CI_PG_PORT_3=$((PG_PORT_BASE + 2))
|
||||
export GBRAIN_CI_PG_PORT_4=$((PG_PORT_BASE + 3))
|
||||
|
||||
# Step 0: gitleaks on the host (no docker, no postgres, no bun needed).
|
||||
# Mirrors test.yml's separate gitleaks job. Fail loudly if not installed.
|
||||
echo "[ci-local] gitleaks detect (host)..."
|
||||
if ! command -v gitleaks >/dev/null 2>&1; then
|
||||
echo "[ci-local] ERROR: gitleaks not installed on host." >&2
|
||||
echo "[ci-local] macOS: brew install gitleaks" >&2
|
||||
echo "[ci-local] Linux: https://github.com/gitleaks/gitleaks/releases" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Two scopes for pre-push:
|
||||
# 1. Working-tree files (catch uncommitted secrets sitting in files)
|
||||
# 2. Branch commits vs origin/master (catch secrets committed on this branch)
|
||||
# Full-history scan is ~4 min on this repo's 3700+ commits; not useful pre-push.
|
||||
gitleaks dir . --redact --no-banner
|
||||
gitleaks git . --redact --no-banner --log-opts="origin/master..HEAD"
|
||||
|
||||
# Step 1: pull. Refreshes pgvector + oven/bun:1 (both are `image:` not `build:`).
|
||||
if [ "$NO_PULL" = "0" ]; then
|
||||
echo "[ci-local] Pulling base images (use --no-pull to skip)..."
|
||||
docker compose -f "$COMPOSE_FILE" pull 2>&1 | tail -5
|
||||
fi
|
||||
|
||||
# Step 2: 4 postgres shards up + wait for healthy.
|
||||
echo "[ci-local] Starting 4 postgres shards..."
|
||||
docker compose -f "$COMPOSE_FILE" up -d postgres-1 postgres-2 postgres-3 postgres-4
|
||||
echo "[ci-local] Waiting for all 4 postgres shards healthy..."
|
||||
for i in {1..40}; do
|
||||
all_healthy=1
|
||||
for shard in 1 2 3 4; do
|
||||
status=$(docker compose -f "$COMPOSE_FILE" ps --format json postgres-$shard 2>/dev/null | grep -o '"Health":"[^"]*"' | head -1 | sed 's/.*":"//;s/"//')
|
||||
if [ "$status" != "healthy" ]; then
|
||||
all_healthy=0
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ "$all_healthy" = "1" ]; then
|
||||
echo "[ci-local] All 4 postgres shards healthy."
|
||||
break
|
||||
fi
|
||||
if [ "$i" = "40" ]; then
|
||||
echo "[ci-local] ERROR: not all postgres shards became healthy in 40 attempts" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Step 3: smoke-test run-e2e.sh argv + shard handling.
|
||||
echo "[ci-local] Smoke: run-e2e.sh argv + shard..."
|
||||
SMOKE_NO_ARGS=$(bash scripts/run-e2e.sh --dry-run-list | wc -l | tr -d ' ')
|
||||
EXPECTED_ALL=$(ls test/e2e/*.test.ts | wc -l | tr -d ' ')
|
||||
if [ "$SMOKE_NO_ARGS" != "$EXPECTED_ALL" ]; then
|
||||
echo "[ci-local] ERROR: --dry-run-list (no args) printed $SMOKE_NO_ARGS, expected $EXPECTED_ALL" >&2
|
||||
exit 1
|
||||
fi
|
||||
SMOKE_ONE_ARG=$(bash scripts/run-e2e.sh --dry-run-list test/e2e/sync.test.ts)
|
||||
if [ "$SMOKE_ONE_ARG" != "test/e2e/sync.test.ts" ]; then
|
||||
echo "[ci-local] ERROR: --dry-run-list with 1 arg printed '$SMOKE_ONE_ARG'" >&2
|
||||
exit 1
|
||||
fi
|
||||
SHARD_TOTAL=$(( $(SHARD=1/4 bash scripts/run-e2e.sh --dry-run-list | wc -l) + \
|
||||
$(SHARD=2/4 bash scripts/run-e2e.sh --dry-run-list | wc -l) + \
|
||||
$(SHARD=3/4 bash scripts/run-e2e.sh --dry-run-list | wc -l) + \
|
||||
$(SHARD=4/4 bash scripts/run-e2e.sh --dry-run-list | wc -l) ))
|
||||
if [ "$SHARD_TOTAL" != "$EXPECTED_ALL" ]; then
|
||||
echo "[ci-local] ERROR: shards 1-4 covered $SHARD_TOTAL files, expected $EXPECTED_ALL" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[ci-local] Smoke OK ($SMOKE_NO_ARGS files no-arg, 1 single-arg, ${SHARD_TOTAL}=4-shard total)."
|
||||
|
||||
# Step 4: build the runner-side command.
|
||||
# Tier 1: 4-shard parallel UNIT + E2E. Each shard runs ~46 unit files + ~9
|
||||
# E2E files against postgres-N. Guards + typecheck run ONCE before fan-out.
|
||||
# --no-shard runs the legacy unsharded flow (debug aid).
|
||||
if [ "$NO_SHARD" = "1" ]; then
|
||||
if [ "$DIFF" = "1" ]; then
|
||||
RUN_PHASES_CMD='echo "[runner] guards + typecheck"
|
||||
bash scripts/check-jsonb-pattern.sh
|
||||
bash scripts/check-progress-to-stdout.sh
|
||||
bash scripts/check-trailing-newline.sh
|
||||
bash scripts/check-wasm-embedded.sh
|
||||
bun run typecheck
|
||||
echo "[runner] unit (unsharded, DATABASE_URL unset)"
|
||||
env -u DATABASE_URL bash scripts/run-unit-shard.sh
|
||||
echo "[runner] e2e (unsharded, --diff selected)"
|
||||
SELECTED=$(bun run scripts/select-e2e.ts)
|
||||
if [ -z "$SELECTED" ]; then
|
||||
echo "[runner] selector emitted nothing (doc-only diff); skipping E2E."
|
||||
else
|
||||
DATABASE_URL=postgresql://postgres:postgres@postgres-1:5432/gbrain_test echo "$SELECTED" | xargs bash scripts/run-e2e.sh
|
||||
fi'
|
||||
else
|
||||
RUN_PHASES_CMD='echo "[runner] guards + typecheck"
|
||||
bash scripts/check-jsonb-pattern.sh
|
||||
bash scripts/check-progress-to-stdout.sh
|
||||
bash scripts/check-trailing-newline.sh
|
||||
bash scripts/check-wasm-embedded.sh
|
||||
bun run typecheck
|
||||
echo "[runner] unit (unsharded, DATABASE_URL unset)"
|
||||
env -u DATABASE_URL bash scripts/run-unit-shard.sh
|
||||
echo "[runner] e2e (unsharded)"
|
||||
DATABASE_URL=postgresql://postgres:postgres@postgres-1:5432/gbrain_test bash scripts/run-e2e.sh'
|
||||
fi
|
||||
else
|
||||
# Tier 1 sharded path. Each shard runs unit+E2E sequentially against its
|
||||
# own postgres-N. Shards run in parallel via xargs -P4.
|
||||
if [ "$DIFF" = "1" ]; then
|
||||
DIFF_E2E_PREP='SELECTED=$(bun run scripts/select-e2e.ts)
|
||||
if [ -z "$SELECTED" ]; then
|
||||
echo "" > /tmp/e2e-selected.txt
|
||||
else
|
||||
echo "$SELECTED" | tr " " "\n" | grep -v "^$" > /tmp/e2e-selected.txt
|
||||
fi'
|
||||
else
|
||||
# Empty file -> run-e2e.sh uses default glob (all 36 E2E files).
|
||||
DIFF_E2E_PREP='> /tmp/e2e-selected.txt'
|
||||
fi
|
||||
RUN_PHASES_CMD="echo \"[runner] guards + typecheck (run once before sharding)\"
|
||||
bash scripts/check-jsonb-pattern.sh
|
||||
bash scripts/check-progress-to-stdout.sh
|
||||
bash scripts/check-trailing-newline.sh
|
||||
bash scripts/check-wasm-embedded.sh
|
||||
bun run typecheck
|
||||
echo \"[runner] Tier 3: building PGLite snapshot fixture (cached across reruns)\"
|
||||
if [ ! -f test/fixtures/pglite-snapshot.tar ] || [ ! -f test/fixtures/pglite-snapshot.version ]; then
|
||||
bun run build:pglite-snapshot
|
||||
else
|
||||
echo \"[runner] snapshot fixture exists; engine will validate hash at load time\"
|
||||
fi
|
||||
export GBRAIN_PGLITE_SNAPSHOT=test/fixtures/pglite-snapshot.tar
|
||||
echo \"[runner] resolving E2E file selection (--diff aware)\"
|
||||
${DIFF_E2E_PREP}
|
||||
mkdir -p /tmp/shard-logs
|
||||
echo \"[runner] Tier 1: 4-shard parallel unit + E2E (xargs -P4)\"
|
||||
set +e
|
||||
printf '%s\\n' 1 2 3 4 | xargs -P4 -I{} sh -c '
|
||||
shard=\$1
|
||||
log=/tmp/shard-logs/shard-\${shard}.log
|
||||
echo \"[shard \${shard}] start\" > \$log
|
||||
echo \"[shard \${shard}] unit phase (SHARD=\${shard}/4, DATABASE_URL unset)\" >> \$log
|
||||
env -u DATABASE_URL SHARD=\${shard}/4 bash scripts/run-unit-shard.sh >> \$log 2>&1
|
||||
unit_exit=\$?
|
||||
if [ \$unit_exit -ne 0 ]; then
|
||||
echo \"[shard \${shard}] UNIT FAILED (exit=\$unit_exit)\" >> \$log
|
||||
exit \$unit_exit
|
||||
fi
|
||||
echo \"[shard \${shard}] e2e phase (SHARD=\${shard}/4, DATABASE_URL=postgres-\${shard})\" >> \$log
|
||||
if [ -s /tmp/e2e-selected.txt ]; then
|
||||
SHARD=\${shard}/4 \\
|
||||
DATABASE_URL=postgresql://postgres:postgres@postgres-\${shard}:5432/gbrain_test \\
|
||||
xargs -a /tmp/e2e-selected.txt bash scripts/run-e2e.sh >> \$log 2>&1
|
||||
else
|
||||
SHARD=\${shard}/4 \\
|
||||
DATABASE_URL=postgresql://postgres:postgres@postgres-\${shard}:5432/gbrain_test \\
|
||||
bash scripts/run-e2e.sh >> \$log 2>&1
|
||||
fi
|
||||
e2e_exit=\$?
|
||||
if [ \$e2e_exit -ne 0 ]; then
|
||||
echo \"[shard \${shard}] E2E FAILED (exit=\$e2e_exit)\" >> \$log
|
||||
exit \$e2e_exit
|
||||
fi
|
||||
echo \"[shard \${shard}] DONE\" >> \$log
|
||||
' _ {}
|
||||
shard_xargs_exit=\$?
|
||||
set -e
|
||||
echo \"\"
|
||||
echo \"=== SHARD LOGS (last 30 lines each + unit/e2e summaries) ===\"
|
||||
for s in 1 2 3 4; do
|
||||
echo \"\"
|
||||
echo \"--- shard \$s ---\"
|
||||
if [ -f /tmp/shard-logs/shard-\$s.log ]; then
|
||||
# Pull the unit + E2E summary lines explicitly so they survive even if
|
||||
# the file is huge. Match: bun's '<N> pass / <N> fail' pairs, run-e2e.sh's
|
||||
# 'Files: ... / Tests: ...' summary, and our own shard markers.
|
||||
grep -E '^\\[shard|^Files: |^Tests: |Ran [0-9]+ tests|^[[:space:]]+[0-9]+ (pass|fail|skip)\$' /tmp/shard-logs/shard-\$s.log || true
|
||||
echo \" (last 30 lines for context)\"
|
||||
tail -30 /tmp/shard-logs/shard-\$s.log
|
||||
else
|
||||
echo \"(no log file written — shard never started)\"
|
||||
fi
|
||||
done
|
||||
echo \"\"
|
||||
if [ \$shard_xargs_exit -ne 0 ]; then
|
||||
echo \"[runner] One or more shards failed (xargs exit=\$shard_xargs_exit). See SHARD LOGS above.\"
|
||||
exit \$shard_xargs_exit
|
||||
fi
|
||||
echo \"[runner] All 4 shards passed.\""
|
||||
fi
|
||||
|
||||
INNER_CMD=$(cat <<'EOF'
|
||||
set -euo pipefail
|
||||
echo "[runner] bun version: $(bun --version)"
|
||||
# oven/bun:1 omits git; many unit tests use mkdtemp + git init for fixtures.
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
echo "[runner] Installing git (debian apt)..."
|
||||
apt-get update -qq >/dev/null
|
||||
apt-get install -y -qq git ca-certificates >/dev/null
|
||||
fi
|
||||
# Container runs as root (uid 0) against a host-uid bind-mount; mark repo +
|
||||
# any worktree gitdir as safe so `git status` etc. don't refuse.
|
||||
git config --global --add safe.directory '*' || true
|
||||
if [ ! -d /app/node_modules ] || [ -z "$(ls -A /app/node_modules 2>/dev/null)" ]; then
|
||||
echo "[runner] First run (or --clean): bun install --frozen-lockfile"
|
||||
bun install --frozen-lockfile
|
||||
fi
|
||||
__RUN_PHASES__
|
||||
EOF
|
||||
)
|
||||
INNER_CMD="${INNER_CMD/__RUN_PHASES__/$RUN_PHASES_CMD}"
|
||||
|
||||
# Conductor / git-worktree support: when `.git` is a file (not a directory),
|
||||
# it points at a host gitdir outside the bind-mount. Without remounting that
|
||||
# path, scripts/check-trailing-newline.sh and any other in-container `git`
|
||||
# call exits 128 ("not a git repository"). Resolve the host gitdir + the
|
||||
# shared common gitdir and bind-mount them at the same absolute paths.
|
||||
EXTRA_MOUNTS=()
|
||||
if [ -f .git ]; then
|
||||
WORKTREE_GITDIR=$(awk '{print $2}' .git)
|
||||
if [ -d "$WORKTREE_GITDIR" ]; then
|
||||
COMMONDIR_FILE="$WORKTREE_GITDIR/commondir"
|
||||
if [ -f "$COMMONDIR_FILE" ]; then
|
||||
COMMON_REL=$(cat "$COMMONDIR_FILE")
|
||||
COMMON_GITDIR=$(cd "$WORKTREE_GITDIR" && cd "$COMMON_REL" && pwd)
|
||||
else
|
||||
COMMON_GITDIR="$WORKTREE_GITDIR"
|
||||
fi
|
||||
# Mount the higher-level common gitdir; covers worktrees/<name> automatically.
|
||||
EXTRA_MOUNTS+=( -v "${COMMON_GITDIR}:${COMMON_GITDIR}:ro" )
|
||||
echo "[ci-local] Worktree detected; mounting shared gitdir: $COMMON_GITDIR"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "[ci-local] Running checks inside runner container..."
|
||||
docker compose -f "$COMPOSE_FILE" run --rm "${EXTRA_MOUNTS[@]:-}" runner bash -c "$INNER_CMD"
|
||||
|
||||
echo ""
|
||||
echo "[ci-local] All checks passed."
|
||||
@@ -1,88 +0,0 @@
|
||||
// scripts/e2e-test-map.ts
|
||||
//
|
||||
// Path-glob -> E2E test files map. Used by scripts/select-e2e.ts.
|
||||
//
|
||||
// CONTRACT: This map can ONLY narrow from "all". When a changed src/ path
|
||||
// matches no glob here, the selector falls back to "run all E2E" (fail-closed).
|
||||
// You can safely add narrowing entries; you cannot break correctness by missing
|
||||
// one. Tune as misses surface (i.e., when ci:local:diff ran more than necessary
|
||||
// and you'd like to narrow that surface area).
|
||||
//
|
||||
// Glob syntax is the minimal subset implemented in select-e2e.ts:
|
||||
// - "**" matches any sequence of path segments (including zero)
|
||||
// - "*" matches any characters within a single path segment
|
||||
// - everything else is literal
|
||||
// No brace expansion, no ?, no [ ].
|
||||
|
||||
export const E2E_TEST_MAP: Record<string, string[]> = {
|
||||
// Source-aware ranking, hybrid search, intent classification.
|
||||
"src/core/search/**": [
|
||||
"test/e2e/search-quality.test.ts",
|
||||
"test/e2e/search-exclude.test.ts",
|
||||
"test/e2e/search-swamp.test.ts",
|
||||
],
|
||||
// Tree-sitter chunkers feed code-indexing E2E.
|
||||
"src/core/chunkers/**": ["test/e2e/code-indexing.test.ts"],
|
||||
// OpenClaw context-engine plugin: engine + entry feed the plugin-shape E2E
|
||||
// (mocked SDK) AND the real-loader Tier 2 E2E that spawns openclaw and
|
||||
// actually installs the plugin into an isolated --profile.
|
||||
"src/core/context-engine.ts": [
|
||||
"test/e2e/openclaw-context-engine-plugin.test.ts",
|
||||
"test/e2e/openclaw-plugin-load-real.test.ts",
|
||||
],
|
||||
"src/openclaw-context-engine.ts": [
|
||||
"test/e2e/openclaw-context-engine-plugin.test.ts",
|
||||
"test/e2e/openclaw-plugin-load-real.test.ts",
|
||||
],
|
||||
// dream.ts is a thin alias over runCycle in cycle.ts.
|
||||
"src/core/cycle.ts": ["test/e2e/cycle.test.ts", "test/e2e/dream.test.ts"],
|
||||
// Multi-source sync writes share the per-source bookmark anchor.
|
||||
"src/core/sync.ts": ["test/e2e/sync.test.ts", "test/e2e/multi-source.test.ts"],
|
||||
// v0.32.8 multi-source bug class regression suite — fires on any cycle
|
||||
// phase, extract, integrity, embed, or migrate-engine change.
|
||||
"src/core/cycle/extract-takes.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/core/cycle/patterns.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/core/cycle/synthesize.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/commands/embed.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/commands/extract.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/commands/migrate-engine.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
// Any minions queue/worker/handler change exercises all minion E2E.
|
||||
"src/core/minions/**": [
|
||||
"test/e2e/minions-concurrency.test.ts",
|
||||
"test/e2e/minions-resilience.test.ts",
|
||||
"test/e2e/minions-shell.test.ts",
|
||||
"test/e2e/minions-shell-pglite.test.ts",
|
||||
"test/e2e/worker-abort-recovery.test.ts",
|
||||
],
|
||||
// postgres.js bind paths + JSONB shapes + parity vs PGLite.
|
||||
"src/core/postgres-engine.ts": [
|
||||
"test/e2e/postgres-bootstrap.test.ts",
|
||||
"test/e2e/postgres-jsonb.test.ts",
|
||||
"test/e2e/jsonb-roundtrip.test.ts",
|
||||
"test/e2e/engine-parity.test.ts",
|
||||
"test/e2e/schema-drift.test.ts",
|
||||
],
|
||||
// PGLite bootstrap path + parity guard.
|
||||
"src/core/pglite-engine.ts": [
|
||||
"test/e2e/postgres-bootstrap.test.ts",
|
||||
"test/e2e/engine-parity.test.ts",
|
||||
"test/e2e/schema-drift.test.ts",
|
||||
],
|
||||
// Schema source of truth: any change must pass the cross-engine drift gate.
|
||||
"src/schema.sql": ["test/e2e/schema-drift.test.ts"],
|
||||
"src/core/pglite-schema.ts": ["test/e2e/schema-drift.test.ts"],
|
||||
"src/core/migrate.ts": ["test/e2e/schema-drift.test.ts", "test/e2e/migrate-chain.test.ts"],
|
||||
// MCP stdio + HTTP transports share dispatch.
|
||||
"src/mcp/**": ["test/e2e/mcp.test.ts", "test/e2e/http-transport.test.ts"],
|
||||
// Integrity batch-load fast path.
|
||||
"src/commands/integrity.ts": ["test/e2e/integrity-batch.test.ts"],
|
||||
// Upgrade chains migration ledger; touches both runners.
|
||||
"src/commands/upgrade.ts": [
|
||||
"test/e2e/upgrade.test.ts",
|
||||
"test/e2e/migrate-chain.test.ts",
|
||||
"test/e2e/migration-flow.test.ts",
|
||||
],
|
||||
"src/commands/doctor.ts": ["test/e2e/doctor-progress.test.ts"],
|
||||
// Knowledge graph layer feeds graph-quality.
|
||||
"src/core/link-extraction.ts": ["test/e2e/graph-quality.test.ts"],
|
||||
};
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user