diff --git a/CHANGELOG.md b/CHANGELOG.md index 53bc4a55a..3f5e7626d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,41 @@ All notable changes to GBrain will be documented in this file. +## [0.35.1.1] - 2026-05-16 + +**Fix wave: `gbrain eval longmemeval` actually runs against the public _s split.** + +A pre-spend smoke for the upcoming embedder shootout caught three tightly-coupled bugs that would have made all 7 cells fail. Shipping the fixes before anyone burns judge tokens. + +### What this fixes + +**The longmemeval adapter accepts the public _s dataset shape.** Pre-v0.35.1.1 assumed every dataset used the oracle `{session_id, turns}` shape, but the HuggingFace _s split serializes sessions as a parallel `haystack_session_ids: string[]` + a `LongMemEvalTurn[][]` (each inner array is one session's turns directly). The old adapter crashed `session.turns is undefined` on every question. New `normalizeSessions` helper accepts both shapes, mirroring the proven path in `gbrain-evals/eval/runner/longmemeval.ts`. + +**Session IDs that contain underscores or uppercase letters now produce valid slugs.** The _s split's IDs look like `sharegpt_yywfIrx_0`, both of which the v0.32.7 CJK-wave slug validator rejects. New `sanitizeSessionIdForSlug` lowercases and rewrites disallowed chars to `-`. The frontmatter `session_id:` line still carries the original verbatim, so downstream JSONL emit + LongMemEval correctness scoring work unchanged — only the slug gets rewritten to satisfy the validator. + +**`gbrain eval longmemeval` now configures the AI gateway before running.** v0.28.8 deliberately skipped `connectEngine()` for this subcommand so it would run on machines without a configured brain. Side effect: the gateway never got configured either, so the first embed call inside `importFromContent` crashed with "AI gateway is not configured." Fix: explicit `configureGateway()` before `runEvalLongMemEval`, reading `~/.gbrain/config.json` if present and falling back to env vars (`GBRAIN_EMBEDDING_MODEL`, `GBRAIN_EMBEDDING_DIMENSIONS`, etc.) when there's no config — preserving the "runs on a fresh machine" property. + +### Itemized changes + +- `src/eval/longmemeval/adapter.ts`: new `normalizeSessions` accepts both oracle (`{session_id, turns}`) and _s (`Turn[][]` + parallel `haystack_session_ids`) shapes; new `sanitizeSessionIdForSlug` rewrites underscores + uppercase + other disallowed chars to `-`; `LongMemEvalQuestion.haystack_sessions` typed as the union, `haystack_session_ids?: string[]` field added with documentation. +- `src/cli.ts`: gateway configure step added before the `eval longmemeval` dispatch path, gated on `--help` short-circuit so help still works without a configured gateway. +- `test/eval-longmemeval.test.ts`: 2 new regression cases pinning the _s shape normalization end-to-end (slugs sanitized, frontmatter preserves original session_id, dates flow through) and the missing-haystack_session_ids fallback to synthesized `lme__` ids. + +## To take advantage of v0.35.1.1 + +`gbrain upgrade` handles the fix transparently. Re-run any LongMemEval-against-_s-split commands that had been crashing. + +1. **Upgrade:** + ```bash + gbrain upgrade + ``` +2. **(If you hit the prior crash) re-run with `--resume-from` to skip any questions already scored:** + ```bash + gbrain eval longmemeval ~/datasets/longmemeval/longmemeval_s.json \ + --output results.jsonl --resume-from results.jsonl --mode tokenmax + ``` +3. **No migration, no schema change, no breaking semantics.** + ## [0.35.1.0] - 2026-05-15 **Embedder shootout prereqs: pricing, public gateway export, and resume-from for long eval runs.** diff --git a/VERSION b/VERSION index e256b0f25..f4b4f188c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.35.1.0 +0.35.1.1 \ No newline at end of file diff --git a/package.json b/package.json index b5036bd9d..49d0b9839 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gbrain", - "version": "0.35.1.0", + "version": "0.35.1.1", "description": "Postgres-native personal knowledge brain with hybrid RAG search", "type": "module", "main": "src/core/index.ts", diff --git a/src/cli.ts b/src/cli.ts index 2a31e1c48..32d2e5081 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -964,8 +964,22 @@ async function handleCliOnly(command: string, args: string[]) { // v0.28.8: longmemeval brings its own in-memory PGLite. Bypassing // connectEngine here keeps `gbrain eval longmemeval --help` and benchmark // runs working on machines that have no `~/.gbrain/config.json` configured. + // + // v0.35.1.1: still need to configureGateway() so the in-memory brain's + // import + hybridSearch can embed via the configured provider. Reads + // ~/.gbrain/config.json when present; falls back to env vars otherwise + // (GBRAIN_EMBEDDING_MODEL / GBRAIN_EMBEDDING_DIMENSIONS). if (command === 'eval' && args[0] === 'longmemeval') { const { runEvalLongMemEval } = await import('./commands/eval-longmemeval.ts'); + if (!(args.length > 1 && (args[1] === '--help' || args[1] === '-h'))) { + const config = loadConfig() ?? ({ + embedding_model: process.env.GBRAIN_EMBEDDING_MODEL, + embedding_dimensions: process.env.GBRAIN_EMBEDDING_DIMENSIONS + ? Number(process.env.GBRAIN_EMBEDDING_DIMENSIONS) : undefined, + } as GBrainConfig); + const { configureGateway } = await import('./core/ai/gateway.ts'); + configureGateway(buildGatewayConfig(config)); + } await runEvalLongMemEval(args.slice(1)); return; } diff --git a/src/eval/longmemeval/adapter.ts b/src/eval/longmemeval/adapter.ts index a64c25150..aeb9a75d8 100644 --- a/src/eval/longmemeval/adapter.ts +++ b/src/eval/longmemeval/adapter.ts @@ -28,7 +28,18 @@ export interface LongMemEvalQuestion { question_type: string; question: string; answer: string; - haystack_sessions: LongMemEvalSession[]; + /** + * Two on-disk shapes are accepted (normalized by `haystackToPages`): + * + * 1. Oracle/structured: `LongMemEvalSession[]` with `{session_id, turns}`. + * 2. _s split (HuggingFace public download as of May 2026): + * `LongMemEvalTurn[][]` — each inner array is the turns of one + * session directly. Session IDs live in a sibling + * `haystack_session_ids: string[]` parallel array. + */ + haystack_sessions: LongMemEvalSession[] | LongMemEvalTurn[][]; + /** Parallel to haystack_sessions in the _s split. Absent in oracle shape. */ + haystack_session_ids?: string[]; /** ISO date strings, parallel to haystack_sessions. Some LongMemEval splits omit this. */ haystack_dates?: string[]; /** Ground truth: which haystack sessions actually contain the answer. */ @@ -61,14 +72,66 @@ function renderSession(session: LongMemEvalSession, date?: string): string { return fm.join('\n') + body.join('\n'); } +/** + * Normalize the on-disk haystack_sessions shape (oracle OR _s) into the + * structured `{session_id, turns}` form `renderSession` consumes. + * + * v0.35.1.1: the public _s split on HuggingFace uses `LongMemEvalTurn[][]` + * for `haystack_sessions` plus a parallel `haystack_session_ids: string[]` + * for the IDs. The pre-v0.35.1.1 adapter assumed only the oracle shape + * and crashed with `session.turns` undefined on the _s split. This + * normalizer accepts both. Mirrors the proven `normalizeSessions` helper + * in gbrain-evals/eval/runner/longmemeval.ts. + */ +function normalizeSessions(question: LongMemEvalQuestion): LongMemEvalSession[] { + const sessions: LongMemEvalSession[] = []; + const ids = question.haystack_session_ids ?? []; + const raw = question.haystack_sessions; + for (let i = 0; i < raw.length; i++) { + const item = raw[i] as unknown; + if (Array.isArray(item)) { + // _s shape: this entry is a turn array directly. + const sid = ids[i] ?? `lme_${question.question_id}_${i}`; + sessions.push({ session_id: sid, turns: item as LongMemEvalTurn[] }); + } else if (item && typeof item === 'object' && Array.isArray((item as LongMemEvalSession).turns)) { + // Oracle shape: {session_id, turns} object. + const sess = item as LongMemEvalSession; + sessions.push({ + session_id: sess.session_id ?? `lme_${question.question_id}_${i}`, + turns: sess.turns, + }); + } + // Silently skip malformed entries — keeps the run progressing on + // mixed/corrupted datasets; the surrounding per-question try/catch + // catches whole-question failures anyway. + } + return sessions; +} + +/** + * Normalize an arbitrary session_id into something `validatePageSlug` accepts. + * + * Validator rules (per v0.32.7 CJK wave): segments are `[a-z0-9CJK\-]+`, + * case-insensitive, forward-slash separated. The HuggingFace _s split uses + * `sharegpt_yywfIrx_0`-style ids with underscores AND uppercase letters, + * both of which are rejected. Lowercase + underscore -> hyphen produces a + * stable, validator-passing alias. Collisions are negligible per question + * (each question's slug-space is reset per benchmark question by the + * harness's resetTables). + */ +function sanitizeSessionIdForSlug(sessionId: string): string { + return sessionId.toLowerCase().replace(/[_.]/g, '-').replace(/[^a-z0-9-]/g, '-'); +} + export function haystackToPages(question: LongMemEvalQuestion): PageInputForImport[] { const pages: PageInputForImport[] = []; const dates = question.haystack_dates ?? []; - for (let i = 0; i < question.haystack_sessions.length; i++) { - const session = question.haystack_sessions[i]; + const sessions = normalizeSessions(question); + for (let i = 0; i < sessions.length; i++) { + const session = sessions[i]; const date = dates[i]; pages.push({ - slug: `chat/${session.session_id}`, + slug: `chat/${sanitizeSessionIdForSlug(session.session_id)}`, content: renderSession(session, date), }); } diff --git a/test/eval-longmemeval.test.ts b/test/eval-longmemeval.test.ts index 9a9758d1a..fafce83ab 100644 --- a/test/eval-longmemeval.test.ts +++ b/test/eval-longmemeval.test.ts @@ -249,6 +249,57 @@ describe('adapter haystackToPages', () => { expect(pages[0].content).toContain('session_id: sess-x'); expect(pages[0].content).not.toContain('date:'); }); + + // v0.35.1.1 regression: the public LongMemEval _s split uses arrays of + // turn-arrays for haystack_sessions plus a parallel haystack_session_ids + // string array. The pre-v0.35.1.1 adapter crashed with `session.turns is + // undefined` on this shape. Pre-v0.35.1.1 the slug validator also + // rejected the underscored, mixed-case session_ids the dataset uses. + test('v0.35.1.1: _s split shape (turn-array + parallel ids) normalizes correctly', () => { + const q: LongMemEvalQuestion = { + question_id: 'q-s-1', + question_type: 'single-session-user', + question: 'q?', + answer: 'a', + haystack_dates: ['2025-01-01', '2025-01-02'], + answer_session_ids: ['sharegpt_AbC_0'], + haystack_session_ids: ['sharegpt_AbC_0', 'sess_DEF_1'], + // No {session_id, turns} — turns directly per the _s shape. + haystack_sessions: [ + [{ role: 'user', content: 'hi' }, { role: 'assistant', content: 'hello' }], + [{ role: 'user', content: 'bye' }], + ], + }; + const pages = haystackToPages(q); + expect(pages.length).toBe(2); + // Slugs got lowercased + underscores became hyphens (validator-safe). + expect(pages[0].slug).toBe('chat/sharegpt-abc-0'); + expect(pages[1].slug).toBe('chat/sess-def-1'); + // Frontmatter keeps the ORIGINAL session_id (no sanitization). The + // _s ids preserve through the round-trip; only the slug got rewritten. + expect(pages[0].content).toContain('session_id: sharegpt_AbC_0'); + expect(pages[0].content).toContain('date: 2025-01-01'); + expect(pages[0].content).toContain('**user:** hi'); + expect(pages[1].content).toContain('**user:** bye'); + }); + + test('v0.35.1.1: missing haystack_session_ids on _s shape synthesizes ids per question', () => { + const q: LongMemEvalQuestion = { + question_id: 'q-s-2', + question_type: 'single-session-user', + question: 'q?', + answer: 'a', + answer_session_ids: [], + // _s shape but the parallel ids array is absent. Adapter falls back + // to a synthesized `lme__` slug. + haystack_sessions: [ + [{ role: 'user', content: 'turn 1' }], + ], + }; + const pages = haystackToPages(q); + expect(pages.length).toBe(1); + expect(pages[0].slug).toBe('chat/lme-q-s-2-0'); + }); }); // ---------------------------------------------------------------------------