Compare commits

..
Author SHA1 Message Date
Garry TanandClaude Fable 5 62bd7fb3b7 fix(schema): block scalars keep '#' as literal content
Inside a YAML block scalar '#' is content, not a comment; parseBlockScalar
was routing lines through stripComment/isBlank, truncating descriptions
like 'see issue #2029' and blanking comment-looking lines. Use the raw
line inside the scalar. Adds a pinning test.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 12:01:58 -07:00
Garry TanandClaude Fable 5 733fcd633a test(schema): point bundled-registry test at bundled.ts source of truth
The bundled pack list moved from load-active.ts to bundled.ts in the
truthful-inspection refactor; the T4 registry test still grepped
load-active.ts source. Assert BUNDLED_PACK_NAMES directly instead.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 10:55:16 -07:00
97e716b01f fix(minions): subagent default client resolves config-stored Anthropic key (#2048)
The legacy subagent path constructed a bare new Anthropic() (env-only), so
launchd/MCP workers whose key lives in gbrain config (anthropic_api_key)
failed auth. anthropic-key.ts now exports resolveAnthropicKey() (env first,
then config; hasAnthropicKey delegates) and makeSubagentHandler passes it as
apiKey.

Partial takeover of #2048 — only the auth patch; the path patches were
superseded by the outputRoot mechanism (#2415).

Co-authored-by: JiraiyaETH <JiraiyaETH@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 14:31:37 -07:00
ff737e4345 fix(schema): make bundled pack inspection truthful (#2029)
Two live bugs:
- parseYamlMini had no block-scalar support, so a 'description: |' swallowed
  every following top-level key — the active gbrain-recommended pack loaded
  with 0 page types. Add parseBlockScalar for |/|-/|+ and >/>-/>+ in both
  mapping and sequence-sibling positions.
- The bundled-pack list was hand-copied in three places (operations.ts had 2
  names, mutate.ts had 3, load-active.ts had 7). New single registry
  src/core/schema-pack/bundled.ts carries all 7 shipped packs; every
  consumer derives from it.

Takeover of #2029, rebased onto master (schema.ts hunks already landed).

Co-authored-by: JiraiyaETH <JiraiyaETH@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 14:31:37 -07:00
20 changed files with 231 additions and 463 deletions
+13 -3
View File
@@ -18,12 +18,22 @@
import { loadConfig } from '../config.ts';
export function hasAnthropicKey(): boolean {
if (process.env.ANTHROPIC_API_KEY) return true;
return resolveAnthropicKey() !== undefined;
}
/**
* Resolve the actual key value: env first, then the gbrain config file.
* Callers constructing an Anthropic client directly (e.g. the legacy
* subagent path) must pass this as `apiKey` — a bare `new Anthropic()`
* only sees env, so launchd/MCP workers with config-stored keys fail.
*/
export function resolveAnthropicKey(): string | undefined {
if (process.env.ANTHROPIC_API_KEY) return process.env.ANTHROPIC_API_KEY;
try {
const cfg = loadConfig();
if (cfg?.anthropic_api_key) return true;
if (cfg?.anthropic_api_key) return cfg.anthropic_api_key;
} catch {
// loadConfig may throw on first-run installs; treat as no key available.
}
return false;
return undefined;
}
+3 -39
View File
@@ -18,9 +18,8 @@
* at runtime.
*/
import { chunkText as recursiveChunk, capByEstimatedTokens, DEFAULT_MAX_EST_TOKENS } from './recursive.ts';
import { chunkText as recursiveChunk } from './recursive.ts';
import { buildQualifiedName } from './qualified-names.ts';
import { estimateEmbeddingTokens } from '../cjk.ts';
// Embed the tree-sitter runtime + per-language grammars as files.
// `with { type: 'file' }` returns a path (string) at runtime. Bun bundles
@@ -112,15 +111,7 @@ import G_ZIG from '../../assets/wasm/grammars/tree-sitter-zig.wasm' with { type:
// chunks get the new columns populated. Without this, the v28 backfill
// gives every existing chunk a search_vector but subsequent Layer 5 AST
// work would silently no-op.
//
// v5: estimated-token hard cap on AST-path chunks (capCodeChunks). A node
// splitLargeNode can't subdivide (giant single-statement function, huge
// literal) previously shipped WHOLE regardless of size and could overflow
// strict per-request embedding-token limits (local llama-server crashes
// past ~2,050 tokens, measured). Mirrors the markdown
// chunker's v4 cap; fallback-path chunks are already capped inside
// recursiveChunk.
export const CHUNKER_VERSION = 5;
export const CHUNKER_VERSION = 4;
// Lazy-loaded tree-sitter module (v0.22.x API: Parser is default export)
let Parser: typeof import('web-tree-sitter') | null = null;
@@ -717,7 +708,7 @@ export async function chunkCodeTextFull(
if (chunks.length === 0) {
return { chunks: fallbackChunks(source, filePath, language, opts), edges: rawEdges };
}
return { chunks: capCodeChunks(mergeSmallSiblings(chunks, chunkTarget)), edges: rawEdges };
return { chunks: mergeSmallSiblings(chunks, chunkTarget), edges: rawEdges };
} catch {
return { chunks: fallbackChunks(source, filePath, language, opts), edges: [] };
} finally {
@@ -800,33 +791,6 @@ function mergeSmallSiblings(chunks: CodeChunk[], chunkTarget: number): CodeChunk
return merged;
}
/**
* v5 final safety pass for AST-path chunks: split any chunk whose
* ESTIMATED embedding tokens (conservative per-char-class heuristic,
* cjk.ts) exceed DEFAULT_MAX_EST_TOKENS. Reaches chunks the AST logic
* can't subdivide — splitLargeNode returns [] for nodes with < 2 body
* children (giant single-statement functions, huge literals), which
* previously shipped whole at any size.
*
* Split pieces inherit the source chunk's metadata verbatim; start/end
* lines become approximate for pieces after the first. Acceptable —
* these chunks exist for embedding + retrieval, and the alternative was
* an embedding request the server rejects (or worse, crashes on).
*/
function capCodeChunks(chunks: CodeChunk[]): CodeChunk[] {
if (chunks.every((c) => estimateEmbeddingTokens(c.text) <= DEFAULT_MAX_EST_TOKENS)) {
return chunks;
}
const out: CodeChunk[] = [];
for (const c of chunks) {
const pieces = capByEstimatedTokens(c.text, DEFAULT_MAX_EST_TOKENS);
for (const piece of pieces) {
out.push({ ...c, text: piece, index: out.length, metadata: { ...c.metadata } });
}
}
return out;
}
function buildMergedChunk(group: CodeChunk[], index: number): CodeChunk {
const first = group[0]!;
const last = group[group.length - 1]!;
+6 -117
View File
@@ -17,13 +17,7 @@
* Lossless invariant: non-overlapping portions reassemble to original.
*/
import {
countCJKAwareWords,
CJK_SENTENCE_DELIMITERS,
CJK_CLAUSE_DELIMITERS,
charEmbedTokenWeight,
estimateEmbeddingTokens,
} from '../cjk.ts';
import { countCJKAwareWords, CJK_SENTENCE_DELIMITERS, CJK_CLAUSE_DELIMITERS } from '../cjk.ts';
/**
* Markdown chunker version. Folded into the per-page chunker_version column
@@ -39,20 +33,8 @@ import {
* re-embed (not re-chunk) so existing pages pick up the wrapper on the
* post-upgrade reembed sweep. See
* `src/core/contextual-retrieval-service.ts`.
*
* v4: estimated-token hard cap + whitespace-word undercount fix. The word
* pipeline counted a 150-char URL as ONE whitespace word, so URL/phone/
* email-dense docs (CJK density < 0.30 → whitespace fallback) produced
* 3-4K-char chunks that overflow strict per-request embedding-token
* limits (measured: local llama-server crashes past ~2,050 tokens; URL
* soup tokenizes at ~1.6 chars/token). Two changes:
* 1. countWords() floors the count at ceil(nonWhitespaceChars/6) so a
* URL counts roughly per-character, not as one word.
* 2. capByEstimatedTokens() final pass guarantees every chunk fits
* `maxTokens` (default 1500) under a conservative per-char-class
* token estimate, regardless of how word counting misjudged it.
*/
export const MARKDOWN_CHUNKER_VERSION = 4;
export const MARKDOWN_CHUNKER_VERSION = 3;
const DELIMITERS: string[][] = [
['\n\n'], // L0: paragraphs
@@ -66,20 +48,8 @@ export interface ChunkOptions {
chunkSize?: number; // target words per chunk (default 300)
chunkOverlap?: number; // overlap words (default 50)
maxChars?: number; // hard cap on any chunk's char length (default 6000)
/**
* v4: hard cap on any chunk's ESTIMATED embedding tokens (default 1500).
* Estimate = conservative per-char-class weights (see cjk.ts
* estimateEmbeddingTokens) — deliberately high, so the real tokenizer
* count stays below this value. Default leaves headroom for the
* contextual-retrieval wrapper (≤ ~630 chars) under a ~2,050-token
* per-request embedding server limit.
*/
maxTokens?: number;
}
/** v4 default for ChunkOptions.maxTokens — see the field doc above. */
export const DEFAULT_MAX_EST_TOKENS = 1500;
export interface TextChunk {
text: string;
index: number;
@@ -103,7 +73,6 @@ export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] {
const chunkSize = opts?.chunkSize || 300;
const chunkOverlap = opts?.chunkOverlap || 50;
const maxChars = opts?.maxChars || 6000;
const maxTokens = opts?.maxTokens || DEFAULT_MAX_EST_TOKENS;
if (!text || text.trim().length === 0) return [];
@@ -120,9 +89,8 @@ export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] {
const wordCount = countWords(stripped);
if (wordCount <= chunkSize) {
// Single-chunk path: still apply the maxChars + maxTokens caps.
const capped = capByChars(stripped.trim(), maxChars)
.flatMap((t) => capByEstimatedTokens(t, maxTokens));
// Single-chunk path: still apply the maxChars cap.
const capped = capByChars(stripped.trim(), maxChars);
return capped.map((t, i) => ({ text: t, index: i }));
}
@@ -133,14 +101,9 @@ export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] {
// v0.32.7: hard char cap. Catches pathological CJK + whitespace-less text
// that the word-level pipeline can't bound (a single Chinese paragraph can
// exceed 8192 OpenAI embedding tokens at any word count).
// v4: estimated-token cap on top — the char cap alone passes token-dense
// content (URL soup at ~1.6 chars/token) that overflows strict embedding
// server limits.
const capped: string[] = [];
for (const chunk of withOverlap) {
for (const piece of capByChars(chunk.trim(), maxChars)) {
capped.push(...capByEstimatedTokens(piece, maxTokens));
}
capped.push(...capByChars(chunk.trim(), maxChars));
}
return capped.map((t, i) => ({ text: t, index: i }));
}
@@ -169,68 +132,6 @@ function capByChars(text: string, maxChars: number): string[] {
return out;
}
/**
* How far back (in chars) the token cap looks for a friendly cut point
* before falling back to a hard cut. 300 covers typical rollup/list line
* lengths so forced splits land at line starts, not mid-URL.
*/
const TOKEN_CAP_CUT_LOOKBACK = 300;
/**
* v4: hard-cap a chunk's ESTIMATED embedding tokens. Final safety pass —
* runs after capByChars on every chunk, so no upstream miscounting
* (whitespace-word fallback, overlap inflation, char-cap survivors) can
* emit a chunk past `maxTokens`.
*
* Cut placement prefers, within the last TOKEN_CAP_CUT_LOOKBACK chars of
* the window: a newline, then any whitespace, then a hard cut. This keeps
* forced splits off mid-line/mid-URL positions for list-shaped content
* and inside code fences. No overlap is added (pieces stay lossless
* modulo the trims the char cap already applies).
*
* @internal exported for the code chunker (code.ts) and tests.
*/
export function capByEstimatedTokens(text: string, maxTokens: number): string[] {
if (text.length === 0) return [];
if (estimateEmbeddingTokens(text) <= maxTokens) return [text];
const out: string[] = [];
let start = 0;
while (start < text.length) {
// Greedily extend the window until the next char would break the cap.
// Always take at least one char so the loop makes forward progress.
let est = 0;
let end = start;
while (end < text.length) {
const w = charEmbedTokenWeight(text.charCodeAt(end));
if (est + w > maxTokens && end > start) break;
est += w;
end++;
}
if (end < text.length) {
const windowStart = Math.max(start + 1, end - TOKEN_CAP_CUT_LOOKBACK);
let cut = text.lastIndexOf('\n', end - 1);
if (cut < windowStart) {
cut = -1;
for (let i = end - 1; i >= windowStart; i--) {
const code = text.charCodeAt(i);
if (code === 0x20 || (code >= 0x09 && code <= 0x0d)) {
cut = i;
break;
}
}
}
if (cut >= windowStart) end = cut + 1;
}
const slice = text.slice(start, end).trim();
if (slice.length > 0) out.push(slice);
start = end;
}
return out;
}
function recursiveSplit(text: string, level: number, target: number): string[] {
if (level >= DELIMITERS.length) {
// Level 4: split on whitespace
@@ -416,19 +317,7 @@ function extractTrailingContext(text: string, targetWords: number): string {
* Delegated to src/core/cjk.ts so the slugify whitelist, expansion
* detection, and PGLite keyword fallback all agree on what "CJK enough"
* means.
*
* v4: floored at ceil(nonWhitespaceChars/6). The whitespace fallback
* counts a 150-char URL as ONE word, so URL/phone/email-dense docs
* (whose ASCII mass pushes CJK density below the 0.30 threshold) were
* sized at a fraction of their real bulk and merged into 3-4K-char
* chunks. The floor makes long whitespace-less runs count roughly
* per-character while leaving normal Latin prose untouched (average
* English word ≈ 5 chars < 6, so the whitespace count still wins).
* Kept local to the chunker — search/expansion.ts keeps the original
* countCJKAwareWords semantics for its query-length check.
*/
function countWords(text: string): number {
const cjkAware = countCJKAwareWords(text);
const nonWhitespace = text.replace(/\s/g, '').length;
return Math.max(cjkAware, Math.ceil(nonWhitespace / 6));
return countCJKAwareWords(text);
}
-61
View File
@@ -65,64 +65,3 @@ export function countCJKAwareWords(s: string): number {
export function escapeLikePattern(s: string): string {
return s.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_');
}
/**
* Conservative per-char-class embedding-token weights (markdown chunker v4).
*
* Why this exists: the chunker's "word" counting drastically UNDER-counts
* whitespace-less ASCII runs (a 150-char URL = 1 whitespace word), so
* word-based size targets can emit chunks that overflow an embedding
* server's per-request token limit. Measured on a local Qwen3-embedding
* llama-server stack:
* - URL/phone/email-dense text tokenizes at ~1.6 chars/token
* - base64-ish / minified blobs approach ~1.3 chars/token (worst case)
* - Korean prose tokenizes NO WORSE than 1 char/token in practice
*
* Weights are deliberately HIGH (tokens are overestimated) so any cap
* based on this estimate is safe against real tokenizers:
* - CJK char → 1.0 token (real CJK prose is cheaper)
* - other non-space → 0.75 token (≈1.33 chars/token, covers base64)
* - whitespace → 0.1 token (mostly folds into neighbor tokens)
*/
export const EMBED_TOKEN_WEIGHT_CJK = 1.0;
export const EMBED_TOKEN_WEIGHT_OTHER = 0.75;
export const EMBED_TOKEN_WEIGHT_WS = 0.1;
/** BMP CJK check by UTF-16 code unit — same ranges as CJK_SLUG_CHARS. */
export function isCJKCodeUnit(code: number): boolean {
return (
(code >= 0x4e00 && code <= 0x9fff) || // Han
(code >= 0x3040 && code <= 0x309f) || // Hiragana
(code >= 0x30a0 && code <= 0x30ff) || // Katakana
(code >= 0xac00 && code <= 0xd7af) // Hangul Syllables
);
}
/**
* Per-code-unit token weight. Unrecognized whitespace (exotic Unicode
* spaces) intentionally falls into OTHER — that only overestimates.
*/
export function charEmbedTokenWeight(code: number): number {
if (isCJKCodeUnit(code)) return EMBED_TOKEN_WEIGHT_CJK;
if (
code === 0x20 || (code >= 0x09 && code <= 0x0d) ||
code === 0xa0 || code === 0x3000
) {
return EMBED_TOKEN_WEIGHT_WS;
}
return EMBED_TOKEN_WEIGHT_OTHER;
}
/**
* Tokenizer-free embedding-token estimate (conservative overestimate).
* See weight docs above. Astral chars count as 2 OTHER code units —
* another overestimate, which is the safe direction.
*/
export function estimateEmbeddingTokens(s: string): number {
if (s.length === 0) return 0;
let est = 0;
for (let i = 0; i < s.length; i++) {
est += charEmbedTokenWeight(s.charCodeAt(i));
}
return Math.ceil(est);
}
+5 -1
View File
@@ -48,6 +48,7 @@ import {
logSubagentHeartbeat,
} from './subagent-audit.ts';
import { resolveModel, isAnthropicProvider, TIER_DEFAULTS } from '../../model-config.ts';
import { resolveAnthropicKey } from '../../ai/anthropic-key.ts';
import { buildSystemPrompt, DEFAULT_SUBAGENT_SYSTEM } from '../system-prompt.ts';
import { toolLoop as gatewayToolLoop } from '../../ai/gateway.ts';
import type { ChatToolDef, ChatMessage, ChatBlock, ChatResult, ToolHandler } from '../../ai/gateway.ts';
@@ -186,7 +187,10 @@ export function makeSubagentHandler(deps: SubagentDeps) {
// lives at sdk.messages.create. Assigning sdk.messages directly gets the
// right object; JS method-call semantics preserve `this` at the call
// site (subagent.ts invokes client.create(...) with client === sdk.messages).
const makeAnthropic = deps.makeAnthropic ?? (() => new Anthropic());
// Resolve the key env-first, then config (anthropic_api_key) — a bare
// new Anthropic() only reads env, so launchd/MCP workers whose key lives
// in the gbrain config file would fail auth (#2048).
const makeAnthropic = deps.makeAnthropic ?? (() => new Anthropic({ apiKey: resolveAnthropicKey() }));
const client: MessagesClient = deps.client ?? makeAnthropic().messages;
const config = deps.config ?? loadConfig() ?? ({ engine: 'postgres' } as GBrainConfig);
const rateLeaseKey = deps.rateLeaseKey ?? DEFAULT_RATE_KEY;
+2 -1
View File
@@ -4562,7 +4562,8 @@ const list_schema_packs: Operation = {
const { existsSync, readdirSync } = await import('node:fs');
const { join } = await import('node:path');
const { gbrainPath } = await import('./config.ts');
const bundled = ['gbrain-base', 'gbrain-recommended'];
const { BUNDLED_PACK_NAMES } = await import('./schema-pack/bundled.ts');
const bundled = [...BUNDLED_PACK_NAMES];
const installedDir = gbrainPath('schema-packs');
const installed: string[] = [];
if (existsSync(installedDir)) {
+24
View File
@@ -0,0 +1,24 @@
// Bundled schema-pack registry — single source of truth for the packs that
// ship in src/core/schema-pack/base/. Keep every bundled-pack consumer
// (CLI/MCP inspection, active-pack loading, mutation guards, upgrade
// discovery) on this one list so they cannot drift.
//
// v0.39 T8 — gbrain-base + gbrain-recommended.
// v0.41 T4 — lens packs: creator, investor, engineer, everything (meta-pack).
// v0.42 type-unification — gbrain-base-v2, the 15-type canonical successor.
export const BUNDLED_PACK_NAMES = [
'gbrain-base',
'gbrain-recommended',
'gbrain-creator',
'gbrain-investor',
'gbrain-engineer',
'gbrain-everything',
'gbrain-base-v2',
] as const;
export type BundledPackName = typeof BUNDLED_PACK_NAMES[number];
export function isBundledPackName(name: string): name is BundledPackName {
return (BUNDLED_PACK_NAMES as readonly string[]).includes(name);
}
+2 -22
View File
@@ -37,6 +37,7 @@ import {
type ResolutionInput,
type ResolutionResult,
} from './registry.ts';
import { isBundledPackName } from './bundled.ts';
/**
* Inputs the caller (operations.ts handler / engine query path) provides.
@@ -92,28 +93,7 @@ export function _resetPackLocatorForTests(): void {
* throwing UnknownPackError with a paste-ready install hint.
*/
function defaultPackLocator(name: string): string | null {
// v0.39 T8 — bundled packs registry. gbrain-base + gbrain-recommended
// ship in src/core/schema-pack/base/. Add a new entry here to bundle
// additional canonical packs.
//
// v0.41 T4 — lens packs join the bundle: creator (atoms + concepts +
// extract_atoms/synthesize_concepts phases), investor (theses + bet
// resolution + 3 calibration domains), engineer (gstack-learnings bridge
// + 3 calibration domains), everything (meta-pack stacking all three
// via extends + borrow_from). Each ships as a real YAML at base/<name>.yaml.
const BUNDLED: ReadonlyArray<string> = [
'gbrain-base',
'gbrain-recommended',
'gbrain-creator',
'gbrain-investor',
'gbrain-engineer',
'gbrain-everything',
// v0.42 type-unification: 15-type canonical successor to gbrain-base.
// Ships as install default (Lane E T17) + via gbrain onboard pack
// upgrade flow (the unify-types Minion handler).
'gbrain-base-v2',
];
if (BUNDLED.includes(name)) {
if (isBundledPackName(name)) {
// Resolve bundled YAML relative to this source file. Works in both
// direct-bun execution and bun --compile binaries.
const here = dirname(fileURLToPath(import.meta.url));
+31
View File
@@ -159,6 +159,29 @@ export function parseYamlMini(content: string): unknown {
return parseMapping(baseIndent);
}
function parseBlockScalar(parentIndent: number, folded: boolean): string {
const contentIndent = parentIndent + 2;
const out: string[] = [];
while (i < lines.length) {
const raw = lines[i];
// Inside a block scalar everything is literal content — '#' is NOT a
// comment here, so use the raw line (no stripComment / isBlank).
if (raw.trim() === '') {
out.push('');
i++;
continue;
}
const indent = indentOf(raw);
if (indent <= parentIndent) break;
out.push(raw.slice(Math.min(contentIndent, indent)));
i++;
}
if (folded) {
return out.join(' ').replace(/\s+$/u, '');
}
return out.join('\n').replace(/\n+$/u, '');
}
function parseSequence(baseIndent: number): unknown[] {
const result: unknown[] = [];
while (i < lines.length) {
@@ -227,6 +250,10 @@ export function parseYamlMini(content: string): unknown {
i++;
if (rest2 === '') {
map[key2] = parseBlock(nextIndent + 2);
} else if (rest2 === '|' || rest2 === '|-' || rest2 === '|+') {
map[key2] = parseBlockScalar(nextIndent, false);
} else if (rest2 === '>' || rest2 === '>-' || rest2 === '>+') {
map[key2] = parseBlockScalar(nextIndent, true);
} else {
map[key2] = parseScalar(rest2);
}
@@ -257,6 +284,10 @@ export function parseYamlMini(content: string): unknown {
i++;
if (rest === '') {
result[key] = parseBlock(indent + 2);
} else if (rest === '|' || rest === '|-' || rest === '|+') {
result[key] = parseBlockScalar(indent, false);
} else if (rest === '>' || rest === '>-' || rest === '>+') {
result[key] = parseBlockScalar(indent, true);
} else {
result[key] = parseScalar(rest);
}
+2 -1
View File
@@ -65,6 +65,7 @@ import { invalidateQueryCache } from './query-cache-invalidator.ts';
import { logMutationFailure, logMutationSuccess, type MutationActor, type MutationOp } from './mutate-audit.ts';
import { runFilePlaneLintRules } from './lint-rules.ts';
import { withPackLock, type PackLockOpts } from './pack-lock.ts';
import { BUNDLED_PACK_NAMES as BUNDLED_PACK_NAME_LIST } from './bundled.ts';
import type { BrainEngine } from '../engine.ts';
export type PackFileFormat = 'json' | 'yaml';
@@ -93,7 +94,7 @@ export class SchemaPackMutationError extends Error {
}
}
export const BUNDLED_PACK_NAMES = new Set(['gbrain-base', 'gbrain-recommended', 'gbrain-base-v2']);
export const BUNDLED_PACK_NAMES = new Set<string>(BUNDLED_PACK_NAME_LIST);
export interface MutateResult {
/** Pack name that was mutated. */
+33 -1
View File
@@ -10,7 +10,7 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { withEnv } from '../helpers/with-env.ts';
import { hasAnthropicKey } from '../../src/core/ai/anthropic-key.ts';
import { hasAnthropicKey, resolveAnthropicKey } from '../../src/core/ai/anthropic-key.ts';
const tmpDirs: string[] = [];
function freshHome(withConfig?: Record<string, unknown>): string {
@@ -62,3 +62,35 @@ describe('hasAnthropicKey', () => {
);
});
});
describe('resolveAnthropicKey (#2048 — subagent config-key auth)', () => {
test('env wins over config', async () => {
const home = freshHome({ anthropic_api_key: 'sk-from-config' });
await withEnv(
{ ANTHROPIC_API_KEY: 'sk-from-env', GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined },
async () => {
expect(resolveAnthropicKey()).toBe('sk-from-env');
},
);
});
test('config key returned when env unset', async () => {
const home = freshHome({ anthropic_api_key: 'sk-from-config' });
await withEnv(
{ ANTHROPIC_API_KEY: undefined, GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined },
async () => {
expect(resolveAnthropicKey()).toBe('sk-from-config');
},
);
});
test('neither → undefined', async () => {
const home = freshHome();
await withEnv(
{ ANTHROPIC_API_KEY: undefined, GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined },
async () => {
expect(resolveAnthropicKey()).toBeUndefined();
},
);
});
});
+3 -6
View File
@@ -15,22 +15,19 @@ import { describe, test, expect } from 'bun:test';
import { CHUNKER_VERSION } from '../src/core/chunkers/code.ts';
describe('Layer 12 — CHUNKER_VERSION constant', () => {
test('bumped to 5 for the estimated-token hard cap', () => {
test('bumped to 4 for Cathedral II', () => {
// v3: v0.19.0 Chonkie parity (tokenizer + small-sibling merge).
// v4: v0.20.0 Cathedral II (qualified names + parent scope + doc_comment
// + fence extraction + chunk-grain FTS). Folded into content_hash
// so any bump forces clean re-chunks on next sync.
// v5: estimated-token hard cap on AST-path chunks (capCodeChunks) so
// un-subdividable giant nodes can't overflow strict embedding
// server token limits.
expect(CHUNKER_VERSION).toBe(5);
expect(CHUNKER_VERSION).toBe(4);
});
test('is stable across imports (not recomputed at call time)', async () => {
const a = (await import('../src/core/chunkers/code.ts')).CHUNKER_VERSION;
const b = (await import('../src/core/chunkers/code.ts')).CHUNKER_VERSION;
expect(a).toBe(b);
expect(a).toBe(5);
expect(a).toBe(4);
});
});
+2 -2
View File
@@ -10,8 +10,8 @@ import { describe, test, expect } from 'bun:test';
import { chunkCodeText, detectCodeLanguage, CHUNKER_VERSION } from '../../src/core/chunkers/code.ts';
describe('CHUNKER_VERSION', () => {
test('v5: estimated-token hard cap on AST-path chunks', () => {
expect(CHUNKER_VERSION).toBe(5);
test('v0.20.0 Cathedral II Layer 12 bumped to 4', () => {
expect(CHUNKER_VERSION).toBe(4);
});
});
+5 -6
View File
@@ -135,14 +135,13 @@ describe('Recursive Text Chunker', () => {
});
describe('CJK chunking (v0.32.7)', () => {
test('MARKDOWN_CHUNKER_VERSION is 4', async () => {
test('MARKDOWN_CHUNKER_VERSION is 3', async () => {
// v0.40.3.0: bumped 2→3 to signal the post-upgrade reembed sweep that
// contextual retrieval wrapping is now applied at embed time.
// v4: estimated-token hard cap + whitespace-word undercount floor
// (URL-dense docs produced chunks past strict embedding server token
// limits). Boundary change → forces re-chunk for chunker_version < 4.
// contextual retrieval wrapping is now applied at embed time. Chunk
// boundaries themselves are unchanged; the bump forces re-embed for
// pages where chunker_version < 3.
const mod = await import('../../src/core/chunkers/recursive.ts');
expect(mod.MARKDOWN_CHUNKER_VERSION).toBe(4);
expect(mod.MARKDOWN_CHUNKER_VERSION).toBe(3);
});
test('long pure-Chinese paragraph splits into multiple chunks', () => {
-195
View File
@@ -1,195 +0,0 @@
/**
* Markdown chunker v4 / code chunker v5 estimated-token hard cap
* regression tests.
*
* Reproduces a field failure: a local llama-server embedding backend
* (`-ub 2048`) crashes deterministically (trace/BPT trap EOF at the
* client) when a single chunk exceeds ~2,050 real tokens. Two content
* shapes triggered it:
*
* 1. Korean docs carrying one long source URL per line.
* The URLs' ASCII mass pushes CJK density below 0.30, flipping
* countCJKAwareWords to whitespace counting, where a 150-char URL
* counts as ONE word chunks ballooned to 3-4K chars 2,000+
* real tokens (URL soup tokenizes at ~1.6 chars/token).
*
* 2. Large JSON code blocks (~7K chars) that the word pipeline
* undercounts the same way (few whitespace tokens).
*
* The fix: every emitted chunk must satisfy
* estimateEmbeddingTokens(chunk) <= maxTokens (default 1500)
* where the estimate deliberately OVERSTATES real tokenizer counts.
*/
import { describe, test, expect } from 'bun:test';
import { chunkText, capByEstimatedTokens, DEFAULT_MAX_EST_TOKENS } from '../../src/core/chunkers/recursive.ts';
import { chunkCodeText } from '../../src/core/chunkers/code.ts';
import { estimateEmbeddingTokens } from '../../src/core/cjk.ts';
/** Synthesize the failing shape: Korean rollup lines each ending in a long Notion URL. */
function urlDenseKoreanRollup(lines: number): string {
const out: string[] = ['# 링크가 줄마다 붙는 한국어 예시 문서', ''];
for (let i = 0; i < lines; i++) {
const hex32 = (i * 2654435761 >>> 0).toString(16).padStart(8, '0').repeat(4);
out.push(
`- **항목 ${i}**: 이 줄은 청커 동작 검증을 위한 의미 없는 한국어 예시 문장입니다 · 전화 000-0000-${String(1000 + i)} · ` +
`이메일 user${i}@example.com · 링크: https://docs.example.com/pages/${hex32}?v=abcdef0123456789&ref=sample`,
);
}
return out.join('\n');
}
/** Synthesize a large pretty-printed JSON block with CJK values. */
function bigJsonBlock(targetChars: number): string {
const entries: string[] = [];
let i = 0;
let len = 0;
while (len < targetChars) {
const row =
` "item_${i}": { "name": "예시-${i}", "url": "https://example.com/api/v2/items/${i}?token=abc${i}def", "qty": ${i % 100}, "memo": "한국어 값이 섞인 예시 데이터" }`;
entries.push(row);
len += row.length;
i++;
}
return `{\n${entries.join(',\n')}\n}`;
}
describe('v4 estimated-token cap — URL-dense Korean doc (field-failure shape)', () => {
test('every chunk stays under the estimated-token cap', () => {
const md = urlDenseKoreanRollup(60);
const chunks = chunkText(md);
expect(chunks.length).toBeGreaterThan(0);
for (const c of chunks) {
expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS);
}
});
test('no chunk reaches the measured 3K-char danger zone for URL soup', () => {
const md = urlDenseKoreanRollup(60);
const chunks = chunkText(md);
// 1500 est tokens at the OTHER weight (0.75/char) bounds chunks to
// ~2,000 chars for pure ASCII — well under the ~3,300 chars where
// URL-dense content crosses ~2,050 real tokens (1.6 chars/token).
for (const c of chunks) {
expect(c.text.length).toBeLessThanOrEqual(2600);
}
});
test('content is preserved (no lines dropped by the cap)', () => {
const md = urlDenseKoreanRollup(60);
const chunks = chunkText(md);
const joined = chunks.map((c) => c.text).join('\n');
// Spot-check first / middle / last rollup lines survive chunking.
for (const marker of ['항목 0', '항목 30', '항목 59']) {
expect(joined).toContain(marker);
}
});
});
describe('v4 estimated-token cap — large JSON blocks', () => {
test('7K-char pretty JSON through the prose path stays under the cap', () => {
const md = `설정 파일 원문 보존:\n\n\`\`\`\n${bigJsonBlock(7000)}\n\`\`\`\n`;
const chunks = chunkText(md);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) {
expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS);
}
});
test('7K-char minified JSON (single whitespace-less token) stays under the cap', () => {
const minified = bigJsonBlock(7000).replace(/\n\s*/g, '');
const chunks = chunkText(minified);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) {
expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS);
}
});
test('json fence via the code chunker stays under the cap (+header slack)', async () => {
const chunks = await chunkCodeText(bigJsonBlock(7000), 'fence.json');
expect(chunks.length).toBeGreaterThan(0);
for (const c of chunks) {
// buildChunk prepends a short "[JSON] fence.json:…" header AFTER the
// body-level cap; allow ~60 est tokens of header slack. Real-token
// safety margin (2,050 overestimated 1,500) absorbs this easily.
expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS + 60);
}
});
});
describe('v4 word-count floor — behavior preserved for normal content', () => {
test('Latin prose chunking is unchanged by the floor (avg word < 6 chars)', () => {
const prose = Array.from({ length: 120 }, (_, i) =>
`This is sentence number ${i} and it talks about ordinary things in plain words.`,
).join(' ');
const chunks = chunkText(prose);
// Historical behavior: ~1,560 whitespace words → multiple ~300-word chunks.
expect(chunks.length).toBeGreaterThan(3);
for (const c of chunks) {
const words = c.text.split(/\s+/).length;
expect(words).toBeLessThanOrEqual(300 * 1.5 + 50); // merge cap + overlap
}
});
test('Korean prose (CJK-dense, no URLs) never triggers the token cap', () => {
const prose = Array.from({ length: 80 }, (_, i) =>
`이 문장은 순수 한국어 산문의 청킹 동작을 확인하기 위한 ${i}번째 예시 문장입니다.`,
).join(' ');
const chunks = chunkText(prose);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) {
// CJK-dense chunks are char-counted (≈450 max) — nowhere near 1500.
expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(700);
}
});
});
describe('capByEstimatedTokens unit behavior', () => {
test('returns input unchanged when under the cap', () => {
expect(capByEstimatedTokens('short text', 1500)).toEqual(['short text']);
expect(capByEstimatedTokens('', 1500)).toEqual([]);
});
test('prefers newline cut points within the lookback window', () => {
const line = 'x'.repeat(100);
const text = Array.from({ length: 40 }, () => line).join('\n');
const pieces = capByEstimatedTokens(text, 1000);
expect(pieces.length).toBeGreaterThan(1);
for (const p of pieces) {
// Every piece should be whole lines (multiples of the 100-char line).
for (const l of p.split('\n')) {
expect(l).toBe(line);
}
}
});
test('makes forward progress on whitespace-less input (hard cut)', () => {
const blob = 'a'.repeat(10_000);
const pieces = capByEstimatedTokens(blob, 1000);
expect(pieces.length).toBeGreaterThan(1);
expect(pieces.join('')).toBe(blob);
for (const p of pieces) {
expect(estimateEmbeddingTokens(p)).toBeLessThanOrEqual(1000);
}
});
});
describe('estimateEmbeddingTokens — weight sanity', () => {
test('overestimates URL-dense ASCII (0.75/char ≥ measured ~0.63/char)', () => {
const url = 'https://docs.example.com/pages/a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4?v=abc&ref=sample';
const est = estimateEmbeddingTokens(url);
expect(est).toBeGreaterThanOrEqual(Math.floor(url.length * 0.7));
});
test('counts CJK at 1 token/char', () => {
expect(estimateEmbeddingTokens('가나다라마')).toBe(5);
});
test('whitespace is nearly free', () => {
expect(estimateEmbeddingTokens(' \n\t ')).toBeLessThanOrEqual(1);
});
test('empty string is 0', () => {
expect(estimateEmbeddingTokens('')).toBe(0);
});
});
+6 -6
View File
@@ -55,13 +55,13 @@ describe('v0.41 T4: all 4 bundled lens packs parse cleanly', () => {
});
describe('v0.41 T4: bundled registry includes lens packs', () => {
test('load-active.ts BUNDLED array source includes the 4 lens pack names', () => {
const loadActiveSrc = readFileSync(
join(here, '..', 'src', 'core', 'schema-pack', 'load-active.ts'),
'utf-8',
);
test('BUNDLED_PACK_NAMES includes the 4 lens pack names', async () => {
// The bundled list moved from load-active.ts to bundled.ts (the
// single source of truth); assert the array directly instead of
// grepping source text.
const { BUNDLED_PACK_NAMES } = await import('../src/core/schema-pack/bundled.ts');
for (const name of PACK_NAMES) {
expect(loadActiveSrc).toContain(`'${name}'`);
expect(BUNDLED_PACK_NAMES).toContain(name);
}
});
});
+3
View File
@@ -149,6 +149,9 @@ describe('list_schema_packs', () => {
seedPack('mine');
const result = await operationsByName.list_schema_packs!.handler(ctxOf(), {}) as { bundled: string[]; installed: string[] };
expect(result.bundled).toContain('gbrain-base');
expect(result.bundled).toContain('gbrain-recommended');
expect(result.bundled).toContain('gbrain-base-v2');
expect(result.bundled).toContain('gbrain-investor');
expect(result.installed).toContain('mine');
});
});
+38 -1
View File
@@ -64,11 +64,14 @@ describe('gbrain schema CLI (Phase C)', () => {
expect(r.stdout + r.stderr).toMatch(/schema|active|list|show|validate|use/i);
});
test('schema list shows gbrain-base bundled', () => {
test('schema list shows all bundled packs', () => {
const r = gbrain(['schema', 'list']);
expect(r.code).toBe(0);
expect(r.stdout).toContain('Bundled packs:');
expect(r.stdout).toContain('gbrain-base');
expect(r.stdout).toContain('gbrain-recommended');
expect(r.stdout).toContain('gbrain-base-v2');
expect(r.stdout).toContain('gbrain-investor');
});
test('schema show gbrain-base prints manifest details', () => {
@@ -97,6 +100,40 @@ describe('gbrain schema CLI (Phase C)', () => {
expect(r.stdout).toContain('valid manifest');
});
test('schema show/validate exposes bundled gbrain-recommended', () => {
const show = gbrain(['schema', 'show', 'gbrain-recommended']);
expect(show.code).toBe(0);
expect(show.stdout).toContain('gbrain-recommended v1.0.0');
expect(show.stdout).toContain('Page types (');
expect(show.stdout).toContain('meeting :: temporal');
const validate = gbrain(['schema', 'validate', 'gbrain-recommended']);
expect(validate.code).toBe(0);
expect(validate.stdout).toContain('valid manifest');
});
test('schema show exposes bundled gbrain-base-v2 successor pack', () => {
const r = gbrain(['schema', 'show', 'gbrain-base-v2']);
expect(r.code).toBe(0);
expect(r.stdout).toContain('gbrain-base-v2 v1.0.0');
expect(r.stdout).toContain('Page types (');
expect(r.stdout).toContain('Link verbs (14)');
});
test('schema active loads configured gbrain-recommended with real types', () => {
const home = mkdtempSync(join(tmpdir(), 'gbrain-schema-active-recommended-'));
try {
mkdirSync(join(home, '.gbrain'), { recursive: true });
writeFileSync(join(home, '.gbrain', 'config.json'), JSON.stringify({ schema_pack: 'gbrain-recommended' }), 'utf-8');
const r = gbrain(['schema', 'active'], { GBRAIN_HOME: home });
expect(r.code).toBe(0);
expect(r.stdout).toContain('Active pack: gbrain-recommended');
expect(r.stdout).not.toContain('Page types: 0');
} finally {
rmSync(home, { recursive: true, force: true });
}
});
test('schema active reports default resolution', () => {
const r = gbrain(['schema', 'active']);
expect(r.code).toBe(0);
+49
View File
@@ -345,6 +345,34 @@ describe('YAML mini-parser', () => {
expect(result.types[1].weight).toBe(2);
});
test('parses block scalar without swallowing following keys', () => {
const yaml = `name: blocky
description: |
First line.
Second line.
page_types:
- name: meeting
primitive: temporal
path_prefixes:
- meetings/
aliases: []
extractable: true
expert_routing: false`;
const result = parseYamlMini(yaml) as { description: string; page_types: Array<Record<string, unknown>> };
expect(result.description).toBe('First line.\nSecond line.');
expect(result.page_types).toHaveLength(1);
expect(result.page_types[0].name).toBe('meeting');
});
test('block scalar keeps # as literal content, not a comment', () => {
const yaml = `description: |
See issue #2029 for context.
name: hashy`;
const result = parseYamlMini(yaml) as Record<string, unknown>;
expect(result.description).toBe('See issue #2029 for context.');
expect(result.name).toBe('hashy');
});
test('strips comments', () => {
const result = parseYamlMini('# top comment\nname: value # inline comment') as Record<string, unknown>;
expect(result.name).toBe('value');
@@ -374,6 +402,27 @@ extends: null`;
const pack = loadPackFromString(json, 'fixture.json');
expect(pack.name).toBe('json-pack');
});
test('loads block-scalar pack descriptions without losing page types', () => {
const pack = loadPackFromString(`api_version: gbrain-schema-pack-v1
name: recommended-fixture
version: 1.0.0
extends: gbrain-base
description: |
Operational starter pack.
page_types:
- name: meeting
primitive: temporal
path_prefixes:
- meetings/
aliases: []
extractable: true
expert_routing: false
link_types: []`, 'fixture.yaml');
expect(pack.name).toBe('recommended-fixture');
expect(pack.extends).toBe('gbrain-base');
expect(pack.page_types.map((t) => t.name)).toContain('meeting');
});
});
describe('ReDoS guard', () => {
+4 -1
View File
@@ -103,7 +103,10 @@ describe('locateMutablePackFile — bundled guard', () => {
expect(BUNDLED_PACK_NAMES.has('gbrain-recommended')).toBe(true);
// v0.42 (T22): gbrain-base-v2 joins the bundled set.
expect(BUNDLED_PACK_NAMES.has('gbrain-base-v2')).toBe(true);
expect(BUNDLED_PACK_NAMES.size).toBe(3);
// Derived from the single bundled registry — the lens packs (creator,
// investor, engineer, everything) are read-only too.
expect(BUNDLED_PACK_NAMES.has('gbrain-investor')).toBe(true);
expect(BUNDLED_PACK_NAMES.size).toBe(7);
});
it('rejects gbrain-base-v2 with PACK_READONLY (bundled guard)', () => {