fix(search): honor sources.config.federated in unqualified local CLI search/query (#2561)

A source registered with `gbrain sources add --federated` was invisible to
an unqualified `gbrain search`/`gbrain query`: the local CLI always emitted
a scalar {sourceId} scope, and nothing on the read path ever consulted
sources.config.federated — contradicting docs/guides/multi-source-brains.md
('Source participates in unqualified gbrain search results').

Fix, at the trusted-local boundary only:
- src/cli.ts makeContext resolves the source WITH its tier and, when the
  tier is non-explicit (local_path / brain_default / sole_non_default /
  seed_default), computes ctx.localFederatedSourceIds = [resolved source,
  ...other config.federated=true sources] (archived excluded).
- New federatedSearchScope (operations.ts) delegates to
  resolveRequestedScope, then widens an unqualified trusted-local scalar
  scope to that set. Used by the search + query handlers only.
- Expansion NEVER applies when ctx.remote !== false (fail-closed source
  isolation), when a per-call source_id/__all__ is passed, when an OAuth
  grant (allowedSources) is present, or when --source/GBRAIN_SOURCE/dotfile
  named the source explicitly.

Deliberately NOT inside sourceScopeOpts: code-intel ops reject multi-source
scopes (resolveCodeIntelScope) and non-search reads keep their scalar
behavior. Cache contamination is already handled — cacheScopeKey folds
sourceIds sets into the query-cache key.

Fixes #2561

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/cli.ts

@@ -808,12 +808,20 @@ async function makeContext(engine: BrainEngine, params: Record<string, unknown>)
   // 'default'. Wrapped in try/catch so a doctor / single-source brain that
   // never set up sources still returns 'default' silently.
   let sourceId: string | undefined;
+  // #2561: when the source resolved via a NON-explicit tier (path-match /
+  // brain default / sole-non-default / seed default), unqualified search-shaped
+  // reads span every `config.federated = true` source. Computed here (the
+  // trusted local boundary) and consumed by federatedSearchScope in
+  // operations.ts, which additionally gates on ctx.remote === false.
+  let localFederated: string[] | undefined;
   try {
-    const { resolveSourceId } = await import('./core/source-resolver.ts');
+    const { resolveSourceWithTier, localFederatedSourceIds } = await import('./core/source-resolver.ts');
     // params.source is set when a CLI flag was parsed for the op (rare; most
     // CLI ops don't take --source). Falls through to env/dotfile/path-match.
     const explicit = (params.source as string | undefined) ?? null;
-    sourceId = await resolveSourceId(engine, explicit);
+    const resolved = await resolveSourceWithTier(engine, explicit);
+    sourceId = resolved.source_id;
+    localFederated = await localFederatedSourceIds(engine, resolved.source_id, resolved.tier);
   } catch {
     // Source resolution failed (e.g. sources table doesn't exist on a fresh
     // pre-init brain). Leave sourceId unset; engine read methods fall through
@@ -834,6 +842,7 @@ async function makeContext(engine: BrainEngine, params: Record<string, unknown>)
     // table). Matches dispatch.ts's auto-fill so the contract holds across
     // every transport.
     sourceId: sourceId ?? 'default',
+    ...(localFederated ? { localFederatedSourceIds: localFederated } : {}),
   };
 }
 

src/core/chunkers/code.ts

@@ -18,9 +18,8 @@
  * at runtime.
  */
 
-import { chunkText as recursiveChunk, capByEstimatedTokens, DEFAULT_MAX_EST_TOKENS } from './recursive.ts';
+import { chunkText as recursiveChunk } from './recursive.ts';
 import { buildQualifiedName } from './qualified-names.ts';
-import { estimateEmbeddingTokens } from '../cjk.ts';
 
 // Embed the tree-sitter runtime + per-language grammars as files.
 // `with { type: 'file' }` returns a path (string) at runtime. Bun bundles
@@ -112,15 +111,7 @@ import G_ZIG from '../../assets/wasm/grammars/tree-sitter-zig.wasm' with { type:
 // chunks get the new columns populated. Without this, the v28 backfill
 // gives every existing chunk a search_vector but subsequent Layer 5 AST
 // work would silently no-op.
-//
-// v5: estimated-token hard cap on AST-path chunks (capCodeChunks). A node
-// splitLargeNode can't subdivide (giant single-statement function, huge
-// literal) previously shipped WHOLE regardless of size and could overflow
-// strict per-request embedding-token limits (local llama-server crashes
-// past ~2,050 tokens, measured). Mirrors the markdown
-// chunker's v4 cap; fallback-path chunks are already capped inside
-// recursiveChunk.
-export const CHUNKER_VERSION = 5;
+export const CHUNKER_VERSION = 4;
 
 // Lazy-loaded tree-sitter module (v0.22.x API: Parser is default export)
 let Parser: typeof import('web-tree-sitter') | null = null;
@@ -717,7 +708,7 @@ export async function chunkCodeTextFull(
     if (chunks.length === 0) {
       return { chunks: fallbackChunks(source, filePath, language, opts), edges: rawEdges };
     }
-    return { chunks: capCodeChunks(mergeSmallSiblings(chunks, chunkTarget)), edges: rawEdges };
+    return { chunks: mergeSmallSiblings(chunks, chunkTarget), edges: rawEdges };
   } catch {
     return { chunks: fallbackChunks(source, filePath, language, opts), edges: [] };
   } finally {
@@ -800,33 +791,6 @@ function mergeSmallSiblings(chunks: CodeChunk[], chunkTarget: number): CodeChunk
   return merged;
 }
 
-/**
- * v5 final safety pass for AST-path chunks: split any chunk whose
- * ESTIMATED embedding tokens (conservative per-char-class heuristic,
- * cjk.ts) exceed DEFAULT_MAX_EST_TOKENS. Reaches chunks the AST logic
- * can't subdivide — splitLargeNode returns [] for nodes with < 2 body
- * children (giant single-statement functions, huge literals), which
- * previously shipped whole at any size.
- *
- * Split pieces inherit the source chunk's metadata verbatim; start/end
- * lines become approximate for pieces after the first. Acceptable —
- * these chunks exist for embedding + retrieval, and the alternative was
- * an embedding request the server rejects (or worse, crashes on).
- */
-function capCodeChunks(chunks: CodeChunk[]): CodeChunk[] {
-  if (chunks.every((c) => estimateEmbeddingTokens(c.text) <= DEFAULT_MAX_EST_TOKENS)) {
-    return chunks;
-  }
-  const out: CodeChunk[] = [];
-  for (const c of chunks) {
-    const pieces = capByEstimatedTokens(c.text, DEFAULT_MAX_EST_TOKENS);
-    for (const piece of pieces) {
-      out.push({ ...c, text: piece, index: out.length, metadata: { ...c.metadata } });
-    }
-  }
-  return out;
-}
-
 function buildMergedChunk(group: CodeChunk[], index: number): CodeChunk {
   const first = group[0]!;
   const last = group[group.length - 1]!;

src/core/chunkers/recursive.ts

@@ -17,13 +17,7 @@
  * Lossless invariant: non-overlapping portions reassemble to original.
  */
 
-import {
-  countCJKAwareWords,
-  CJK_SENTENCE_DELIMITERS,
-  CJK_CLAUSE_DELIMITERS,
-  charEmbedTokenWeight,
-  estimateEmbeddingTokens,
-} from '../cjk.ts';
+import { countCJKAwareWords, CJK_SENTENCE_DELIMITERS, CJK_CLAUSE_DELIMITERS } from '../cjk.ts';
 
 /**
  * Markdown chunker version. Folded into the per-page chunker_version column
@@ -39,20 +33,8 @@ import {
  * re-embed (not re-chunk) so existing pages pick up the wrapper on the
  * post-upgrade reembed sweep. See
  * `src/core/contextual-retrieval-service.ts`.
- *
- * v4: estimated-token hard cap + whitespace-word undercount fix. The word
- * pipeline counted a 150-char URL as ONE whitespace word, so URL/phone/
- * email-dense docs (CJK density < 0.30 → whitespace fallback) produced
- * 3-4K-char chunks that overflow strict per-request embedding-token
- * limits (measured: local llama-server crashes past ~2,050 tokens; URL
- * soup tokenizes at ~1.6 chars/token). Two changes:
- *   1. countWords() floors the count at ceil(nonWhitespaceChars/6) so a
- *      URL counts roughly per-character, not as one word.
- *   2. capByEstimatedTokens() final pass guarantees every chunk fits
- *      `maxTokens` (default 1500) under a conservative per-char-class
- *      token estimate, regardless of how word counting misjudged it.
  */
-export const MARKDOWN_CHUNKER_VERSION = 4;
+export const MARKDOWN_CHUNKER_VERSION = 3;
 
 const DELIMITERS: string[][] = [
   ['\n\n'],                          // L0: paragraphs
@@ -66,20 +48,8 @@ export interface ChunkOptions {
   chunkSize?: number;    // target words per chunk (default 300)
   chunkOverlap?: number; // overlap words (default 50)
   maxChars?: number;     // hard cap on any chunk's char length (default 6000)
-  /**
-   * v4: hard cap on any chunk's ESTIMATED embedding tokens (default 1500).
-   * Estimate = conservative per-char-class weights (see cjk.ts
-   * estimateEmbeddingTokens) — deliberately high, so the real tokenizer
-   * count stays below this value. Default leaves headroom for the
-   * contextual-retrieval wrapper (≤ ~630 chars) under a ~2,050-token
-   * per-request embedding server limit.
-   */
-  maxTokens?: number;
 }
 
-/** v4 default for ChunkOptions.maxTokens — see the field doc above. */
-export const DEFAULT_MAX_EST_TOKENS = 1500;
-
 export interface TextChunk {
   text: string;
   index: number;
@@ -103,7 +73,6 @@ export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] {
   const chunkSize = opts?.chunkSize || 300;
   const chunkOverlap = opts?.chunkOverlap || 50;
   const maxChars = opts?.maxChars || 6000;
-  const maxTokens = opts?.maxTokens || DEFAULT_MAX_EST_TOKENS;
 
   if (!text || text.trim().length === 0) return [];
 
@@ -120,9 +89,8 @@ export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] {
 
   const wordCount = countWords(stripped);
   if (wordCount <= chunkSize) {
-    // Single-chunk path: still apply the maxChars + maxTokens caps.
-    const capped = capByChars(stripped.trim(), maxChars)
-      .flatMap((t) => capByEstimatedTokens(t, maxTokens));
+    // Single-chunk path: still apply the maxChars cap.
+    const capped = capByChars(stripped.trim(), maxChars);
     return capped.map((t, i) => ({ text: t, index: i }));
   }
 
@@ -133,14 +101,9 @@ export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] {
   // v0.32.7: hard char cap. Catches pathological CJK + whitespace-less text
   // that the word-level pipeline can't bound (a single Chinese paragraph can
   // exceed 8192 OpenAI embedding tokens at any word count).
-  // v4: estimated-token cap on top — the char cap alone passes token-dense
-  // content (URL soup at ~1.6 chars/token) that overflows strict embedding
-  // server limits.
   const capped: string[] = [];
   for (const chunk of withOverlap) {
-    for (const piece of capByChars(chunk.trim(), maxChars)) {
-      capped.push(...capByEstimatedTokens(piece, maxTokens));
-    }
+    capped.push(...capByChars(chunk.trim(), maxChars));
   }
   return capped.map((t, i) => ({ text: t, index: i }));
 }
@@ -169,68 +132,6 @@ function capByChars(text: string, maxChars: number): string[] {
   return out;
 }
 
-/**
- * How far back (in chars) the token cap looks for a friendly cut point
- * before falling back to a hard cut. 300 covers typical rollup/list line
- * lengths so forced splits land at line starts, not mid-URL.
- */
-const TOKEN_CAP_CUT_LOOKBACK = 300;
-
-/**
- * v4: hard-cap a chunk's ESTIMATED embedding tokens. Final safety pass —
- * runs after capByChars on every chunk, so no upstream miscounting
- * (whitespace-word fallback, overlap inflation, char-cap survivors) can
- * emit a chunk past `maxTokens`.
- *
- * Cut placement prefers, within the last TOKEN_CAP_CUT_LOOKBACK chars of
- * the window: a newline, then any whitespace, then a hard cut. This keeps
- * forced splits off mid-line/mid-URL positions for list-shaped content
- * and inside code fences. No overlap is added (pieces stay lossless
- * modulo the trims the char cap already applies).
- *
- * @internal exported for the code chunker (code.ts) and tests.
- */
-export function capByEstimatedTokens(text: string, maxTokens: number): string[] {
-  if (text.length === 0) return [];
-  if (estimateEmbeddingTokens(text) <= maxTokens) return [text];
-
-  const out: string[] = [];
-  let start = 0;
-  while (start < text.length) {
-    // Greedily extend the window until the next char would break the cap.
-    // Always take at least one char so the loop makes forward progress.
-    let est = 0;
-    let end = start;
-    while (end < text.length) {
-      const w = charEmbedTokenWeight(text.charCodeAt(end));
-      if (est + w > maxTokens && end > start) break;
-      est += w;
-      end++;
-    }
-
-    if (end < text.length) {
-      const windowStart = Math.max(start + 1, end - TOKEN_CAP_CUT_LOOKBACK);
-      let cut = text.lastIndexOf('\n', end - 1);
-      if (cut < windowStart) {
-        cut = -1;
-        for (let i = end - 1; i >= windowStart; i--) {
-          const code = text.charCodeAt(i);
-          if (code === 0x20 || (code >= 0x09 && code <= 0x0d)) {
-            cut = i;
-            break;
-          }
-        }
-      }
-      if (cut >= windowStart) end = cut + 1;
-    }
-
-    const slice = text.slice(start, end).trim();
-    if (slice.length > 0) out.push(slice);
-    start = end;
-  }
-  return out;
-}
-
 function recursiveSplit(text: string, level: number, target: number): string[] {
   if (level >= DELIMITERS.length) {
     // Level 4: split on whitespace
@@ -416,19 +317,7 @@ function extractTrailingContext(text: string, targetWords: number): string {
  * Delegated to src/core/cjk.ts so the slugify whitelist, expansion
  * detection, and PGLite keyword fallback all agree on what "CJK enough"
  * means.
- *
- * v4: floored at ceil(nonWhitespaceChars/6). The whitespace fallback
- * counts a 150-char URL as ONE word, so URL/phone/email-dense docs
- * (whose ASCII mass pushes CJK density below the 0.30 threshold) were
- * sized at a fraction of their real bulk and merged into 3-4K-char
- * chunks. The floor makes long whitespace-less runs count roughly
- * per-character while leaving normal Latin prose untouched (average
- * English word ≈ 5 chars < 6, so the whitespace count still wins).
- * Kept local to the chunker — search/expansion.ts keeps the original
- * countCJKAwareWords semantics for its query-length check.
  */
 function countWords(text: string): number {
-  const cjkAware = countCJKAwareWords(text);
-  const nonWhitespace = text.replace(/\s/g, '').length;
-  return Math.max(cjkAware, Math.ceil(nonWhitespace / 6));
+  return countCJKAwareWords(text);
 }

src/core/cjk.ts

@@ -65,64 +65,3 @@ export function countCJKAwareWords(s: string): number {
 export function escapeLikePattern(s: string): string {
   return s.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_');
 }
-
-/**
- * Conservative per-char-class embedding-token weights (markdown chunker v4).
- *
- * Why this exists: the chunker's "word" counting drastically UNDER-counts
- * whitespace-less ASCII runs (a 150-char URL = 1 whitespace word), so
- * word-based size targets can emit chunks that overflow an embedding
- * server's per-request token limit. Measured on a local Qwen3-embedding
- * llama-server stack:
- *   - URL/phone/email-dense text tokenizes at ~1.6 chars/token
- *   - base64-ish / minified blobs approach ~1.3 chars/token (worst case)
- *   - Korean prose tokenizes NO WORSE than 1 char/token in practice
- *
- * Weights are deliberately HIGH (tokens are overestimated) so any cap
- * based on this estimate is safe against real tokenizers:
- *   - CJK char        → 1.0 token  (real CJK prose is cheaper)
- *   - other non-space → 0.75 token (≈1.33 chars/token, covers base64)
- *   - whitespace      → 0.1 token  (mostly folds into neighbor tokens)
- */
-export const EMBED_TOKEN_WEIGHT_CJK = 1.0;
-export const EMBED_TOKEN_WEIGHT_OTHER = 0.75;
-export const EMBED_TOKEN_WEIGHT_WS = 0.1;
-
-/** BMP CJK check by UTF-16 code unit — same ranges as CJK_SLUG_CHARS. */
-export function isCJKCodeUnit(code: number): boolean {
-  return (
-    (code >= 0x4e00 && code <= 0x9fff) || // Han
-    (code >= 0x3040 && code <= 0x309f) || // Hiragana
-    (code >= 0x30a0 && code <= 0x30ff) || // Katakana
-    (code >= 0xac00 && code <= 0xd7af)    // Hangul Syllables
-  );
-}
-
-/**
- * Per-code-unit token weight. Unrecognized whitespace (exotic Unicode
- * spaces) intentionally falls into OTHER — that only overestimates.
- */
-export function charEmbedTokenWeight(code: number): number {
-  if (isCJKCodeUnit(code)) return EMBED_TOKEN_WEIGHT_CJK;
-  if (
-    code === 0x20 || (code >= 0x09 && code <= 0x0d) ||
-    code === 0xa0 || code === 0x3000
-  ) {
-    return EMBED_TOKEN_WEIGHT_WS;
-  }
-  return EMBED_TOKEN_WEIGHT_OTHER;
-}
-
-/**
- * Tokenizer-free embedding-token estimate (conservative overestimate).
- * See weight docs above. Astral chars count as 2 OTHER code units —
- * another overestimate, which is the safe direction.
- */
-export function estimateEmbeddingTokens(s: string): number {
-  if (s.length === 0) return 0;
-  let est = 0;
-  for (let i = 0; i < s.length; i++) {
-    est += charEmbedTokenWeight(s.charCodeAt(i));
-  }
-  return Math.ceil(est);
-}

src/core/operations.ts

@@ -424,6 +424,23 @@ export interface OperationContext {
    * satisfied even on single-source brains.
    */
   sourceId: string;
+  /**
+   * #2561 — federated read scope for UNQUALIFIED local CLI reads.
+   *
+   * Set ONLY by the local CLI's context builder (src/cli.ts makeContext), and
+   * only when the source resolved via a non-explicit tier (local_path /
+   * brain_default / sole_non_default / seed_default — NOT --source, NOT
+   * GBRAIN_SOURCE, NOT a .gbrain-source dotfile). Contains the resolved
+   * source first, then every other `config.federated = true` source, so an
+   * unqualified `gbrain search "X"` spans federated sources as
+   * docs/guides/multi-source-brains.md promises.
+   *
+   * Consumed exclusively by `federatedSearchScope` and ONLY when
+   * `ctx.remote === false` — a remote caller's scope stays governed by
+   * `ctx.auth.allowedSources` / scalar `ctx.sourceId` (source-isolation
+   * invariant, fail-closed).
+   */
+  localFederatedSourceIds?: string[];
 }
 
 /**
@@ -539,6 +556,45 @@ export function resolveRequestedScope(
   return sourceScopeOpts(ctx);
 }
 
+/**
+ * #2561 — source scope for the search-shaped read ops (`search`, `query`).
+ *
+ * Delegates to `resolveRequestedScope` (the single trust+grant resolver), then
+ * widens an UNQUALIFIED trusted-local scalar scope to the CLI-computed
+ * federated set (`ctx.localFederatedSourceIds`, resolved source first). This is
+ * what makes `sources add --federated` mean something for local search: a
+ * federated source participates in unqualified `gbrain search "X"` results.
+ *
+ * The expansion NEVER applies when:
+ *   - the caller is not strictly trusted-local (`ctx.remote !== false`) —
+ *     remote scope stays grant-governed (fail-closed source isolation);
+ *   - a per-call `source_id` was passed (explicit wins, including `__all__`);
+ *   - the resolver already produced a federated array (OAuth grant);
+ *   - the CLI resolved the source from an explicit signal (--source / env /
+ *     dotfile) — makeContext leaves `localFederatedSourceIds` unset then.
+ *
+ * Deliberately NOT inside `sourceScopeOpts`: code-intel ops collapse a
+ * multi-element scope to an error (`resolveCodeIntelScope`), and non-search
+ * reads (get_page, get_links, …) keep their long-standing scalar behavior.
+ */
+export function federatedSearchScope(
+  ctx: OperationContext,
+  sourceIdParam?: string,
+): { sourceId?: string; sourceIds?: string[] } {
+  const scope = resolveRequestedScope(ctx, sourceIdParam);
+  if (
+    ctx.remote === false &&
+    sourceIdParam === undefined &&
+    scope.sourceId !== undefined &&
+    scope.sourceIds === undefined &&
+    ctx.localFederatedSourceIds !== undefined &&
+    ctx.localFederatedSourceIds.length > 1
+  ) {
+    return { sourceIds: ctx.localFederatedSourceIds };
+  }
+  return scope;
+}
+
 /**
  * Code-intel adapter for `resolveRequestedScope`. Graph traversal
  * (code_callers/code_callees/code_blast/code_flow) is single-source by design —
@@ -1448,7 +1504,8 @@ const search: Operation = {
     const queryText = p.query as string;
     const limit = (p.limit as number) || 20;
     const offset = (p.offset as number) || 0;
-    const scope = sourceScopeOpts(ctx);
+    // #2561: unqualified trusted-local search spans federated sources.
+    const scope = federatedSearchScope(ctx);
 
     // T4/D5 — per-call mode honored ONLY for trusted/local callers so a remote
     // OAuth client can't escalate to the costly tokenmax bundle. Local + unknown
@@ -1610,7 +1667,9 @@ const query: Operation = {
     // is spread into BOTH the image-similarity searchVector path and the text
     // hybridSearch path below, so both honor the same grant.
     const sourceIdParam = typeof p.source_id === 'string' ? p.source_id : undefined;
-    const querySourceScope = resolveRequestedScope(ctx, sourceIdParam);
+    // #2561: unqualified trusted-local query spans federated sources (per-call
+    // source_id / remote grants still resolve through resolveRequestedScope).
+    const querySourceScope = federatedSearchScope(ctx, sourceIdParam);
 
     // v0.27.1: image-similarity branch. Bypasses hybridSearch (which is
     // text-only); embeds the image via embedMultimodal and runs a direct

src/core/source-resolver.ts

@@ -353,6 +353,45 @@ export async function resolveSourceWithTier(
   return { source_id: 'default', tier: 'seed_default' };
 }
 
+/**
+ * #2561 — compute the federated read scope for an UNQUALIFIED local CLI call.
+ *
+ * `sources add --federated` promises that a `config.federated = true` source
+ * "participates in unqualified `gbrain search` results"
+ * (docs/guides/multi-source-brains.md). This helper turns that promise into a
+ * scope: given the resolved source and WHICH tier resolved it, return
+ * `[resolvedSource, ...other federated source ids]` — or `undefined` when the
+ * expansion must not apply:
+ *
+ *   - explicit tiers (`flag` / `env` / `dotfile`): the user named a source;
+ *     scalar scope stands (that IS the qualified case);
+ *   - no other federated source exists: keep the scalar fast path unchanged.
+ *
+ * Archived sources are excluded (same rationale as pickSoleNonDefaultSource);
+ * the archived column is v34+, so fall back to the un-archived query on older
+ * brains. Callers put the result on `OperationContext.localFederatedSourceIds`
+ * — consumed only by `federatedSearchScope` and only when `remote === false`.
+ */
+export async function localFederatedSourceIds(
+  engine: BrainEngine,
+  sourceId: string,
+  tier: SourceTier,
+): Promise<string[] | undefined> {
+  if (tier === 'flag' || tier === 'env' || tier === 'dotfile') return undefined;
+  let rows: Array<{ id: string }>;
+  try {
+    rows = await engine.executeRaw<{ id: string }>(
+      `SELECT id FROM sources WHERE config->>'federated' = 'true' AND archived = false ORDER BY id`,
+    );
+  } catch {
+    rows = await engine.executeRaw<{ id: string }>(
+      `SELECT id FROM sources WHERE config->>'federated' = 'true' ORDER BY id`,
+    );
+  }
+  const ids = [sourceId, ...rows.map((r) => r.id).filter((id) => id !== sourceId)];
+  return ids.length > 1 ? ids : undefined;
+}
+
 /** Exposed for tests. */
 export const __testing = {
   readDotfileWalk,

test/chunker-version-gate.test.ts

@@ -15,22 +15,19 @@ import { describe, test, expect } from 'bun:test';
 import { CHUNKER_VERSION } from '../src/core/chunkers/code.ts';
 
 describe('Layer 12 — CHUNKER_VERSION constant', () => {
-  test('bumped to 5 for the estimated-token hard cap', () => {
+  test('bumped to 4 for Cathedral II', () => {
     // v3: v0.19.0 Chonkie parity (tokenizer + small-sibling merge).
     // v4: v0.20.0 Cathedral II (qualified names + parent scope + doc_comment
     //     + fence extraction + chunk-grain FTS). Folded into content_hash
     //     so any bump forces clean re-chunks on next sync.
-    // v5: estimated-token hard cap on AST-path chunks (capCodeChunks) so
-    //     un-subdividable giant nodes can't overflow strict embedding
-    //     server token limits.
-    expect(CHUNKER_VERSION).toBe(5);
+    expect(CHUNKER_VERSION).toBe(4);
   });
 
   test('is stable across imports (not recomputed at call time)', async () => {
     const a = (await import('../src/core/chunkers/code.ts')).CHUNKER_VERSION;
     const b = (await import('../src/core/chunkers/code.ts')).CHUNKER_VERSION;
     expect(a).toBe(b);
-    expect(a).toBe(5);
+    expect(a).toBe(4);
   });
 });
 

test/chunkers/code.test.ts

@@ -10,8 +10,8 @@ import { describe, test, expect } from 'bun:test';
 import { chunkCodeText, detectCodeLanguage, CHUNKER_VERSION } from '../../src/core/chunkers/code.ts';
 
 describe('CHUNKER_VERSION', () => {
-  test('v5: estimated-token hard cap on AST-path chunks', () => {
-    expect(CHUNKER_VERSION).toBe(5);
+  test('v0.20.0 Cathedral II Layer 12 bumped to 4', () => {
+    expect(CHUNKER_VERSION).toBe(4);
   });
 });
 

test/chunkers/recursive.test.ts

@@ -135,14 +135,13 @@ describe('Recursive Text Chunker', () => {
 });
 
 describe('CJK chunking (v0.32.7)', () => {
-  test('MARKDOWN_CHUNKER_VERSION is 4', async () => {
+  test('MARKDOWN_CHUNKER_VERSION is 3', async () => {
     // v0.40.3.0: bumped 2→3 to signal the post-upgrade reembed sweep that
-    // contextual retrieval wrapping is now applied at embed time.
-    // v4: estimated-token hard cap + whitespace-word undercount floor
-    // (URL-dense docs produced chunks past strict embedding server token
-    // limits). Boundary change → forces re-chunk for chunker_version < 4.
+    // contextual retrieval wrapping is now applied at embed time. Chunk
+    // boundaries themselves are unchanged; the bump forces re-embed for
+    // pages where chunker_version < 3.
     const mod = await import('../../src/core/chunkers/recursive.ts');
-    expect(mod.MARKDOWN_CHUNKER_VERSION).toBe(4);
+    expect(mod.MARKDOWN_CHUNKER_VERSION).toBe(3);
   });
 
   test('long pure-Chinese paragraph splits into multiple chunks', () => {

test/chunkers/token-cap.test.ts

@@ -1,195 +0,0 @@
-/**
- * Markdown chunker v4 / code chunker v5 — estimated-token hard cap
- * regression tests.
- *
- * Reproduces a field failure: a local llama-server embedding backend
- * (`-ub 2048`) crashes deterministically (trace/BPT trap → EOF at the
- * client) when a single chunk exceeds ~2,050 real tokens. Two content
- * shapes triggered it:
- *
- *   1. Korean docs carrying one long source URL per line.
- *      The URLs' ASCII mass pushes CJK density below 0.30, flipping
- *      countCJKAwareWords to whitespace counting, where a 150-char URL
- *      counts as ONE word → chunks ballooned to 3-4K chars ≈ 2,000+
- *      real tokens (URL soup tokenizes at ~1.6 chars/token).
- *
- *   2. Large JSON code blocks (~7K chars) that the word pipeline
- *      undercounts the same way (few whitespace tokens).
- *
- * The fix: every emitted chunk must satisfy
- *   estimateEmbeddingTokens(chunk) <= maxTokens (default 1500)
- * where the estimate deliberately OVERSTATES real tokenizer counts.
- */
-
-import { describe, test, expect } from 'bun:test';
-import { chunkText, capByEstimatedTokens, DEFAULT_MAX_EST_TOKENS } from '../../src/core/chunkers/recursive.ts';
-import { chunkCodeText } from '../../src/core/chunkers/code.ts';
-import { estimateEmbeddingTokens } from '../../src/core/cjk.ts';
-
-/** Synthesize the failing shape: Korean rollup lines each ending in a long Notion URL. */
-function urlDenseKoreanRollup(lines: number): string {
-  const out: string[] = ['# 링크가 줄마다 붙는 한국어 예시 문서', ''];
-  for (let i = 0; i < lines; i++) {
-    const hex32 = (i * 2654435761 >>> 0).toString(16).padStart(8, '0').repeat(4);
-    out.push(
-      `- **항목 ${i}**: 이 줄은 청커 동작 검증을 위한 의미 없는 한국어 예시 문장입니다 · 전화 000-0000-${String(1000 + i)} · ` +
-      `이메일 user${i}@example.com · 링크: https://docs.example.com/pages/${hex32}?v=abcdef0123456789&ref=sample`,
-    );
-  }
-  return out.join('\n');
-}
-
-/** Synthesize a large pretty-printed JSON block with CJK values. */
-function bigJsonBlock(targetChars: number): string {
-  const entries: string[] = [];
-  let i = 0;
-  let len = 0;
-  while (len < targetChars) {
-    const row =
-      `  "item_${i}": { "name": "예시-${i}", "url": "https://example.com/api/v2/items/${i}?token=abc${i}def", "qty": ${i % 100}, "memo": "한국어 값이 섞인 예시 데이터" }`;
-    entries.push(row);
-    len += row.length;
-    i++;
-  }
-  return `{\n${entries.join(',\n')}\n}`;
-}
-
-describe('v4 estimated-token cap — URL-dense Korean doc (field-failure shape)', () => {
-  test('every chunk stays under the estimated-token cap', () => {
-    const md = urlDenseKoreanRollup(60);
-    const chunks = chunkText(md);
-    expect(chunks.length).toBeGreaterThan(0);
-    for (const c of chunks) {
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS);
-    }
-  });
-
-  test('no chunk reaches the measured 3K-char danger zone for URL soup', () => {
-    const md = urlDenseKoreanRollup(60);
-    const chunks = chunkText(md);
-    // 1500 est tokens at the OTHER weight (0.75/char) bounds chunks to
-    // ~2,000 chars for pure ASCII — well under the ~3,300 chars where
-    // URL-dense content crosses ~2,050 real tokens (1.6 chars/token).
-    for (const c of chunks) {
-      expect(c.text.length).toBeLessThanOrEqual(2600);
-    }
-  });
-
-  test('content is preserved (no lines dropped by the cap)', () => {
-    const md = urlDenseKoreanRollup(60);
-    const chunks = chunkText(md);
-    const joined = chunks.map((c) => c.text).join('\n');
-    // Spot-check first / middle / last rollup lines survive chunking.
-    for (const marker of ['항목 0', '항목 30', '항목 59']) {
-      expect(joined).toContain(marker);
-    }
-  });
-});
-
-describe('v4 estimated-token cap — large JSON blocks', () => {
-  test('7K-char pretty JSON through the prose path stays under the cap', () => {
-    const md = `설정 파일 원문 보존:\n\n\`\`\`\n${bigJsonBlock(7000)}\n\`\`\`\n`;
-    const chunks = chunkText(md);
-    expect(chunks.length).toBeGreaterThan(1);
-    for (const c of chunks) {
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS);
-    }
-  });
-
-  test('7K-char minified JSON (single whitespace-less token) stays under the cap', () => {
-    const minified = bigJsonBlock(7000).replace(/\n\s*/g, '');
-    const chunks = chunkText(minified);
-    expect(chunks.length).toBeGreaterThan(1);
-    for (const c of chunks) {
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS);
-    }
-  });
-
-  test('json fence via the code chunker stays under the cap (+header slack)', async () => {
-    const chunks = await chunkCodeText(bigJsonBlock(7000), 'fence.json');
-    expect(chunks.length).toBeGreaterThan(0);
-    for (const c of chunks) {
-      // buildChunk prepends a short "[JSON] fence.json:…" header AFTER the
-      // body-level cap; allow ~60 est tokens of header slack. Real-token
-      // safety margin (2,050 − overestimated 1,500) absorbs this easily.
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS + 60);
-    }
-  });
-});
-
-describe('v4 word-count floor — behavior preserved for normal content', () => {
-  test('Latin prose chunking is unchanged by the floor (avg word < 6 chars)', () => {
-    const prose = Array.from({ length: 120 }, (_, i) =>
-      `This is sentence number ${i} and it talks about ordinary things in plain words.`,
-    ).join(' ');
-    const chunks = chunkText(prose);
-    // Historical behavior: ~1,560 whitespace words → multiple ~300-word chunks.
-    expect(chunks.length).toBeGreaterThan(3);
-    for (const c of chunks) {
-      const words = c.text.split(/\s+/).length;
-      expect(words).toBeLessThanOrEqual(300 * 1.5 + 50); // merge cap + overlap
-    }
-  });
-
-  test('Korean prose (CJK-dense, no URLs) never triggers the token cap', () => {
-    const prose = Array.from({ length: 80 }, (_, i) =>
-      `이 문장은 순수 한국어 산문의 청킹 동작을 확인하기 위한 ${i}번째 예시 문장입니다.`,
-    ).join(' ');
-    const chunks = chunkText(prose);
-    expect(chunks.length).toBeGreaterThan(1);
-    for (const c of chunks) {
-      // CJK-dense chunks are char-counted (≈450 max) — nowhere near 1500.
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(700);
-    }
-  });
-});
-
-describe('capByEstimatedTokens unit behavior', () => {
-  test('returns input unchanged when under the cap', () => {
-    expect(capByEstimatedTokens('short text', 1500)).toEqual(['short text']);
-    expect(capByEstimatedTokens('', 1500)).toEqual([]);
-  });
-
-  test('prefers newline cut points within the lookback window', () => {
-    const line = 'x'.repeat(100);
-    const text = Array.from({ length: 40 }, () => line).join('\n');
-    const pieces = capByEstimatedTokens(text, 1000);
-    expect(pieces.length).toBeGreaterThan(1);
-    for (const p of pieces) {
-      // Every piece should be whole lines (multiples of the 100-char line).
-      for (const l of p.split('\n')) {
-        expect(l).toBe(line);
-      }
-    }
-  });
-
-  test('makes forward progress on whitespace-less input (hard cut)', () => {
-    const blob = 'a'.repeat(10_000);
-    const pieces = capByEstimatedTokens(blob, 1000);
-    expect(pieces.length).toBeGreaterThan(1);
-    expect(pieces.join('')).toBe(blob);
-    for (const p of pieces) {
-      expect(estimateEmbeddingTokens(p)).toBeLessThanOrEqual(1000);
-    }
-  });
-});
-
-describe('estimateEmbeddingTokens — weight sanity', () => {
-  test('overestimates URL-dense ASCII (0.75/char ≥ measured ~0.63/char)', () => {
-    const url = 'https://docs.example.com/pages/a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4?v=abc&ref=sample';
-    const est = estimateEmbeddingTokens(url);
-    expect(est).toBeGreaterThanOrEqual(Math.floor(url.length * 0.7));
-  });
-
-  test('counts CJK at 1 token/char', () => {
-    expect(estimateEmbeddingTokens('가나다라마')).toBe(5);
-  });
-
-  test('whitespace is nearly free', () => {
-    expect(estimateEmbeddingTokens('   \n\t  ')).toBeLessThanOrEqual(1);
-  });
-
-  test('empty string is 0', () => {
-    expect(estimateEmbeddingTokens('')).toBe(0);
-  });
-});

test/local-federated-search-scope.test.ts

@@ -0,0 +1,154 @@
+/**
+ * #2561 — sources.config.federated participates in UNQUALIFIED local CLI
+ * search/query.
+ *
+ * Pre-fix: the local CLI always emitted a scalar `{sourceId}` scope (required
+ * field, auto-filled 'default'), so a source registered with
+ * `gbrain sources add --federated` was invisible to an unqualified
+ * `gbrain search "X"` — contradicting docs/guides/multi-source-brains.md
+ * ("Source participates in unqualified `gbrain search` results").
+ *
+ * Fix: the CLI context builder computes `ctx.localFederatedSourceIds`
+ * (resolved source + every other federated source) whenever the source
+ * resolved via a NON-explicit tier; `federatedSearchScope` widens the scalar
+ * scope to that set for the `search` / `query` ops — trusted-local only
+ * (`ctx.remote === false`), never for remote callers, never when a per-call
+ * `source_id` or an explicit --source/env/dotfile was given.
+ */
+import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { localFederatedSourceIds } from '../src/core/source-resolver.ts';
+import {
+  federatedSearchScope,
+  operations,
+  type OperationContext,
+} from '../src/core/operations.ts';
+
+let engine: PGLiteEngine;
+const search = operations.find((o) => o.name === 'search')!;
+
+function ctxOf(overrides: Partial<OperationContext> = {}): OperationContext {
+  return {
+    engine: engine as any,
+    config: {} as any,
+    logger: console as any,
+    dryRun: false,
+    remote: false,
+    sourceId: 'default',
+    ...overrides,
+  };
+}
+
+beforeAll(async () => {
+  engine = new PGLiteEngine();
+  await engine.connect({});
+  await engine.initSchema();
+  // Seeded 'default' source is federated=true. Add:
+  //   wiki    — federated (must join unqualified search)
+  //   private — NOT federated (must stay invisible unless explicitly named)
+  //   oldnews — federated but archived (must stay excluded)
+  await engine.executeRaw(
+    `INSERT INTO sources (id, name, local_path, config) VALUES ('wiki', 'wiki', '/tmp/wiki', '{"federated": true}'::jsonb)`,
+  );
+  await engine.executeRaw(
+    `INSERT INTO sources (id, name, local_path, config) VALUES ('private', 'private', '/tmp/private', '{}'::jsonb)`,
+  );
+  await engine.executeRaw(
+    `INSERT INTO sources (id, name, local_path, config, archived) VALUES ('oldnews', 'oldnews', '/tmp/oldnews', '{"federated": true}'::jsonb, true)`,
+  );
+  const pages: Array<[slug: string, sourceId: string, where: string]> = [
+    ['notes/home', 'default', 'default'],
+    ['wiki/topic', 'wiki', 'wiki'],
+    ['private/topic', 'private', 'private'],
+    ['old/topic', 'oldnews', 'oldnews'],
+  ];
+  for (const [slug, sourceId, where] of pages) {
+    await engine.putPage(slug, {
+      type: 'note', title: `Topic in ${where}`, compiled_truth: `the zebra telescope in ${where}`, frontmatter: {},
+    }, { sourceId });
+    await engine.upsertChunks(slug, [
+      { chunk_index: 0, chunk_text: `the zebra telescope in ${where}`, chunk_source: 'compiled_truth' },
+    ], { sourceId });
+  }
+  // Keyword-only search path: no embedding provider needed in tests.
+  await engine.setConfig('search.mcp_keyword_only', 'true');
+}, 60_000);
+
+afterAll(async () => {
+  if (engine) await engine.disconnect();
+}, 60_000);
+
+describe('localFederatedSourceIds — CLI-side scope computation', () => {
+  test('non-explicit tier: resolved source first, then other federated, archived excluded', async () => {
+    expect(await localFederatedSourceIds(engine, 'default', 'seed_default')).toEqual(['default', 'wiki']);
+  });
+
+  test('non-federated resolved source still joins its own scope', async () => {
+    expect(await localFederatedSourceIds(engine, 'private', 'brain_default')).toEqual(['private', 'default', 'wiki']);
+  });
+
+  test('explicit tiers (--source / env / dotfile) never expand', async () => {
+    expect(await localFederatedSourceIds(engine, 'default', 'flag')).toBeUndefined();
+    expect(await localFederatedSourceIds(engine, 'default', 'env')).toBeUndefined();
+    expect(await localFederatedSourceIds(engine, 'default', 'dotfile')).toBeUndefined();
+  });
+
+  test('single federated source (the resolved one) keeps the scalar fast path', async () => {
+    const solo = { executeRaw: async () => [{ id: 'default' }] } as any;
+    expect(await localFederatedSourceIds(solo, 'default', 'seed_default')).toBeUndefined();
+  });
+});
+
+describe('federatedSearchScope — trust + explicitness matrix', () => {
+  test('trusted local + unqualified widens to the federated set', () => {
+    const ctx = ctxOf({ localFederatedSourceIds: ['default', 'wiki'] });
+    expect(federatedSearchScope(ctx)).toEqual({ sourceIds: ['default', 'wiki'] });
+  });
+
+  test('remote caller NEVER widens (fail-closed), even if the field is set', () => {
+    const ctx = ctxOf({ remote: true, localFederatedSourceIds: ['default', 'wiki'] });
+    expect(federatedSearchScope(ctx)).toEqual({ sourceId: 'default' });
+  });
+
+  test('per-call source_id wins over the federated set', () => {
+    const ctx = ctxOf({ localFederatedSourceIds: ['default', 'wiki'] });
+    expect(federatedSearchScope(ctx, 'wiki')).toEqual({ sourceId: 'wiki' });
+  });
+
+  test('per-call __all__ keeps the whole-brain semantics for trusted local', () => {
+    const ctx = ctxOf({ localFederatedSourceIds: ['default', 'wiki'] });
+    expect(federatedSearchScope(ctx, '__all__')).toEqual({});
+  });
+
+  test('a federated OAuth grant wins over the local set', () => {
+    const ctx = ctxOf({
+      localFederatedSourceIds: ['default', 'wiki'],
+      auth: { allowedSources: ['a', 'b'] } as OperationContext['auth'],
+    });
+    expect(federatedSearchScope(ctx)).toEqual({ sourceIds: ['a', 'b'] });
+  });
+
+  test('no local federated set → unchanged scalar scope', () => {
+    expect(federatedSearchScope(ctxOf())).toEqual({ sourceId: 'default' });
+  });
+});
+
+describe('search op — unqualified local search spans federated sources', () => {
+  test('federated source results appear; non-federated + archived stay invisible', async () => {
+    const ctx = ctxOf({
+      localFederatedSourceIds: await localFederatedSourceIds(engine, 'default', 'seed_default'),
+    });
+    const results = (await search.handler(ctx, { query: 'zebra telescope' })) as Array<{ slug: string }>;
+    const slugs = results.map((r) => r.slug);
+    expect(slugs).toContain('notes/home');
+    expect(slugs).toContain('wiki/topic'); // pre-#2561 this was missing
+    expect(slugs).not.toContain('private/topic');
+    expect(slugs).not.toContain('old/topic');
+  });
+
+  test('explicit source resolution (no federated set on ctx) stays single-source', async () => {
+    const results = (await search.handler(ctxOf(), { query: 'zebra telescope' })) as Array<{ slug: string }>;
+    const slugs = results.map((r) => r.slug);
+    expect(slugs).toEqual(['notes/home']);
+  });
+});