Compare commits

..
Author SHA1 Message Date
Garry TanandClaude Fable 5 c80b8b6757 fix(search): honor sources.config.federated in unqualified local CLI search/query (#2561)
A source registered with `gbrain sources add --federated` was invisible to
an unqualified `gbrain search`/`gbrain query`: the local CLI always emitted
a scalar {sourceId} scope, and nothing on the read path ever consulted
sources.config.federated — contradicting docs/guides/multi-source-brains.md
('Source participates in unqualified gbrain search results').

Fix, at the trusted-local boundary only:
- src/cli.ts makeContext resolves the source WITH its tier and, when the
  tier is non-explicit (local_path / brain_default / sole_non_default /
  seed_default), computes ctx.localFederatedSourceIds = [resolved source,
  ...other config.federated=true sources] (archived excluded).
- New federatedSearchScope (operations.ts) delegates to
  resolveRequestedScope, then widens an unqualified trusted-local scalar
  scope to that set. Used by the search + query handlers only.
- Expansion NEVER applies when ctx.remote !== false (fail-closed source
  isolation), when a per-call source_id/__all__ is passed, when an OAuth
  grant (allowedSources) is present, or when --source/GBRAIN_SOURCE/dotfile
  named the source explicitly.

Deliberately NOT inside sourceScopeOpts: code-intel ops reject multi-source
scopes (resolveCodeIntelScope) and non-search reads keep their scalar
behavior. Cache contamination is already handled — cacheScopeKey folds
sourceIds sets into the query-cache key.

Fixes #2561

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 14:43:36 -07:00
8 changed files with 274 additions and 108 deletions
+11 -2
View File
@@ -808,12 +808,20 @@ async function makeContext(engine: BrainEngine, params: Record<string, unknown>)
// 'default'. Wrapped in try/catch so a doctor / single-source brain that
// never set up sources still returns 'default' silently.
let sourceId: string | undefined;
// #2561: when the source resolved via a NON-explicit tier (path-match /
// brain default / sole-non-default / seed default), unqualified search-shaped
// reads span every `config.federated = true` source. Computed here (the
// trusted local boundary) and consumed by federatedSearchScope in
// operations.ts, which additionally gates on ctx.remote === false.
let localFederated: string[] | undefined;
try {
const { resolveSourceId } = await import('./core/source-resolver.ts');
const { resolveSourceWithTier, localFederatedSourceIds } = await import('./core/source-resolver.ts');
// params.source is set when a CLI flag was parsed for the op (rare; most
// CLI ops don't take --source). Falls through to env/dotfile/path-match.
const explicit = (params.source as string | undefined) ?? null;
sourceId = await resolveSourceId(engine, explicit);
const resolved = await resolveSourceWithTier(engine, explicit);
sourceId = resolved.source_id;
localFederated = await localFederatedSourceIds(engine, resolved.source_id, resolved.tier);
} catch {
// Source resolution failed (e.g. sources table doesn't exist on a fresh
// pre-init brain). Leave sourceId unset; engine read methods fall through
@@ -834,6 +842,7 @@ async function makeContext(engine: BrainEngine, params: Record<string, unknown>)
// table). Matches dispatch.ts's auto-fill so the contract holds across
// every transport.
sourceId: sourceId ?? 'default',
...(localFederated ? { localFederatedSourceIds: localFederated } : {}),
};
}
+5 -15
View File
@@ -43,7 +43,7 @@ import {
} from '../core/link-extraction.ts';
import { createProgress } from '../core/progress.ts';
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
import { pathToSlug, slugifyPath, pruneDir, isSyncable } from '../core/sync.ts';
import { pathToSlug, pruneDir, isSyncable } from '../core/sync.ts';
// v0.41.18.0: withRetry + isRetryableConnError + WithRetryOpts moved to
// src/core/retry.ts as the canonical primitive. Engine methods
// (addLinksBatch/addTimelineEntriesBatch/upsertChunks) now self-retry via
@@ -269,24 +269,14 @@ export function extractMarkdownLinks(content: string): { name: string; relTarget
export function resolveSlug(fileDir: string, relTarget: string, allSlugs: Set<string>): string | null {
const targetNoExt = relTarget.endsWith('.md') ? relTarget.slice(0, -3) : relTarget;
// Issue #1964: wikilinks carry raw Obsidian paths (`[[llm-wiki/entities/AI 3.0]]`)
// but allSlugs holds sync-slugified slugs (`llm-wiki/entities/ai-3.0`). Try the
// raw candidate first (back-compat), then the sync-consistent slugified form.
const hit = (candidate: string): string | null => {
if (allSlugs.has(candidate)) return candidate;
const slugified = slugifyPath(candidate);
if (slugified !== candidate && allSlugs.has(slugified)) return slugified;
return null;
};
const s1 = hit(join(fileDir, targetNoExt));
if (s1) return s1;
const s1 = join(fileDir, targetNoExt);
if (allSlugs.has(s1)) return s1;
const parts = fileDir.split('/').filter(Boolean);
for (let strip = 1; strip <= parts.length; strip++) {
const ancestor = parts.slice(0, parts.length - strip).join('/');
const candidate = hit(ancestor ? join(ancestor, targetNoExt) : targetNoExt);
if (candidate) return candidate;
const candidate = ancestor ? join(ancestor, targetNoExt) : targetNoExt;
if (allSlugs.has(candidate)) return candidate;
}
return null;
+4 -20
View File
@@ -14,7 +14,6 @@
import type { BrainEngine } from './engine.ts';
import type { PageType } from './types.ts';
import { ensureWellFormed } from './text-safe.ts';
import { slugifyPath } from './sync.ts';
/**
* v0.42.7 — link-extraction version stamp. Bump this ISO timestamp whenever the
@@ -483,29 +482,14 @@ export async function extractPageLinks(
// pre-v0.40.8.2 behavior of dropping bare wikilinks outside
// DIR_PATTERN.
if (ref.needsResolution) {
if (typeof resolver.resolveBasenameMatches !== 'function') continue;
if (!opts.globalBasename || typeof resolver.resolveBasenameMatches !== 'function') {
continue;
}
// Issue #972 (codex): resolve by the wikilink TARGET (ref.slug — the
// text inside `[[...]]` before any `|`), NOT the display alias
// (ref.name = match[2]). `[[struktura|the project]]` must resolve
// `struktura`, not "the project". The display text is for context only.
//
// Issue #1964: a dir-qualified wikilink (`[[llm-wiki/entities/AI 3.0]]`)
// carries a raw Obsidian path while page slugs are sync-slugified
// (`llm-wiki/entities/ai-3.0`). Slugify the path the same way sync does,
// then match by exact slug or path-suffix (wiki-root-relative authoring).
// This runs regardless of global_basename — it's dir-qualified, so the
// cross-dir false-positive risk the flag guards against doesn't apply.
// Mirrors the FS path's resolveSlug ancestor walk. Bare `[[name]]`
// wikilinks still require the global_basename flag.
let matches: string[] = [];
const slugified = ref.slug.includes('/') ? slugifyPath(ref.slug) : '';
if (slugified.includes('/')) {
const tail = slugified.slice(slugified.lastIndexOf('/') + 1);
matches = (await resolver.resolveBasenameMatches(tail))
.filter(m => m === slugified || m.endsWith(`/${slugified}`));
} else if (opts.globalBasename) {
matches = await resolver.resolveBasenameMatches(ref.slug);
}
const matches = await resolver.resolveBasenameMatches(ref.slug);
if (matches.length === 0) continue;
const idx = content.indexOf(ref.slug);
const context = idx >= 0 ? excerpt(content, idx, 240) : ref.name;
+61 -2
View File
@@ -424,6 +424,23 @@ export interface OperationContext {
* satisfied even on single-source brains.
*/
sourceId: string;
/**
* #2561 — federated read scope for UNQUALIFIED local CLI reads.
*
* Set ONLY by the local CLI's context builder (src/cli.ts makeContext), and
* only when the source resolved via a non-explicit tier (local_path /
* brain_default / sole_non_default / seed_default — NOT --source, NOT
* GBRAIN_SOURCE, NOT a .gbrain-source dotfile). Contains the resolved
* source first, then every other `config.federated = true` source, so an
* unqualified `gbrain search "X"` spans federated sources as
* docs/guides/multi-source-brains.md promises.
*
* Consumed exclusively by `federatedSearchScope` and ONLY when
* `ctx.remote === false` — a remote caller's scope stays governed by
* `ctx.auth.allowedSources` / scalar `ctx.sourceId` (source-isolation
* invariant, fail-closed).
*/
localFederatedSourceIds?: string[];
}
/**
@@ -539,6 +556,45 @@ export function resolveRequestedScope(
return sourceScopeOpts(ctx);
}
/**
* #2561 — source scope for the search-shaped read ops (`search`, `query`).
*
* Delegates to `resolveRequestedScope` (the single trust+grant resolver), then
* widens an UNQUALIFIED trusted-local scalar scope to the CLI-computed
* federated set (`ctx.localFederatedSourceIds`, resolved source first). This is
* what makes `sources add --federated` mean something for local search: a
* federated source participates in unqualified `gbrain search "X"` results.
*
* The expansion NEVER applies when:
* - the caller is not strictly trusted-local (`ctx.remote !== false`) —
* remote scope stays grant-governed (fail-closed source isolation);
* - a per-call `source_id` was passed (explicit wins, including `__all__`);
* - the resolver already produced a federated array (OAuth grant);
* - the CLI resolved the source from an explicit signal (--source / env /
* dotfile) — makeContext leaves `localFederatedSourceIds` unset then.
*
* Deliberately NOT inside `sourceScopeOpts`: code-intel ops collapse a
* multi-element scope to an error (`resolveCodeIntelScope`), and non-search
* reads (get_page, get_links, …) keep their long-standing scalar behavior.
*/
export function federatedSearchScope(
ctx: OperationContext,
sourceIdParam?: string,
): { sourceId?: string; sourceIds?: string[] } {
const scope = resolveRequestedScope(ctx, sourceIdParam);
if (
ctx.remote === false &&
sourceIdParam === undefined &&
scope.sourceId !== undefined &&
scope.sourceIds === undefined &&
ctx.localFederatedSourceIds !== undefined &&
ctx.localFederatedSourceIds.length > 1
) {
return { sourceIds: ctx.localFederatedSourceIds };
}
return scope;
}
/**
* Code-intel adapter for `resolveRequestedScope`. Graph traversal
* (code_callers/code_callees/code_blast/code_flow) is single-source by design —
@@ -1448,7 +1504,8 @@ const search: Operation = {
const queryText = p.query as string;
const limit = (p.limit as number) || 20;
const offset = (p.offset as number) || 0;
const scope = sourceScopeOpts(ctx);
// #2561: unqualified trusted-local search spans federated sources.
const scope = federatedSearchScope(ctx);
// T4/D5 — per-call mode honored ONLY for trusted/local callers so a remote
// OAuth client can't escalate to the costly tokenmax bundle. Local + unknown
@@ -1610,7 +1667,9 @@ const query: Operation = {
// is spread into BOTH the image-similarity searchVector path and the text
// hybridSearch path below, so both honor the same grant.
const sourceIdParam = typeof p.source_id === 'string' ? p.source_id : undefined;
const querySourceScope = resolveRequestedScope(ctx, sourceIdParam);
// #2561: unqualified trusted-local query spans federated sources (per-call
// source_id / remote grants still resolve through resolveRequestedScope).
const querySourceScope = federatedSearchScope(ctx, sourceIdParam);
// v0.27.1: image-similarity branch. Bypasses hybridSearch (which is
// text-only); embeds the image via embedMultimodal and runs a direct
+39
View File
@@ -353,6 +353,45 @@ export async function resolveSourceWithTier(
return { source_id: 'default', tier: 'seed_default' };
}
/**
* #2561 compute the federated read scope for an UNQUALIFIED local CLI call.
*
* `sources add --federated` promises that a `config.federated = true` source
* "participates in unqualified `gbrain search` results"
* (docs/guides/multi-source-brains.md). This helper turns that promise into a
* scope: given the resolved source and WHICH tier resolved it, return
* `[resolvedSource, ...other federated source ids]` or `undefined` when the
* expansion must not apply:
*
* - explicit tiers (`flag` / `env` / `dotfile`): the user named a source;
* scalar scope stands (that IS the qualified case);
* - no other federated source exists: keep the scalar fast path unchanged.
*
* Archived sources are excluded (same rationale as pickSoleNonDefaultSource);
* the archived column is v34+, so fall back to the un-archived query on older
* brains. Callers put the result on `OperationContext.localFederatedSourceIds`
* consumed only by `federatedSearchScope` and only when `remote === false`.
*/
export async function localFederatedSourceIds(
engine: BrainEngine,
sourceId: string,
tier: SourceTier,
): Promise<string[] | undefined> {
if (tier === 'flag' || tier === 'env' || tier === 'dotfile') return undefined;
let rows: Array<{ id: string }>;
try {
rows = await engine.executeRaw<{ id: string }>(
`SELECT id FROM sources WHERE config->>'federated' = 'true' AND archived = false ORDER BY id`,
);
} catch {
rows = await engine.executeRaw<{ id: string }>(
`SELECT id FROM sources WHERE config->>'federated' = 'true' ORDER BY id`,
);
}
const ids = [sourceId, ...rows.map((r) => r.id).filter((id) => id !== sourceId)];
return ids.length > 1 ? ids : undefined;
}
/** Exposed for tests. */
export const __testing = {
readDotfileWalk,
-27
View File
@@ -389,33 +389,6 @@ describe('resolveSlugAll', () => {
});
});
// ─── issue #1964: cross-directory wikilinks — slug/path mismatch ──────────
describe('issue #1964: raw Obsidian wikilink paths resolve to sync-slugified slugs', () => {
test('resolveSlug slugifies the candidate (sync-consistent), no flag needed', () => {
const all = new Set(['llm-wiki/entities/ai-3.0']);
// Wikilink literal `[[llm-wiki/entities/AI 3.0]]` — spaces + uppercase.
expect(resolveSlug('llm-wiki/notes', 'llm-wiki/entities/AI 3.0.md', all))
.toBe('llm-wiki/entities/ai-3.0');
});
test('resolveSlug slugifies raw (unslugified) fileDir too', () => {
const all = new Set(['llm-wiki/entities/ai-3.0']);
// fileDir comes from dirname(relPath) — the raw on-disk directory.
expect(resolveSlug('LLM Wiki/Notes', 'entities/AI 3.0.md', all))
.toBe('llm-wiki/entities/ai-3.0');
});
test('extractLinksFromFile resolves cross-directory wikilink with flag OFF as a typed edge', async () => {
const allSlugs = new Set(['llm-wiki/entities/ai-3.0', 'llm-wiki/notes/roadmap']);
const content = '---\ntitle: Roadmap\ntype: concept\n---\n\nSee [[llm-wiki/entities/AI 3.0]].\n';
const links = await extractLinksFromFile(content, 'llm-wiki/notes/roadmap.md', allSlugs);
expect(links.map(l => l.to_slug)).toEqual(['llm-wiki/entities/ai-3.0']);
// Dir-qualified path resolution is exact, NOT the basename fallback.
expect(links[0].link_type).not.toBe('wikilink_basename');
});
});
describe('issue #972 repro: bare wikilinks resolve when flag is on', () => {
// End-to-end: reproduces the issue's exact repro inside a tempdir +
// PGLite, then asserts edge count under both flag states.
-42
View File
@@ -9,8 +9,6 @@ import {
parseTimelineEntries,
isAutoLinkEnabled,
FRONTMATTER_LINK_MAP,
buildBasenameIndex,
queryBasenameIndex,
type SlugResolver,
} from '../src/core/link-extraction.ts';
import type { BrainEngine } from '../src/core/engine.ts';
@@ -426,46 +424,6 @@ describe('extractPageLinks', () => {
expect(strk!.linkType).toBe('wikilink_basename');
});
// ─── issue #1964: dir-qualified wikilinks with raw Obsidian paths ────────
test('#1964: dir-qualified wikilink resolves via sync-consistent slugification (flag OFF)', async () => {
// `[[llm-wiki/entities/AI 3.0]]` is a raw Obsidian path; the page slug
// is the sync-slugified `llm-wiki/entities/ai-3.0`. Must resolve WITHOUT
// global_basename (it's dir-qualified) and must NOT leak to a same-tail
// page in a different directory.
const resolver: SlugResolver = {
resolve: async () => null,
resolveBasenameMatches: async (name) =>
name === 'ai-3.0' ? ['other/ai-3.0', 'llm-wiki/entities/ai-3.0'] : [],
};
const { candidates } = await extractPageLinks(
'llm-wiki/notes/roadmap',
'See [[llm-wiki/entities/AI 3.0]] for the model.',
{}, 'concept', resolver,
// opts.globalBasename omitted (= false) — path is dir-qualified
);
expect(candidates.map(c => c.targetSlug)).toEqual(['llm-wiki/entities/ai-3.0']);
expect(candidates[0].linkType).toBe('wikilink_basename');
expect(candidates[0].linkSource).toBe('wikilink-resolved');
});
test('#1964: path-suffix match resolves wiki-root-relative paths against a real index', async () => {
// Author writes `[[llm-wiki/entities/AI 3.0]]` but the brain nests the
// wiki under a vault dir. Suffix match rescues it; queried through the
// REAL basename index so the tail-key lookup is exercised end to end.
const idx = buildBasenameIndex(['vault/llm-wiki/entities/ai-3.0', 'people/ai-3.0']);
const resolver: SlugResolver = {
resolve: async () => null,
resolveBasenameMatches: async (name) => queryBasenameIndex(idx, name),
};
const { candidates } = await extractPageLinks(
'vault/llm-wiki/notes/roadmap',
'See [[llm-wiki/entities/AI 3.0]].',
{}, 'concept', resolver,
);
expect(candidates.map(c => c.targetSlug)).toEqual(['vault/llm-wiki/entities/ai-3.0']);
});
test('opts.skipFrontmatter suppresses the frontmatter pass', async () => {
// Real resolver shape that WOULD resolve frontmatter source: too,
// but skipFrontmatter blocks the path entirely.
+154
View File
@@ -0,0 +1,154 @@
/**
* #2561 sources.config.federated participates in UNQUALIFIED local CLI
* search/query.
*
* Pre-fix: the local CLI always emitted a scalar `{sourceId}` scope (required
* field, auto-filled 'default'), so a source registered with
* `gbrain sources add --federated` was invisible to an unqualified
* `gbrain search "X"` contradicting docs/guides/multi-source-brains.md
* ("Source participates in unqualified `gbrain search` results").
*
* Fix: the CLI context builder computes `ctx.localFederatedSourceIds`
* (resolved source + every other federated source) whenever the source
* resolved via a NON-explicit tier; `federatedSearchScope` widens the scalar
* scope to that set for the `search` / `query` ops trusted-local only
* (`ctx.remote === false`), never for remote callers, never when a per-call
* `source_id` or an explicit --source/env/dotfile was given.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { localFederatedSourceIds } from '../src/core/source-resolver.ts';
import {
federatedSearchScope,
operations,
type OperationContext,
} from '../src/core/operations.ts';
let engine: PGLiteEngine;
const search = operations.find((o) => o.name === 'search')!;
function ctxOf(overrides: Partial<OperationContext> = {}): OperationContext {
return {
engine: engine as any,
config: {} as any,
logger: console as any,
dryRun: false,
remote: false,
sourceId: 'default',
...overrides,
};
}
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
// Seeded 'default' source is federated=true. Add:
// wiki — federated (must join unqualified search)
// private — NOT federated (must stay invisible unless explicitly named)
// oldnews — federated but archived (must stay excluded)
await engine.executeRaw(
`INSERT INTO sources (id, name, local_path, config) VALUES ('wiki', 'wiki', '/tmp/wiki', '{"federated": true}'::jsonb)`,
);
await engine.executeRaw(
`INSERT INTO sources (id, name, local_path, config) VALUES ('private', 'private', '/tmp/private', '{}'::jsonb)`,
);
await engine.executeRaw(
`INSERT INTO sources (id, name, local_path, config, archived) VALUES ('oldnews', 'oldnews', '/tmp/oldnews', '{"federated": true}'::jsonb, true)`,
);
const pages: Array<[slug: string, sourceId: string, where: string]> = [
['notes/home', 'default', 'default'],
['wiki/topic', 'wiki', 'wiki'],
['private/topic', 'private', 'private'],
['old/topic', 'oldnews', 'oldnews'],
];
for (const [slug, sourceId, where] of pages) {
await engine.putPage(slug, {
type: 'note', title: `Topic in ${where}`, compiled_truth: `the zebra telescope in ${where}`, frontmatter: {},
}, { sourceId });
await engine.upsertChunks(slug, [
{ chunk_index: 0, chunk_text: `the zebra telescope in ${where}`, chunk_source: 'compiled_truth' },
], { sourceId });
}
// Keyword-only search path: no embedding provider needed in tests.
await engine.setConfig('search.mcp_keyword_only', 'true');
}, 60_000);
afterAll(async () => {
if (engine) await engine.disconnect();
}, 60_000);
describe('localFederatedSourceIds — CLI-side scope computation', () => {
test('non-explicit tier: resolved source first, then other federated, archived excluded', async () => {
expect(await localFederatedSourceIds(engine, 'default', 'seed_default')).toEqual(['default', 'wiki']);
});
test('non-federated resolved source still joins its own scope', async () => {
expect(await localFederatedSourceIds(engine, 'private', 'brain_default')).toEqual(['private', 'default', 'wiki']);
});
test('explicit tiers (--source / env / dotfile) never expand', async () => {
expect(await localFederatedSourceIds(engine, 'default', 'flag')).toBeUndefined();
expect(await localFederatedSourceIds(engine, 'default', 'env')).toBeUndefined();
expect(await localFederatedSourceIds(engine, 'default', 'dotfile')).toBeUndefined();
});
test('single federated source (the resolved one) keeps the scalar fast path', async () => {
const solo = { executeRaw: async () => [{ id: 'default' }] } as any;
expect(await localFederatedSourceIds(solo, 'default', 'seed_default')).toBeUndefined();
});
});
describe('federatedSearchScope — trust + explicitness matrix', () => {
test('trusted local + unqualified widens to the federated set', () => {
const ctx = ctxOf({ localFederatedSourceIds: ['default', 'wiki'] });
expect(federatedSearchScope(ctx)).toEqual({ sourceIds: ['default', 'wiki'] });
});
test('remote caller NEVER widens (fail-closed), even if the field is set', () => {
const ctx = ctxOf({ remote: true, localFederatedSourceIds: ['default', 'wiki'] });
expect(federatedSearchScope(ctx)).toEqual({ sourceId: 'default' });
});
test('per-call source_id wins over the federated set', () => {
const ctx = ctxOf({ localFederatedSourceIds: ['default', 'wiki'] });
expect(federatedSearchScope(ctx, 'wiki')).toEqual({ sourceId: 'wiki' });
});
test('per-call __all__ keeps the whole-brain semantics for trusted local', () => {
const ctx = ctxOf({ localFederatedSourceIds: ['default', 'wiki'] });
expect(federatedSearchScope(ctx, '__all__')).toEqual({});
});
test('a federated OAuth grant wins over the local set', () => {
const ctx = ctxOf({
localFederatedSourceIds: ['default', 'wiki'],
auth: { allowedSources: ['a', 'b'] } as OperationContext['auth'],
});
expect(federatedSearchScope(ctx)).toEqual({ sourceIds: ['a', 'b'] });
});
test('no local federated set → unchanged scalar scope', () => {
expect(federatedSearchScope(ctxOf())).toEqual({ sourceId: 'default' });
});
});
describe('search op — unqualified local search spans federated sources', () => {
test('federated source results appear; non-federated + archived stay invisible', async () => {
const ctx = ctxOf({
localFederatedSourceIds: await localFederatedSourceIds(engine, 'default', 'seed_default'),
});
const results = (await search.handler(ctx, { query: 'zebra telescope' })) as Array<{ slug: string }>;
const slugs = results.map((r) => r.slug);
expect(slugs).toContain('notes/home');
expect(slugs).toContain('wiki/topic'); // pre-#2561 this was missing
expect(slugs).not.toContain('private/topic');
expect(slugs).not.toContain('old/topic');
});
test('explicit source resolution (no federated set on ctx) stays single-source', async () => {
const results = (await search.handler(ctxOf(), { query: 'zebra telescope' })) as Array<{ slug: string }>;
const slugs = results.map((r) => r.slug);
expect(slugs).toEqual(['notes/home']);
});
});