mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-15 01:12:20 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
921048827a | ||
|
|
0dff84b16a |
+2
-11
@@ -808,20 +808,12 @@ async function makeContext(engine: BrainEngine, params: Record<string, unknown>)
|
||||
// 'default'. Wrapped in try/catch so a doctor / single-source brain that
|
||||
// never set up sources still returns 'default' silently.
|
||||
let sourceId: string | undefined;
|
||||
// #2561: when the source resolved via a NON-explicit tier (path-match /
|
||||
// brain default / sole-non-default / seed default), unqualified search-shaped
|
||||
// reads span every `config.federated = true` source. Computed here (the
|
||||
// trusted local boundary) and consumed by federatedSearchScope in
|
||||
// operations.ts, which additionally gates on ctx.remote === false.
|
||||
let localFederated: string[] | undefined;
|
||||
try {
|
||||
const { resolveSourceWithTier, localFederatedSourceIds } = await import('./core/source-resolver.ts');
|
||||
const { resolveSourceId } = await import('./core/source-resolver.ts');
|
||||
// params.source is set when a CLI flag was parsed for the op (rare; most
|
||||
// CLI ops don't take --source). Falls through to env/dotfile/path-match.
|
||||
const explicit = (params.source as string | undefined) ?? null;
|
||||
const resolved = await resolveSourceWithTier(engine, explicit);
|
||||
sourceId = resolved.source_id;
|
||||
localFederated = await localFederatedSourceIds(engine, resolved.source_id, resolved.tier);
|
||||
sourceId = await resolveSourceId(engine, explicit);
|
||||
} catch {
|
||||
// Source resolution failed (e.g. sources table doesn't exist on a fresh
|
||||
// pre-init brain). Leave sourceId unset; engine read methods fall through
|
||||
@@ -842,7 +834,6 @@ async function makeContext(engine: BrainEngine, params: Record<string, unknown>)
|
||||
// table). Matches dispatch.ts's auto-fill so the contract holds across
|
||||
// every transport.
|
||||
sourceId: sourceId ?? 'default',
|
||||
...(localFederated ? { localFederatedSourceIds: localFederated } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -686,6 +686,7 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
try {
|
||||
const { MinionQueue } = await import('../core/minions/queue.ts');
|
||||
const { computeRecommendations, embeddingProviderConfigured, HOSTED_EMBED_KEY_CONFIG } = await import('../core/brain-score-recommendations.ts');
|
||||
const { countExtractionLag } = await import('../core/remediation/context.ts');
|
||||
const queue = new MinionQueue(engine);
|
||||
const slotMs = Math.floor(Date.now() / (baseInterval * 1000)) * baseInterval * 1000;
|
||||
const slot = new Date(slotMs).toISOString();
|
||||
@@ -877,6 +878,9 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
return !!(process.env[envVar] || (cfgField ? embedKeyCfg[cfgField] : undefined));
|
||||
}),
|
||||
hasChatApiKey: !!(process.env.ANTHROPIC_API_KEY || await engine.getConfig('anthropic_api_key')),
|
||||
// Real extraction-lag gate for sync.repo/extract.all — same counter
|
||||
// loadRecommendationContext uses (replaces the health.stale_pages proxy).
|
||||
extractionLagPages: await countExtractionLag(engine),
|
||||
};
|
||||
// v0.41.18.0 (A5 + A19 + A22, T15): consult onboard recommendations
|
||||
// ALONGSIDE doctor's brain-score recommendations. Onboard's 4 new
|
||||
|
||||
@@ -146,6 +146,16 @@ export interface RecommendationContext {
|
||||
chatModel?: string;
|
||||
/** Whether the chat provider has a usable API key. */
|
||||
hasChatApiKey?: boolean;
|
||||
/**
|
||||
* Count of pages needing link/timeline extraction — the SAME staleness the
|
||||
* `gbrain extract --stale` walk and doctor's `links_extraction_lag` check use
|
||||
* (`engine.countStalePagesForExtraction`). Gates the sync→extract pipeline
|
||||
* (sync.repo / extract.all). Replaces the old `health.stale_pages` gate, which
|
||||
* counted "pages whose updated_at predates their newest timeline entry" — a
|
||||
* proxy that broke when the updated_at-on-timeline-insert trigger was dropped
|
||||
* (migration v10) and never reflected real extraction work.
|
||||
*/
|
||||
extractionLagPages?: number;
|
||||
}
|
||||
|
||||
/** Triage result for one check. */
|
||||
@@ -192,20 +202,28 @@ export function computeRecommendations(
|
||||
const source = ctx.sourceId ?? 'default';
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// sync.repo — fires when sync hasn't run recently OR pages are stale
|
||||
// sync.repo + extract.all — the materialization pipeline, gated on the REAL
|
||||
// extraction lag (pages whose link/timeline edges are stale), NOT on the
|
||||
// legacy `health.stale_pages` proxy. `extractionLagPages` comes from the same
|
||||
// counter the `extract --stale` walk + doctor's `links_extraction_lag` use, so
|
||||
// the recommendation can only fire when running extract will actually reduce
|
||||
// it (and clear the rec). See RecommendationContext.extractionLagPages.
|
||||
// sync.repo is the prerequisite: re-sync so pages are current before extract
|
||||
// materializes their edges.
|
||||
// ---------------------------------------------------------------------
|
||||
if (ctx.repoPath && health.stale_pages > 0) {
|
||||
const extractionLag = ctx.extractionLagPages ?? 0;
|
||||
if (ctx.repoPath && extractionLag > 0) {
|
||||
const params = { repoPath: ctx.repoPath, sourceId: ctx.sourceId, noEmbed: true };
|
||||
out.push({
|
||||
id: 'sync.repo',
|
||||
job: 'sync',
|
||||
params,
|
||||
idempotency_key: idemKey(source, 'sync', params),
|
||||
severity: health.stale_pages > 50 ? 'high' : 'medium',
|
||||
est_seconds: Math.min(600, 30 + health.stale_pages * 0.5),
|
||||
severity: extractionLag > 50 ? 'high' : 'medium',
|
||||
est_seconds: Math.min(600, 30 + extractionLag * 0.5),
|
||||
est_usd_cost: 0, // sync is fs+DB only
|
||||
depends_on: [],
|
||||
rationale: `${health.stale_pages} stale page${health.stale_pages === 1 ? '' : 's'} on disk`,
|
||||
rationale: `Sync before extracting ${extractionLag} page${extractionLag === 1 ? '' : 's'} with stale link/timeline edges`,
|
||||
status: 'remediable',
|
||||
});
|
||||
}
|
||||
@@ -237,7 +255,7 @@ export function computeRecommendations(
|
||||
est_seconds: Math.min(3600, 5 + health.missing_embeddings * 0.05),
|
||||
est_usd_cost,
|
||||
// sync should run first so embed sees fresh pages.
|
||||
depends_on: ctx.repoPath && health.stale_pages > 0 ? ['sync.repo'] : [],
|
||||
depends_on: ctx.repoPath && extractionLag > 0 ? ['sync.repo'] : [],
|
||||
rationale: `${health.missing_embeddings} chunk${health.missing_embeddings === 1 ? '' : 's'} invisible to vector search`,
|
||||
status: 'remediable',
|
||||
});
|
||||
@@ -267,7 +285,7 @@ export function computeRecommendations(
|
||||
// Triggered when sync.repo fires (because sync was set to noEmbed:true,
|
||||
// and noExtract:true after T5 lands → extract job is the materializer).
|
||||
// ---------------------------------------------------------------------
|
||||
if (ctx.repoPath && health.stale_pages > 0) {
|
||||
if (ctx.repoPath && extractionLag > 0) {
|
||||
const params = { mode: 'all', dir: ctx.repoPath };
|
||||
out.push({
|
||||
id: 'extract.all',
|
||||
@@ -278,7 +296,7 @@ export function computeRecommendations(
|
||||
est_seconds: Math.min(600, 30 + health.page_count * 0.01),
|
||||
est_usd_cost: 0,
|
||||
depends_on: ['sync.repo'],
|
||||
rationale: 'Materialize link + timeline edges from fresh pages',
|
||||
rationale: `Materialize link + timeline edges for ${extractionLag} page${extractionLag === 1 ? '' : 's'} with stale extraction`,
|
||||
status: 'remediable',
|
||||
});
|
||||
}
|
||||
|
||||
+2
-61
@@ -424,23 +424,6 @@ export interface OperationContext {
|
||||
* satisfied even on single-source brains.
|
||||
*/
|
||||
sourceId: string;
|
||||
/**
|
||||
* #2561 — federated read scope for UNQUALIFIED local CLI reads.
|
||||
*
|
||||
* Set ONLY by the local CLI's context builder (src/cli.ts makeContext), and
|
||||
* only when the source resolved via a non-explicit tier (local_path /
|
||||
* brain_default / sole_non_default / seed_default — NOT --source, NOT
|
||||
* GBRAIN_SOURCE, NOT a .gbrain-source dotfile). Contains the resolved
|
||||
* source first, then every other `config.federated = true` source, so an
|
||||
* unqualified `gbrain search "X"` spans federated sources as
|
||||
* docs/guides/multi-source-brains.md promises.
|
||||
*
|
||||
* Consumed exclusively by `federatedSearchScope` and ONLY when
|
||||
* `ctx.remote === false` — a remote caller's scope stays governed by
|
||||
* `ctx.auth.allowedSources` / scalar `ctx.sourceId` (source-isolation
|
||||
* invariant, fail-closed).
|
||||
*/
|
||||
localFederatedSourceIds?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -556,45 +539,6 @@ export function resolveRequestedScope(
|
||||
return sourceScopeOpts(ctx);
|
||||
}
|
||||
|
||||
/**
|
||||
* #2561 — source scope for the search-shaped read ops (`search`, `query`).
|
||||
*
|
||||
* Delegates to `resolveRequestedScope` (the single trust+grant resolver), then
|
||||
* widens an UNQUALIFIED trusted-local scalar scope to the CLI-computed
|
||||
* federated set (`ctx.localFederatedSourceIds`, resolved source first). This is
|
||||
* what makes `sources add --federated` mean something for local search: a
|
||||
* federated source participates in unqualified `gbrain search "X"` results.
|
||||
*
|
||||
* The expansion NEVER applies when:
|
||||
* - the caller is not strictly trusted-local (`ctx.remote !== false`) —
|
||||
* remote scope stays grant-governed (fail-closed source isolation);
|
||||
* - a per-call `source_id` was passed (explicit wins, including `__all__`);
|
||||
* - the resolver already produced a federated array (OAuth grant);
|
||||
* - the CLI resolved the source from an explicit signal (--source / env /
|
||||
* dotfile) — makeContext leaves `localFederatedSourceIds` unset then.
|
||||
*
|
||||
* Deliberately NOT inside `sourceScopeOpts`: code-intel ops collapse a
|
||||
* multi-element scope to an error (`resolveCodeIntelScope`), and non-search
|
||||
* reads (get_page, get_links, …) keep their long-standing scalar behavior.
|
||||
*/
|
||||
export function federatedSearchScope(
|
||||
ctx: OperationContext,
|
||||
sourceIdParam?: string,
|
||||
): { sourceId?: string; sourceIds?: string[] } {
|
||||
const scope = resolveRequestedScope(ctx, sourceIdParam);
|
||||
if (
|
||||
ctx.remote === false &&
|
||||
sourceIdParam === undefined &&
|
||||
scope.sourceId !== undefined &&
|
||||
scope.sourceIds === undefined &&
|
||||
ctx.localFederatedSourceIds !== undefined &&
|
||||
ctx.localFederatedSourceIds.length > 1
|
||||
) {
|
||||
return { sourceIds: ctx.localFederatedSourceIds };
|
||||
}
|
||||
return scope;
|
||||
}
|
||||
|
||||
/**
|
||||
* Code-intel adapter for `resolveRequestedScope`. Graph traversal
|
||||
* (code_callers/code_callees/code_blast/code_flow) is single-source by design —
|
||||
@@ -1504,8 +1448,7 @@ const search: Operation = {
|
||||
const queryText = p.query as string;
|
||||
const limit = (p.limit as number) || 20;
|
||||
const offset = (p.offset as number) || 0;
|
||||
// #2561: unqualified trusted-local search spans federated sources.
|
||||
const scope = federatedSearchScope(ctx);
|
||||
const scope = sourceScopeOpts(ctx);
|
||||
|
||||
// T4/D5 — per-call mode honored ONLY for trusted/local callers so a remote
|
||||
// OAuth client can't escalate to the costly tokenmax bundle. Local + unknown
|
||||
@@ -1667,9 +1610,7 @@ const query: Operation = {
|
||||
// is spread into BOTH the image-similarity searchVector path and the text
|
||||
// hybridSearch path below, so both honor the same grant.
|
||||
const sourceIdParam = typeof p.source_id === 'string' ? p.source_id : undefined;
|
||||
// #2561: unqualified trusted-local query spans federated sources (per-call
|
||||
// source_id / remote grants still resolve through resolveRequestedScope).
|
||||
const querySourceScope = federatedSearchScope(ctx, sourceIdParam);
|
||||
const querySourceScope = resolveRequestedScope(ctx, sourceIdParam);
|
||||
|
||||
// v0.27.1: image-similarity branch. Bypasses hybridSearch (which is
|
||||
// text-only); embeds the image via embedMultimodal and runs a direct
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
import type { RecommendationContext } from '../brain-score-recommendations.ts';
|
||||
import { LINK_EXTRACTOR_VERSION_TS } from '../link-extraction.ts';
|
||||
|
||||
// Re-export so consumers can `import { RecommendationContext } from '../remediation'`
|
||||
// — the canonical RecommendationContext type still lives in
|
||||
@@ -68,5 +69,29 @@ export async function loadRecommendationContext(
|
||||
embeddingDimensions,
|
||||
embeddingProviderConfigured: embeddingConfigured,
|
||||
hasChatApiKey: !!(process.env.ANTHROPIC_API_KEY || fileCfg?.anthropic_api_key),
|
||||
extractionLagPages: await countExtractionLag(engine),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Real extraction-lag count — the SAME staleness `gbrain extract --stale`
|
||||
* processes (engine.countStalePagesForExtraction with
|
||||
* versionTs=LINK_EXTRACTOR_VERSION_TS, matching doctor's links_extraction_lag
|
||||
* check — without versionTs, pages stamped before an extractor version bump
|
||||
* would lag for doctor/extract but never trip this gate). Drives the
|
||||
* sync→extract recommendation pipeline; replaces the legacy
|
||||
* `health.stale_pages` proxy that no longer reflected real extraction work
|
||||
* after the v10 trigger drop.
|
||||
*
|
||||
* Shared by loadRecommendationContext AND the D7 per-step recheck in
|
||||
* runRemediation — the recheck MUST refresh this gate alongside getHealth,
|
||||
* or a completed extract step keeps re-firing off the frozen initial count.
|
||||
*/
|
||||
export async function countExtractionLag(engine: BrainEngine): Promise<number> {
|
||||
try {
|
||||
return await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS });
|
||||
} catch {
|
||||
/* counter unavailable (very old brain / mid-migration) — treat as 0 */
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ import {
|
||||
computeRecommendations,
|
||||
} from '../brain-score-recommendations.ts';
|
||||
import type { RemediationStep } from '../remediation-step.ts';
|
||||
import { loadRecommendationContext } from './context.ts';
|
||||
import { countExtractionLag, loadRecommendationContext } from './context.ts';
|
||||
import { computeRemediationPlan } from './plan.ts';
|
||||
import type {
|
||||
RemediationHooks,
|
||||
@@ -65,7 +65,7 @@ export async function runRemediation(
|
||||
clearRemediationCheckpoint,
|
||||
} = await import('../remediation-checkpoint.ts');
|
||||
|
||||
const ctx = await loadRecommendationContext(engine);
|
||||
let ctx = await loadRecommendationContext(engine);
|
||||
|
||||
// Pre-flight ceiling check via the shared plan computation.
|
||||
const initialPlan = await computeRemediationPlan(engine, { targetScore });
|
||||
@@ -305,6 +305,11 @@ export async function runRemediation(
|
||||
// steps with bumped retry suffix (D1).
|
||||
if (recs.length === 0 || stepCount >= maxJobs) break;
|
||||
const freshHealth = await engine.getHealth();
|
||||
// Refresh the extraction-lag gate alongside health: ctx was loaded once
|
||||
// before the loop, and a completed sync/extract step is exactly what
|
||||
// drives the count down. Reusing the frozen initial count would re-fire
|
||||
// sync.repo/extract.all every recheck until maxJobs.
|
||||
ctx = { ...ctx, extractionLagPages: await countExtractionLag(engine) };
|
||||
recs = computeRecommendations(freshHealth, ctx).filter((r) => r.status === 'remediable');
|
||||
}
|
||||
};
|
||||
|
||||
@@ -353,45 +353,6 @@ export async function resolveSourceWithTier(
|
||||
return { source_id: 'default', tier: 'seed_default' };
|
||||
}
|
||||
|
||||
/**
|
||||
* #2561 — compute the federated read scope for an UNQUALIFIED local CLI call.
|
||||
*
|
||||
* `sources add --federated` promises that a `config.federated = true` source
|
||||
* "participates in unqualified `gbrain search` results"
|
||||
* (docs/guides/multi-source-brains.md). This helper turns that promise into a
|
||||
* scope: given the resolved source and WHICH tier resolved it, return
|
||||
* `[resolvedSource, ...other federated source ids]` — or `undefined` when the
|
||||
* expansion must not apply:
|
||||
*
|
||||
* - explicit tiers (`flag` / `env` / `dotfile`): the user named a source;
|
||||
* scalar scope stands (that IS the qualified case);
|
||||
* - no other federated source exists: keep the scalar fast path unchanged.
|
||||
*
|
||||
* Archived sources are excluded (same rationale as pickSoleNonDefaultSource);
|
||||
* the archived column is v34+, so fall back to the un-archived query on older
|
||||
* brains. Callers put the result on `OperationContext.localFederatedSourceIds`
|
||||
* — consumed only by `federatedSearchScope` and only when `remote === false`.
|
||||
*/
|
||||
export async function localFederatedSourceIds(
|
||||
engine: BrainEngine,
|
||||
sourceId: string,
|
||||
tier: SourceTier,
|
||||
): Promise<string[] | undefined> {
|
||||
if (tier === 'flag' || tier === 'env' || tier === 'dotfile') return undefined;
|
||||
let rows: Array<{ id: string }>;
|
||||
try {
|
||||
rows = await engine.executeRaw<{ id: string }>(
|
||||
`SELECT id FROM sources WHERE config->>'federated' = 'true' AND archived = false ORDER BY id`,
|
||||
);
|
||||
} catch {
|
||||
rows = await engine.executeRaw<{ id: string }>(
|
||||
`SELECT id FROM sources WHERE config->>'federated' = 'true' ORDER BY id`,
|
||||
);
|
||||
}
|
||||
const ids = [sourceId, ...rows.map((r) => r.id).filter((id) => id !== sourceId)];
|
||||
return ids.length > 1 ? ids : undefined;
|
||||
}
|
||||
|
||||
/** Exposed for tests. */
|
||||
export const __testing = {
|
||||
readDotfileWalk,
|
||||
|
||||
@@ -1423,6 +1423,15 @@ export interface BrainStats {
|
||||
export interface BrainHealth {
|
||||
page_count: number;
|
||||
embed_coverage: number;
|
||||
/**
|
||||
* LEGACY proxy: count of pages whose `updated_at` predates their newest
|
||||
* timeline entry. This bumped meaningfully only while a trigger updated
|
||||
* `pages.updated_at` on timeline insert; that trigger was dropped in
|
||||
* migration v10, so the metric no longer reflects real "needs work" state.
|
||||
* NO LONGER gates remediations — the sync→extract pipeline now gates on
|
||||
* `RecommendationContext.extractionLagPages` (the real extraction-lag from
|
||||
* `countStalePagesForExtraction`). Retained for the CLI health line + back-compat.
|
||||
*/
|
||||
stale_pages: number;
|
||||
/**
|
||||
* Islanded pages — zero inbound AND zero outbound links. A hub page
|
||||
|
||||
@@ -119,13 +119,12 @@ describe('computeRecommendations', () => {
|
||||
expect(recs.find((r) => r.id === 'embed.stale')).toBeUndefined();
|
||||
});
|
||||
|
||||
test('stale pages + dead links produce sync + backlinks + extract', () => {
|
||||
test('extraction lag + dead links produce sync + backlinks + extract', () => {
|
||||
const health = makeHealth({
|
||||
stale_pages: 25,
|
||||
dead_links: 8,
|
||||
brain_score: 70,
|
||||
});
|
||||
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true });
|
||||
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true, extractionLagPages: 25 });
|
||||
const ids = recs.map((r) => r.id);
|
||||
expect(ids).toContain('sync.repo');
|
||||
expect(ids).toContain('backlinks.fix');
|
||||
@@ -133,18 +132,17 @@ describe('computeRecommendations', () => {
|
||||
});
|
||||
|
||||
test('extract.all depends on sync.repo (D14: stable ids)', () => {
|
||||
const health = makeHealth({ stale_pages: 10 });
|
||||
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true });
|
||||
const health = makeHealth();
|
||||
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true, extractionLagPages: 10 });
|
||||
const extract = recs.find((r) => r.id === 'extract.all');
|
||||
expect(extract?.depends_on).toContain('sync.repo');
|
||||
});
|
||||
|
||||
test('embed.stale depends on sync.repo when sync also needed', () => {
|
||||
test('embed.stale depends on sync.repo when extraction also needed', () => {
|
||||
const health = makeHealth({
|
||||
stale_pages: 10,
|
||||
missing_embeddings: 100,
|
||||
});
|
||||
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true });
|
||||
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true, extractionLagPages: 10 });
|
||||
const embed = recs.find((r) => r.id === 'embed.stale');
|
||||
expect(embed?.depends_on).toContain('sync.repo');
|
||||
});
|
||||
@@ -159,9 +157,9 @@ describe('computeRecommendations', () => {
|
||||
test('severity ordering: critical before high before medium', () => {
|
||||
const health = makeHealth({
|
||||
missing_embeddings: 100, // critical
|
||||
stale_pages: 80, // high
|
||||
});
|
||||
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true });
|
||||
// extractionLagPages > 50 → sync.repo fires at 'high' severity.
|
||||
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true, extractionLagPages: 80 });
|
||||
const critIdx = recs.findIndex((r) => r.severity === 'critical');
|
||||
const highIdx = recs.findIndex((r) => r.severity === 'high');
|
||||
expect(critIdx).toBeLessThan(highIdx);
|
||||
@@ -170,11 +168,10 @@ describe('computeRecommendations', () => {
|
||||
// D6 #5 — THE critical regression test for the agent contract.
|
||||
test('D6 #5: determinism — same input twice produces identical output', () => {
|
||||
const health = makeHealth({
|
||||
stale_pages: 10,
|
||||
missing_embeddings: 50,
|
||||
dead_links: 3,
|
||||
});
|
||||
const ctx = { repoPath: '/brain', embeddingProviderConfigured: true, sourceId: 'default' };
|
||||
const ctx = { repoPath: '/brain', embeddingProviderConfigured: true, sourceId: 'default', extractionLagPages: 10 };
|
||||
const run1 = computeRecommendations(health, ctx);
|
||||
const run2 = computeRecommendations(health, ctx);
|
||||
expect(JSON.stringify(run1)).toBe(JSON.stringify(run2));
|
||||
|
||||
@@ -1,154 +0,0 @@
|
||||
/**
|
||||
* #2561 — sources.config.federated participates in UNQUALIFIED local CLI
|
||||
* search/query.
|
||||
*
|
||||
* Pre-fix: the local CLI always emitted a scalar `{sourceId}` scope (required
|
||||
* field, auto-filled 'default'), so a source registered with
|
||||
* `gbrain sources add --federated` was invisible to an unqualified
|
||||
* `gbrain search "X"` — contradicting docs/guides/multi-source-brains.md
|
||||
* ("Source participates in unqualified `gbrain search` results").
|
||||
*
|
||||
* Fix: the CLI context builder computes `ctx.localFederatedSourceIds`
|
||||
* (resolved source + every other federated source) whenever the source
|
||||
* resolved via a NON-explicit tier; `federatedSearchScope` widens the scalar
|
||||
* scope to that set for the `search` / `query` ops — trusted-local only
|
||||
* (`ctx.remote === false`), never for remote callers, never when a per-call
|
||||
* `source_id` or an explicit --source/env/dotfile was given.
|
||||
*/
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { localFederatedSourceIds } from '../src/core/source-resolver.ts';
|
||||
import {
|
||||
federatedSearchScope,
|
||||
operations,
|
||||
type OperationContext,
|
||||
} from '../src/core/operations.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
const search = operations.find((o) => o.name === 'search')!;
|
||||
|
||||
function ctxOf(overrides: Partial<OperationContext> = {}): OperationContext {
|
||||
return {
|
||||
engine: engine as any,
|
||||
config: {} as any,
|
||||
logger: console as any,
|
||||
dryRun: false,
|
||||
remote: false,
|
||||
sourceId: 'default',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
// Seeded 'default' source is federated=true. Add:
|
||||
// wiki — federated (must join unqualified search)
|
||||
// private — NOT federated (must stay invisible unless explicitly named)
|
||||
// oldnews — federated but archived (must stay excluded)
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, local_path, config) VALUES ('wiki', 'wiki', '/tmp/wiki', '{"federated": true}'::jsonb)`,
|
||||
);
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, local_path, config) VALUES ('private', 'private', '/tmp/private', '{}'::jsonb)`,
|
||||
);
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, local_path, config, archived) VALUES ('oldnews', 'oldnews', '/tmp/oldnews', '{"federated": true}'::jsonb, true)`,
|
||||
);
|
||||
const pages: Array<[slug: string, sourceId: string, where: string]> = [
|
||||
['notes/home', 'default', 'default'],
|
||||
['wiki/topic', 'wiki', 'wiki'],
|
||||
['private/topic', 'private', 'private'],
|
||||
['old/topic', 'oldnews', 'oldnews'],
|
||||
];
|
||||
for (const [slug, sourceId, where] of pages) {
|
||||
await engine.putPage(slug, {
|
||||
type: 'note', title: `Topic in ${where}`, compiled_truth: `the zebra telescope in ${where}`, frontmatter: {},
|
||||
}, { sourceId });
|
||||
await engine.upsertChunks(slug, [
|
||||
{ chunk_index: 0, chunk_text: `the zebra telescope in ${where}`, chunk_source: 'compiled_truth' },
|
||||
], { sourceId });
|
||||
}
|
||||
// Keyword-only search path: no embedding provider needed in tests.
|
||||
await engine.setConfig('search.mcp_keyword_only', 'true');
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (engine) await engine.disconnect();
|
||||
}, 60_000);
|
||||
|
||||
describe('localFederatedSourceIds — CLI-side scope computation', () => {
|
||||
test('non-explicit tier: resolved source first, then other federated, archived excluded', async () => {
|
||||
expect(await localFederatedSourceIds(engine, 'default', 'seed_default')).toEqual(['default', 'wiki']);
|
||||
});
|
||||
|
||||
test('non-federated resolved source still joins its own scope', async () => {
|
||||
expect(await localFederatedSourceIds(engine, 'private', 'brain_default')).toEqual(['private', 'default', 'wiki']);
|
||||
});
|
||||
|
||||
test('explicit tiers (--source / env / dotfile) never expand', async () => {
|
||||
expect(await localFederatedSourceIds(engine, 'default', 'flag')).toBeUndefined();
|
||||
expect(await localFederatedSourceIds(engine, 'default', 'env')).toBeUndefined();
|
||||
expect(await localFederatedSourceIds(engine, 'default', 'dotfile')).toBeUndefined();
|
||||
});
|
||||
|
||||
test('single federated source (the resolved one) keeps the scalar fast path', async () => {
|
||||
const solo = { executeRaw: async () => [{ id: 'default' }] } as any;
|
||||
expect(await localFederatedSourceIds(solo, 'default', 'seed_default')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('federatedSearchScope — trust + explicitness matrix', () => {
|
||||
test('trusted local + unqualified widens to the federated set', () => {
|
||||
const ctx = ctxOf({ localFederatedSourceIds: ['default', 'wiki'] });
|
||||
expect(federatedSearchScope(ctx)).toEqual({ sourceIds: ['default', 'wiki'] });
|
||||
});
|
||||
|
||||
test('remote caller NEVER widens (fail-closed), even if the field is set', () => {
|
||||
const ctx = ctxOf({ remote: true, localFederatedSourceIds: ['default', 'wiki'] });
|
||||
expect(federatedSearchScope(ctx)).toEqual({ sourceId: 'default' });
|
||||
});
|
||||
|
||||
test('per-call source_id wins over the federated set', () => {
|
||||
const ctx = ctxOf({ localFederatedSourceIds: ['default', 'wiki'] });
|
||||
expect(federatedSearchScope(ctx, 'wiki')).toEqual({ sourceId: 'wiki' });
|
||||
});
|
||||
|
||||
test('per-call __all__ keeps the whole-brain semantics for trusted local', () => {
|
||||
const ctx = ctxOf({ localFederatedSourceIds: ['default', 'wiki'] });
|
||||
expect(federatedSearchScope(ctx, '__all__')).toEqual({});
|
||||
});
|
||||
|
||||
test('a federated OAuth grant wins over the local set', () => {
|
||||
const ctx = ctxOf({
|
||||
localFederatedSourceIds: ['default', 'wiki'],
|
||||
auth: { allowedSources: ['a', 'b'] } as OperationContext['auth'],
|
||||
});
|
||||
expect(federatedSearchScope(ctx)).toEqual({ sourceIds: ['a', 'b'] });
|
||||
});
|
||||
|
||||
test('no local federated set → unchanged scalar scope', () => {
|
||||
expect(federatedSearchScope(ctxOf())).toEqual({ sourceId: 'default' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('search op — unqualified local search spans federated sources', () => {
|
||||
test('federated source results appear; non-federated + archived stay invisible', async () => {
|
||||
const ctx = ctxOf({
|
||||
localFederatedSourceIds: await localFederatedSourceIds(engine, 'default', 'seed_default'),
|
||||
});
|
||||
const results = (await search.handler(ctx, { query: 'zebra telescope' })) as Array<{ slug: string }>;
|
||||
const slugs = results.map((r) => r.slug);
|
||||
expect(slugs).toContain('notes/home');
|
||||
expect(slugs).toContain('wiki/topic'); // pre-#2561 this was missing
|
||||
expect(slugs).not.toContain('private/topic');
|
||||
expect(slugs).not.toContain('old/topic');
|
||||
});
|
||||
|
||||
test('explicit source resolution (no federated set on ctx) stays single-source', async () => {
|
||||
const results = (await search.handler(ctxOf(), { query: 'zebra telescope' })) as Array<{ slug: string }>;
|
||||
const slugs = results.map((r) => r.slug);
|
||||
expect(slugs).toEqual(['notes/home']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
// test/remediation-context-extraction-lag.test.ts
|
||||
//
|
||||
// Pins the v-next fix: the sync→extract remediation pipeline gates on REAL
|
||||
// extraction lag, not the legacy `health.stale_pages` proxy (which counted
|
||||
// "updated_at predates newest timeline entry" — meaningless after the v10
|
||||
// trigger drop). loadRecommendationContext now populates `extractionLagPages`
|
||||
// from `engine.countStalePagesForExtraction` — the SAME counter the
|
||||
// `gbrain extract --stale` walk and doctor's `links_extraction_lag` use — so a
|
||||
// recommendation can only fire when running extract will actually reduce it.
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { loadRecommendationContext } from '../src/core/remediation/context.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
describe('loadRecommendationContext — extractionLagPages wiring', () => {
|
||||
it('is 0 on an empty brain (nothing to extract)', async () => {
|
||||
const ctx = await loadRecommendationContext(engine);
|
||||
expect(ctx.extractionLagPages).toBe(0);
|
||||
});
|
||||
|
||||
it('reflects the real extraction-lag count once a page needs extraction', async () => {
|
||||
// A freshly-imported page has links_extracted_at = NULL, which the canonical
|
||||
// countStalePagesForExtraction predicate counts as stale-for-extraction.
|
||||
await engine.putPage('p0', {
|
||||
title: 'p0',
|
||||
type: 'note' as never,
|
||||
compiled_truth: 'body that is long enough to pass any minimum-length guards in the codebase',
|
||||
timeline: '',
|
||||
frontmatter: {},
|
||||
source_path: 'p0.md',
|
||||
});
|
||||
const ctx = await loadRecommendationContext(engine);
|
||||
expect(ctx.extractionLagPages).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('counts pages stamped before LINK_EXTRACTOR_VERSION_TS (version-bump arm)', async () => {
|
||||
// Backdate p0 so BOTH the NULL arm and the updated_at arm are quiet:
|
||||
// updated_at < links_extracted_at, but links_extracted_at predates the
|
||||
// extractor version stamp. doctor's links_extraction_lag and
|
||||
// `extract --stale` both count this page; the remediation gate must too.
|
||||
await engine.executeRaw(
|
||||
`UPDATE pages SET updated_at = '2020-01-01T00:00:00Z'::timestamptz,
|
||||
links_extracted_at = '2020-01-02T00:00:00Z'::timestamptz
|
||||
WHERE slug = 'p0'`,
|
||||
[],
|
||||
);
|
||||
const ctx = await loadRecommendationContext(engine);
|
||||
expect(ctx.extractionLagPages).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
// test/remediation-run-d7-refresh.serial.test.ts
|
||||
//
|
||||
// Pins the D7-recheck half of the extraction-lag gate fix: runRemediation
|
||||
// loads RecommendationContext ONCE before the step loop, and the per-step
|
||||
// recheck (D7) must REFRESH ctx.extractionLagPages alongside getHealth.
|
||||
// Without the refresh, a completed sync/extract step keeps re-firing off
|
||||
// the frozen initial count — the plan never converges and the loop burns
|
||||
// steps until maxJobs.
|
||||
//
|
||||
// SERIAL (R2): uses top-level mock.module for the minion queue +
|
||||
// wait-for-completion so no real worker is needed — mocks leak across
|
||||
// files in a shard process, so this file must run in its own process.
|
||||
|
||||
import { describe, expect, mock, test } from 'bun:test';
|
||||
|
||||
// The fake brain: sync.repo clears the extraction lag when it "runs"
|
||||
// (today's sync materializes link/timeline edges; extract.all is the
|
||||
// explicit re-materializer). The frozen-ctx bug makes runRemediation
|
||||
// ignore that and resubmit sync.repo on every D7 recheck.
|
||||
let extractionLag = 25;
|
||||
const submittedJobs: string[] = [];
|
||||
|
||||
mock.module('../src/core/minions/queue.ts', () => ({
|
||||
MinionQueue: class {
|
||||
constructor(_engine: unknown) {}
|
||||
async add(job: string): Promise<{ id: number }> {
|
||||
submittedJobs.push(job);
|
||||
if (job === 'sync' || job === 'extract') extractionLag = 0;
|
||||
return { id: submittedJobs.length };
|
||||
}
|
||||
},
|
||||
}));
|
||||
|
||||
mock.module('../src/core/minions/wait-for-completion.ts', () => ({
|
||||
waitForCompletion: async () => ({ status: 'completed' }),
|
||||
}));
|
||||
|
||||
const health = () => ({
|
||||
page_count: 100,
|
||||
embed_coverage: 1.0,
|
||||
stale_pages: 0, // legacy proxy stays 0 — the real counter drives the gate
|
||||
orphan_pages: 0,
|
||||
missing_embeddings: 0,
|
||||
brain_score: 70,
|
||||
dead_links: 0,
|
||||
link_coverage: 1.0,
|
||||
timeline_coverage: 1.0,
|
||||
most_connected: [],
|
||||
embed_coverage_score: 35,
|
||||
link_density_score: 25,
|
||||
timeline_coverage_score: 15,
|
||||
no_orphans_score: 15,
|
||||
no_dead_links_score: 10,
|
||||
});
|
||||
|
||||
const fakeEngine = {
|
||||
kind: 'pglite' as const,
|
||||
getHealth: async () => health(),
|
||||
getConfig: async (key: string) =>
|
||||
key === 'sync.repo_path' ? '/tmp/brain-example' : null,
|
||||
countStalePagesForExtraction: async () => extractionLag,
|
||||
};
|
||||
|
||||
describe('runRemediation D7 recheck — extraction-lag gate refresh', () => {
|
||||
test('a completed materializer step clears the gate; the pipeline is not resubmitted', async () => {
|
||||
const { runRemediation } = await import('../src/core/remediation/run.ts');
|
||||
const result = await runRemediation(
|
||||
// Only the methods the orchestrator touches are needed.
|
||||
fakeEngine as never,
|
||||
{ targetScore: 0, maxJobs: 6 },
|
||||
);
|
||||
// Frozen-ctx bug: extractionLagPages stays 25 forever, so every D7
|
||||
// recheck re-introduces the sync/extract pipeline and the loop burns
|
||||
// all 6 maxJobs. With the refresh, the plan converges after the first
|
||||
// completed step: no step id is ever submitted twice.
|
||||
const ids = result.submitted.map((s) => s.id);
|
||||
expect(new Set(ids).size).toBe(ids.length);
|
||||
expect(submittedJobs.length).toBeLessThan(3);
|
||||
expect(extractionLag).toBe(0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user