Compare commits

..
Author SHA1 Message Date
Garry TanandClaude Fable 5 921048827a fix(remediation): pass LINK_EXTRACTOR_VERSION_TS to the extraction-lag gate
countExtractionLag omitted versionTs, so its predicate diverged from the
counter it claims to share with doctor's links_extraction_lag check and
the extract --stale walk: pages stamped before an extractor version bump
(links_extracted_at < LINK_EXTRACTOR_VERSION_TS) lagged for doctor and
extract but never tripped the sync.repo/extract.all gate. Pass the stamp;
pin the version-bump arm with a backdated-page test (fails without it).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 10:53:55 -07:00
0dff84b16a fix(remediation): gate sync/extract recs on real extraction lag, refresh at D7 recheck
Takeover of #2363. The sync.repo/extract.all recommendations gated on
health.stale_pages — a proxy (updated_at predates newest timeline entry)
that stopped meaning anything after migration v10 dropped the trigger
behind it. Gate them on the honest counter instead:
engine.countStalePagesForExtraction, the same staleness `gbrain extract
--stale` and doctor's links_extraction_lag use.

On top of the original PR, two repairs:

- runRemediation loads RecommendationContext once, but the D7 per-step
  recheck reused the frozen extractionLagPages — a completed sync/extract
  step could never clear the gate, so the pipeline re-fired every recheck
  until maxJobs. The recheck now refreshes the gate alongside getHealth
  via the shared countExtractionLag() helper (extracted into
  remediation/context.ts). Pinned by
  test/remediation-run-d7-refresh.serial.test.ts (serial: mock.module).
- autopilot builds its own RecommendationContext by hand; without wiring,
  it would silently never fire sync.repo/extract.all again. It now
  populates extractionLagPages from the same helper.

Co-authored-by: DarkNightForge <DarkNightForge@users.noreply.github.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 15:06:34 -07:00
19 changed files with 285 additions and 348 deletions
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1 -1
View File
@@ -7,7 +7,7 @@
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
<script type="module" crossorigin src="/admin/assets/index-CzLjRij_.js"></script>
<script type="module" crossorigin src="/admin/assets/index-CoGEje3-.js"></script>
<link rel="stylesheet" crossorigin href="/admin/assets/index-GxkWX7v3.css">
</head>
<body>
-1
View File
@@ -39,7 +39,6 @@ export const api = {
stats: () => apiFetch('/admin/api/stats'),
health: () => apiFetch('/admin/api/health-indicators'),
agents: () => apiFetch('/admin/api/agents'),
sources: () => apiFetch('/admin/api/sources'),
requests: (page = 1, qs = '') => apiFetch(`/admin/api/requests?page=${page}${qs}`),
apiKeys: () => apiFetch('/admin/api/api-keys'),
createApiKey: (name: string) => apiFetch('/admin/api/api-keys', { method: 'POST', body: JSON.stringify({ name }) }),
+1 -87
View File
@@ -23,16 +23,9 @@ interface Agent {
total_requests: number;
requests_today: number;
token_ttl: number | null;
source_id: string | null;
federated_read: string[] | null;
status: 'active' | 'revoked';
}
interface SourceOption {
id: string;
name: string | null;
}
interface ApiKey {
id: string;
name: string;
@@ -95,7 +88,6 @@ export function AgentsPage() {
<th>Name</th>
<th>Type</th>
<th>Scopes</th>
<th>Source</th>
<th>Status</th>
<th>Requests</th>
<th>Last Used</th>
@@ -116,9 +108,6 @@ export function AgentsPage() {
<span key={s} className={`badge badge-${s}`} style={{ marginRight: 4 }}>{s}</span>
))}
</td>
<td className="mono" style={{ fontSize: 12, color: 'var(--text-secondary)' }}>
{a.auth_type === 'oauth' ? (a.source_id || 'default') : 'all'}
</td>
<td>
<span className={`badge ${a.status === 'active' ? 'badge-success' : 'badge-danger'}`}>{a.status}</span>
</td>
@@ -268,29 +257,9 @@ function RegisterModal({ onClose, onRegistered }: {
Object.fromEntries(ALLOWED_SCOPES_LIST.map(s => [s, s === 'read'])) as Record<Scope, boolean>,
);
const [ttl, setTtl] = useState('86400'); // 24h default
const [redirectUris, setRedirectUris] = useState('');
const [sources, setSources] = useState<SourceOption[]>([{ id: 'default', name: 'default' }]);
const [sourceId, setSourceId] = useState('default');
const [readSources, setReadSources] = useState<Record<string, boolean>>({ default: true });
const [loading, setLoading] = useState(false);
const [error, setError] = useState('');
useEffect(() => {
api.sources()
.then((rows: SourceOption[]) => {
const nextSources = rows.length > 0 ? rows : [{ id: 'default', name: 'default' }];
const nextSourceId = nextSources.some(s => s.id === sourceId) ? sourceId : nextSources[0].id;
setSources(nextSources);
setSourceId(nextSourceId);
setReadSources(prev => {
const next = Object.fromEntries(nextSources.map(s => [s.id, Boolean(prev[s.id])])) as Record<string, boolean>;
next[nextSourceId] = true;
return next;
});
})
.catch(() => {});
}, []);
const ttlOptions = [
{ label: '1 hour', value: '3600' },
{ label: '24 hours', value: '86400' },
@@ -308,24 +277,11 @@ function RegisterModal({ onClose, onRegistered }: {
try {
// Use the CLI registration endpoint (POST to admin API)
const selectedScopes = Object.entries(scopes).filter(([, v]) => v).map(([k]) => k).join(' ');
const federatedRead = sources.map(s => s.id).filter(id => readSources[id]);
if (federatedRead.length === 0) { setError('Select at least one read source'); setLoading(false); return; }
// #1036: redirect URIs, one per line. Mirrors the CLI convention
// (src/commands/auth.ts): presence of redirect URIs implies
// authorization_code + refresh_token grants unless the user had none.
const uris = redirectUris.split('\n').map(u => u.trim()).filter(Boolean);
const res = await fetch('/admin/api/register-client', {
method: 'POST',
credentials: 'same-origin',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
name: name.trim(),
scopes: selectedScopes,
tokenTtl: ttl === '0' ? 315360000 : Number(ttl),
source_id: sourceId,
federated_read: federatedRead,
...(uris.length > 0 ? { redirectUris: uris, grantTypes: ['authorization_code', 'refresh_token'] } : {}),
}),
body: JSON.stringify({ name: name.trim(), scopes: selectedScopes, tokenTtl: ttl === '0' ? 315360000 : Number(ttl) }),
});
if (!res.ok) throw new Error('Registration failed');
const data = await res.json();
@@ -356,42 +312,6 @@ function RegisterModal({ onClose, onRegistered }: {
))}
</div>
</div>
<div style={{ marginBottom: 16 }}>
<label>Write Source</label>
<select value={sourceId} onChange={e => {
const id = e.target.value;
setSourceId(id);
setReadSources(p => ({ ...p, [id]: true }));
}}
style={{ width: '100%', background: 'var(--bg-secondary)', color: 'var(--text-primary)', border: '1px solid var(--border)', borderRadius: 6, padding: '6px 10px', fontSize: 14 }}>
{sources.map(s => <option key={s.id} value={s.id}>{s.name && s.name !== s.id ? `${s.id} (${s.name})` : s.id}</option>)}
</select>
</div>
<div style={{ marginBottom: 16 }}>
<label>Read Sources</label>
<div className="checkbox-group">
{sources.map(s => (
<label key={s.id} className="checkbox-label">
<input
type="checkbox"
checked={Boolean(readSources[s.id])}
onChange={e => setReadSources(p => ({ ...p, [s.id]: e.target.checked }))}
/>
{s.name && s.name !== s.id ? `${s.id} (${s.name})` : s.id}
</label>
))}
</div>
</div>
<div style={{ marginBottom: 16 }}>
<label>Redirect URIs <span style={{ color: 'var(--text-secondary)', fontWeight: 400 }}>(optional, one per line enables authorization_code flow)</span></label>
<textarea
placeholder={'https://example.com/callback'}
value={redirectUris}
onChange={e => setRedirectUris(e.target.value)}
rows={2}
style={{ width: '100%', background: 'var(--bg-secondary)', color: 'var(--text-primary)', border: '1px solid var(--border)', borderRadius: 6, padding: '6px 10px', fontSize: 13, fontFamily: 'inherit', resize: 'vertical' }}
/>
</div>
<div style={{ marginBottom: 20 }}>
<label>Token Lifetime</label>
<select value={ttl} onChange={e => setTtl(e.target.value)}
@@ -608,8 +528,6 @@ function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: ()
client_name: agentName,
auth_type: agent.auth_type,
scope: agent.scope,
source_id: agent.source_id,
federated_read: agent.federated_read,
}, null, 2),
};
@@ -629,10 +547,6 @@ function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: ()
<span>{(agent.scope || '').split(' ').filter(Boolean).map(s => (
<span key={s} className={`badge badge-${s}`} style={{ marginRight: 4 }}>{s}</span>
))}</span>
<span style={{ color: 'var(--text-secondary)' }}>Write Source</span>
<span className="mono">{agent.source_id || (isOAuth ? 'default' : 'all')}</span>
<span style={{ color: 'var(--text-secondary)' }}>Read Sources</span>
<span className="mono">{isOAuth ? (agent.federated_read && agent.federated_read.length > 0 ? agent.federated_read.join(', ') : (agent.source_id || 'default')) : 'all'}</span>
<span style={{ color: 'var(--text-secondary)' }}>Registered</span>
<span>{new Date(agent.created_at).toLocaleDateString()}</span>
<span style={{ color: 'var(--text-secondary)' }}>Token TTL</span>
+3 -3
View File
@@ -1,13 +1,13 @@
// AUTO-GENERATED — do not edit by hand.
// Run `bun run scripts/build-admin-embedded.ts` to regenerate.
// Source: admin/dist/ at 2026-07-21.
// Source: admin/dist/ at 2026-05-27.
//
// Bun resolves the file: imports to a path that works at runtime even
// inside a compiled binary (`bun build --compile`). The manifest maps
// the request path the express handler sees to (resolved-path, mime).
// @ts-ignore — type: 'file' is Bun ESM, not in lib.d.ts
import A_0_assets_index_CzLjRij__js from '../admin/dist/assets/index-CzLjRij_.js' with { type: 'file' };
import A_0_assets_index_CoGEje3__js from '../admin/dist/assets/index-CoGEje3-.js' with { type: 'file' };
// @ts-ignore — type: 'file' is Bun ESM, not in lib.d.ts
import A_1_assets_index_GxkWX7v3_css from '../admin/dist/assets/index-GxkWX7v3.css' with { type: 'file' };
// @ts-ignore — type: 'file' is Bun ESM, not in lib.d.ts
@@ -19,7 +19,7 @@ export interface AdminAsset {
}
export const ADMIN_ASSETS: Record<string, AdminAsset> = {
"/admin/assets/index-CzLjRij_.js": { path: A_0_assets_index_CzLjRij__js as unknown as string, mime: "application/javascript; charset=utf-8" },
"/admin/assets/index-CoGEje3-.js": { path: A_0_assets_index_CoGEje3__js as unknown as string, mime: "application/javascript; charset=utf-8" },
"/admin/assets/index-GxkWX7v3.css": { path: A_1_assets_index_GxkWX7v3_css as unknown as string, mime: "text/css; charset=utf-8" },
"/admin/index.html": { path: A_2_index_html as unknown as string, mime: "text/html; charset=utf-8" },
};
+4
View File
@@ -686,6 +686,7 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
try {
const { MinionQueue } = await import('../core/minions/queue.ts');
const { computeRecommendations, embeddingProviderConfigured, HOSTED_EMBED_KEY_CONFIG } = await import('../core/brain-score-recommendations.ts');
const { countExtractionLag } = await import('../core/remediation/context.ts');
const queue = new MinionQueue(engine);
const slotMs = Math.floor(Date.now() / (baseInterval * 1000)) * baseInterval * 1000;
const slot = new Date(slotMs).toISOString();
@@ -877,6 +878,9 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
return !!(process.env[envVar] || (cfgField ? embedKeyCfg[cfgField] : undefined));
}),
hasChatApiKey: !!(process.env.ANTHROPIC_API_KEY || await engine.getConfig('anthropic_api_key')),
// Real extraction-lag gate for sync.repo/extract.all — same counter
// loadRecommendationContext uses (replaces the health.stale_pages proxy).
extractionLagPages: await countExtractionLag(engine),
};
// v0.41.18.0 (A5 + A19 + A22, T15): consult onboard recommendations
// ALONGSIDE doctor's brain-score recommendations. Onboard's 4 new
+1 -32
View File
@@ -39,7 +39,6 @@ import * as db from '../core/db.ts';
import { sqlQueryForEngine, executeRawJsonb } from '../core/sql-query.ts';
import { MinionQueue } from '../core/minions/queue.ts';
import { isRetryableError } from '../core/retry-matcher.ts';
import { normalizeAdminClientSourceScope } from '../core/admin-source-scope.ts';
import {
computeContentHash,
validateIngestionEvent,
@@ -1087,7 +1086,6 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
const oauthClients = await sql`
SELECT c.client_id as id, c.client_name as name, 'oauth' as auth_type,
c.grant_types, c.scope, c.created_at, c.token_ttl,
c.source_id, c.federated_read,
CASE WHEN c.deleted_at IS NOT NULL THEN 'revoked' ELSE 'active' END as status,
(SELECT max(created_at) FROM mcp_request_log WHERE token_name = c.client_id) as last_used_at,
(SELECT count(*)::int FROM mcp_request_log WHERE token_name = c.client_id) as total_requests,
@@ -1097,7 +1095,6 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
const legacyKeys = await sql`
SELECT a.id, a.name, 'api_key' as auth_type,
'{"bearer"}' as grant_types, 'read write admin' as scope, a.created_at, null as token_ttl,
null as source_id, null as federated_read,
CASE WHEN a.revoked_at IS NOT NULL THEN 'revoked' ELSE 'active' END as status,
a.last_used_at,
(SELECT count(*)::int FROM mcp_request_log WHERE token_name = a.name) as total_requests,
@@ -1110,17 +1107,6 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
}
});
app.get('/admin/api/sources', requireAdmin, async (_req: Request, res: Response) => {
try {
const sources = await engine.listAllSources({ includeArchived: false });
const options = sources.map(s => ({ id: s.id, name: s.name }));
if (!options.some(s => s.id === 'default')) options.unshift({ id: 'default', name: 'default' });
res.json(options);
} catch (e) {
res.status(503).json({ error: 'service_unavailable' });
}
});
// v0.38 Slice 4 — per-OAuth-client agent spend viewer. Pre-computes today's
// spend (committed + pending reservations) per client so the Agents tab
// can render a "$X / $Y today" cell. Read-side endpoint only — no mutation.
@@ -1454,8 +1440,6 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
// missing, empty) and rejects the rest with a structured 400.
const { name, tokenTtl, grantTypes, redirectUris, tokenEndpointAuthMethod } = req.body;
const rawScopes = (req.body as Record<string, unknown>).scopes ?? (req.body as Record<string, unknown>).scope;
const rawSourceId = (req.body as Record<string, unknown>).source_id ?? (req.body as Record<string, unknown>).sourceId;
const rawFederatedRead = (req.body as Record<string, unknown>).federated_read ?? (req.body as Record<string, unknown>).federatedRead;
if (!name) { res.status(400).json({ error: 'Name required' }); return; }
let scopeString: string;
try {
@@ -1486,23 +1470,8 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
});
return;
}
let sourceScope: ReturnType<typeof normalizeAdminClientSourceScope>;
try {
const sources = await engine.listAllSources({ includeArchived: false });
sourceScope = normalizeAdminClientSourceScope(
rawSourceId,
rawFederatedRead,
sources.map(s => s.id),
);
} catch (e) {
res.status(400).json({
error: 'invalid_source_scope',
message: e instanceof Error ? e.message : String(e),
});
return;
}
const result = await oauthProvider.registerClientManual(
name, grants, scopeString, uris, sourceScope.sourceId, sourceScope.federatedRead, validatedAuthMethod,
name, grants, scopeString, uris, 'default', undefined, validatedAuthMethod,
);
// Set per-client TTL if specified
if (tokenTtl && Number(tokenTtl) > 0) {
-51
View File
@@ -1,51 +0,0 @@
import { isValidSourceId } from './source-id.ts';
export interface AdminClientSourceScope {
sourceId: string;
federatedRead: string[];
}
function parseFederatedReadInput(raw: unknown, sourceId: string): string[] {
if (raw === undefined || raw === null) return [sourceId];
if (typeof raw === 'string') {
const values = raw.split(',').map(s => s.trim()).filter(Boolean);
return values.length > 0 ? values : [sourceId];
}
if (Array.isArray(raw)) {
if (!raw.every(v => typeof v === 'string')) {
throw new Error('federated_read must be an array of source ids');
}
const values = raw.map(v => v.trim()).filter(Boolean);
return values.length > 0 ? values : [sourceId];
}
throw new Error('federated_read must be a string or array');
}
export function normalizeAdminClientSourceScope(
rawSourceId: unknown,
rawFederatedRead: unknown,
availableSourceIds: string[],
): AdminClientSourceScope {
const knownSources = new Set(availableSourceIds);
const fallbackSourceId = knownSources.has('default') ? 'default' : (availableSourceIds[0] ?? 'default');
const sourceId = rawSourceId === undefined || rawSourceId === null || rawSourceId === ''
? fallbackSourceId
: rawSourceId;
if (!isValidSourceId(sourceId)) {
throw new Error(`Invalid source_id: ${JSON.stringify(sourceId)}`);
}
if (knownSources.size > 0 && !knownSources.has(sourceId)) {
throw new Error(`Unknown source_id: ${sourceId}`);
}
const federatedRead = Array.from(new Set(parseFederatedReadInput(rawFederatedRead, sourceId)));
for (const id of federatedRead) {
if (!isValidSourceId(id)) {
throw new Error(`Invalid federated_read source_id: ${JSON.stringify(id)}`);
}
if (knownSources.size > 0 && !knownSources.has(id)) {
throw new Error(`Unknown federated_read source_id: ${id}`);
}
}
return { sourceId, federatedRead };
}
+26 -8
View File
@@ -146,6 +146,16 @@ export interface RecommendationContext {
chatModel?: string;
/** Whether the chat provider has a usable API key. */
hasChatApiKey?: boolean;
/**
* Count of pages needing link/timeline extraction the SAME staleness the
* `gbrain extract --stale` walk and doctor's `links_extraction_lag` check use
* (`engine.countStalePagesForExtraction`). Gates the syncextract pipeline
* (sync.repo / extract.all). Replaces the old `health.stale_pages` gate, which
* counted "pages whose updated_at predates their newest timeline entry" a
* proxy that broke when the updated_at-on-timeline-insert trigger was dropped
* (migration v10) and never reflected real extraction work.
*/
extractionLagPages?: number;
}
/** Triage result for one check. */
@@ -192,20 +202,28 @@ export function computeRecommendations(
const source = ctx.sourceId ?? 'default';
// ---------------------------------------------------------------------
// sync.repo — fires when sync hasn't run recently OR pages are stale
// sync.repo + extract.all — the materialization pipeline, gated on the REAL
// extraction lag (pages whose link/timeline edges are stale), NOT on the
// legacy `health.stale_pages` proxy. `extractionLagPages` comes from the same
// counter the `extract --stale` walk + doctor's `links_extraction_lag` use, so
// the recommendation can only fire when running extract will actually reduce
// it (and clear the rec). See RecommendationContext.extractionLagPages.
// sync.repo is the prerequisite: re-sync so pages are current before extract
// materializes their edges.
// ---------------------------------------------------------------------
if (ctx.repoPath && health.stale_pages > 0) {
const extractionLag = ctx.extractionLagPages ?? 0;
if (ctx.repoPath && extractionLag > 0) {
const params = { repoPath: ctx.repoPath, sourceId: ctx.sourceId, noEmbed: true };
out.push({
id: 'sync.repo',
job: 'sync',
params,
idempotency_key: idemKey(source, 'sync', params),
severity: health.stale_pages > 50 ? 'high' : 'medium',
est_seconds: Math.min(600, 30 + health.stale_pages * 0.5),
severity: extractionLag > 50 ? 'high' : 'medium',
est_seconds: Math.min(600, 30 + extractionLag * 0.5),
est_usd_cost: 0, // sync is fs+DB only
depends_on: [],
rationale: `${health.stale_pages} stale page${health.stale_pages === 1 ? '' : 's'} on disk`,
rationale: `Sync before extracting ${extractionLag} page${extractionLag === 1 ? '' : 's'} with stale link/timeline edges`,
status: 'remediable',
});
}
@@ -237,7 +255,7 @@ export function computeRecommendations(
est_seconds: Math.min(3600, 5 + health.missing_embeddings * 0.05),
est_usd_cost,
// sync should run first so embed sees fresh pages.
depends_on: ctx.repoPath && health.stale_pages > 0 ? ['sync.repo'] : [],
depends_on: ctx.repoPath && extractionLag > 0 ? ['sync.repo'] : [],
rationale: `${health.missing_embeddings} chunk${health.missing_embeddings === 1 ? '' : 's'} invisible to vector search`,
status: 'remediable',
});
@@ -267,7 +285,7 @@ export function computeRecommendations(
// Triggered when sync.repo fires (because sync was set to noEmbed:true,
// and noExtract:true after T5 lands → extract job is the materializer).
// ---------------------------------------------------------------------
if (ctx.repoPath && health.stale_pages > 0) {
if (ctx.repoPath && extractionLag > 0) {
const params = { mode: 'all', dir: ctx.repoPath };
out.push({
id: 'extract.all',
@@ -278,7 +296,7 @@ export function computeRecommendations(
est_seconds: Math.min(600, 30 + health.page_count * 0.01),
est_usd_cost: 0,
depends_on: ['sync.repo'],
rationale: 'Materialize link + timeline edges from fresh pages',
rationale: `Materialize link + timeline edges for ${extractionLag} page${extractionLag === 1 ? '' : 's'} with stale extraction`,
status: 'remediable',
});
}
+25
View File
@@ -8,6 +8,7 @@
import type { BrainEngine } from '../engine.ts';
import type { RecommendationContext } from '../brain-score-recommendations.ts';
import { LINK_EXTRACTOR_VERSION_TS } from '../link-extraction.ts';
// Re-export so consumers can `import { RecommendationContext } from '../remediation'`
// — the canonical RecommendationContext type still lives in
@@ -68,5 +69,29 @@ export async function loadRecommendationContext(
embeddingDimensions,
embeddingProviderConfigured: embeddingConfigured,
hasChatApiKey: !!(process.env.ANTHROPIC_API_KEY || fileCfg?.anthropic_api_key),
extractionLagPages: await countExtractionLag(engine),
};
}
/**
* Real extraction-lag count the SAME staleness `gbrain extract --stale`
* processes (engine.countStalePagesForExtraction with
* versionTs=LINK_EXTRACTOR_VERSION_TS, matching doctor's links_extraction_lag
* check without versionTs, pages stamped before an extractor version bump
* would lag for doctor/extract but never trip this gate). Drives the
* syncextract recommendation pipeline; replaces the legacy
* `health.stale_pages` proxy that no longer reflected real extraction work
* after the v10 trigger drop.
*
* Shared by loadRecommendationContext AND the D7 per-step recheck in
* runRemediation the recheck MUST refresh this gate alongside getHealth,
* or a completed extract step keeps re-firing off the frozen initial count.
*/
export async function countExtractionLag(engine: BrainEngine): Promise<number> {
try {
return await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS });
} catch {
/* counter unavailable (very old brain / mid-migration) — treat as 0 */
return 0;
}
}
+7 -2
View File
@@ -16,7 +16,7 @@ import {
computeRecommendations,
} from '../brain-score-recommendations.ts';
import type { RemediationStep } from '../remediation-step.ts';
import { loadRecommendationContext } from './context.ts';
import { countExtractionLag, loadRecommendationContext } from './context.ts';
import { computeRemediationPlan } from './plan.ts';
import type {
RemediationHooks,
@@ -65,7 +65,7 @@ export async function runRemediation(
clearRemediationCheckpoint,
} = await import('../remediation-checkpoint.ts');
const ctx = await loadRecommendationContext(engine);
let ctx = await loadRecommendationContext(engine);
// Pre-flight ceiling check via the shared plan computation.
const initialPlan = await computeRemediationPlan(engine, { targetScore });
@@ -305,6 +305,11 @@ export async function runRemediation(
// steps with bumped retry suffix (D1).
if (recs.length === 0 || stepCount >= maxJobs) break;
const freshHealth = await engine.getHealth();
// Refresh the extraction-lag gate alongside health: ctx was loaded once
// before the loop, and a completed sync/extract step is exactly what
// drives the count down. Reusing the frozen initial count would re-fire
// sync.repo/extract.all every recheck until maxJobs.
ctx = { ...ctx, extractionLagPages: await countExtractionLag(engine) };
recs = computeRecommendations(freshHealth, ctx).filter((r) => r.status === 'remediable');
}
};
+9
View File
@@ -1423,6 +1423,15 @@ export interface BrainStats {
export interface BrainHealth {
page_count: number;
embed_coverage: number;
/**
* LEGACY proxy: count of pages whose `updated_at` predates their newest
* timeline entry. This bumped meaningfully only while a trigger updated
* `pages.updated_at` on timeline insert; that trigger was dropped in
* migration v10, so the metric no longer reflects real "needs work" state.
* NO LONGER gates remediations the syncextract pipeline now gates on
* `RecommendationContext.extractionLagPages` (the real extraction-lag from
* `countStalePagesForExtraction`). Retained for the CLI health line + back-compat.
*/
stale_pages: number;
/**
* Islanded pages zero inbound AND zero outbound links. A hub page
-44
View File
@@ -1,44 +0,0 @@
import { describe, expect, test } from 'bun:test';
import { normalizeAdminClientSourceScope } from '../src/core/admin-source-scope.ts';
describe('admin register-client source scope normalization', () => {
const sources = ['default', 'team-a', 'team-b'];
test('defaults write and read scope to default source', () => {
expect(normalizeAdminClientSourceScope(undefined, undefined, sources)).toEqual({
sourceId: 'default',
federatedRead: ['default'],
});
});
test('accepts snake-case body fields with multiple read sources', () => {
expect(normalizeAdminClientSourceScope('team-a', ['team-a', 'team-b'], sources)).toEqual({
sourceId: 'team-a',
federatedRead: ['team-a', 'team-b'],
});
});
test('accepts comma-separated federated_read for API callers', () => {
expect(normalizeAdminClientSourceScope('team-a', 'team-b, default,team-b', sources)).toEqual({
sourceId: 'team-a',
federatedRead: ['team-b', 'default'],
});
});
test('falls back to selected write source when federated_read is empty', () => {
expect(normalizeAdminClientSourceScope('team-b', [], sources)).toEqual({
sourceId: 'team-b',
federatedRead: ['team-b'],
});
});
test('rejects malformed source ids before registration', () => {
expect(() => normalizeAdminClientSourceScope('../secret', undefined, sources)).toThrow(/Invalid source_id/);
expect(() => normalizeAdminClientSourceScope('team-a', ['snake_case'], sources)).toThrow(/Invalid federated_read/);
});
test('rejects valid-looking but unregistered source ids', () => {
expect(() => normalizeAdminClientSourceScope('ghost', undefined, sources)).toThrow(/Unknown source_id/);
expect(() => normalizeAdminClientSourceScope('team-a', ['ghost'], sources)).toThrow(/Unknown federated_read/);
});
});
+9 -12
View File
@@ -119,13 +119,12 @@ describe('computeRecommendations', () => {
expect(recs.find((r) => r.id === 'embed.stale')).toBeUndefined();
});
test('stale pages + dead links produce sync + backlinks + extract', () => {
test('extraction lag + dead links produce sync + backlinks + extract', () => {
const health = makeHealth({
stale_pages: 25,
dead_links: 8,
brain_score: 70,
});
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true });
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true, extractionLagPages: 25 });
const ids = recs.map((r) => r.id);
expect(ids).toContain('sync.repo');
expect(ids).toContain('backlinks.fix');
@@ -133,18 +132,17 @@ describe('computeRecommendations', () => {
});
test('extract.all depends on sync.repo (D14: stable ids)', () => {
const health = makeHealth({ stale_pages: 10 });
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true });
const health = makeHealth();
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true, extractionLagPages: 10 });
const extract = recs.find((r) => r.id === 'extract.all');
expect(extract?.depends_on).toContain('sync.repo');
});
test('embed.stale depends on sync.repo when sync also needed', () => {
test('embed.stale depends on sync.repo when extraction also needed', () => {
const health = makeHealth({
stale_pages: 10,
missing_embeddings: 100,
});
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true });
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true, extractionLagPages: 10 });
const embed = recs.find((r) => r.id === 'embed.stale');
expect(embed?.depends_on).toContain('sync.repo');
});
@@ -159,9 +157,9 @@ describe('computeRecommendations', () => {
test('severity ordering: critical before high before medium', () => {
const health = makeHealth({
missing_embeddings: 100, // critical
stale_pages: 80, // high
});
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true });
// extractionLagPages > 50 → sync.repo fires at 'high' severity.
const recs = computeRecommendations(health, { repoPath: '/brain', embeddingProviderConfigured: true, extractionLagPages: 80 });
const critIdx = recs.findIndex((r) => r.severity === 'critical');
const highIdx = recs.findIndex((r) => r.severity === 'high');
expect(critIdx).toBeLessThan(highIdx);
@@ -170,11 +168,10 @@ describe('computeRecommendations', () => {
// D6 #5 — THE critical regression test for the agent contract.
test('D6 #5: determinism — same input twice produces identical output', () => {
const health = makeHealth({
stale_pages: 10,
missing_embeddings: 50,
dead_links: 3,
});
const ctx = { repoPath: '/brain', embeddingProviderConfigured: true, sourceId: 'default' };
const ctx = { repoPath: '/brain', embeddingProviderConfigured: true, sourceId: 'default', extractionLagPages: 10 };
const run1 = computeRecommendations(health, ctx);
const run2 = computeRecommendations(health, ctx);
expect(JSON.stringify(run1)).toBe(JSON.stringify(run2));
-26
View File
@@ -78,8 +78,6 @@ describe('v0.36.1.x #1077 — admin register-client supports PKCE public clients
// (under either name) from req.body. Pin the fallback pattern so the
// PKCE-fix regression contract stays load-bearing.
expect(src).toMatch(/req\.body[^;]*scopes\s*\?\?\s*[^;]*scope\b/);
expect(src).toMatch(/req\.body[^;]*source_id\s*\?\?\s*[^;]*sourceId\b/);
expect(src).toMatch(/req\.body[^;]*federated_read\s*\?\?\s*[^;]*federatedRead\b/);
// v0.41.3 (T4 atomicity fix, codex F4): admin endpoint now validates
// tokenEndpointAuthMethod via the shared validator and passes it to
// registerClientManual as a positional arg. Pre-v0.41.3 the route did
@@ -89,7 +87,6 @@ describe('v0.36.1.x #1077 — admin register-client supports PKCE public clients
// UPDATE block (the regex deliberately asserts the post-insert UPDATE
// is GONE).
expect(src).toMatch(/validateTokenEndpointAuthMethod\(tokenEndpointAuthMethod\)/);
expect(src).toMatch(/normalizeAdminClientSourceScope\(/);
expect(src).toMatch(/registerClientManual\([^)]*validatedAuthMethod[^)]*\)/);
// Regression guard: post-insert UPDATE flipping client_secret_hash to
// NULL based on a runtime check is exactly the non-atomic pattern T4
@@ -270,26 +267,3 @@ describe('v0.42.43.0 #2095 — volunteer-events sink + cycle purge wiring (struc
expect(src).toMatch(/purged_volunteer_events_count/);
});
});
describe('#1490 + #1036 — admin Register Agent form exposes source scope + redirect URIs', () => {
test('Agents.tsx RegisterModal sends source_id + federated_read in the POST body (#1490)', () => {
const src = readFileSync('admin/src/pages/Agents.tsx', 'utf8');
expect(src).toMatch(/source_id:\s*sourceId/);
expect(src).toMatch(/federated_read:\s*federatedRead/);
// The source list comes from the requireAdmin-gated endpoint, not a hardcoded 'default'.
expect(src).toMatch(/api\.sources\(\)/);
});
test('serve-http.ts serves /admin/api/sources behind requireAdmin (#1490)', () => {
const src = readFileSync('src/commands/serve-http.ts', 'utf8');
expect(src).toMatch(/app\.get\('\/admin\/api\/sources',\s*requireAdmin/);
});
test('Agents.tsx RegisterModal sends redirectUris + authorization_code grants when URIs are entered (#1036)', () => {
const src = readFileSync('admin/src/pages/Agents.tsx', 'utf8');
// One-per-line textarea split into an array...
expect(src).toMatch(/redirectUris\.split\('\\n'\)/);
// ...included in the body with the CLI's grant-type inference convention.
expect(src).toMatch(/redirectUris:\s*uris,\s*grantTypes:\s*\['authorization_code',\s*'refresh_token'\]/);
});
});
-24
View File
@@ -125,30 +125,6 @@ describe('client registration', () => {
expect(client!.client_name).toBe('test-agent');
});
test('registerClientManual persists source_id and federated_read into AuthInfo', async () => {
await sql`
INSERT INTO sources (id, name) VALUES (${'team-a'}, ${'Team A'}), (${'team-b'}, ${'Team B'})
ON CONFLICT DO NOTHING
`;
const { clientId, clientSecret } = await provider.registerClientManual(
'source-scoped-test',
['client_credentials'],
'read write',
[],
'team-a',
['team-a', 'team-b'],
);
const tokens = await provider.exchangeClientCredentials(clientId, clientSecret!, 'read');
const authInfo = await provider.verifyAccessToken(tokens.access_token) as {
sourceId?: string;
allowedSources?: string[];
};
expect(authInfo.sourceId).toBe('team-a');
expect(authInfo.allowedSources).toEqual(['team-a', 'team-b']);
});
test('getClient returns undefined for unknown client', async () => {
const client = await provider.clientsStore.getClient('nonexistent');
expect(client).toBeUndefined();
@@ -0,0 +1,62 @@
// test/remediation-context-extraction-lag.test.ts
//
// Pins the v-next fix: the sync→extract remediation pipeline gates on REAL
// extraction lag, not the legacy `health.stale_pages` proxy (which counted
// "updated_at predates newest timeline entry" — meaningless after the v10
// trigger drop). loadRecommendationContext now populates `extractionLagPages`
// from `engine.countStalePagesForExtraction` — the SAME counter the
// `gbrain extract --stale` walk and doctor's `links_extraction_lag` use — so a
// recommendation can only fire when running extract will actually reduce it.
import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { loadRecommendationContext } from '../src/core/remediation/context.ts';
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
});
afterAll(async () => {
await engine.disconnect();
});
describe('loadRecommendationContext — extractionLagPages wiring', () => {
it('is 0 on an empty brain (nothing to extract)', async () => {
const ctx = await loadRecommendationContext(engine);
expect(ctx.extractionLagPages).toBe(0);
});
it('reflects the real extraction-lag count once a page needs extraction', async () => {
// A freshly-imported page has links_extracted_at = NULL, which the canonical
// countStalePagesForExtraction predicate counts as stale-for-extraction.
await engine.putPage('p0', {
title: 'p0',
type: 'note' as never,
compiled_truth: 'body that is long enough to pass any minimum-length guards in the codebase',
timeline: '',
frontmatter: {},
source_path: 'p0.md',
});
const ctx = await loadRecommendationContext(engine);
expect(ctx.extractionLagPages).toBeGreaterThan(0);
});
it('counts pages stamped before LINK_EXTRACTOR_VERSION_TS (version-bump arm)', async () => {
// Backdate p0 so BOTH the NULL arm and the updated_at arm are quiet:
// updated_at < links_extracted_at, but links_extracted_at predates the
// extractor version stamp. doctor's links_extraction_lag and
// `extract --stale` both count this page; the remediation gate must too.
await engine.executeRaw(
`UPDATE pages SET updated_at = '2020-01-01T00:00:00Z'::timestamptz,
links_extracted_at = '2020-01-02T00:00:00Z'::timestamptz
WHERE slug = 'p0'`,
[],
);
const ctx = await loadRecommendationContext(engine);
expect(ctx.extractionLagPages).toBeGreaterThan(0);
});
});
@@ -0,0 +1,81 @@
// test/remediation-run-d7-refresh.serial.test.ts
//
// Pins the D7-recheck half of the extraction-lag gate fix: runRemediation
// loads RecommendationContext ONCE before the step loop, and the per-step
// recheck (D7) must REFRESH ctx.extractionLagPages alongside getHealth.
// Without the refresh, a completed sync/extract step keeps re-firing off
// the frozen initial count — the plan never converges and the loop burns
// steps until maxJobs.
//
// SERIAL (R2): uses top-level mock.module for the minion queue +
// wait-for-completion so no real worker is needed — mocks leak across
// files in a shard process, so this file must run in its own process.
import { describe, expect, mock, test } from 'bun:test';
// The fake brain: sync.repo clears the extraction lag when it "runs"
// (today's sync materializes link/timeline edges; extract.all is the
// explicit re-materializer). The frozen-ctx bug makes runRemediation
// ignore that and resubmit sync.repo on every D7 recheck.
let extractionLag = 25;
const submittedJobs: string[] = [];
mock.module('../src/core/minions/queue.ts', () => ({
MinionQueue: class {
constructor(_engine: unknown) {}
async add(job: string): Promise<{ id: number }> {
submittedJobs.push(job);
if (job === 'sync' || job === 'extract') extractionLag = 0;
return { id: submittedJobs.length };
}
},
}));
mock.module('../src/core/minions/wait-for-completion.ts', () => ({
waitForCompletion: async () => ({ status: 'completed' }),
}));
const health = () => ({
page_count: 100,
embed_coverage: 1.0,
stale_pages: 0, // legacy proxy stays 0 — the real counter drives the gate
orphan_pages: 0,
missing_embeddings: 0,
brain_score: 70,
dead_links: 0,
link_coverage: 1.0,
timeline_coverage: 1.0,
most_connected: [],
embed_coverage_score: 35,
link_density_score: 25,
timeline_coverage_score: 15,
no_orphans_score: 15,
no_dead_links_score: 10,
});
const fakeEngine = {
kind: 'pglite' as const,
getHealth: async () => health(),
getConfig: async (key: string) =>
key === 'sync.repo_path' ? '/tmp/brain-example' : null,
countStalePagesForExtraction: async () => extractionLag,
};
describe('runRemediation D7 recheck — extraction-lag gate refresh', () => {
test('a completed materializer step clears the gate; the pipeline is not resubmitted', async () => {
const { runRemediation } = await import('../src/core/remediation/run.ts');
const result = await runRemediation(
// Only the methods the orchestrator touches are needed.
fakeEngine as never,
{ targetScore: 0, maxJobs: 6 },
);
// Frozen-ctx bug: extractionLagPages stays 25 forever, so every D7
// recheck re-introduces the sync/extract pipeline and the loop burns
// all 6 maxJobs. With the refresh, the plan converges after the first
// completed step: no step id is ever submitted twice.
const ids = result.submitted.map((s) => s.id);
expect(new Set(ids).size).toBe(ids.length);
expect(submittedJobs.length).toBeLessThan(3);
expect(extractionLag).toBe(0);
});
});