fix(extract): stop silently dropping links from non-whitelisted directories (#2576) (#3560)

Co-Authored-By: Time Attakc <89218912+time-attack@users.noreply.github.com>
This commit is contained in:
Garry Tan
2026-08-01 10:06:17 +08:00
committed by Sina Matian
co-authored by Time Attakc
parent 7cbb99ffef
commit f08a51d9de
7 changed files with 254 additions and 38 deletions
File diff suppressed because one or more lines are too long
+5 -4
View File
@@ -154,10 +154,11 @@ these are the densest source of real bugs in the whole backlog.
`aliases:`), first-H1-title, and basename fallback resolution (path-equality-only gives
~5.5% edge recall on real vaults). Master shipped global-basename (#1388); the alias/
title fallbacks are the still-novel part.
- **Schema-pack-aware link extraction** (#1547, @billy-armstrong) — **OPEN, high.** The
link extractor's `DIR_PATTERN` is a frozen 16-prefix const that ignores pack-declared
`path_prefixes`, so default-pack installs silently lose wikilinks to `person/`,
`writing/`, `wiki/*`. Resolve prefixes from the active pack.
- **Schema-pack-aware link extraction** (#1547, @billy-armstrong) — **RESOLVED via #2576.**
The extractor no longer gates on the frozen `DIR_PATTERN` whitelist: any dir-shaped
path produces a candidate and the persist paths' page-existence checks decide, so
pack-declared directories (`person/`, `writing/`, `wiki/*`, `ops/`) link without a
prefix registry.
- **DB-source extraction** (#1539, @afshaker) — **OPEN, high.** The cycle's extract phase
only walks the filesystem, so DB-resident pages (imported transcripts, remote-DB brains)
never get links/timeline and `brain_score` is capped. Thread `source:'db'`.
+17 -4
View File
@@ -1458,6 +1458,8 @@ async function extractLinksFromDB(
slugToSources.set(ref.slug, list);
}
let processed = 0, created = 0;
// #2576: skipped-candidate counter — see extractStaleFromDB's twin.
let skippedMissingTarget = 0;
// v0.42.7 (#1696): pages whose links we extracted this run — stamped after
// the loop so a manual `gbrain extract links|all --source db` clears the
// links_extraction_lag doctor signal. Non-dry-run only.
@@ -1514,7 +1516,7 @@ async function extractLinksFromDB(
// endpoint-validation + from/to source-id picking (null = skip: missing
// endpoint OR target only in a non-origin/non-default source).
const resolved = resolveCandidateSources(c, slug, source_id, allSlugs, slugToSources);
if (!resolved) continue;
if (!resolved) { skippedMissingTarget++; continue; }
const { fromSlug, fromSourceId, toSourceId } = resolved;
if (dryRunSeen) {
@@ -1571,6 +1573,9 @@ async function extractLinksFromDB(
if (!jsonMode) {
const label = dryRun ? '(dry run) would create' : 'created';
console.log(`Links: ${label} ${created} from ${processed} pages (db source)`);
if (skippedMissingTarget > 0) {
console.log(`Skipped ${skippedMissingTarget} candidate(s) whose target page doesn't exist (references to non-pages are never persisted).`);
}
if (includeFrontmatter && unresolved.length > 0) {
// Top-20 preview of unresolvable frontmatter names so the user can
// see where the graph has holes (codex tension 6.4).
@@ -1716,7 +1721,7 @@ export async function extractStaleFromDB(
sourceIdFilter?: string;
catchUp: boolean;
},
): Promise<{ linksCreated: number; timelineCreated: number; pagesProcessed: number; staleRemaining: number }> {
): Promise<{ linksCreated: number; timelineCreated: number; pagesProcessed: number; staleRemaining: number; skippedMissingTarget?: number }> {
const { dryRun, jsonMode, includeFrontmatter, sourceIdFilter, catchUp } = opts;
const versionTs = LINK_EXTRACTOR_VERSION_TS;
@@ -1768,6 +1773,10 @@ export async function extractStaleFromDB(
let afterPageId = 0;
let linksCreated = 0, timelineCreated = 0, pagesProcessed = 0;
let budgetHit = false;
// #2576: candidates whose endpoint pages don't exist are skipped, not
// persisted. Counted so a dropped reference is observable in the summary
// instead of vanishing silently (the failure mode that hid bug 2).
let skippedMissingTarget = 0;
for (;;) {
const rows = await engine.listStalePagesForExtraction({
@@ -1787,7 +1796,7 @@ export async function extractStaleFromDB(
);
for (const c of extracted.candidates) {
const r = resolveCandidateSources(c, page.slug, page.source_id, allSlugs, slugToSources);
if (!r) continue;
if (!r) { skippedMissingTarget++; continue; }
linkRows.push({
from_slug: r.fromSlug, to_slug: c.targetSlug, link_type: c.linkType,
context: c.context, link_source: c.linkSource, origin_slug: c.originSlug,
@@ -1848,6 +1857,9 @@ export async function extractStaleFromDB(
if (!jsonMode) {
console.log(`Extract --stale: ${linksCreated} link(s) + ${timelineCreated} timeline entr(ies) from ${pagesProcessed} page(s).`);
if (skippedMissingTarget > 0) {
console.log(`Skipped ${skippedMissingTarget} candidate(s) whose target page doesn't exist (references to non-pages are never persisted).`);
}
if (budgetHit && staleRemaining > 0) {
console.log(`Time budget reached — ${staleRemaining} page(s) still stale. Re-run 'gbrain extract --stale' (or pass --catch-up) to continue.`);
}
@@ -1855,9 +1867,10 @@ export async function extractStaleFromDB(
process.stdout.write(JSON.stringify({
action: 'extract_stale_done', links_created: linksCreated, timeline_created: timelineCreated,
pages_processed: pagesProcessed, stale_remaining: staleRemaining, budget_hit: budgetHit,
skipped_missing_target: skippedMissingTarget,
}) + '\n');
}
return { linksCreated, timelineCreated, pagesProcessed, staleRemaining };
return { linksCreated, timelineCreated, pagesProcessed, staleRemaining, skippedMissingTarget };
}
/**
+63 -16
View File
@@ -28,11 +28,13 @@ import { ensureWellFormed } from './text-safe.ts';
* OR updated_at > links_extracted_at`. It is an ISO-8601 string (NOT a number) —
* the column is TIMESTAMPTZ and the predicate binds it as `::timestamptz`.
*/
// 2026-07-30: bumped for the #3466 inferTypeByDir fix — unevidenced
// people/ -> companies/ adjacency now infers 'mentions' instead of
// 'works_at'; the bump re-flags stamped pages so the next --stale sweep
// re-extracts them under the corrected inference.
export const LINK_EXTRACTOR_VERSION_TS = '2026-07-30T00:00:00Z';
// 2026-08-01: bumped for the fix-wave-i extraction batch — the #3466
// inferTypeByDir fix (unevidenced people/ -> companies/ adjacency now infers
// 'mentions' instead of 'works_at') AND the #2576 bug-2 fix (the DIR_PATTERN
// whitelist no longer drops markdown links / bare-slug refs / slash-shaped
// wikilinks in non-whitelisted directories). Pages stamped by earlier sweeps
// are re-flagged so the next --stale sweep re-extracts under both fixes.
export const LINK_EXTRACTOR_VERSION_TS = '2026-08-01T00:00:00Z';
// ─── Entity references ──────────────────────────────────────────
@@ -81,16 +83,30 @@ export const WIKILINK_BASENAME_LINK_TYPE = 'wikilink_basename';
export type LinkResolutionType = 'qualified' | 'unqualified';
/**
* Directory prefix whitelist. These are the top-level slug dirs the extractor
* recognizes as entity references. Upstream canonical + our extensions:
* - Gbrain canonical: people, companies, meetings, concepts, deal, civic, project, source, media, yc, projects, reference
* - Our domain extensions: tech, finance, personal, openclaw (domain-organized wikis)
* - Our entity prefix: entities (we kept some legacy entities/projects/ pages)
* Directory prefix whitelist. These are the canonical top-level slug dirs
* (gbrain-base pack dirs + historical extensions). #2576 (bug 2): this list
* is NO LONGER a drop-gate for markdown links, bare-slug prose refs, or
* slash-shaped wikilinks — those now match ANY_DIR_SEGMENT and rely on the
* downstream page-existence checks that every persist path already runs
* (resolveCandidateSources in extract.ts, the allSlugs filter in put_page
* auto-link, and addLinksBatch's INNER JOINs as the final backstop). The
* whitelist survives only as the typed fast-path for pass-2b wikilinks;
* non-whitelisted `[[dir/...]]` get equivalent treatment in pass 2c.
*/
const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities|reference)';
/**
* Match `[Name](path)` markdown links pointing to entity directories.
* #2576 (bug 2): a plausible top-level slug directory — lowercase alnum
* with dashes/underscores, digit-leading allowed (`90-people`). Used where
* the hardcoded DIR_PATTERN whitelist used to silently drop every
* user-invented directory (`ops/`, `notes/`, custom schema-pack dirs).
* Candidates matched through this are validated for page existence
* downstream, so a wider net creates no dead edges — only candidates.
*/
const ANY_DIR_SEGMENT = '[a-z0-9][a-z0-9_-]*';
/**
* Match `[Name](path)` markdown links pointing at page-shaped paths.
* Accepts both filesystem-relative format (`[Name](../people/slug.md)`)
* AND engine-slug format (`[Name](people/slug)`).
*
@@ -98,9 +114,14 @@ const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|pr
*
* The regex permits an optional `../` prefix (any number) and an optional
* `.md` suffix so the same function works for both filesystem and DB content.
*
* #2576 (bug 2): the first segment is ANY_DIR_SEGMENT, not the DIR_PATTERN
* whitelist — `[Pointer](../ops/services/pointer-agent.md)` must produce a
* candidate for a brain that has an `ops/` directory. Nonexistent targets
* are dropped by the callers' existence checks, exactly as before.
*/
const ENTITY_REF_RE = new RegExp(
`\\[([^\\]]+)\\]\\((?:\\.\\.\\/)*(${DIR_PATTERN}\\/[^)\\s]+?)(?:\\.md)?\\)`,
`\\[([^\\]]+)\\]\\((?:\\.\\.\\/)*(${ANY_DIR_SEGMENT}\\/[^)\\s]+?)(?:\\.md)?\\)`,
'g',
);
@@ -486,6 +507,26 @@ export async function extractPageLinks(
// pre-v0.40.8.2 behavior of dropping bare wikilinks outside
// DIR_PATTERN.
if (ref.needsResolution) {
const slashIdx = ref.slug.lastIndexOf('/');
// #2576 (bug 2): a slash-shaped wikilink outside DIR_PATTERN
// (`[[ops/services/pointer-agent]]`) gets the SAME treatment a
// whitelisted dir gets from pass 2b — a direct, verb-typed candidate
// for the literal path, emitted regardless of the global_basename
// flag. Downstream existence checks (resolveCandidateSources /
// put_page's allSlugs filter / addLinksBatch's INNER JOINs) drop it
// when no such page exists, exactly as they do for 2b candidates.
// Pre-fix these refs were silently dropped (flag off) or demoted to
// untyped wikilink_basename edges (flag on).
if (slashIdx !== -1 && ref.slug !== slug) {
const litIdx = content.indexOf(ref.slug);
const litContext = litIdx >= 0 ? excerpt(content, litIdx, 240) : ref.name;
candidates.push({
targetSlug: ref.slug,
linkType: inferLinkType(pageType, litContext, content, ref.slug),
context: litContext,
linkSource: 'markdown',
});
}
if (!opts.globalBasename || typeof resolver.resolveBasenameMatches !== 'function') {
continue;
}
@@ -503,11 +544,12 @@ export async function extractPageLinks(
// (the analogue of the FS ancestor walk honoring the written path):
// a match must end with the literal, so `[[notes/struktura]]` can
// resolve to `vault/notes/struktura` but never to `wiki/struktura`.
const slashIdx = ref.slug.lastIndexOf('/');
// The EXACT literal is excluded here — the direct typed candidate
// above already covers it (#2576), so keeping it would double-emit.
const basename = slashIdx === -1 ? ref.slug : ref.slug.slice(slashIdx + 1);
let matches = await resolver.resolveBasenameMatches(basename);
if (slashIdx !== -1) {
matches = matches.filter(m => m === ref.slug || m.endsWith(`/${ref.slug}`));
matches = matches.filter(m => m !== ref.slug && m.endsWith(`/${ref.slug}`));
}
if (matches.length === 0) continue;
const idx = content.indexOf(ref.slug);
@@ -540,11 +582,14 @@ export async function extractPageLinks(
}
// 2. Bare slug references (e.g. "see people/alice-chen for context").
// Limited to the same entity directories ENTITY_REF_RE covers.
// #2576 (bug 2): any dir-shaped path, not just the DIR_PATTERN whitelist —
// `see ops/services/pointer-agent` must produce a candidate. Prose noise
// that happens to look like a path (`on/off`, `com/foo/bar` inside a URL)
// is dropped by the callers' page-existence checks, never persisted.
// Code blocks are stripped first — slugs in code samples are not real refs.
const strippedContent = stripCodeBlocks(content);
const bareRe = new RegExp(
`\\b(${DIR_PATTERN}\\/[a-z0-9][a-z0-9/-]*[a-z0-9])\\b`,
`\\b(${ANY_DIR_SEGMENT}\\/[a-z0-9][a-z0-9/-]*[a-z0-9])\\b`,
'g',
);
let m: RegExpExecArray | null;
@@ -552,6 +597,8 @@ export async function extractPageLinks(
// Skip matches that are part of a markdown link (already handled above).
const charBefore = m.index > 0 ? strippedContent[m.index - 1] : '';
if (charBefore === '/' || charBefore === '(') continue;
// #2576: never emit a self-loop for a page mentioning its own slug.
if (m[1] === slug) continue;
const context = excerpt(strippedContent, m.index, 240);
candidates.push({
targetSlug: m[1],
+10 -6
View File
@@ -194,8 +194,11 @@ describe('issue #972 — DB-source (gbrain extract links --source db)', () => {
const outLinks = await engine.getLinks('concepts/knowledge-graph');
const strk = outLinks.find(l => l.to_slug === 'notes/struktura');
expect(strk).toBeDefined();
expect(strk!.link_type).toBe('wikilink_basename');
expect(strk!.link_source).toBe('wikilink-resolved');
// #2576: an exact-path wikilink to an existing page now produces the
// direct verb-typed edge (parity with whitelisted dirs), no longer a
// wikilink_basename demotion.
expect(strk!.link_type).toBe('mentions');
expect(strk!.link_source).toBe('markdown');
});
test('path-qualified wikilink never attaches to a basename-only sibling', async () => {
@@ -219,10 +222,11 @@ describe('issue #972 — DB-source (gbrain extract links --source db)', () => {
await runExtract(engine, ['links', '--source', 'db']);
const outLinks = await engine.getLinks('concepts/x');
const basenameLinks = outLinks
.filter(l => l.link_type === 'wikilink_basename')
.map(l => l.to_slug);
expect(basenameLinks).toEqual(['notes/struktura']);
// #2576: the exact-path edge is now direct + verb-typed. The invariant
// under test is unchanged: the written path binds to notes/struktura and
// NEVER to the basename-only sibling wiki/struktura.
expect(outLinks.map(l => l.to_slug)).toContain('notes/struktura');
expect(outLinks.map(l => l.to_slug)).not.toContain('wiki/struktura');
});
test('flag OFF → no basename edges via DB path (back-compat)', async () => {
@@ -0,0 +1,139 @@
/**
* #2576 (bug 2) — link extraction must not silently drop edges for
* non-whitelisted directories.
*
* The hardcoded DIR_PATTERN whitelist gated three reference shapes on the
* DB-source path (extractPageLinks): markdown links, bare-slug prose refs,
* and (via the pass-2c flag gate) slash-shaped wikilinks. A brain with an
* `ops/` directory (or any user-invented dir a custom schema pack declares)
* had 5 of 6 reference shapes DROPPED with no counter — while identical
* `people/` references resolved in all 6.
*
* Post-fix, extraction emits candidates for ANY dir-shaped path and relies
* on the page-existence checks every persist path already runs
* (resolveCandidateSources in extract.ts, put_page's allSlugs filter,
* addLinksBatch's INNER JOINs). These tests exercise the pure extraction
* core the DB paths (`extract --stale`, `extract links --source db`,
* put_page auto-link) all share — every "ops" case below FAILS on master.
*/
import { describe, test, expect } from 'bun:test';
import {
extractPageLinks,
extractEntityRefs,
LINK_EXTRACTOR_VERSION_TS,
type SlugResolver,
} from '../src/core/link-extraction.ts';
const nullResolver: SlugResolver = { resolve: async () => null };
/** Resolver backed by a fixed slug set, tail-keyed like makeResolver's index. */
function setResolver(slugs: string[]): SlugResolver {
return {
resolve: async () => null,
resolveBasenameMatches: async (name: string) =>
slugs.filter(s => s.slice(s.lastIndexOf('/') + 1) === name),
};
}
describe('#2576 bug 2 — non-whitelisted dirs produce candidates (ops/ = people/ parity)', () => {
test('markdown link into ops/ produces a typed candidate (was: dropped)', async () => {
const { candidates } = await extractPageLinks(
'notes/index', '[Pointer](../ops/services/pointer-agent.md) runs the fleet.',
{}, 'concept', nullResolver, { skipFrontmatter: true },
);
const c = candidates.find(x => x.targetSlug === 'ops/services/pointer-agent');
expect(c).toBeDefined();
expect(c!.linkSource).toBe('markdown');
expect(c!.linkType).toBe('mentions');
});
test('bare-slug prose ref into ops/ produces a candidate (was: dropped)', async () => {
const { candidates } = await extractPageLinks(
'notes/index', 'see ops/services/pointer-agent for details.',
{}, 'concept', nullResolver, { skipFrontmatter: true },
);
expect(candidates.map(c => c.targetSlug)).toContain('ops/services/pointer-agent');
});
test('[[ops/...]] wikilink with global_basename OFF produces a typed candidate (was: dropped)', async () => {
const { candidates } = await extractPageLinks(
'notes/index', '[[ops/services/pointer-agent]] runs the fleet.',
{}, 'concept', nullResolver, { skipFrontmatter: true },
);
const c = candidates.find(x => x.targetSlug === 'ops/services/pointer-agent');
expect(c).toBeDefined();
expect(c!.linkSource).toBe('markdown');
});
test('[[ops/...]] with global_basename ON yields ONE typed candidate, not a wikilink_basename demotion', async () => {
const resolver = setResolver(['ops/services/pointer-agent']);
const { candidates } = await extractPageLinks(
'notes/index', '[[ops/services/pointer-agent]] runs the fleet.',
{}, 'concept', resolver, { skipFrontmatter: true, globalBasename: true },
);
const hits = candidates.filter(c => c.targetSlug === 'ops/services/pointer-agent');
expect(hits).toHaveLength(1);
expect(hits[0].linkType).toBe('mentions'); // typed, like people/
expect(hits[0].linkSource).toBe('markdown'); // NOT 'wikilink-resolved'
});
test('verb inference works for non-whitelisted dirs (typed edge, not just mentions)', async () => {
const { candidates } = await extractPageLinks(
'people/carol', 'Carol founded [Widget Co](../startups/widget-co.md) in 2024.',
{}, 'person', nullResolver, { skipFrontmatter: true },
);
const c = candidates.find(x => x.targetSlug === 'startups/widget-co');
expect(c).toBeDefined();
expect(c!.linkType).toBe('founded');
});
test('extractEntityRefs surfaces non-whitelisted markdown refs', () => {
const refs = extractEntityRefs('[Pointer](ops/services/pointer-agent)');
expect(refs.map(r => r.slug)).toContain('ops/services/pointer-agent');
});
// ── regression pins: what must NOT change ─────────────────────────────
test('suffix rescue is preserved: [[notes/struktura]] still finds vault/notes/struktura (flag ON)', async () => {
const resolver = setResolver(['vault/notes/struktura', 'wiki/struktura']);
const { candidates } = await extractPageLinks(
'concepts/x', 'See [[notes/struktura]].',
{}, 'concept', resolver, { skipFrontmatter: true, globalBasename: true },
);
const rescue = candidates.find(c => c.targetSlug === 'vault/notes/struktura');
expect(rescue).toBeDefined();
expect(rescue!.linkType).toBe('wikilink_basename');
// wiki/struktura does not end with the written path — still excluded.
expect(candidates.map(c => c.targetSlug)).not.toContain('wiki/struktura');
});
test('slash-shaped self-link is never emitted', async () => {
const { candidates } = await extractPageLinks(
'ops/runbook', 'See [[ops/runbook]] for the checklist.',
{}, 'concept', nullResolver, { skipFrontmatter: true },
);
expect(candidates).toEqual([]);
});
test('bare [[name]] wikilinks (no slash) keep the flag-gated behavior', async () => {
const resolver = setResolver(['projects/struktura']);
const off = await extractPageLinks(
'concepts/x', 'This relates to [[struktura]].',
{}, 'concept', resolver, { skipFrontmatter: true },
);
expect(off.candidates).toEqual([]);
const on = await extractPageLinks(
'concepts/x', 'This relates to [[struktura]].',
{}, 'concept', resolver, { skipFrontmatter: true, globalBasename: true },
);
expect(on.candidates.map(c => c.targetSlug)).toEqual(['projects/struktura']);
expect(on.candidates[0].linkType).toBe('wikilink_basename');
});
test('LINK_EXTRACTOR_VERSION_TS was bumped so stamped pages re-extract', () => {
// Pages stamped by pre-fix sweeps had their non-whitelisted-dir edges
// silently dropped; the watermark bump re-flags them as stale.
expect(LINK_EXTRACTOR_VERSION_TS > '2026-07-10T00:00:00Z').toBe(true);
});
});
+18 -6
View File
@@ -100,10 +100,14 @@ describe('extractEntityRefs', () => {
expect(extractEntityRefs('[Alice(people/alice)')).toEqual([]);
});
test('skips non-entity dirs (notes/, ideas/ stay if added later but are accepted now)', () => {
// Current regex targets entity dirs explicitly. Notes/ shouldn't match.
test('#2576: non-whitelisted dirs (notes/, ops/) ARE extracted as candidates', () => {
// Pre-#2576 the DIR_PATTERN whitelist silently dropped these. Now any
// dir-shaped path is a candidate; page-existence checks downstream
// (resolveCandidateSources / put_page allSlugs / addLinksBatch JOIN)
// decide whether an edge is persisted.
const refs = extractEntityRefs('See [random](notes/random).');
expect(refs).toEqual([]);
expect(refs.map(r => r.slug)).toEqual(['notes/random']);
expect(refs[0].dir).toBe('notes');
});
test('extracts meeting refs', () => {
@@ -484,8 +488,10 @@ describe('extractPageLinks', () => {
expect(seen).toContain('struktura');
expect(seen).not.toContain('notes/struktura');
expect(candidates.map(c => c.targetSlug)).toEqual(['notes/struktura']);
expect(candidates[0].linkType).toBe('wikilink_basename');
expect(candidates[0].linkSource).toBe('wikilink-resolved');
// #2576: the literal path now yields the direct verb-typed candidate
// (parity with whitelisted dirs), not a wikilink_basename demotion.
expect(candidates[0].linkType).toBe('mentions');
expect(candidates[0].linkSource).toBe('markdown');
});
test('path-qualified wikilink keeps only matches ending with the written path', async () => {
@@ -516,7 +522,13 @@ describe('extractPageLinks', () => {
'concepts/x', 'See [[notes/struktura]].',
{}, 'concept', resolver, { globalBasename: true },
);
expect(candidates.map(c => c.targetSlug)).toEqual(['vault/notes/struktura']);
// #2576: the literal path is ALSO emitted as a direct candidate (typed,
// linkSource 'markdown') — downstream existence checks drop it when no
// `notes/struktura` page exists, so only the suffix match persists.
expect(candidates.map(c => c.targetSlug)).toEqual(['notes/struktura', 'vault/notes/struktura']);
const suffixMatch = candidates.find(c => c.targetSlug === 'vault/notes/struktura')!;
expect(suffixMatch.linkType).toBe('wikilink_basename');
expect(suffixMatch.linkSource).toBe('wikilink-resolved');
});
test('path-qualified self-link is dropped like the bare form', async () => {