Compare commits

..
Author SHA1 Message Date
Garry TanandClaude Fable 5 72b9e9333f fix(extract): resolve cross-directory wikilinks via sync-consistent slugification (#1964)
Raw Obsidian wikilink paths ([[llm-wiki/entities/AI 3.0]]) never matched
sync-slugified slugs (llm-wiki/entities/ai-3.0) on either extraction path.

- FS path: resolveSlug now slugifies each candidate (slugifyPath, the same
  function sync uses) when the raw path-equality check misses, in both the
  direct join and the ancestor walk. Dir-qualified wikilinks resolve as
  typed edges without the global_basename flag.
- DB path: extractPageLinks resolves dir-qualified generic wikilinks by
  slugifying the path and matching by exact slug or path suffix against
  the basename index — regardless of global_basename (the flag guards
  bare-name cross-dir ambiguity, which doesn't apply to dir-qualified
  paths). A same-tail page in a different directory can no longer be the
  only candidate and is filtered out.

Fixes #1964

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 14:57:58 -07:00
5 changed files with 109 additions and 67 deletions
+16 -14
View File
@@ -43,7 +43,7 @@ import {
} from '../core/link-extraction.ts';
import { createProgress } from '../core/progress.ts';
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
import { pathToSlug, pruneDir, isSyncable } from '../core/sync.ts';
import { pathToSlug, slugifyPath, pruneDir, isSyncable } from '../core/sync.ts';
// v0.41.18.0: withRetry + isRetryableConnError + WithRetryOpts moved to
// src/core/retry.ts as the canonical primitive. Engine methods
// (addLinksBatch/addTimelineEntriesBatch/upsertChunks) now self-retry via
@@ -269,14 +269,24 @@ export function extractMarkdownLinks(content: string): { name: string; relTarget
export function resolveSlug(fileDir: string, relTarget: string, allSlugs: Set<string>): string | null {
const targetNoExt = relTarget.endsWith('.md') ? relTarget.slice(0, -3) : relTarget;
const s1 = join(fileDir, targetNoExt);
if (allSlugs.has(s1)) return s1;
// Issue #1964: wikilinks carry raw Obsidian paths (`[[llm-wiki/entities/AI 3.0]]`)
// but allSlugs holds sync-slugified slugs (`llm-wiki/entities/ai-3.0`). Try the
// raw candidate first (back-compat), then the sync-consistent slugified form.
const hit = (candidate: string): string | null => {
if (allSlugs.has(candidate)) return candidate;
const slugified = slugifyPath(candidate);
if (slugified !== candidate && allSlugs.has(slugified)) return slugified;
return null;
};
const s1 = hit(join(fileDir, targetNoExt));
if (s1) return s1;
const parts = fileDir.split('/').filter(Boolean);
for (let strip = 1; strip <= parts.length; strip++) {
const ancestor = parts.slice(0, parts.length - strip).join('/');
const candidate = ancestor ? join(ancestor, targetNoExt) : targetNoExt;
if (allSlugs.has(candidate)) return candidate;
const candidate = hit(ancestor ? join(ancestor, targetNoExt) : targetNoExt);
if (candidate) return candidate;
}
return null;
@@ -1743,15 +1753,7 @@ async function extractStaleFromDB(
// `page.updated_at.toISOString()` — the JS Date is ms-truncated, so the
// µs-precision DB updated_at stayed strictly greater and the page never
// cleared on Postgres. Stamping the exact value makes them equal.
//
// Version-arm floor: a page last edited BEFORE LINK_EXTRACTOR_VERSION_TS
// would otherwise be stamped below the version watermark and stay
// permanently stale (`links_extracted_at < versionTs` re-fires every run).
// Stamp max(updated_at, versionTs) — versionTs is always a past release
// date, so a concurrent edit's now() still exceeds the stamp and D4 holds.
// Tie at ms precision picks updated_at_iso (its µs ≥ versionTs's .000000).
const stampTs = new Date(page.updated_at_iso) >= new Date(versionTs) ? page.updated_at_iso : versionTs;
processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: stampTs });
processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: page.updated_at_iso });
}
// Flush NON-swallowing (CDX-4): a throw here propagates out of the sweep so
+24 -17
View File
@@ -14,6 +14,7 @@
import type { BrainEngine } from './engine.ts';
import type { PageType } from './types.ts';
import { ensureWellFormed } from './text-safe.ts';
import { slugifyPath } from './sync.ts';
/**
* v0.42.7 — link-extraction version stamp. Bump this ISO timestamp whenever the
@@ -28,7 +29,7 @@ import { ensureWellFormed } from './text-safe.ts';
* OR updated_at > links_extracted_at`. It is an ISO-8601 string (NOT a number) —
* the column is TIMESTAMPTZ and the predicate binds it as `::timestamptz`.
*/
export const LINK_EXTRACTOR_VERSION_TS = '2026-07-21T00:00:00Z';
export const LINK_EXTRACTOR_VERSION_TS = '2026-05-31T00:00:00Z';
// ─── Entity references ──────────────────────────────────────────
@@ -80,10 +81,10 @@ export type LinkResolutionType = 'qualified' | 'unqualified';
* Directory prefix whitelist. These are the top-level slug dirs the extractor
* recognizes as entity references. Upstream canonical + our extensions:
* - Gbrain canonical: people, companies, meetings, concepts, deal, civic, project, source, media, yc, projects
* - Our domain extensions: tech, finance, personal, openclaw, ops (domain-organized wikis)
* - Our domain extensions: tech, finance, personal, openclaw (domain-organized wikis)
* - Our entity prefix: entities (we kept some legacy entities/projects/ pages)
*/
const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities|ops)';
const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities)';
/**
* Match `[Name](path)` markdown links pointing to entity directories.
@@ -482,14 +483,29 @@ export async function extractPageLinks(
// pre-v0.40.8.2 behavior of dropping bare wikilinks outside
// DIR_PATTERN.
if (ref.needsResolution) {
if (!opts.globalBasename || typeof resolver.resolveBasenameMatches !== 'function') {
continue;
}
if (typeof resolver.resolveBasenameMatches !== 'function') continue;
// Issue #972 (codex): resolve by the wikilink TARGET (ref.slug — the
// text inside `[[...]]` before any `|`), NOT the display alias
// (ref.name = match[2]). `[[struktura|the project]]` must resolve
// `struktura`, not "the project". The display text is for context only.
const matches = await resolver.resolveBasenameMatches(ref.slug);
//
// Issue #1964: a dir-qualified wikilink (`[[llm-wiki/entities/AI 3.0]]`)
// carries a raw Obsidian path while page slugs are sync-slugified
// (`llm-wiki/entities/ai-3.0`). Slugify the path the same way sync does,
// then match by exact slug or path-suffix (wiki-root-relative authoring).
// This runs regardless of global_basename — it's dir-qualified, so the
// cross-dir false-positive risk the flag guards against doesn't apply.
// Mirrors the FS path's resolveSlug ancestor walk. Bare `[[name]]`
// wikilinks still require the global_basename flag.
let matches: string[] = [];
const slugified = ref.slug.includes('/') ? slugifyPath(ref.slug) : '';
if (slugified.includes('/')) {
const tail = slugified.slice(slugified.lastIndexOf('/') + 1);
matches = (await resolver.resolveBasenameMatches(tail))
.filter(m => m === slugified || m.endsWith(`/${slugified}`));
} else if (opts.globalBasename) {
matches = await resolver.resolveBasenameMatches(ref.slug);
}
if (matches.length === 0) continue;
const idx = content.indexOf(ref.slug);
const context = idx >= 0 ? excerpt(content, idx, 240) : ref.name;
@@ -865,16 +881,7 @@ export function queryBasenameIndex(idx: Map<string, string[]>, name: string): st
if (!name || typeof name !== 'string') return [];
const trimmed = name.trim();
if (!trimmed) return [];
let hit = idx.get(trimmed) ?? idx.get(trimmed.toLowerCase()) ?? idx.get(normalizeBasename(trimmed));
// Issue #2576 bug 2: path-style refs (`runbooks/2026-05-01-x`) from dirs
// outside DIR_PATTERN reach here, but normalizeBasename strips slashes
// into a garbage key (`runbooks2026-05-01-x`) that can never hit the
// tail-keyed index. Fall back to the path tail so qualified refs resolve
// by basename like everything else.
if (!hit && trimmed.includes('/')) {
const tail = trimmed.slice(trimmed.lastIndexOf('/') + 1).trim();
if (tail) hit = idx.get(tail) ?? idx.get(tail.toLowerCase()) ?? idx.get(normalizeBasename(tail));
}
const hit = idx.get(trimmed) ?? idx.get(trimmed.toLowerCase()) ?? idx.get(normalizeBasename(trimmed));
return hit ? [...hit].sort(basenameSort) : [];
}
+27
View File
@@ -389,6 +389,33 @@ describe('resolveSlugAll', () => {
});
});
// ─── issue #1964: cross-directory wikilinks — slug/path mismatch ──────────
describe('issue #1964: raw Obsidian wikilink paths resolve to sync-slugified slugs', () => {
test('resolveSlug slugifies the candidate (sync-consistent), no flag needed', () => {
const all = new Set(['llm-wiki/entities/ai-3.0']);
// Wikilink literal `[[llm-wiki/entities/AI 3.0]]` — spaces + uppercase.
expect(resolveSlug('llm-wiki/notes', 'llm-wiki/entities/AI 3.0.md', all))
.toBe('llm-wiki/entities/ai-3.0');
});
test('resolveSlug slugifies raw (unslugified) fileDir too', () => {
const all = new Set(['llm-wiki/entities/ai-3.0']);
// fileDir comes from dirname(relPath) — the raw on-disk directory.
expect(resolveSlug('LLM Wiki/Notes', 'entities/AI 3.0.md', all))
.toBe('llm-wiki/entities/ai-3.0');
});
test('extractLinksFromFile resolves cross-directory wikilink with flag OFF as a typed edge', async () => {
const allSlugs = new Set(['llm-wiki/entities/ai-3.0', 'llm-wiki/notes/roadmap']);
const content = '---\ntitle: Roadmap\ntype: concept\n---\n\nSee [[llm-wiki/entities/AI 3.0]].\n';
const links = await extractLinksFromFile(content, 'llm-wiki/notes/roadmap.md', allSlugs);
expect(links.map(l => l.to_slug)).toEqual(['llm-wiki/entities/ai-3.0']);
// Dir-qualified path resolution is exact, NOT the basename fallback.
expect(links[0].link_type).not.toBe('wikilink_basename');
});
});
describe('issue #972 repro: bare wikilinks resolve when flag is on', () => {
// End-to-end: reproduces the issue's exact repro inside a tempdir +
// PGLite, then asserts edge count under both flag states.
-18
View File
@@ -209,24 +209,6 @@ describe('gbrain extract --stale', () => {
expect(usRows[0]?.eq).toBe(true);
});
test('version-arm floor: page edited BEFORE LINK_EXTRACTOR_VERSION_TS clears after --stale (issue #2576 bug 3)', async () => {
// A page whose updated_at predates the version watermark used to be
// stamped at its updated_at (< versionTs), so the version arm re-fired
// every run — permanently stale. The sweep now floors the stamp at
// versionTs. (The #1768 test above also covers this since the v0.42.x
// VERSION_TS bump moved its date below the watermark, but this pins the
// behavior explicitly so a date "repair" there can't drop coverage.)
await engine.putPage('people/alice', personPage('Alice'));
await engine.executeRaw(`UPDATE pages SET updated_at = '2000-01-01T00:00:00Z' WHERE slug = 'people/alice'`);
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(1);
await runExtract(engine, ['--stale']);
// Pre-floor this stayed 1 forever (stamp < versionTs → version arm re-fires).
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
await runExtract(engine, ['--stale']);
expect(await engine.countStalePagesForExtraction({ versionTs: LINK_EXTRACTOR_VERSION_TS })).toBe(0);
});
test('CDX-4 (D2): a link-flush throw aborts the sweep and leaves pages UNSTAMPED', async () => {
await engine.putPage('people/alice', personPage('Alice'));
await engine.putPage('companies/acme', companyPage('Acme', '[Alice](people/alice) founded [Acme](companies/acme).'));
+42 -18
View File
@@ -9,6 +9,8 @@ import {
parseTimelineEntries,
isAutoLinkEnabled,
FRONTMATTER_LINK_MAP,
buildBasenameIndex,
queryBasenameIndex,
type SlugResolver,
} from '../src/core/link-extraction.ts';
import type { BrainEngine } from '../src/core/engine.ts';
@@ -140,15 +142,6 @@ describe('extractEntityRefs', () => {
expect(wikiRefs[0].needsResolution).toBe(true);
});
test('recognizes ops/ qualified wikilinks (issue #2576 bug 2)', () => {
// `ops` was missing from DIR_PATTERN, so [[ops/...]] fell through to
// the generic 2c pass (needsResolution) instead of being a real ref.
const refs = extractEntityRefs('Deployed via [[ops/services/pointer-agent]].');
expect(refs.length).toBe(1);
expect(refs[0].slug).toBe('ops/services/pointer-agent');
expect(refs[0].needsResolution).toBeUndefined();
});
test('skips qualified-syntax tokens (those belong to 2a)', () => {
// [[wiki:topics/ai]] looks like 2a's qualified shape — even though
// it wouldn't satisfy DIR_PATTERN, 2c must not claim it either
@@ -433,6 +426,46 @@ describe('extractPageLinks', () => {
expect(strk!.linkType).toBe('wikilink_basename');
});
// ─── issue #1964: dir-qualified wikilinks with raw Obsidian paths ────────
test('#1964: dir-qualified wikilink resolves via sync-consistent slugification (flag OFF)', async () => {
// `[[llm-wiki/entities/AI 3.0]]` is a raw Obsidian path; the page slug
// is the sync-slugified `llm-wiki/entities/ai-3.0`. Must resolve WITHOUT
// global_basename (it's dir-qualified) and must NOT leak to a same-tail
// page in a different directory.
const resolver: SlugResolver = {
resolve: async () => null,
resolveBasenameMatches: async (name) =>
name === 'ai-3.0' ? ['other/ai-3.0', 'llm-wiki/entities/ai-3.0'] : [],
};
const { candidates } = await extractPageLinks(
'llm-wiki/notes/roadmap',
'See [[llm-wiki/entities/AI 3.0]] for the model.',
{}, 'concept', resolver,
// opts.globalBasename omitted (= false) — path is dir-qualified
);
expect(candidates.map(c => c.targetSlug)).toEqual(['llm-wiki/entities/ai-3.0']);
expect(candidates[0].linkType).toBe('wikilink_basename');
expect(candidates[0].linkSource).toBe('wikilink-resolved');
});
test('#1964: path-suffix match resolves wiki-root-relative paths against a real index', async () => {
// Author writes `[[llm-wiki/entities/AI 3.0]]` but the brain nests the
// wiki under a vault dir. Suffix match rescues it; queried through the
// REAL basename index so the tail-key lookup is exercised end to end.
const idx = buildBasenameIndex(['vault/llm-wiki/entities/ai-3.0', 'people/ai-3.0']);
const resolver: SlugResolver = {
resolve: async () => null,
resolveBasenameMatches: async (name) => queryBasenameIndex(idx, name),
};
const { candidates } = await extractPageLinks(
'vault/llm-wiki/notes/roadmap',
'See [[llm-wiki/entities/AI 3.0]].',
{}, 'concept', resolver,
);
expect(candidates.map(c => c.targetSlug)).toEqual(['vault/llm-wiki/entities/ai-3.0']);
});
test('opts.skipFrontmatter suppresses the frontmatter pass', async () => {
// Real resolver shape that WOULD resolve frontmatter source: too,
// but skipFrontmatter blocks the path entirely.
@@ -1078,15 +1111,6 @@ describe('makeResolver — fallback chain', () => {
]);
});
test('resolveBasenameMatches: path-style ref falls back to the tail (issue #2576 bug 2)', async () => {
// normalizeBasename strips slashes, so `runbooks/2026-05-01-pointer-agent`
// used to normalize to a garbage key that never hit the tail-keyed index.
const engine = makeFakeEngineWithSlugs(['ops/changes/2026-05-01-pointer-agent']);
const r = makeResolver(engine);
expect(await r.resolveBasenameMatches!('runbooks/2026-05-01-pointer-agent'))
.toEqual(['ops/changes/2026-05-01-pointer-agent']);
});
test('resolveBasenameMatches: case-insensitive fallback', async () => {
const engine = makeFakeEngineWithSlugs(['companies/fast-weigh']);
const r = makeResolver(engine);