fix(extract): resolve cross-directory wikilinks via sync-consistent slugification (#1964)

Raw Obsidian wikilink paths ([[llm-wiki/entities/AI 3.0]]) never matched
sync-slugified slugs (llm-wiki/entities/ai-3.0) on either extraction path.

- FS path: resolveSlug now slugifies each candidate (slugifyPath, the same
  function sync uses) when the raw path-equality check misses, in both the
  direct join and the ancestor walk. Dir-qualified wikilinks resolve as
  typed edges without the global_basename flag.
- DB path: extractPageLinks resolves dir-qualified generic wikilinks by
  slugifying the path and matching by exact slug or path suffix against
  the basename index — regardless of global_basename (the flag guards
  bare-name cross-dir ambiguity, which doesn't apply to dir-qualified
  paths). A same-tail page in a different directory can no longer be the
  only candidate and is filtered out.

Fixes #1964

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/commands/extract.ts

@@ -43,7 +43,7 @@ import {
 } from '../core/link-extraction.ts';
 import { createProgress } from '../core/progress.ts';
 import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
-import { pathToSlug, pruneDir, isSyncable } from '../core/sync.ts';
+import { pathToSlug, slugifyPath, pruneDir, isSyncable } from '../core/sync.ts';
 // v0.41.18.0: withRetry + isRetryableConnError + WithRetryOpts moved to
 // src/core/retry.ts as the canonical primitive. Engine methods
 // (addLinksBatch/addTimelineEntriesBatch/upsertChunks) now self-retry via
@@ -269,14 +269,24 @@ export function extractMarkdownLinks(content: string): { name: string; relTarget
 export function resolveSlug(fileDir: string, relTarget: string, allSlugs: Set<string>): string | null {
   const targetNoExt = relTarget.endsWith('.md') ? relTarget.slice(0, -3) : relTarget;
 
-  const s1 = join(fileDir, targetNoExt);
-  if (allSlugs.has(s1)) return s1;
+  // Issue #1964: wikilinks carry raw Obsidian paths (`[[llm-wiki/entities/AI 3.0]]`)
+  // but allSlugs holds sync-slugified slugs (`llm-wiki/entities/ai-3.0`). Try the
+  // raw candidate first (back-compat), then the sync-consistent slugified form.
+  const hit = (candidate: string): string | null => {
+    if (allSlugs.has(candidate)) return candidate;
+    const slugified = slugifyPath(candidate);
+    if (slugified !== candidate && allSlugs.has(slugified)) return slugified;
+    return null;
+  };
+
+  const s1 = hit(join(fileDir, targetNoExt));
+  if (s1) return s1;
 
   const parts = fileDir.split('/').filter(Boolean);
   for (let strip = 1; strip <= parts.length; strip++) {
     const ancestor = parts.slice(0, parts.length - strip).join('/');
-    const candidate = ancestor ? join(ancestor, targetNoExt) : targetNoExt;
-    if (allSlugs.has(candidate)) return candidate;
+    const candidate = hit(ancestor ? join(ancestor, targetNoExt) : targetNoExt);
+    if (candidate) return candidate;
   }
 
   return null;

src/core/chunkers/code.ts

@@ -18,9 +18,8 @@
  * at runtime.
  */
 
-import { chunkText as recursiveChunk, capByEstimatedTokens, DEFAULT_MAX_EST_TOKENS } from './recursive.ts';
+import { chunkText as recursiveChunk } from './recursive.ts';
 import { buildQualifiedName } from './qualified-names.ts';
-import { estimateEmbeddingTokens } from '../cjk.ts';
 
 // Embed the tree-sitter runtime + per-language grammars as files.
 // `with { type: 'file' }` returns a path (string) at runtime. Bun bundles
@@ -112,15 +111,7 @@ import G_ZIG from '../../assets/wasm/grammars/tree-sitter-zig.wasm' with { type:
 // chunks get the new columns populated. Without this, the v28 backfill
 // gives every existing chunk a search_vector but subsequent Layer 5 AST
 // work would silently no-op.
-//
-// v5: estimated-token hard cap on AST-path chunks (capCodeChunks). A node
-// splitLargeNode can't subdivide (giant single-statement function, huge
-// literal) previously shipped WHOLE regardless of size and could overflow
-// strict per-request embedding-token limits (local llama-server crashes
-// past ~2,050 tokens, measured). Mirrors the markdown
-// chunker's v4 cap; fallback-path chunks are already capped inside
-// recursiveChunk.
-export const CHUNKER_VERSION = 5;
+export const CHUNKER_VERSION = 4;
 
 // Lazy-loaded tree-sitter module (v0.22.x API: Parser is default export)
 let Parser: typeof import('web-tree-sitter') | null = null;
@@ -717,7 +708,7 @@ export async function chunkCodeTextFull(
     if (chunks.length === 0) {
       return { chunks: fallbackChunks(source, filePath, language, opts), edges: rawEdges };
     }
-    return { chunks: capCodeChunks(mergeSmallSiblings(chunks, chunkTarget)), edges: rawEdges };
+    return { chunks: mergeSmallSiblings(chunks, chunkTarget), edges: rawEdges };
   } catch {
     return { chunks: fallbackChunks(source, filePath, language, opts), edges: [] };
   } finally {
@@ -800,33 +791,6 @@ function mergeSmallSiblings(chunks: CodeChunk[], chunkTarget: number): CodeChunk
   return merged;
 }
 
-/**
- * v5 final safety pass for AST-path chunks: split any chunk whose
- * ESTIMATED embedding tokens (conservative per-char-class heuristic,
- * cjk.ts) exceed DEFAULT_MAX_EST_TOKENS. Reaches chunks the AST logic
- * can't subdivide — splitLargeNode returns [] for nodes with < 2 body
- * children (giant single-statement functions, huge literals), which
- * previously shipped whole at any size.
- *
- * Split pieces inherit the source chunk's metadata verbatim; start/end
- * lines become approximate for pieces after the first. Acceptable —
- * these chunks exist for embedding + retrieval, and the alternative was
- * an embedding request the server rejects (or worse, crashes on).
- */
-function capCodeChunks(chunks: CodeChunk[]): CodeChunk[] {
-  if (chunks.every((c) => estimateEmbeddingTokens(c.text) <= DEFAULT_MAX_EST_TOKENS)) {
-    return chunks;
-  }
-  const out: CodeChunk[] = [];
-  for (const c of chunks) {
-    const pieces = capByEstimatedTokens(c.text, DEFAULT_MAX_EST_TOKENS);
-    for (const piece of pieces) {
-      out.push({ ...c, text: piece, index: out.length, metadata: { ...c.metadata } });
-    }
-  }
-  return out;
-}
-
 function buildMergedChunk(group: CodeChunk[], index: number): CodeChunk {
   const first = group[0]!;
   const last = group[group.length - 1]!;

src/core/chunkers/recursive.ts

@@ -17,13 +17,7 @@
  * Lossless invariant: non-overlapping portions reassemble to original.
  */
 
-import {
-  countCJKAwareWords,
-  CJK_SENTENCE_DELIMITERS,
-  CJK_CLAUSE_DELIMITERS,
-  charEmbedTokenWeight,
-  estimateEmbeddingTokens,
-} from '../cjk.ts';
+import { countCJKAwareWords, CJK_SENTENCE_DELIMITERS, CJK_CLAUSE_DELIMITERS } from '../cjk.ts';
 
 /**
  * Markdown chunker version. Folded into the per-page chunker_version column
@@ -39,20 +33,8 @@ import {
  * re-embed (not re-chunk) so existing pages pick up the wrapper on the
  * post-upgrade reembed sweep. See
  * `src/core/contextual-retrieval-service.ts`.
- *
- * v4: estimated-token hard cap + whitespace-word undercount fix. The word
- * pipeline counted a 150-char URL as ONE whitespace word, so URL/phone/
- * email-dense docs (CJK density < 0.30 → whitespace fallback) produced
- * 3-4K-char chunks that overflow strict per-request embedding-token
- * limits (measured: local llama-server crashes past ~2,050 tokens; URL
- * soup tokenizes at ~1.6 chars/token). Two changes:
- *   1. countWords() floors the count at ceil(nonWhitespaceChars/6) so a
- *      URL counts roughly per-character, not as one word.
- *   2. capByEstimatedTokens() final pass guarantees every chunk fits
- *      `maxTokens` (default 1500) under a conservative per-char-class
- *      token estimate, regardless of how word counting misjudged it.
  */
-export const MARKDOWN_CHUNKER_VERSION = 4;
+export const MARKDOWN_CHUNKER_VERSION = 3;
 
 const DELIMITERS: string[][] = [
   ['\n\n'],                          // L0: paragraphs
@@ -66,20 +48,8 @@ export interface ChunkOptions {
   chunkSize?: number;    // target words per chunk (default 300)
   chunkOverlap?: number; // overlap words (default 50)
   maxChars?: number;     // hard cap on any chunk's char length (default 6000)
-  /**
-   * v4: hard cap on any chunk's ESTIMATED embedding tokens (default 1500).
-   * Estimate = conservative per-char-class weights (see cjk.ts
-   * estimateEmbeddingTokens) — deliberately high, so the real tokenizer
-   * count stays below this value. Default leaves headroom for the
-   * contextual-retrieval wrapper (≤ ~630 chars) under a ~2,050-token
-   * per-request embedding server limit.
-   */
-  maxTokens?: number;
 }
 
-/** v4 default for ChunkOptions.maxTokens — see the field doc above. */
-export const DEFAULT_MAX_EST_TOKENS = 1500;
-
 export interface TextChunk {
   text: string;
   index: number;
@@ -103,7 +73,6 @@ export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] {
   const chunkSize = opts?.chunkSize || 300;
   const chunkOverlap = opts?.chunkOverlap || 50;
   const maxChars = opts?.maxChars || 6000;
-  const maxTokens = opts?.maxTokens || DEFAULT_MAX_EST_TOKENS;
 
   if (!text || text.trim().length === 0) return [];
 
@@ -120,9 +89,8 @@ export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] {
 
   const wordCount = countWords(stripped);
   if (wordCount <= chunkSize) {
-    // Single-chunk path: still apply the maxChars + maxTokens caps.
-    const capped = capByChars(stripped.trim(), maxChars)
-      .flatMap((t) => capByEstimatedTokens(t, maxTokens));
+    // Single-chunk path: still apply the maxChars cap.
+    const capped = capByChars(stripped.trim(), maxChars);
     return capped.map((t, i) => ({ text: t, index: i }));
   }
 
@@ -133,14 +101,9 @@ export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] {
   // v0.32.7: hard char cap. Catches pathological CJK + whitespace-less text
   // that the word-level pipeline can't bound (a single Chinese paragraph can
   // exceed 8192 OpenAI embedding tokens at any word count).
-  // v4: estimated-token cap on top — the char cap alone passes token-dense
-  // content (URL soup at ~1.6 chars/token) that overflows strict embedding
-  // server limits.
   const capped: string[] = [];
   for (const chunk of withOverlap) {
-    for (const piece of capByChars(chunk.trim(), maxChars)) {
-      capped.push(...capByEstimatedTokens(piece, maxTokens));
-    }
+    capped.push(...capByChars(chunk.trim(), maxChars));
   }
   return capped.map((t, i) => ({ text: t, index: i }));
 }
@@ -169,68 +132,6 @@ function capByChars(text: string, maxChars: number): string[] {
   return out;
 }
 
-/**
- * How far back (in chars) the token cap looks for a friendly cut point
- * before falling back to a hard cut. 300 covers typical rollup/list line
- * lengths so forced splits land at line starts, not mid-URL.
- */
-const TOKEN_CAP_CUT_LOOKBACK = 300;
-
-/**
- * v4: hard-cap a chunk's ESTIMATED embedding tokens. Final safety pass —
- * runs after capByChars on every chunk, so no upstream miscounting
- * (whitespace-word fallback, overlap inflation, char-cap survivors) can
- * emit a chunk past `maxTokens`.
- *
- * Cut placement prefers, within the last TOKEN_CAP_CUT_LOOKBACK chars of
- * the window: a newline, then any whitespace, then a hard cut. This keeps
- * forced splits off mid-line/mid-URL positions for list-shaped content
- * and inside code fences. No overlap is added (pieces stay lossless
- * modulo the trims the char cap already applies).
- *
- * @internal exported for the code chunker (code.ts) and tests.
- */
-export function capByEstimatedTokens(text: string, maxTokens: number): string[] {
-  if (text.length === 0) return [];
-  if (estimateEmbeddingTokens(text) <= maxTokens) return [text];
-
-  const out: string[] = [];
-  let start = 0;
-  while (start < text.length) {
-    // Greedily extend the window until the next char would break the cap.
-    // Always take at least one char so the loop makes forward progress.
-    let est = 0;
-    let end = start;
-    while (end < text.length) {
-      const w = charEmbedTokenWeight(text.charCodeAt(end));
-      if (est + w > maxTokens && end > start) break;
-      est += w;
-      end++;
-    }
-
-    if (end < text.length) {
-      const windowStart = Math.max(start + 1, end - TOKEN_CAP_CUT_LOOKBACK);
-      let cut = text.lastIndexOf('\n', end - 1);
-      if (cut < windowStart) {
-        cut = -1;
-        for (let i = end - 1; i >= windowStart; i--) {
-          const code = text.charCodeAt(i);
-          if (code === 0x20 || (code >= 0x09 && code <= 0x0d)) {
-            cut = i;
-            break;
-          }
-        }
-      }
-      if (cut >= windowStart) end = cut + 1;
-    }
-
-    const slice = text.slice(start, end).trim();
-    if (slice.length > 0) out.push(slice);
-    start = end;
-  }
-  return out;
-}
-
 function recursiveSplit(text: string, level: number, target: number): string[] {
   if (level >= DELIMITERS.length) {
     // Level 4: split on whitespace
@@ -416,19 +317,7 @@ function extractTrailingContext(text: string, targetWords: number): string {
  * Delegated to src/core/cjk.ts so the slugify whitelist, expansion
  * detection, and PGLite keyword fallback all agree on what "CJK enough"
  * means.
- *
- * v4: floored at ceil(nonWhitespaceChars/6). The whitespace fallback
- * counts a 150-char URL as ONE word, so URL/phone/email-dense docs
- * (whose ASCII mass pushes CJK density below the 0.30 threshold) were
- * sized at a fraction of their real bulk and merged into 3-4K-char
- * chunks. The floor makes long whitespace-less runs count roughly
- * per-character while leaving normal Latin prose untouched (average
- * English word ≈ 5 chars < 6, so the whitespace count still wins).
- * Kept local to the chunker — search/expansion.ts keeps the original
- * countCJKAwareWords semantics for its query-length check.
  */
 function countWords(text: string): number {
-  const cjkAware = countCJKAwareWords(text);
-  const nonWhitespace = text.replace(/\s/g, '').length;
-  return Math.max(cjkAware, Math.ceil(nonWhitespace / 6));
+  return countCJKAwareWords(text);
 }

src/core/cjk.ts

@@ -65,64 +65,3 @@ export function countCJKAwareWords(s: string): number {
 export function escapeLikePattern(s: string): string {
   return s.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_');
 }
-
-/**
- * Conservative per-char-class embedding-token weights (markdown chunker v4).
- *
- * Why this exists: the chunker's "word" counting drastically UNDER-counts
- * whitespace-less ASCII runs (a 150-char URL = 1 whitespace word), so
- * word-based size targets can emit chunks that overflow an embedding
- * server's per-request token limit. Measured on a local Qwen3-embedding
- * llama-server stack:
- *   - URL/phone/email-dense text tokenizes at ~1.6 chars/token
- *   - base64-ish / minified blobs approach ~1.3 chars/token (worst case)
- *   - Korean prose tokenizes NO WORSE than 1 char/token in practice
- *
- * Weights are deliberately HIGH (tokens are overestimated) so any cap
- * based on this estimate is safe against real tokenizers:
- *   - CJK char        → 1.0 token  (real CJK prose is cheaper)
- *   - other non-space → 0.75 token (≈1.33 chars/token, covers base64)
- *   - whitespace      → 0.1 token  (mostly folds into neighbor tokens)
- */
-export const EMBED_TOKEN_WEIGHT_CJK = 1.0;
-export const EMBED_TOKEN_WEIGHT_OTHER = 0.75;
-export const EMBED_TOKEN_WEIGHT_WS = 0.1;
-
-/** BMP CJK check by UTF-16 code unit — same ranges as CJK_SLUG_CHARS. */
-export function isCJKCodeUnit(code: number): boolean {
-  return (
-    (code >= 0x4e00 && code <= 0x9fff) || // Han
-    (code >= 0x3040 && code <= 0x309f) || // Hiragana
-    (code >= 0x30a0 && code <= 0x30ff) || // Katakana
-    (code >= 0xac00 && code <= 0xd7af)    // Hangul Syllables
-  );
-}
-
-/**
- * Per-code-unit token weight. Unrecognized whitespace (exotic Unicode
- * spaces) intentionally falls into OTHER — that only overestimates.
- */
-export function charEmbedTokenWeight(code: number): number {
-  if (isCJKCodeUnit(code)) return EMBED_TOKEN_WEIGHT_CJK;
-  if (
-    code === 0x20 || (code >= 0x09 && code <= 0x0d) ||
-    code === 0xa0 || code === 0x3000
-  ) {
-    return EMBED_TOKEN_WEIGHT_WS;
-  }
-  return EMBED_TOKEN_WEIGHT_OTHER;
-}
-
-/**
- * Tokenizer-free embedding-token estimate (conservative overestimate).
- * See weight docs above. Astral chars count as 2 OTHER code units —
- * another overestimate, which is the safe direction.
- */
-export function estimateEmbeddingTokens(s: string): number {
-  if (s.length === 0) return 0;
-  let est = 0;
-  for (let i = 0; i < s.length; i++) {
-    est += charEmbedTokenWeight(s.charCodeAt(i));
-  }
-  return Math.ceil(est);
-}

src/core/link-extraction.ts

@@ -14,6 +14,7 @@
 import type { BrainEngine } from './engine.ts';
 import type { PageType } from './types.ts';
 import { ensureWellFormed } from './text-safe.ts';
+import { slugifyPath } from './sync.ts';
 
 /**
  * v0.42.7 — link-extraction version stamp. Bump this ISO timestamp whenever the
@@ -482,14 +483,29 @@ export async function extractPageLinks(
     // pre-v0.40.8.2 behavior of dropping bare wikilinks outside
     // DIR_PATTERN.
     if (ref.needsResolution) {
-      if (!opts.globalBasename || typeof resolver.resolveBasenameMatches !== 'function') {
-        continue;
-      }
+      if (typeof resolver.resolveBasenameMatches !== 'function') continue;
       // Issue #972 (codex): resolve by the wikilink TARGET (ref.slug — the
       // text inside `[[...]]` before any `|`), NOT the display alias
       // (ref.name = match[2]). `[[struktura|the project]]` must resolve
       // `struktura`, not "the project". The display text is for context only.
-      const matches = await resolver.resolveBasenameMatches(ref.slug);
+      //
+      // Issue #1964: a dir-qualified wikilink (`[[llm-wiki/entities/AI 3.0]]`)
+      // carries a raw Obsidian path while page slugs are sync-slugified
+      // (`llm-wiki/entities/ai-3.0`). Slugify the path the same way sync does,
+      // then match by exact slug or path-suffix (wiki-root-relative authoring).
+      // This runs regardless of global_basename — it's dir-qualified, so the
+      // cross-dir false-positive risk the flag guards against doesn't apply.
+      // Mirrors the FS path's resolveSlug ancestor walk. Bare `[[name]]`
+      // wikilinks still require the global_basename flag.
+      let matches: string[] = [];
+      const slugified = ref.slug.includes('/') ? slugifyPath(ref.slug) : '';
+      if (slugified.includes('/')) {
+        const tail = slugified.slice(slugified.lastIndexOf('/') + 1);
+        matches = (await resolver.resolveBasenameMatches(tail))
+          .filter(m => m === slugified || m.endsWith(`/${slugified}`));
+      } else if (opts.globalBasename) {
+        matches = await resolver.resolveBasenameMatches(ref.slug);
+      }
       if (matches.length === 0) continue;
       const idx = content.indexOf(ref.slug);
       const context = idx >= 0 ? excerpt(content, idx, 240) : ref.name;

test/chunker-version-gate.test.ts

@@ -15,22 +15,19 @@ import { describe, test, expect } from 'bun:test';
 import { CHUNKER_VERSION } from '../src/core/chunkers/code.ts';
 
 describe('Layer 12 — CHUNKER_VERSION constant', () => {
-  test('bumped to 5 for the estimated-token hard cap', () => {
+  test('bumped to 4 for Cathedral II', () => {
     // v3: v0.19.0 Chonkie parity (tokenizer + small-sibling merge).
     // v4: v0.20.0 Cathedral II (qualified names + parent scope + doc_comment
     //     + fence extraction + chunk-grain FTS). Folded into content_hash
     //     so any bump forces clean re-chunks on next sync.
-    // v5: estimated-token hard cap on AST-path chunks (capCodeChunks) so
-    //     un-subdividable giant nodes can't overflow strict embedding
-    //     server token limits.
-    expect(CHUNKER_VERSION).toBe(5);
+    expect(CHUNKER_VERSION).toBe(4);
   });
 
   test('is stable across imports (not recomputed at call time)', async () => {
     const a = (await import('../src/core/chunkers/code.ts')).CHUNKER_VERSION;
     const b = (await import('../src/core/chunkers/code.ts')).CHUNKER_VERSION;
     expect(a).toBe(b);
-    expect(a).toBe(5);
+    expect(a).toBe(4);
   });
 });
 

test/chunkers/code.test.ts

@@ -10,8 +10,8 @@ import { describe, test, expect } from 'bun:test';
 import { chunkCodeText, detectCodeLanguage, CHUNKER_VERSION } from '../../src/core/chunkers/code.ts';
 
 describe('CHUNKER_VERSION', () => {
-  test('v5: estimated-token hard cap on AST-path chunks', () => {
-    expect(CHUNKER_VERSION).toBe(5);
+  test('v0.20.0 Cathedral II Layer 12 bumped to 4', () => {
+    expect(CHUNKER_VERSION).toBe(4);
   });
 });
 

test/chunkers/recursive.test.ts

@@ -135,14 +135,13 @@ describe('Recursive Text Chunker', () => {
 });
 
 describe('CJK chunking (v0.32.7)', () => {
-  test('MARKDOWN_CHUNKER_VERSION is 4', async () => {
+  test('MARKDOWN_CHUNKER_VERSION is 3', async () => {
     // v0.40.3.0: bumped 2→3 to signal the post-upgrade reembed sweep that
-    // contextual retrieval wrapping is now applied at embed time.
-    // v4: estimated-token hard cap + whitespace-word undercount floor
-    // (URL-dense docs produced chunks past strict embedding server token
-    // limits). Boundary change → forces re-chunk for chunker_version < 4.
+    // contextual retrieval wrapping is now applied at embed time. Chunk
+    // boundaries themselves are unchanged; the bump forces re-embed for
+    // pages where chunker_version < 3.
     const mod = await import('../../src/core/chunkers/recursive.ts');
-    expect(mod.MARKDOWN_CHUNKER_VERSION).toBe(4);
+    expect(mod.MARKDOWN_CHUNKER_VERSION).toBe(3);
   });
 
   test('long pure-Chinese paragraph splits into multiple chunks', () => {

test/chunkers/token-cap.test.ts

@@ -1,195 +0,0 @@
-/**
- * Markdown chunker v4 / code chunker v5 — estimated-token hard cap
- * regression tests.
- *
- * Reproduces a field failure: a local llama-server embedding backend
- * (`-ub 2048`) crashes deterministically (trace/BPT trap → EOF at the
- * client) when a single chunk exceeds ~2,050 real tokens. Two content
- * shapes triggered it:
- *
- *   1. Korean docs carrying one long source URL per line.
- *      The URLs' ASCII mass pushes CJK density below 0.30, flipping
- *      countCJKAwareWords to whitespace counting, where a 150-char URL
- *      counts as ONE word → chunks ballooned to 3-4K chars ≈ 2,000+
- *      real tokens (URL soup tokenizes at ~1.6 chars/token).
- *
- *   2. Large JSON code blocks (~7K chars) that the word pipeline
- *      undercounts the same way (few whitespace tokens).
- *
- * The fix: every emitted chunk must satisfy
- *   estimateEmbeddingTokens(chunk) <= maxTokens (default 1500)
- * where the estimate deliberately OVERSTATES real tokenizer counts.
- */
-
-import { describe, test, expect } from 'bun:test';
-import { chunkText, capByEstimatedTokens, DEFAULT_MAX_EST_TOKENS } from '../../src/core/chunkers/recursive.ts';
-import { chunkCodeText } from '../../src/core/chunkers/code.ts';
-import { estimateEmbeddingTokens } from '../../src/core/cjk.ts';
-
-/** Synthesize the failing shape: Korean rollup lines each ending in a long Notion URL. */
-function urlDenseKoreanRollup(lines: number): string {
-  const out: string[] = ['# 링크가 줄마다 붙는 한국어 예시 문서', ''];
-  for (let i = 0; i < lines; i++) {
-    const hex32 = (i * 2654435761 >>> 0).toString(16).padStart(8, '0').repeat(4);
-    out.push(
-      `- **항목 ${i}**: 이 줄은 청커 동작 검증을 위한 의미 없는 한국어 예시 문장입니다 · 전화 000-0000-${String(1000 + i)} · ` +
-      `이메일 user${i}@example.com · 링크: https://docs.example.com/pages/${hex32}?v=abcdef0123456789&ref=sample`,
-    );
-  }
-  return out.join('\n');
-}
-
-/** Synthesize a large pretty-printed JSON block with CJK values. */
-function bigJsonBlock(targetChars: number): string {
-  const entries: string[] = [];
-  let i = 0;
-  let len = 0;
-  while (len < targetChars) {
-    const row =
-      `  "item_${i}": { "name": "예시-${i}", "url": "https://example.com/api/v2/items/${i}?token=abc${i}def", "qty": ${i % 100}, "memo": "한국어 값이 섞인 예시 데이터" }`;
-    entries.push(row);
-    len += row.length;
-    i++;
-  }
-  return `{\n${entries.join(',\n')}\n}`;
-}
-
-describe('v4 estimated-token cap — URL-dense Korean doc (field-failure shape)', () => {
-  test('every chunk stays under the estimated-token cap', () => {
-    const md = urlDenseKoreanRollup(60);
-    const chunks = chunkText(md);
-    expect(chunks.length).toBeGreaterThan(0);
-    for (const c of chunks) {
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS);
-    }
-  });
-
-  test('no chunk reaches the measured 3K-char danger zone for URL soup', () => {
-    const md = urlDenseKoreanRollup(60);
-    const chunks = chunkText(md);
-    // 1500 est tokens at the OTHER weight (0.75/char) bounds chunks to
-    // ~2,000 chars for pure ASCII — well under the ~3,300 chars where
-    // URL-dense content crosses ~2,050 real tokens (1.6 chars/token).
-    for (const c of chunks) {
-      expect(c.text.length).toBeLessThanOrEqual(2600);
-    }
-  });
-
-  test('content is preserved (no lines dropped by the cap)', () => {
-    const md = urlDenseKoreanRollup(60);
-    const chunks = chunkText(md);
-    const joined = chunks.map((c) => c.text).join('\n');
-    // Spot-check first / middle / last rollup lines survive chunking.
-    for (const marker of ['항목 0', '항목 30', '항목 59']) {
-      expect(joined).toContain(marker);
-    }
-  });
-});
-
-describe('v4 estimated-token cap — large JSON blocks', () => {
-  test('7K-char pretty JSON through the prose path stays under the cap', () => {
-    const md = `설정 파일 원문 보존:\n\n\`\`\`\n${bigJsonBlock(7000)}\n\`\`\`\n`;
-    const chunks = chunkText(md);
-    expect(chunks.length).toBeGreaterThan(1);
-    for (const c of chunks) {
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS);
-    }
-  });
-
-  test('7K-char minified JSON (single whitespace-less token) stays under the cap', () => {
-    const minified = bigJsonBlock(7000).replace(/\n\s*/g, '');
-    const chunks = chunkText(minified);
-    expect(chunks.length).toBeGreaterThan(1);
-    for (const c of chunks) {
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS);
-    }
-  });
-
-  test('json fence via the code chunker stays under the cap (+header slack)', async () => {
-    const chunks = await chunkCodeText(bigJsonBlock(7000), 'fence.json');
-    expect(chunks.length).toBeGreaterThan(0);
-    for (const c of chunks) {
-      // buildChunk prepends a short "[JSON] fence.json:…" header AFTER the
-      // body-level cap; allow ~60 est tokens of header slack. Real-token
-      // safety margin (2,050 − overestimated 1,500) absorbs this easily.
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(DEFAULT_MAX_EST_TOKENS + 60);
-    }
-  });
-});
-
-describe('v4 word-count floor — behavior preserved for normal content', () => {
-  test('Latin prose chunking is unchanged by the floor (avg word < 6 chars)', () => {
-    const prose = Array.from({ length: 120 }, (_, i) =>
-      `This is sentence number ${i} and it talks about ordinary things in plain words.`,
-    ).join(' ');
-    const chunks = chunkText(prose);
-    // Historical behavior: ~1,560 whitespace words → multiple ~300-word chunks.
-    expect(chunks.length).toBeGreaterThan(3);
-    for (const c of chunks) {
-      const words = c.text.split(/\s+/).length;
-      expect(words).toBeLessThanOrEqual(300 * 1.5 + 50); // merge cap + overlap
-    }
-  });
-
-  test('Korean prose (CJK-dense, no URLs) never triggers the token cap', () => {
-    const prose = Array.from({ length: 80 }, (_, i) =>
-      `이 문장은 순수 한국어 산문의 청킹 동작을 확인하기 위한 ${i}번째 예시 문장입니다.`,
-    ).join(' ');
-    const chunks = chunkText(prose);
-    expect(chunks.length).toBeGreaterThan(1);
-    for (const c of chunks) {
-      // CJK-dense chunks are char-counted (≈450 max) — nowhere near 1500.
-      expect(estimateEmbeddingTokens(c.text)).toBeLessThanOrEqual(700);
-    }
-  });
-});
-
-describe('capByEstimatedTokens unit behavior', () => {
-  test('returns input unchanged when under the cap', () => {
-    expect(capByEstimatedTokens('short text', 1500)).toEqual(['short text']);
-    expect(capByEstimatedTokens('', 1500)).toEqual([]);
-  });
-
-  test('prefers newline cut points within the lookback window', () => {
-    const line = 'x'.repeat(100);
-    const text = Array.from({ length: 40 }, () => line).join('\n');
-    const pieces = capByEstimatedTokens(text, 1000);
-    expect(pieces.length).toBeGreaterThan(1);
-    for (const p of pieces) {
-      // Every piece should be whole lines (multiples of the 100-char line).
-      for (const l of p.split('\n')) {
-        expect(l).toBe(line);
-      }
-    }
-  });
-
-  test('makes forward progress on whitespace-less input (hard cut)', () => {
-    const blob = 'a'.repeat(10_000);
-    const pieces = capByEstimatedTokens(blob, 1000);
-    expect(pieces.length).toBeGreaterThan(1);
-    expect(pieces.join('')).toBe(blob);
-    for (const p of pieces) {
-      expect(estimateEmbeddingTokens(p)).toBeLessThanOrEqual(1000);
-    }
-  });
-});
-
-describe('estimateEmbeddingTokens — weight sanity', () => {
-  test('overestimates URL-dense ASCII (0.75/char ≥ measured ~0.63/char)', () => {
-    const url = 'https://docs.example.com/pages/a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4?v=abc&ref=sample';
-    const est = estimateEmbeddingTokens(url);
-    expect(est).toBeGreaterThanOrEqual(Math.floor(url.length * 0.7));
-  });
-
-  test('counts CJK at 1 token/char', () => {
-    expect(estimateEmbeddingTokens('가나다라마')).toBe(5);
-  });
-
-  test('whitespace is nearly free', () => {
-    expect(estimateEmbeddingTokens('   \n\t  ')).toBeLessThanOrEqual(1);
-  });
-
-  test('empty string is 0', () => {
-    expect(estimateEmbeddingTokens('')).toBe(0);
-  });
-});

test/extract-fs.test.ts

@@ -389,6 +389,33 @@ describe('resolveSlugAll', () => {
   });
 });
 
+// ─── issue #1964: cross-directory wikilinks — slug/path mismatch ──────────
+
+describe('issue #1964: raw Obsidian wikilink paths resolve to sync-slugified slugs', () => {
+  test('resolveSlug slugifies the candidate (sync-consistent), no flag needed', () => {
+    const all = new Set(['llm-wiki/entities/ai-3.0']);
+    // Wikilink literal `[[llm-wiki/entities/AI 3.0]]` — spaces + uppercase.
+    expect(resolveSlug('llm-wiki/notes', 'llm-wiki/entities/AI 3.0.md', all))
+      .toBe('llm-wiki/entities/ai-3.0');
+  });
+
+  test('resolveSlug slugifies raw (unslugified) fileDir too', () => {
+    const all = new Set(['llm-wiki/entities/ai-3.0']);
+    // fileDir comes from dirname(relPath) — the raw on-disk directory.
+    expect(resolveSlug('LLM Wiki/Notes', 'entities/AI 3.0.md', all))
+      .toBe('llm-wiki/entities/ai-3.0');
+  });
+
+  test('extractLinksFromFile resolves cross-directory wikilink with flag OFF as a typed edge', async () => {
+    const allSlugs = new Set(['llm-wiki/entities/ai-3.0', 'llm-wiki/notes/roadmap']);
+    const content = '---\ntitle: Roadmap\ntype: concept\n---\n\nSee [[llm-wiki/entities/AI 3.0]].\n';
+    const links = await extractLinksFromFile(content, 'llm-wiki/notes/roadmap.md', allSlugs);
+    expect(links.map(l => l.to_slug)).toEqual(['llm-wiki/entities/ai-3.0']);
+    // Dir-qualified path resolution is exact, NOT the basename fallback.
+    expect(links[0].link_type).not.toBe('wikilink_basename');
+  });
+});
+
 describe('issue #972 repro: bare wikilinks resolve when flag is on', () => {
   // End-to-end: reproduces the issue's exact repro inside a tempdir +
   // PGLite, then asserts edge count under both flag states.

test/link-extraction.test.ts

@@ -9,6 +9,8 @@ import {
   parseTimelineEntries,
   isAutoLinkEnabled,
   FRONTMATTER_LINK_MAP,
+  buildBasenameIndex,
+  queryBasenameIndex,
   type SlugResolver,
 } from '../src/core/link-extraction.ts';
 import type { BrainEngine } from '../src/core/engine.ts';
@@ -424,6 +426,46 @@ describe('extractPageLinks', () => {
     expect(strk!.linkType).toBe('wikilink_basename');
   });
 
+  // ─── issue #1964: dir-qualified wikilinks with raw Obsidian paths ────────
+
+  test('#1964: dir-qualified wikilink resolves via sync-consistent slugification (flag OFF)', async () => {
+    // `[[llm-wiki/entities/AI 3.0]]` is a raw Obsidian path; the page slug
+    // is the sync-slugified `llm-wiki/entities/ai-3.0`. Must resolve WITHOUT
+    // global_basename (it's dir-qualified) and must NOT leak to a same-tail
+    // page in a different directory.
+    const resolver: SlugResolver = {
+      resolve: async () => null,
+      resolveBasenameMatches: async (name) =>
+        name === 'ai-3.0' ? ['other/ai-3.0', 'llm-wiki/entities/ai-3.0'] : [],
+    };
+    const { candidates } = await extractPageLinks(
+      'llm-wiki/notes/roadmap',
+      'See [[llm-wiki/entities/AI 3.0]] for the model.',
+      {}, 'concept', resolver,
+      // opts.globalBasename omitted (= false) — path is dir-qualified
+    );
+    expect(candidates.map(c => c.targetSlug)).toEqual(['llm-wiki/entities/ai-3.0']);
+    expect(candidates[0].linkType).toBe('wikilink_basename');
+    expect(candidates[0].linkSource).toBe('wikilink-resolved');
+  });
+
+  test('#1964: path-suffix match resolves wiki-root-relative paths against a real index', async () => {
+    // Author writes `[[llm-wiki/entities/AI 3.0]]` but the brain nests the
+    // wiki under a vault dir. Suffix match rescues it; queried through the
+    // REAL basename index so the tail-key lookup is exercised end to end.
+    const idx = buildBasenameIndex(['vault/llm-wiki/entities/ai-3.0', 'people/ai-3.0']);
+    const resolver: SlugResolver = {
+      resolve: async () => null,
+      resolveBasenameMatches: async (name) => queryBasenameIndex(idx, name),
+    };
+    const { candidates } = await extractPageLinks(
+      'vault/llm-wiki/notes/roadmap',
+      'See [[llm-wiki/entities/AI 3.0]].',
+      {}, 'concept', resolver,
+    );
+    expect(candidates.map(c => c.targetSlug)).toEqual(['vault/llm-wiki/entities/ai-3.0']);
+  });
+
   test('opts.skipFrontmatter suppresses the frontmatter pass', async () => {
     // Real resolver shape that WOULD resolve frontmatter source: too,
     // but skipFrontmatter blocks the path entirely.