mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-17 02:12:40 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2ba63ddec5 | ||
|
|
214449b190 |
@@ -21,17 +21,14 @@ GBrain is tuned for the Supabase **Transaction pooler** (port 6543): it
|
||||
auto-disables prepared statements there and routes `engine.transaction()`
|
||||
(migrations, DDL, sync imports) to a derived **direct** connection
|
||||
(`db.<ref>.supabase.co:5432`). That direct host is IPv6-only, so on an
|
||||
IPv4-only host it is unreachable. When that happens gbrain now falls back to
|
||||
the pooler automatically (one stderr warning, then single-pool mode for the
|
||||
rest of the process) — but the pooler's ~2-min statement timeout can truncate
|
||||
very long migrations or bulk imports.
|
||||
IPv4-only host, reads work but sync **silently skips most pages**. This is the
|
||||
number one cause of "sync ran but nothing happened."
|
||||
|
||||
Fix: make the direct connection reachable over IPv4. Either set
|
||||
`GBRAIN_DIRECT_DATABASE_URL` to the **Session pooler** string (port 5432 on the
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on.
|
||||
`GBRAIN_DISABLE_DIRECT_POOL=1` skips the direct pool (and the fallback warning)
|
||||
entirely. Verify by running `gbrain sync` and checking that the page count in
|
||||
`gbrain stats` matches the syncable file count in the repo.
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on. Verify by
|
||||
running `gbrain sync` and checking that the page count in `gbrain stats` matches
|
||||
the syncable file count in the repo.
|
||||
|
||||
### The Primitives
|
||||
|
||||
|
||||
+5
-8
@@ -2720,17 +2720,14 @@ GBrain is tuned for the Supabase **Transaction pooler** (port 6543): it
|
||||
auto-disables prepared statements there and routes `engine.transaction()`
|
||||
(migrations, DDL, sync imports) to a derived **direct** connection
|
||||
(`db.<ref>.supabase.co:5432`). That direct host is IPv6-only, so on an
|
||||
IPv4-only host it is unreachable. When that happens gbrain now falls back to
|
||||
the pooler automatically (one stderr warning, then single-pool mode for the
|
||||
rest of the process) — but the pooler's ~2-min statement timeout can truncate
|
||||
very long migrations or bulk imports.
|
||||
IPv4-only host, reads work but sync **silently skips most pages**. This is the
|
||||
number one cause of "sync ran but nothing happened."
|
||||
|
||||
Fix: make the direct connection reachable over IPv4. Either set
|
||||
`GBRAIN_DIRECT_DATABASE_URL` to the **Session pooler** string (port 5432 on the
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on.
|
||||
`GBRAIN_DISABLE_DIRECT_POOL=1` skips the direct pool (and the fallback warning)
|
||||
entirely. Verify by running `gbrain sync` and checking that the page count in
|
||||
`gbrain stats` matches the syncable file count in the repo.
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on. Verify by
|
||||
running `gbrain sync` and checking that the page count in `gbrain stats` matches
|
||||
the syncable file count in the repo.
|
||||
|
||||
### The Primitives
|
||||
|
||||
|
||||
+22
-3
@@ -433,7 +433,10 @@ export async function extractLinksFromFile(
|
||||
async resolve(name: string, dirHint?: string | string[]): Promise<string | null> {
|
||||
if (!name) return null;
|
||||
const trimmed = name.trim();
|
||||
if (/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(trimmed) && allSlugs.has(trimmed)) {
|
||||
// Same broadened slug-shape as makeResolver step 1: accepts
|
||||
// digit-leading folders (`90-people/nicolai`) and nested paths.
|
||||
// Exact Set membership guards it — no false positives.
|
||||
if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed) && allSlugs.has(trimmed)) {
|
||||
return trimmed;
|
||||
}
|
||||
const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []);
|
||||
@@ -582,6 +585,17 @@ export interface ExtractOpts {
|
||||
* before (single-'default'-source brains unaffected).
|
||||
*/
|
||||
sourceId?: string;
|
||||
/**
|
||||
* v0.42 — also extract frontmatter links on the incremental (slugs) path.
|
||||
* `extractForSlugs` extracts BODY links only by default; set this true to also
|
||||
* parse each changed page's frontmatter so `sources:`/`related:` edges stay fresh
|
||||
* when YAML is edited externally and synced in. Applied PER changed page, so the
|
||||
* incremental walk stays bounded (no switch to a full DB scan). Only honored on
|
||||
* the incremental path (`slugs` defined); the full-walk path already covers
|
||||
* frontmatter via its own dispatch. Gated upstream by the config key
|
||||
* `autopilot.incremental_extract_include_frontmatter` (default off).
|
||||
*/
|
||||
includeFrontmatter?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -620,7 +634,7 @@ export async function runExtractCore(engine: BrainEngine, opts: ExtractOpts): Pr
|
||||
// Nothing changed — skip entirely.
|
||||
return result;
|
||||
}
|
||||
const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId);
|
||||
const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId, opts.includeFrontmatter);
|
||||
result.links_created = r.links_created;
|
||||
result.timeline_entries_created = r.timeline_created;
|
||||
result.pages_processed = r.pages;
|
||||
@@ -1011,6 +1025,11 @@ async function extractForSlugs(
|
||||
signal?: AbortSignal,
|
||||
// #1747/#1503: stamp resolved brain source id on batch rows (see ExtractOpts.sourceId).
|
||||
sourceId?: string,
|
||||
// v0.42: when true, also extract frontmatter links per changed page so
|
||||
// externally-edited YAML (`sources:`/`related:`) stays fresh on the cycle.
|
||||
// Default false preserves the body-only incremental behavior. Gated upstream
|
||||
// by `autopilot.incremental_extract_include_frontmatter`.
|
||||
includeFrontmatter: boolean = false,
|
||||
): Promise<{ links_created: number; timeline_created: number; pages: number }> {
|
||||
// Build the full slug set for link resolution (fast: just readdir, no file reads)
|
||||
const allFiles = walkMarkdownFiles(brainDir);
|
||||
@@ -1085,7 +1104,7 @@ async function extractForSlugs(
|
||||
const content = readFileSync(fullPath, 'utf-8');
|
||||
|
||||
if (doLinks) {
|
||||
const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename });
|
||||
const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename, includeFrontmatter });
|
||||
for (const link of links) {
|
||||
if (dryRun) {
|
||||
if (!jsonMode) console.log(` ${link.from_slug} → ${link.to_slug} (${link.link_type})`);
|
||||
|
||||
@@ -1078,9 +1078,6 @@ async function initPostgres(opts: {
|
||||
console.warn(' Direct connections are IPv6 only and fail in many environments.');
|
||||
console.warn(' Use the Transaction pooler connection string instead (port 6543):');
|
||||
console.warn(' Supabase Dashboard > Connect (top bar) > Connection String > Transaction pooler');
|
||||
console.warn(' (With a pooler URL, gbrain derives a direct connection for DDL and falls back');
|
||||
console.warn(' to the pooler automatically if that host is unreachable. Power users:');
|
||||
console.warn(' GBRAIN_DIRECT_DATABASE_URL overrides the derived URL; GBRAIN_DISABLE_DIRECT_POOL=1 disables it.)');
|
||||
console.warn('');
|
||||
}
|
||||
|
||||
@@ -1094,9 +1091,6 @@ async function initPostgres(opts: {
|
||||
if (databaseUrl.includes('supabase.co') && (msg.includes('ECONNREFUSED') || msg.includes('ETIMEDOUT'))) {
|
||||
console.error('Connection failed. Supabase direct connections (db.*.supabase.co:5432) are IPv6 only.');
|
||||
console.error('Use the Transaction pooler connection string instead (port 6543).');
|
||||
console.error('(gbrain derives its own direct connection from pooler URLs for DDL; if that host is');
|
||||
console.error('unreachable it falls back to the pooler. GBRAIN_DIRECT_DATABASE_URL overrides the');
|
||||
console.error('derived URL; GBRAIN_DISABLE_DIRECT_POOL=1 disables the direct pool entirely.)');
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
@@ -121,6 +121,18 @@ export interface GBrainConfig {
|
||||
/** Daily spend cap (USD); bounds drains/day = floor(cap / ~$0.30). Default 2.0. */
|
||||
max_usd_per_day?: number;
|
||||
};
|
||||
/**
|
||||
* v0.42 — keep frontmatter links fresh on the incremental cycle. The cycle's
|
||||
* extract phase re-extracts only the slugs a sync changed, but `extractForSlugs`
|
||||
* extracts BODY links only — frontmatter (`sources:`/`related:` etc.) link edges
|
||||
* silently drift stale when a page's YAML is edited externally and synced in.
|
||||
* Set true to also extract frontmatter links per changed page each cycle, keeping
|
||||
* externally-edited YAML edges fresh without a full rescan. Default false
|
||||
* (preserves current behavior). Read via the file/env/DB plane in the cycle's
|
||||
* extract dispatch. Disable/enable with
|
||||
* `gbrain config set autopilot.incremental_extract_include_frontmatter <bool>`.
|
||||
*/
|
||||
incremental_extract_include_frontmatter?: boolean;
|
||||
};
|
||||
eval?: {
|
||||
/** false disables capture entirely. Defaults to true. */
|
||||
|
||||
@@ -167,25 +167,6 @@ export function deriveDirectUrl(url: string): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error codes that mean "the direct host is unreachable from this network"
|
||||
* (#1641). The auto-derived db.<ref>.supabase.co host is IPv6-only without
|
||||
* the paid IPv4 add-on, so ENOTFOUND/ECONNREFUSED here is expected on
|
||||
* IPv4-only networks — we fall back to the pooler instead of failing init.
|
||||
*/
|
||||
const NETWORK_UNREACHABLE_CODES = [
|
||||
'ENOTFOUND', 'ECONNREFUSED', 'ENETUNREACH', 'EHOSTUNREACH',
|
||||
'ETIMEDOUT', 'CONNECT_TIMEOUT',
|
||||
];
|
||||
|
||||
/** True when err looks like a network-unreachable failure (not auth/SQL). */
|
||||
export function isNetworkUnreachableError(err: unknown): boolean {
|
||||
const code = (err as { code?: unknown } | null)?.code;
|
||||
if (typeof code === 'string' && NETWORK_UNREACHABLE_CODES.includes(code)) return true;
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return NETWORK_UNREACHABLE_CODES.some(c => msg.includes(c));
|
||||
}
|
||||
|
||||
/**
|
||||
* Read kill-switch state from env. Subordinate to parent manager's state
|
||||
* when present (A2 inheritance).
|
||||
@@ -338,30 +319,7 @@ export class ConnectionManager {
|
||||
throw err;
|
||||
});
|
||||
}
|
||||
let pool: Sql | null;
|
||||
try {
|
||||
pool = await this._directInit;
|
||||
} catch (err) {
|
||||
// #1641: the derived direct host (db.<ref>.supabase.co) is IPv6-only
|
||||
// without Supabase's IPv4 add-on. On IPv4-only networks the direct
|
||||
// pool can never connect — permanently fall back to the read pool
|
||||
// (self-activating kill-switch) instead of failing init/migrations.
|
||||
// Non-network errors (auth, SQL) still throw: they mean misconfig,
|
||||
// not unreachability.
|
||||
if (isNetworkUnreachableError(err)) {
|
||||
const alreadyWarned = this._killSwitch;
|
||||
this._killSwitch = true;
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
if (!alreadyWarned) console.error(
|
||||
`gbrain: direct connection to ${this._directUrl ? this.hostOnly(this._directUrl) : 'unknown host'} unreachable (${msg}); ` +
|
||||
'falling back to the pooler for DDL/bulk (long migrations may hit the pooler statement timeout). ' +
|
||||
'Set GBRAIN_DIRECT_DATABASE_URL to a reachable direct URL (e.g. the Session pooler, port 5432) or enable the Supabase IPv4 add-on; ' +
|
||||
'GBRAIN_DISABLE_DIRECT_POOL=1 silences this.',
|
||||
);
|
||||
return this.getReadPool();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const pool = await this._directInit;
|
||||
if (!pool) {
|
||||
// Defensive — initDirectPool should have thrown.
|
||||
throw new Error('connection-manager: direct pool init returned null');
|
||||
@@ -392,9 +350,8 @@ export class ConnectionManager {
|
||||
},
|
||||
};
|
||||
const t0 = Date.now();
|
||||
let pool: Sql | null = null;
|
||||
try {
|
||||
pool = postgres(this._directUrl, opts);
|
||||
const pool = postgres(this._directUrl, opts);
|
||||
// Probe to validate connectivity early.
|
||||
await pool`SELECT 1`;
|
||||
logConnectionEvent({
|
||||
@@ -405,9 +362,6 @@ export class ConnectionManager {
|
||||
});
|
||||
return pool;
|
||||
} catch (err) {
|
||||
// Don't leak the failed pool's sockets/timers (#1641 fallback keeps
|
||||
// the process running afterward).
|
||||
if (pool) await endPoolBounded(pool);
|
||||
logConnectionEvent({
|
||||
pool: 'ddl',
|
||||
op: 'error',
|
||||
|
||||
@@ -996,6 +996,21 @@ async function runPhaseExtract(
|
||||
): Promise<PhaseResult> {
|
||||
try {
|
||||
const { runExtractCore } = await import('../commands/extract.ts');
|
||||
const { loadConfig } = await import('./config.ts');
|
||||
// Default off: the incremental cycle extracts body links only unless the
|
||||
// operator opts in to keeping externally-edited frontmatter links fresh too.
|
||||
// Both planes, file wins (env > file > DB precedence, per loadConfigWithEngine):
|
||||
// `gbrain config set autopilot.incremental_extract_include_frontmatter true`
|
||||
// writes the DB plane (engine.setConfig), so a file-plane-only read here
|
||||
// would make the documented enable command a silent no-op (#2120 class).
|
||||
const fileVal = loadConfig()?.autopilot?.incremental_extract_include_frontmatter;
|
||||
let includeFrontmatter = fileVal === true;
|
||||
if (fileVal === undefined) {
|
||||
try {
|
||||
includeFrontmatter =
|
||||
(await engine.getConfig('autopilot.incremental_extract_include_frontmatter')) === 'true';
|
||||
} catch { /* config table unreadable → default off */ }
|
||||
}
|
||||
// Extract is read-mostly against the filesystem + write to links table.
|
||||
// Honor dryRun by skipping with a 'skipped' entry: extract doesn't have
|
||||
// a clean dry-run mode today and runCycle should be honest about it.
|
||||
@@ -1016,6 +1031,7 @@ async function runPhaseExtract(
|
||||
slugs: changedSlugs, // undefined = full walk (first run / manual)
|
||||
signal,
|
||||
sourceId,
|
||||
includeFrontmatter, // honored on the incremental (slugs) path only
|
||||
});
|
||||
const linksCreated = result?.links_created ?? 0;
|
||||
const timelineCreated = result?.timeline_entries_created ?? 0;
|
||||
|
||||
@@ -942,8 +942,17 @@ export function makeResolver(
|
||||
|
||||
const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []);
|
||||
|
||||
// Step 1: already a slug? (dir/name shape, lowercase, hyphenated)
|
||||
if (/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(trimmed)) {
|
||||
// Step 1: already a slug? Try an exact page lookup for any slug-shaped
|
||||
// value (contains '/', slug charset). Broadened beyond the original
|
||||
// single-segment lowercase-leading form (`^[a-z][a-z0-9-]*\/[a-z0-9]...`)
|
||||
// to also accept digit-leading folders (`90-people/nicolai`,
|
||||
// `01-trading/...`) and nested paths (`a/b/c`) — common in PARA-numbered
|
||||
// vaults. This is an EXACT getPage match only — no fuzzy — so it never
|
||||
// produces a false positive; a non-existent slug just falls through to
|
||||
// the steps below. Fixes frontmatter `related: [[dir/slug]]` values
|
||||
// (unwrapped by unwrapWikilink) that name a real page the strict regex
|
||||
// could not reach and whose full-path fuzzy score is below threshold.
|
||||
if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed)) {
|
||||
const page = await engine.getPage(trimmed);
|
||||
if (page) {
|
||||
cache.set(cacheKey, trimmed);
|
||||
@@ -1003,6 +1012,25 @@ export function makeResolver(
|
||||
|
||||
// ─── Frontmatter extractor ──────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Unwrap an Obsidian `[[wikilink]]` frontmatter value to its bare link
|
||||
* target so the resolver (which expects bare titles / dir slugs) can match
|
||||
* it. Mainstream Obsidian authors frontmatter links as `related: ["[[Page]]"]`;
|
||||
* without this, the resolver treats the brackets as part of the value and a
|
||||
* `[[90-people/nicolai]]` is normalized into `90peoplenicolai`, so it never
|
||||
* resolves. Strips a trailing `|alias`, `#heading`, or `^block` suffix — the
|
||||
* link target only. The regex is anchored to a wholly-wrapped value
|
||||
* (`^\s*\[\[…\]\]\s*$`), so bare titles and any value not fully wrapped pass
|
||||
* through unchanged and existing behavior is preserved exactly.
|
||||
*/
|
||||
export function unwrapWikilink(value: string): string {
|
||||
const match = /^\s*\[\[(.+?)\]\]\s*$/.exec(value);
|
||||
if (!match) return value;
|
||||
// Take the link target: drop |alias, then #heading / ^block suffixes.
|
||||
const target = match[1].split('|')[0].split('#')[0].split('^')[0];
|
||||
return target.trim();
|
||||
}
|
||||
|
||||
export interface UnresolvedFrontmatterRef {
|
||||
/** The frontmatter field name. */
|
||||
field: string;
|
||||
@@ -1060,7 +1088,12 @@ export async function extractFrontmatterLinks(
|
||||
}
|
||||
if (!name) continue; // skip numbers, nulls, malformed objects
|
||||
|
||||
const resolved = await resolver.resolve(name, mapping.dirHint);
|
||||
// Accept Obsidian `[[wikilink]]` values in frontmatter link fields by
|
||||
// unwrapping to the bare target before resolution. Bare titles pass
|
||||
// through unchanged; the original `name` is preserved for the
|
||||
// unresolved report and edge context.
|
||||
const linkTarget = unwrapWikilink(name);
|
||||
const resolved = await resolver.resolve(linkTarget, mapping.dirHint);
|
||||
if (!resolved) {
|
||||
unresolved.push({ field, name });
|
||||
continue;
|
||||
|
||||
@@ -3,7 +3,6 @@ import {
|
||||
isSupabasePoolerUrl,
|
||||
deriveDirectUrl,
|
||||
readKillSwitchEnv,
|
||||
isNetworkUnreachableError,
|
||||
resolveDirectPoolSize,
|
||||
ConnectionManager,
|
||||
DEFAULT_DIRECT_POOL_SIZE,
|
||||
@@ -239,65 +238,3 @@ describe('ConnectionManager — parent inheritance (A2)', () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('isNetworkUnreachableError (#1641)', () => {
|
||||
test('classifies network codes as unreachable', () => {
|
||||
for (const code of ['ENOTFOUND', 'ECONNREFUSED', 'ENETUNREACH', 'EHOSTUNREACH', 'ETIMEDOUT', 'CONNECT_TIMEOUT']) {
|
||||
const err = Object.assign(new Error('connect failed'), { code });
|
||||
expect(isNetworkUnreachableError(err)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('classifies by message when code absent', () => {
|
||||
expect(isNetworkUnreachableError(new Error('getaddrinfo ENOTFOUND db.abc.supabase.co'))).toBe(true);
|
||||
});
|
||||
|
||||
test('auth/SQL errors are NOT unreachable', () => {
|
||||
expect(isNetworkUnreachableError(new Error('password authentication failed for user "postgres"'))).toBe(false);
|
||||
expect(isNetworkUnreachableError(new Error('syntax error at or near "SELEC"'))).toBe(false);
|
||||
expect(isNetworkUnreachableError(null)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ConnectionManager — direct-pool fallback on unreachable host (#1641)', () => {
|
||||
let originalKillSwitch: string | undefined;
|
||||
let originalError: typeof console.error;
|
||||
let errLines: string[];
|
||||
beforeEach(() => {
|
||||
originalKillSwitch = process.env.GBRAIN_DISABLE_DIRECT_POOL;
|
||||
delete process.env.GBRAIN_DISABLE_DIRECT_POOL;
|
||||
originalError = console.error;
|
||||
errLines = [];
|
||||
console.error = (...args: unknown[]) => { errLines.push(args.join(' ')); };
|
||||
});
|
||||
afterEach(() => {
|
||||
console.error = originalError;
|
||||
if (originalKillSwitch === undefined) delete process.env.GBRAIN_DISABLE_DIRECT_POOL;
|
||||
else process.env.GBRAIN_DISABLE_DIRECT_POOL = originalKillSwitch;
|
||||
});
|
||||
|
||||
test('ddl() falls back to the read pool when the direct host is unreachable', async () => {
|
||||
const cm = new ConnectionManager({
|
||||
url: 'postgresql://postgres.abc:p@aws.pooler.supabase.com:6543/db',
|
||||
// 127.0.0.1:9 (discard) → instant ECONNREFUSED, the IPv4-only-network shape.
|
||||
directUrl: 'postgresql://postgres:p@127.0.0.1:9/db',
|
||||
});
|
||||
const fakeReadPool = {} as ReturnType<typeof ConnectionManager.prototype.read>;
|
||||
cm.setReadPool(fakeReadPool);
|
||||
expect(cm.isDualPoolActive()).toBe(true);
|
||||
|
||||
const pool = await cm.ddl(); // without the fix this throws ECONNREFUSED
|
||||
expect(pool).toBe(fakeReadPool);
|
||||
// Self-activating kill-switch: subsequent calls skip the direct pool.
|
||||
expect(cm.isKillSwitchActive()).toBe(true);
|
||||
expect(cm.isDualPoolActive()).toBe(false);
|
||||
expect(cm.describeMode().mode).toBe('single (kill-switch)');
|
||||
// One stderr line mentioning the power-user override.
|
||||
const warning = errLines.filter(l => l.includes('GBRAIN_DIRECT_DATABASE_URL'));
|
||||
expect(warning.length).toBe(1);
|
||||
|
||||
const again = await cm.ddl();
|
||||
expect(again).toBe(fakeReadPool);
|
||||
expect(errLines.filter(l => l.includes('GBRAIN_DIRECT_DATABASE_URL')).length).toBe(1);
|
||||
}, 20000);
|
||||
});
|
||||
|
||||
@@ -191,3 +191,37 @@ describe('runExtractCore — incremental cycle path (#417)', () => {
|
||||
expect(result.links_created).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
describe('runExtractCore — incremental frontmatter gate (includeFrontmatter)', () => {
|
||||
// alice has a `source:` frontmatter edge but NO body links. The incremental
|
||||
// path extracts body links only by default, so the frontmatter edge is the
|
||||
// sole signal that distinguishes the gate off vs on.
|
||||
const aliceFm = '---\nsource: companies/acme-example\n---\n# alice';
|
||||
|
||||
test('9. default (flag omitted) does NOT extract frontmatter links on the incremental path', async () => {
|
||||
await seedPage('companies/acme-example', '# acme');
|
||||
await seedPage('people/alice-example', aliceFm);
|
||||
const result = await runExtractCore(engine as unknown as BrainEngine, {
|
||||
mode: 'all',
|
||||
dir: tempDir,
|
||||
slugs: ['people/alice-example'],
|
||||
});
|
||||
// alice's only potential edge is her frontmatter `source:`; with the gate off
|
||||
// it must not be extracted (preserves the body-only incremental behavior).
|
||||
expect(result.pages_processed).toBe(1);
|
||||
expect(result.links_created).toBe(0);
|
||||
});
|
||||
|
||||
test('10. includeFrontmatter: true extracts the frontmatter link on the incremental path', async () => {
|
||||
await seedPage('companies/acme-example', '# acme');
|
||||
await seedPage('people/alice-example', aliceFm);
|
||||
const result = await runExtractCore(engine as unknown as BrainEngine, {
|
||||
mode: 'all',
|
||||
dir: tempDir,
|
||||
slugs: ['people/alice-example'],
|
||||
includeFrontmatter: true,
|
||||
});
|
||||
// Same page, gate on → the `source:` frontmatter edge is now extracted.
|
||||
expect(result.pages_processed).toBe(1);
|
||||
expect(result.links_created).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -76,6 +76,18 @@ describe('extractLinksFromFile', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('resolves wrapped [[wikilink]] digit-leading slug-path in frontmatter (fs resolver, broadened step 1)', async () => {
|
||||
// Same bug class as makeResolver step 1 (#1983): the fs resolver's strict
|
||||
// `^[a-z]…` slug regex rejected digit-leading / nested paths, so a PARA-vault
|
||||
// `related: "[[90-people/nicolai]]"` never resolved even though the page exists.
|
||||
const content = '---\nrelated: "[[90-people/nicolai]]"\ntype: concept\n---\nContent.';
|
||||
const allSlugs = new Set(['wiki/note', '90-people/nicolai']);
|
||||
const links = await extractLinksFromFile(content, 'wiki/note.md', allSlugs, { includeFrontmatter: true });
|
||||
const related = links.filter(l => l.link_type === 'related_to');
|
||||
expect(related).toHaveLength(1);
|
||||
expect(related[0].to_slug).toBe('90-people/nicolai');
|
||||
});
|
||||
|
||||
it('frontmatter extraction is default OFF (back-compat)', async () => {
|
||||
// Without includeFrontmatter, fs-source no longer auto-extracts frontmatter.
|
||||
// Matches db-source behavior. User opts in with --include-frontmatter flag.
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
parseTimelineEntries,
|
||||
isAutoLinkEnabled,
|
||||
FRONTMATTER_LINK_MAP,
|
||||
unwrapWikilink,
|
||||
type SlugResolver,
|
||||
} from '../src/core/link-extraction.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
@@ -1291,3 +1292,175 @@ describe('parseTimelineEntries — Format 3: inline [Source: ..., YYYY-MM-DD] ci
|
||||
expect(parseTimelineEntries('[Source: import batch, 2025-07-01]')).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
// ─── Frontmatter [[wikilink]] + slug-path resolution ──────────────────────
|
||||
// Mainstream Obsidian authors frontmatter links as `related: ["[[Page]]"]`,
|
||||
// and PARA-numbered vaults use digit-leading / nested slug paths like
|
||||
// `[[90-people/nicolai]]`. Both were silently dropped: brackets were treated
|
||||
// as part of the value and the step-1 slug regex (`^[a-z]…`) rejected
|
||||
// digit-leading / nested paths, while full-path fuzzy scored below threshold.
|
||||
// Fix: unwrapWikilink() before resolution + an exact getPage() for any
|
||||
// slug-shaped value (exact-match only → no false positives).
|
||||
|
||||
describe('unwrapWikilink', () => {
|
||||
test('wrapped title → bare title', () => {
|
||||
expect(unwrapWikilink('[[Monday Range]]')).toBe('Monday Range');
|
||||
});
|
||||
test('wrapped slug-path (digit-leading folder) → bare slug', () => {
|
||||
expect(unwrapWikilink('[[90-people/nicolai]]')).toBe('90-people/nicolai');
|
||||
});
|
||||
test('wrapped nested slug-path → bare slug', () => {
|
||||
expect(unwrapWikilink('[[01-trading/wiki/strategies/opening-range-breakout]]'))
|
||||
.toBe('01-trading/wiki/strategies/opening-range-breakout');
|
||||
});
|
||||
test('strips |alias', () => {
|
||||
expect(unwrapWikilink('[[90-people/nicolai|Nicolai]]')).toBe('90-people/nicolai');
|
||||
});
|
||||
test('strips #heading', () => {
|
||||
expect(unwrapWikilink('[[Page#Section]]')).toBe('Page');
|
||||
});
|
||||
test('strips ^block', () => {
|
||||
expect(unwrapWikilink('[[Page^abc123]]')).toBe('Page');
|
||||
});
|
||||
test('surrounding whitespace tolerated', () => {
|
||||
expect(unwrapWikilink(' [[Page]] ')).toBe('Page');
|
||||
});
|
||||
test('bare title passes through unchanged', () => {
|
||||
expect(unwrapWikilink('Monday Range')).toBe('Monday Range');
|
||||
});
|
||||
test('bare slug passes through unchanged', () => {
|
||||
expect(unwrapWikilink('90-people/nicolai')).toBe('90-people/nicolai');
|
||||
});
|
||||
test('partially-wrapped value is NOT unwrapped (anchored)', () => {
|
||||
// Not a wholly-wrapped value → left intact so existing behavior is exact.
|
||||
expect(unwrapWikilink('see [[Page]] for detail')).toBe('see [[Page]] for detail');
|
||||
});
|
||||
});
|
||||
|
||||
describe('makeResolver — slug-path exact getPage (step 1 broadened)', () => {
|
||||
function fakeEngine(
|
||||
slugs: string[],
|
||||
fuzzyMap: Map<string, { slug: string; similarity: number }> = new Map(),
|
||||
): BrainEngine {
|
||||
const lookup = new Set(slugs);
|
||||
return {
|
||||
async getPage(slug: string) { return lookup.has(slug) ? { slug } as any : null; },
|
||||
async findByTitleFuzzy(name: string) { return fuzzyMap.get(name) ?? null; },
|
||||
async searchKeyword() { return []; },
|
||||
} as unknown as BrainEngine;
|
||||
}
|
||||
|
||||
test('digit-leading folder slug resolves via exact getPage', async () => {
|
||||
const r = makeResolver(fakeEngine(['90-people/nicolai']));
|
||||
expect(await r.resolve('90-people/nicolai')).toBe('90-people/nicolai');
|
||||
});
|
||||
|
||||
test('nested (>2 segment) slug resolves via exact getPage', async () => {
|
||||
const r = makeResolver(fakeEngine(['01-trading/wiki/strategies/opening-range-breakout']));
|
||||
expect(await r.resolve('01-trading/wiki/strategies/opening-range-breakout'))
|
||||
.toBe('01-trading/wiki/strategies/opening-range-breakout');
|
||||
});
|
||||
|
||||
test('regression: single-segment lowercase slug still resolves', async () => {
|
||||
const r = makeResolver(fakeEngine(['people/pedro']));
|
||||
expect(await r.resolve('people/pedro')).toBe('people/pedro');
|
||||
});
|
||||
|
||||
test('exact-only: slug-shaped value with no matching page falls through (no false positive)', async () => {
|
||||
// `90-people/ghost` is slug-shaped but absent → step-1 getPage misses,
|
||||
// no fuzzy hit → null. Never invents an edge.
|
||||
const r = makeResolver(fakeEngine(['90-people/nicolai']));
|
||||
expect(await r.resolve('90-people/ghost')).toBeNull();
|
||||
});
|
||||
|
||||
test('non-slug value still routes to fuzzy', async () => {
|
||||
const r = makeResolver(fakeEngine(
|
||||
['01-trading/monday-range'],
|
||||
new Map([['Monday Range', { slug: '01-trading/monday-range', similarity: 1 }]]),
|
||||
));
|
||||
expect(await r.resolve('Monday Range')).toBe('01-trading/monday-range');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractFrontmatterLinks — [[wikilink]] related: values (end-to-end)', () => {
|
||||
function fakeEngine(
|
||||
slugs: string[],
|
||||
fuzzyMap: Map<string, { slug: string; similarity: number }> = new Map(),
|
||||
): BrainEngine {
|
||||
const lookup = new Set(slugs);
|
||||
return {
|
||||
async getPage(slug: string) { return lookup.has(slug) ? { slug } as any : null; },
|
||||
async findByTitleFuzzy(name: string) { return fuzzyMap.get(name) ?? null; },
|
||||
async searchKeyword() { return []; },
|
||||
} as unknown as BrainEngine;
|
||||
}
|
||||
|
||||
test('wrapped slug-path related: resolves (the core win)', async () => {
|
||||
const resolver = makeResolver(fakeEngine(['90-people/nicolai']));
|
||||
const { candidates, unresolved } = await extractFrontmatterLinks(
|
||||
'wiki/originals/ideas/note', 'note' as never,
|
||||
{ related: '[[90-people/nicolai]]' }, resolver,
|
||||
);
|
||||
expect(unresolved).toHaveLength(0);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0]).toMatchObject({
|
||||
fromSlug: 'wiki/originals/ideas/note',
|
||||
targetSlug: '90-people/nicolai',
|
||||
linkType: 'related_to',
|
||||
linkSource: 'frontmatter',
|
||||
});
|
||||
});
|
||||
|
||||
test('wrapped nested slug-path related: resolves', async () => {
|
||||
const resolver = makeResolver(fakeEngine(['01-trading/wiki/strategies/opening-range-breakout']));
|
||||
const { candidates } = await extractFrontmatterLinks(
|
||||
'wiki/note', 'note' as never,
|
||||
{ related: ['[[01-trading/wiki/strategies/opening-range-breakout]]'] }, resolver,
|
||||
);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].targetSlug).toBe('01-trading/wiki/strategies/opening-range-breakout');
|
||||
});
|
||||
|
||||
test('wrapped value with |alias resolves to the target', async () => {
|
||||
const resolver = makeResolver(fakeEngine(['90-people/nicolai']));
|
||||
const { candidates } = await extractFrontmatterLinks(
|
||||
'wiki/note', 'note' as never,
|
||||
{ related: '[[90-people/nicolai|Nicolai]]' }, resolver,
|
||||
);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].targetSlug).toBe('90-people/nicolai');
|
||||
});
|
||||
|
||||
test('regression: bare slug related: still resolves', async () => {
|
||||
const resolver = makeResolver(fakeEngine(['90-people/nicolai']));
|
||||
const { candidates } = await extractFrontmatterLinks(
|
||||
'wiki/note', 'note' as never,
|
||||
{ related: '90-people/nicolai' }, resolver,
|
||||
);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].targetSlug).toBe('90-people/nicolai');
|
||||
});
|
||||
|
||||
test('regression: wrapped title resolves via fuzzy (brackets harmless)', async () => {
|
||||
const resolver = makeResolver(fakeEngine(
|
||||
['01-trading/monday-range'],
|
||||
new Map([['Monday Range', { slug: '01-trading/monday-range', similarity: 1 }]]),
|
||||
));
|
||||
const { candidates } = await extractFrontmatterLinks(
|
||||
'wiki/note', 'note' as never,
|
||||
{ related: '[[Monday Range]]' }, resolver,
|
||||
);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].targetSlug).toBe('01-trading/monday-range');
|
||||
});
|
||||
|
||||
test('unknown wrapped slug → unresolved (no crash), original value preserved', async () => {
|
||||
const resolver = makeResolver(fakeEngine(['90-people/nicolai']));
|
||||
const { candidates, unresolved } = await extractFrontmatterLinks(
|
||||
'wiki/note', 'note' as never,
|
||||
{ related: '[[99-archive/does-not-exist]]' }, resolver,
|
||||
);
|
||||
expect(candidates).toHaveLength(0);
|
||||
expect(unresolved).toHaveLength(1);
|
||||
expect(unresolved[0]).toEqual({ field: 'related', name: '[[99-archive/does-not-exist]]' });
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user