mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 08:53:22 +00:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d45a5ddedc | ||
|
|
91674d59c2 | ||
|
|
c428500a4c | ||
|
|
1afae80f7a |
+10
-1
@@ -43,6 +43,8 @@ import {
|
||||
} from '../core/link-extraction.ts';
|
||||
import { createProgress } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
import { loadActivePackBestEffort } from '../core/schema-pack/best-effort.ts';
|
||||
import { packLinkExtractionView } from '../core/schema-pack/link-inference.ts';
|
||||
import { pathToSlug, pruneDir, isSyncable } from '../core/sync.ts';
|
||||
// v0.41.18.0: withRetry + isRetryableConnError + WithRetryOpts moved to
|
||||
// src/core/retry.ts as the canonical primitive. Engine methods
|
||||
@@ -1381,6 +1383,9 @@ async function extractLinksFromDB(
|
||||
// Issue #972: opt-in global-basename wikilink resolution. Read once
|
||||
// per extract run; threaded into each extractPageLinks call.
|
||||
const globalBasename = await isGlobalBasenameEnabled(engine);
|
||||
// #3190: active custom pack (null for the codegen'd base packs) so pack
|
||||
// path_prefixes / link verbs / frontmatter_links drive extraction.
|
||||
const pack = packLinkExtractionView(await loadActivePackBestEffort({ engine } as never));
|
||||
// v0.32.8: listAllPageRefs enumerates (slug, source_id) so we can thread
|
||||
// sourceId to getPage AND build a cross-source resolution map for link
|
||||
// disambiguation. Pre-fix used getAllSlugs() which collapsed
|
||||
@@ -1461,7 +1466,7 @@ async function extractLinksFromDB(
|
||||
// basename lookup; off by default for back-compat.
|
||||
const extracted = await extractPageLinks(
|
||||
slug, fullContent, page.frontmatter, page.type, resolver,
|
||||
{ skipFrontmatter: !includeFrontmatter, globalBasename },
|
||||
{ skipFrontmatter: !includeFrontmatter, globalBasename, pack },
|
||||
);
|
||||
unresolved.push(...extracted.unresolved);
|
||||
|
||||
@@ -1687,6 +1692,9 @@ export async function extractStaleFromDB(
|
||||
const resolver = makeResolver(engine, { mode: 'batch' });
|
||||
const nullResolver = { resolve: async () => null as string | null };
|
||||
const activeResolver = includeFrontmatter ? resolver : nullResolver;
|
||||
// #3190: same pack threading as extractLinksFromDB so `extract --stale`
|
||||
// produces the same edges as a manual `extract links --source db`.
|
||||
const pack = packLinkExtractionView(await loadActivePackBestEffort({ engine } as never));
|
||||
const allRefs = await engine.listAllPageRefs();
|
||||
const allSlugs = new Set<string>();
|
||||
const slugToSources = new Map<string, string[]>();
|
||||
@@ -1719,6 +1727,7 @@ export async function extractStaleFromDB(
|
||||
const fullContent = page.compiled_truth + '\n' + page.timeline;
|
||||
const extracted = await extractPageLinks(
|
||||
page.slug, fullContent, page.frontmatter, page.type, activeResolver,
|
||||
{ pack },
|
||||
);
|
||||
for (const c of extracted.candidates) {
|
||||
const r = resolveCandidateSources(c, page.slug, page.source_id, allSlugs, slugToSources);
|
||||
|
||||
+76
-4
@@ -3207,14 +3207,74 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
|
||||
const providerMetadata = (result as any).providerMetadata as Record<string, any> | undefined;
|
||||
const anthropicCache = providerMetadata?.anthropic ?? {};
|
||||
|
||||
const inTok = Number(usage.inputTokens ?? usage.promptTokens ?? 0);
|
||||
const outTok = Number(usage.outputTokens ?? usage.completionTokens ?? 0);
|
||||
const rawInTok = usage.inputTokens ?? usage.promptTokens;
|
||||
const rawOutTok = usage.outputTokens ?? usage.completionTokens;
|
||||
const inTok = Number(rawInTok ?? 0);
|
||||
const outTok = Number(rawOutTok ?? 0);
|
||||
|
||||
const stopReason = mapStopReason((result as any).finishReason, providerMetadata);
|
||||
const text = blocks
|
||||
.filter(b => b.type === 'text')
|
||||
.map(b => (b as { type: 'text'; text: string }).text)
|
||||
.join('');
|
||||
|
||||
// A completion with no usable content must not report success (#3217).
|
||||
// The check is on the CONTENT (text / tool-call blocks), never on reported
|
||||
// usage — providers commonly normalize omitted usage to zero, so a
|
||||
// usage-based gate would wave empty responses through as "free" successes
|
||||
// that silently propagate no result to every chat() caller. Refusal and
|
||||
// content-filter stops stay non-throwing: they are meaningful terminal
|
||||
// signals callers branch on (toolLoop surfaces each as its own stop
|
||||
// reason). A length stop with no content is classified distinctly — the
|
||||
// output budget was exhausted before any text was emitted (thinking
|
||||
// models spend it on internal reasoning first), which is deterministic
|
||||
// for the same call, so it gets a config error with an actionable fix
|
||||
// instead of a futile retry.
|
||||
if (
|
||||
text.trim().length === 0 &&
|
||||
!blocks.some(b => b.type === 'tool-call') &&
|
||||
stopReason !== 'refusal' &&
|
||||
stopReason !== 'content_filter'
|
||||
) {
|
||||
const label = `chat(${recipe.id}:${modelId})`;
|
||||
const emptyErr =
|
||||
stopReason === 'length'
|
||||
? new AIConfigError(
|
||||
`${label}: output budget exhausted before any content was emitted (maxOutputTokens=${maxOutputTokens})`,
|
||||
'Raise maxTokens for this call (or the configured max output tokens). Thinking models spend output budget on internal reasoning before emitting text.',
|
||||
)
|
||||
: new AITransientError(
|
||||
`${label}: provider returned an empty completion (stopReason=${stopReason}, output_tokens=${outTok}) — no text and no tool calls`,
|
||||
);
|
||||
// Mark the contentless-length case so callers with their own
|
||||
// stopReason==='length' truncation handling (facts extract's #2113
|
||||
// double-cap retry, chronicle's #2606 'truncated' failure marker)
|
||||
// can keep that handling reachable — this throw fires before they
|
||||
// can observe the stop reason. See isContentlessLengthError.
|
||||
if (stopReason === 'length') {
|
||||
(emptyErr as { contentlessLength?: boolean }).contentlessLength = true;
|
||||
}
|
||||
// Carry the real usage on the error so the catch-path budget record
|
||||
// charges actual tokens instead of the pessimistic ceiling — but only
|
||||
// the fields the provider actually reported. When usage was omitted
|
||||
// entirely, attach nothing so _extractUsageFromError keeps its
|
||||
// pessimistic fallback (attaching a synthesized {0,0} would record the
|
||||
// spend as free).
|
||||
const reportedUsage: Record<string, number> = {};
|
||||
if (typeof rawInTok === 'number' && Number.isFinite(rawInTok)) reportedUsage.input_tokens = rawInTok;
|
||||
if (typeof rawOutTok === 'number' && Number.isFinite(rawOutTok)) reportedUsage.output_tokens = rawOutTok;
|
||||
if (Object.keys(reportedUsage).length > 0) {
|
||||
(emptyErr as { usage?: unknown }).usage = reportedUsage;
|
||||
}
|
||||
throw emptyErr;
|
||||
}
|
||||
|
||||
_recordBudget(`${recipe.id}:${modelId}`, inTok, outTok);
|
||||
|
||||
return {
|
||||
text: blocks.filter(b => b.type === 'text').map(b => (b as { type: 'text'; text: string }).text).join(''),
|
||||
text,
|
||||
blocks,
|
||||
stopReason: mapStopReason((result as any).finishReason, providerMetadata),
|
||||
stopReason,
|
||||
usage: {
|
||||
input_tokens: inTok,
|
||||
output_tokens: outTok,
|
||||
@@ -3237,6 +3297,18 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* #3217 — true when `err` is the AIConfigError `chat()` throws for a
|
||||
* completion that exhausted its output budget with NO content at all
|
||||
* (the contentless 'length' stop). Callers that had their own
|
||||
* `stopReason === 'length'` truncation handling branch on this instead
|
||||
* of the (unstable) error message.
|
||||
*/
|
||||
export function isContentlessLengthError(err: unknown): boolean {
|
||||
return err instanceof AIConfigError &&
|
||||
(err as { contentlessLength?: boolean }).contentlessLength === true;
|
||||
}
|
||||
|
||||
// ---- Tool loop (v0.38 — D11 + D6/D7 gateway-native subagent path) ----
|
||||
|
||||
/**
|
||||
|
||||
@@ -192,7 +192,7 @@ const DEFAULT_JUDGE_MAX_TOKENS = 4000;
|
||||
|
||||
function defaultJudge(engine: BrainEngine): ChronicleJudge {
|
||||
return async (input) => {
|
||||
const { isAvailable, chat } = await import('../ai/gateway.ts');
|
||||
const { isAvailable, chat, isContentlessLengthError } = await import('../ai/gateway.ts');
|
||||
if (!isAvailable('chat')) return { events: [] };
|
||||
const body = (input.body || '').slice(0, 12_000);
|
||||
// #2606: configurable cap so event-dense pages have headroom.
|
||||
@@ -222,6 +222,10 @@ function defaultJudge(engine: BrainEngine): ChronicleJudge {
|
||||
text = res.text;
|
||||
} catch (err) {
|
||||
if ((err as Error)?.name === 'AbortError') throw err;
|
||||
// #3217 follow-through: a contentless 'length' stop now throws inside
|
||||
// chat() before the stopReason check above can see it — preserve the
|
||||
// #2606 truncation signal instead of degrading to a silent no-op.
|
||||
if (isContentlessLengthError(err)) return { events: [], failure: 'truncated' };
|
||||
return { events: [] };
|
||||
}
|
||||
const parsed = parseJudgeJson(text);
|
||||
|
||||
@@ -191,7 +191,9 @@ export async function runFactsBackstop(
|
||||
// identical writes (idempotent ON CONFLICT returns existing row).
|
||||
idempotency_key: `facts-absorb:${ctx.sourceId}:${parsedPage.slug}:${contentHash}`,
|
||||
max_attempts: 3,
|
||||
timeout_ms: 180_000,
|
||||
// No explicit timeout_ms: let MinionQueue.add stamp the
|
||||
// handler-timeouts.ts default (#3207 — a hardcoded 180s here
|
||||
// overrode the map and wall-clock-killed slow-gateway absorbs).
|
||||
},
|
||||
);
|
||||
return { mode: 'queue', enqueued: true, queueDepth: 0 };
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
* gateway-down errors are absorbed into NULL-embedding rows.
|
||||
*/
|
||||
|
||||
import { chat, embedOne, isAvailable } from '../ai/gateway.ts';
|
||||
import { chat, embedOne, isAvailable, isContentlessLengthError } from '../ai/gateway.ts';
|
||||
import type { ChatResult } from '../ai/gateway.ts';
|
||||
import { INJECTION_PATTERNS } from '../think/sanitize.ts';
|
||||
import { resolveModel } from '../model-config.ts';
|
||||
@@ -190,18 +190,29 @@ export async function extractFactsFromTurn(input: ExtractInput): Promise<Extract
|
||||
}`;
|
||||
let result: ChatResult;
|
||||
try {
|
||||
result = await chat({
|
||||
model,
|
||||
system: EXTRACTOR_SYSTEM,
|
||||
messages: [{ role: 'user', content: userContent }],
|
||||
maxTokens,
|
||||
abortSignal: input.abortSignal,
|
||||
});
|
||||
// #3217 follow-through: a completion that spent the WHOLE cap on
|
||||
// internal reasoning (zero text) now throws inside chat() before this
|
||||
// function can observe stopReason === 'length'. Same root cause as the
|
||||
// partial-truncation branch below → same recovery: one retry at double
|
||||
// the cap, instead of silently extracting zero facts.
|
||||
let first: ChatResult | null = null;
|
||||
try {
|
||||
first = await chat({
|
||||
model,
|
||||
system: EXTRACTOR_SYSTEM,
|
||||
messages: [{ role: 'user', content: userContent }],
|
||||
maxTokens,
|
||||
abortSignal: input.abortSignal,
|
||||
});
|
||||
} catch (err) {
|
||||
if (!isContentlessLengthError(err)) throw err;
|
||||
}
|
||||
result = first as ChatResult;
|
||||
// #2113: never checked pre-fix — a truncated response (stopReason
|
||||
// 'length', e.g. reasoning tokens eating the cap on mandatory-reasoning
|
||||
// models) produced unparseable JSON and silently extracted zero facts.
|
||||
// Retry ONCE at double the cap, then surface the truncation loudly.
|
||||
if (result.stopReason === 'length') {
|
||||
if (first === null || first.stopReason === 'length') {
|
||||
process.stderr.write(
|
||||
`[facts-extract] WARN: extractor output truncated at maxTokens=${maxTokens} ` +
|
||||
`(model=${model}); retrying once at ${maxTokens * 2}\n`,
|
||||
|
||||
+88
-23
@@ -14,6 +14,8 @@
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import type { PageType } from './types.ts';
|
||||
import { ensureWellFormed } from './text-safe.ts';
|
||||
import { inferLinkTypeFromPack, type PackLinkExtraction } from './schema-pack/link-inference.ts';
|
||||
import { PageRegexBudget } from './schema-pack/redos-guard.ts';
|
||||
|
||||
/**
|
||||
* v0.42.7 — link-extraction version stamp. Bump this ISO timestamp whenever the
|
||||
@@ -28,7 +30,7 @@ import { ensureWellFormed } from './text-safe.ts';
|
||||
* OR updated_at > links_extracted_at`. It is an ISO-8601 string (NOT a number) —
|
||||
* the column is TIMESTAMPTZ and the predicate binds it as `::timestamptz`.
|
||||
*/
|
||||
export const LINK_EXTRACTOR_VERSION_TS = '2026-05-31T00:00:00Z';
|
||||
export const LINK_EXTRACTOR_VERSION_TS = '2026-07-23T00:00:00Z';
|
||||
|
||||
// ─── Entity references ──────────────────────────────────────────
|
||||
|
||||
@@ -83,7 +85,22 @@ export type LinkResolutionType = 'qualified' | 'unqualified';
|
||||
* - Our domain extensions: tech, finance, personal, openclaw (domain-organized wikis)
|
||||
* - Our entity prefix: entities (we kept some legacy entities/projects/ pages)
|
||||
*/
|
||||
const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities)';
|
||||
const DIR_ALTERNATIVES = 'people|companies|meetings|concepts|deal|civic|project|projects|source|media|yc|tech|finance|personal|openclaw|entities';
|
||||
const DIR_PATTERN = `(?:${DIR_ALTERNATIVES})`;
|
||||
|
||||
/**
|
||||
* #3190 — union the built-in dir whitelist with pack-declared entity dirs
|
||||
* (from `page_types[].path_prefixes`, pre-validated to a slug-safe charset
|
||||
* by `packLinkExtractionView`). Longer alternatives sort first so a nested
|
||||
* prefix like `wiki/people` wins over a hypothetical `wiki`.
|
||||
*/
|
||||
function dirPatternWith(extraDirs?: string[]): string {
|
||||
if (!extraDirs || extraDirs.length === 0) return DIR_PATTERN;
|
||||
const extras = [...new Set(extraDirs)]
|
||||
.sort((a, b) => (b.length - a.length) || a.localeCompare(b))
|
||||
.map(d => d.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
|
||||
return `(?:${extras.join('|')}|${DIR_ALTERNATIVES})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Match `[Name](path)` markdown links pointing to entity directories.
|
||||
@@ -95,8 +112,8 @@ const DIR_PATTERN = '(?:people|companies|meetings|concepts|deal|civic|project|pr
|
||||
* The regex permits an optional `../` prefix (any number) and an optional
|
||||
* `.md` suffix so the same function works for both filesystem and DB content.
|
||||
*/
|
||||
const ENTITY_REF_RE = new RegExp(
|
||||
`\\[([^\\]]+)\\]\\((?:\\.\\.\\/)*(${DIR_PATTERN}\\/[^)\\s]+?)(?:\\.md)?\\)`,
|
||||
const entityRefRe = (dirPattern: string) => new RegExp(
|
||||
`\\[([^\\]]+)\\]\\((?:\\.\\.\\/)*(${dirPattern}\\/[^)\\s]+?)(?:\\.md)?\\)`,
|
||||
'g',
|
||||
);
|
||||
|
||||
@@ -104,12 +121,12 @@ const ENTITY_REF_RE = new RegExp(
|
||||
* Match Obsidian-style `[[path]]` or `[[path|Display Text]]` wikilinks.
|
||||
* Captures: slug (dir/...), displayName (optional).
|
||||
*
|
||||
* Same dir whitelist as ENTITY_REF_RE. Strips trailing `.md`, strips section
|
||||
* Same dir whitelist as entityRefRe. Strips trailing `.md`, strips section
|
||||
* anchors (`#heading`), skips external URLs. Wiki KBs use this format almost
|
||||
* exclusively so missing it leaves the graph empty.
|
||||
*/
|
||||
const WIKILINK_RE = new RegExp(
|
||||
`\\[\\[(${DIR_PATTERN}\\/[^|\\]#]+?)(?:#[^|\\]]*?)?(?:\\|([^\\]]+?))?\\]\\]`,
|
||||
const wikilinkRe = (dirPattern: string) => new RegExp(
|
||||
`\\[\\[(${dirPattern}\\/[^|\\]#]+?)(?:#[^|\\]]*?)?(?:\\|([^\\]]+?))?\\]\\]`,
|
||||
'g',
|
||||
);
|
||||
|
||||
@@ -121,12 +138,12 @@ const WIKILINK_RE = new RegExp(
|
||||
*
|
||||
* Captures: sourceId, slug (dir/...), displayName (optional).
|
||||
*
|
||||
* Matched BEFORE WIKILINK_RE so `[[wiki:topics/ai]]` isn't mis-parsed by
|
||||
* the unqualified regex (the source prefix would not satisfy DIR_PATTERN
|
||||
* anyway, but the two-pass approach keeps intent crystal-clear).
|
||||
* Matched BEFORE the unqualified wikilink pass so `[[wiki:topics/ai]]` isn't
|
||||
* mis-parsed by the unqualified regex (the source prefix would not satisfy
|
||||
* DIR_PATTERN anyway, but the two-pass approach keeps intent crystal-clear).
|
||||
*/
|
||||
const QUALIFIED_WIKILINK_RE = new RegExp(
|
||||
`\\[\\[([a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?):(${DIR_PATTERN}\\/[^|\\]#]+?)(?:#[^|\\]]*?)?(?:\\|([^\\]]+?))?\\]\\]`,
|
||||
const qualifiedWikilinkRe = (dirPattern: string) => new RegExp(
|
||||
`\\[\\[([a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?):(${dirPattern}\\/[^|\\]#]+?)(?:#[^|\\]]*?)?(?:\\|([^\\]]+?))?\\]\\]`,
|
||||
'g',
|
||||
);
|
||||
|
||||
@@ -298,17 +315,19 @@ export function extractCodeRefs(content: string): CodeRef[] {
|
||||
* here; caller dedups). Slugs appearing inside fenced or inline code blocks
|
||||
* are excluded — those are typically code samples, not real entity references.
|
||||
*/
|
||||
export function extractEntityRefs(content: string): EntityRef[] {
|
||||
export function extractEntityRefs(content: string, extraDirs?: string[]): EntityRef[] {
|
||||
const stripped = stripCodeBlocks(content);
|
||||
const refs: EntityRef[] = [];
|
||||
let match: RegExpExecArray | null;
|
||||
// #3190: pack-declared entity dirs widen the whitelist for this call.
|
||||
const dirPattern = dirPatternWith(extraDirs);
|
||||
|
||||
// 1. Markdown links: [Name](path)
|
||||
// Markdown links have no source-qualification syntax — they're
|
||||
// always unqualified. Omit sourceId so the shape stays compatible
|
||||
// with pre-v0.17 consumers doing strict equality.
|
||||
const markdownRanges: Array<[number, number]> = [];
|
||||
const mdPattern = new RegExp(ENTITY_REF_RE.source, ENTITY_REF_RE.flags);
|
||||
const mdPattern = entityRefRe(dirPattern);
|
||||
while ((match = mdPattern.exec(stripped)) !== null) {
|
||||
const name = match[1];
|
||||
const fullPath = match[2];
|
||||
@@ -322,7 +341,7 @@ export function extractEntityRefs(content: string): EntityRef[] {
|
||||
// Must run BEFORE the unqualified pass or we'd double-emit. We also
|
||||
// mask out the matched spans so pass 2b can't grab them.
|
||||
const qualifiedRanges: Array<[number, number]> = [];
|
||||
const qualPattern = new RegExp(QUALIFIED_WIKILINK_RE.source, QUALIFIED_WIKILINK_RE.flags);
|
||||
const qualPattern = qualifiedWikilinkRe(dirPattern);
|
||||
while ((match = qualPattern.exec(stripped)) !== null) {
|
||||
const sourceId = match[1];
|
||||
let slug = match[2].trim();
|
||||
@@ -339,7 +358,7 @@ export function extractEntityRefs(content: string): EntityRef[] {
|
||||
// Same shape rule: omit sourceId when unqualified.
|
||||
const unqualifiedRanges: Array<[number, number]> = [];
|
||||
const unmasked = maskRanges(stripped, qualifiedRanges);
|
||||
const wikiPattern = new RegExp(WIKILINK_RE.source, WIKILINK_RE.flags);
|
||||
const wikiPattern = wikilinkRe(dirPattern);
|
||||
while ((match = wikiPattern.exec(unmasked)) !== null) {
|
||||
let slug = match[1].trim();
|
||||
if (!slug) continue;
|
||||
@@ -468,12 +487,35 @@ export async function extractPageLinks(
|
||||
frontmatter: Record<string, unknown>,
|
||||
pageType: PageType,
|
||||
resolver: SlugResolver,
|
||||
opts: { globalBasename?: boolean; skipFrontmatter?: boolean } = {},
|
||||
opts: { globalBasename?: boolean; skipFrontmatter?: boolean; pack?: PackLinkExtraction | null } = {},
|
||||
): Promise<PageLinksResult> {
|
||||
const candidates: LinkCandidate[] = [];
|
||||
|
||||
// #3190: pack-aware extraction. When the caller threads the active
|
||||
// (non-base) pack, three things widen:
|
||||
// - entity-dir whitelist unions in pack path_prefixes,
|
||||
// - link-verb inference consults pack link_types BEFORE the legacy
|
||||
// matchers (the documented wrap in schema-pack/link-inference.ts),
|
||||
// - frontmatter_links mappings extend FRONTMATTER_LINK_MAP.
|
||||
// No pack (or a codegen'd base pack) → behavior is byte-identical to
|
||||
// the legacy path.
|
||||
const pack = opts.pack ?? null;
|
||||
const extraDirs = pack?.entity_dirs;
|
||||
// One ReDoS budget per page, shared across every per-candidate inference
|
||||
// call (mirrors extract-ner's per-page budget).
|
||||
const regexBudget = pack ? new PageRegexBudget() : null;
|
||||
const inferType = (context: string, targetSlug: string): string => {
|
||||
if (pack) {
|
||||
try {
|
||||
const verb = inferLinkTypeFromPack(pack, pageType as string, context, regexBudget ?? undefined);
|
||||
if (verb) return verb;
|
||||
} catch { /* pack inference is best-effort; fall through to legacy */ }
|
||||
}
|
||||
return inferLinkType(pageType, context, content, targetSlug);
|
||||
};
|
||||
|
||||
// 1. Markdown entity refs.
|
||||
for (const ref of extractEntityRefs(content)) {
|
||||
for (const ref of extractEntityRefs(content, extraDirs)) {
|
||||
// Issue #972: refs from the generic `[[bare-name]]` pass carry the
|
||||
// literal wikilink text, not a real page slug. When global_basename
|
||||
// mode is on AND the resolver implements basename lookup, resolve
|
||||
@@ -529,7 +571,7 @@ export async function extractPageLinks(
|
||||
const context = idx >= 0 ? excerpt(content, idx, 240) : ref.name;
|
||||
candidates.push({
|
||||
targetSlug: ref.slug,
|
||||
linkType: inferLinkType(pageType, context, content, ref.slug),
|
||||
linkType: inferType(context, ref.slug),
|
||||
context,
|
||||
linkSource: 'markdown',
|
||||
});
|
||||
@@ -540,7 +582,7 @@ export async function extractPageLinks(
|
||||
// Code blocks are stripped first — slugs in code samples are not real refs.
|
||||
const strippedContent = stripCodeBlocks(content);
|
||||
const bareRe = new RegExp(
|
||||
`\\b(${DIR_PATTERN}\\/[a-z0-9][a-z0-9/-]*[a-z0-9])\\b`,
|
||||
`\\b(${dirPatternWith(extraDirs)}\\/[a-z0-9][a-z0-9/-]*[a-z0-9])\\b`,
|
||||
'g',
|
||||
);
|
||||
let m: RegExpExecArray | null;
|
||||
@@ -551,7 +593,7 @@ export async function extractPageLinks(
|
||||
const context = excerpt(strippedContent, m.index, 240);
|
||||
candidates.push({
|
||||
targetSlug: m[1],
|
||||
linkType: inferLinkType(pageType, context, content, m[1]),
|
||||
linkType: inferType(context, m[1]),
|
||||
context,
|
||||
linkSource: 'markdown',
|
||||
});
|
||||
@@ -567,7 +609,7 @@ export async function extractPageLinks(
|
||||
// path needed `resolveBasenameMatches` on the real resolver.
|
||||
let fmUnresolved: UnresolvedFrontmatterRef[] = [];
|
||||
if (!opts.skipFrontmatter) {
|
||||
const fm = await extractFrontmatterLinks(slug, pageType, frontmatter, resolver);
|
||||
const fm = await extractFrontmatterLinks(slug, pageType, frontmatter, resolver, pack);
|
||||
candidates.push(...fm.candidates);
|
||||
fmUnresolved = fm.unresolved;
|
||||
}
|
||||
@@ -1047,11 +1089,34 @@ export async function extractFrontmatterLinks(
|
||||
pageType: PageType,
|
||||
frontmatter: Record<string, unknown>,
|
||||
resolver: SlugResolver,
|
||||
pack?: PackLinkExtraction | null,
|
||||
): Promise<FrontmatterExtractResult> {
|
||||
const candidates: LinkCandidate[] = [];
|
||||
const unresolved: UnresolvedFrontmatterRef[] = [];
|
||||
|
||||
for (const mapping of FRONTMATTER_LINK_MAP) {
|
||||
// #3190: pack-declared frontmatter_links extend the hardcoded map.
|
||||
// The hardcoded map WINS on any (pageType, field) it already covers —
|
||||
// it carries direction + dir hints the pack schema doesn't — so a pack
|
||||
// restating a built-in field can't emit reversed/duplicate edges.
|
||||
// Pack mappings are outgoing (page → resolved value); the pack's
|
||||
// entity dirs serve as resolution hints so bare-name values resolve
|
||||
// into pack-declared layouts (e.g. `wiki/people/<slug>`).
|
||||
const mappings: FrontmatterFieldMapping[] = [...FRONTMATTER_LINK_MAP];
|
||||
for (const fl of pack?.frontmatter_links ?? []) {
|
||||
const freeFields = fl.fields.filter(f =>
|
||||
!FRONTMATTER_LINK_MAP.some(m =>
|
||||
m.fields.includes(f) && (m.pageType === undefined || m.pageType === fl.page_type)));
|
||||
if (freeFields.length === 0) continue;
|
||||
mappings.push({
|
||||
fields: freeFields,
|
||||
pageType: fl.page_type,
|
||||
type: fl.link_type,
|
||||
direction: 'outgoing',
|
||||
dirHint: pack?.entity_dirs ?? '',
|
||||
});
|
||||
}
|
||||
|
||||
for (const mapping of mappings) {
|
||||
if (mapping.pageType && mapping.pageType !== pageType) continue;
|
||||
for (const field of mapping.fields) {
|
||||
const value = frontmatter[field];
|
||||
|
||||
@@ -43,6 +43,9 @@ export const HANDLER_DEFAULT_TIMEOUT_MS: Readonly<Record<string, number>> = {
|
||||
// few writes. Generous 10-min budget (vs the tight null-default) covers a
|
||||
// slow gateway without the 30-min loop budget.
|
||||
chronicle_extract: TEN_MIN_MS,
|
||||
// #3207 — facts absorb: same one-page-one-LLM-call shape as
|
||||
// chronicle_extract; a slow gateway must not wall-clock-kill it.
|
||||
'facts-absorb': TEN_MIN_MS,
|
||||
// Per-page contextual reindex jobs process chunks sequentially with one
|
||||
// rate-leased LLM synopsis call per chunk; large transcript pages need more
|
||||
// than the standard 30-min long-job budget.
|
||||
|
||||
+10
-1
@@ -1161,9 +1161,18 @@ async function runAutoLink(
|
||||
const resolver = makeResolver(engine, { mode: 'live', sourceId: opts?.sourceId });
|
||||
// Issue #972: opt-in bare-wikilink basename resolution. Off by default.
|
||||
const globalBasename = await isGlobalBasenameEnabled(engine);
|
||||
// #3190: consult the active custom pack (path_prefixes / link verbs /
|
||||
// frontmatter_links) so put_page auto-link matches what `gbrain extract
|
||||
// links --source db` produces. Best-effort: null for base packs or on
|
||||
// any load failure.
|
||||
const { loadActivePackBestEffort } = await import('./schema-pack/best-effort.ts');
|
||||
const { packLinkExtractionView } = await import('./schema-pack/link-inference.ts');
|
||||
const pack = packLinkExtractionView(
|
||||
await loadActivePackBestEffort({ engine, sourceId: opts?.sourceId } as never),
|
||||
);
|
||||
const { candidates, unresolved } = await extractPageLinks(
|
||||
slug, fullContent, parsed.frontmatter, parsed.type, resolver,
|
||||
{ globalBasename },
|
||||
{ globalBasename, pack },
|
||||
);
|
||||
|
||||
// Resolve which targets exist (skip refs to non-existent pages to avoid FK
|
||||
|
||||
@@ -116,3 +116,58 @@ export function frontmatterLinkTypeFromPack(
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// #3190 — pack-aware plain link extraction.
|
||||
//
|
||||
// The codegen'd base packs are generated FROM the in-code tables
|
||||
// (inferLinkType's production regexes + FRONTMATTER_LINK_MAP + the
|
||||
// DIR_PATTERN dirs — see scripts/generate-gbrain-base.ts). Consulting them
|
||||
// at extraction time would double-apply WORSE copies of the same semantics:
|
||||
// the YAML carries simplified sketch regexes, and its frontmatter_links
|
||||
// entries have no direction field (the hardcoded map's `incoming` entries
|
||||
// like key_people would come back reversed). Custom packs are the ones
|
||||
// extraction must honor.
|
||||
const CODEGEN_BASE_PACKS = new Set(['gbrain-base', 'gbrain-base-v2']);
|
||||
|
||||
/**
|
||||
* The slice of a resolved pack that plain link extraction
|
||||
* (`extractPageLinks` / `extractFrontmatterLinks`) consumes.
|
||||
*/
|
||||
export interface PackLinkExtraction {
|
||||
link_types: SchemaPackManifest['link_types'];
|
||||
frontmatter_links: SchemaPackManifest['frontmatter_links'];
|
||||
/**
|
||||
* Entity-dir alternatives derived from `page_types[].path_prefixes`
|
||||
* (leading/trailing slashes stripped). Unioned into DIR_PATTERN so
|
||||
* markdown/wikilink targets under pack-declared layouts (e.g.
|
||||
* `wiki/people/…`) become extraction candidates.
|
||||
*/
|
||||
entity_dirs: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Project a resolved active pack down to what link extraction needs.
|
||||
* Returns null (extraction stays purely legacy) when there is no pack or
|
||||
* the pack is one of the codegen'd base packs.
|
||||
*/
|
||||
export function packLinkExtractionView(
|
||||
pack: { manifest: SchemaPackManifest } | null | undefined,
|
||||
): PackLinkExtraction | null {
|
||||
const m = pack?.manifest;
|
||||
if (!m || CODEGEN_BASE_PACKS.has(m.name)) return null;
|
||||
const dirs = new Set<string>();
|
||||
for (const pt of m.page_types) {
|
||||
for (const prefix of pt.path_prefixes ?? []) {
|
||||
const d = prefix.replace(/^\/+/, '').replace(/\/+$/, '');
|
||||
// Only slug-shaped prefixes participate in regex union; anything
|
||||
// else (globs, regex metachars) is skipped rather than escaped so
|
||||
// the entity regexes stay predictable.
|
||||
if (d && /^[A-Za-z0-9][A-Za-z0-9/_-]*$/.test(d)) dirs.add(d);
|
||||
}
|
||||
}
|
||||
return {
|
||||
link_types: m.link_types,
|
||||
frontmatter_links: m.frontmatter_links,
|
||||
entity_dirs: [...dirs],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
/**
|
||||
* #3217 — zero-token empty completions must not report success.
|
||||
*
|
||||
* Root cause: `chat()` returned `{ text: '', blocks: [] }` for a contentless
|
||||
* non-refusal completion, so an empty provider response silently propagated
|
||||
* "no result" to every caller as if it were a valid answer. Providers commonly
|
||||
* normalize omitted usage to zero output tokens, so any usage-based gate would
|
||||
* treat those responses as free successes — the guard therefore validates
|
||||
* usable CONTENT (text / tool-call blocks) independently of reported usage.
|
||||
*
|
||||
* Pinned behavior:
|
||||
* - contentless + stop → AITransientError (retryable provider blip)
|
||||
* - contentless + usage omitted entirely → same (usage-independent check)
|
||||
* - contentless + length → AIConfigError with an actionable fix
|
||||
* (output budget exhausted before any text — deterministic, not retryable)
|
||||
* - refusal / content_filter → still returned, NOT thrown (callers branch
|
||||
* on these stop reasons; toolLoop surfaces each as its own stop reason)
|
||||
* - non-empty text + zero usage → success (zero usage alone is not failure)
|
||||
* - tool-call-only completion → success (a tool call IS usable content)
|
||||
* - budget: the throw carries real usage so the catch-path record charges
|
||||
* actual tokens, not the pessimistic maxOutputTokens ceiling
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
chat,
|
||||
configureGateway,
|
||||
isContentlessLengthError,
|
||||
resetGateway,
|
||||
withBudgetTracker,
|
||||
__setGenerateTextTransportForTests,
|
||||
} from '../../src/core/ai/gateway.ts';
|
||||
import { AIConfigError, AITransientError } from '../../src/core/ai/errors.ts';
|
||||
import {
|
||||
BudgetTracker,
|
||||
_resetBudgetTrackerWarningsForTest,
|
||||
} from '../../src/core/budget/budget-tracker.ts';
|
||||
|
||||
function installTransport(response: Record<string, unknown>): void {
|
||||
__setGenerateTextTransportForTests(async () => response as any);
|
||||
}
|
||||
|
||||
function callChat() {
|
||||
return chat({
|
||||
model: 'anthropic:claude-sonnet-4-6',
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
});
|
||||
}
|
||||
|
||||
describe('#3217 — empty-completion guard in chat()', () => {
|
||||
beforeEach(() => {
|
||||
resetGateway();
|
||||
__setGenerateTextTransportForTests(null);
|
||||
configureGateway({
|
||||
chat_model: 'anthropic:claude-sonnet-4-6',
|
||||
env: { ANTHROPIC_API_KEY: 'fake' },
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
__setGenerateTextTransportForTests(null);
|
||||
resetGateway();
|
||||
});
|
||||
|
||||
test('contentless completion with zero-normalized usage throws AITransientError', async () => {
|
||||
installTransport({
|
||||
content: [],
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 42, outputTokens: 0 },
|
||||
});
|
||||
await expect(callChat()).rejects.toThrow(AITransientError);
|
||||
});
|
||||
|
||||
test('contentless completion with usage omitted entirely still throws (check is usage-independent)', async () => {
|
||||
installTransport({
|
||||
content: [],
|
||||
finishReason: 'stop',
|
||||
// no `usage` field at all — the provider omitted it
|
||||
});
|
||||
await expect(callChat()).rejects.toThrow(AITransientError);
|
||||
});
|
||||
|
||||
test('whitespace-only text is not usable content', async () => {
|
||||
installTransport({
|
||||
content: [{ type: 'text', text: '\n \n' }],
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 42, outputTokens: 3 },
|
||||
});
|
||||
await expect(callChat()).rejects.toThrow(AITransientError);
|
||||
});
|
||||
|
||||
test('contentless length stop is classified distinctly as AIConfigError with a fix', async () => {
|
||||
installTransport({
|
||||
content: [],
|
||||
finishReason: 'length',
|
||||
usage: { inputTokens: 42, outputTokens: 4096 },
|
||||
});
|
||||
try {
|
||||
await callChat();
|
||||
throw new Error('should have thrown');
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(AIConfigError);
|
||||
expect((e as AIConfigError).message).toContain('output budget exhausted');
|
||||
expect((e as AIConfigError).fix).toBeDefined();
|
||||
// Callers with their own length-stop truncation handling (facts
|
||||
// extract #2113, chronicle #2606) branch on this predicate.
|
||||
expect(isContentlessLengthError(e)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('isContentlessLengthError is false for the non-length empty-completion throw', async () => {
|
||||
installTransport({
|
||||
content: [],
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 42, outputTokens: 0 },
|
||||
});
|
||||
try {
|
||||
await callChat();
|
||||
throw new Error('should have thrown');
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(AITransientError);
|
||||
expect(isContentlessLengthError(e)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('refusal stop with empty content is returned, not thrown', async () => {
|
||||
installTransport({
|
||||
content: [],
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 42, outputTokens: 0 },
|
||||
providerMetadata: { anthropic: { stopReason: 'refusal' } },
|
||||
});
|
||||
const result = await callChat();
|
||||
expect(result.stopReason).toBe('refusal');
|
||||
expect(result.text).toBe('');
|
||||
});
|
||||
|
||||
test('content_filter stop with empty content is returned, not thrown', async () => {
|
||||
installTransport({
|
||||
content: [],
|
||||
finishReason: 'content-filter',
|
||||
usage: { inputTokens: 42, outputTokens: 0 },
|
||||
});
|
||||
const result = await callChat();
|
||||
expect(result.stopReason).toBe('content_filter');
|
||||
expect(result.text).toBe('');
|
||||
});
|
||||
|
||||
test('non-empty text with zero-reported usage succeeds (zero usage alone is not failure)', async () => {
|
||||
installTransport({
|
||||
content: [{ type: 'text', text: 'a real answer' }],
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 0, outputTokens: 0 },
|
||||
});
|
||||
const result = await callChat();
|
||||
expect(result.text).toBe('a real answer');
|
||||
expect(result.usage.output_tokens).toBe(0);
|
||||
});
|
||||
|
||||
test('tool-call-only completion (no text) succeeds', async () => {
|
||||
installTransport({
|
||||
content: [
|
||||
{ type: 'tool-call', toolCallId: 'tc_1', toolName: 'lookup', input: { q: 'x' } },
|
||||
],
|
||||
finishReason: 'tool-calls',
|
||||
usage: { inputTokens: 42, outputTokens: 12 },
|
||||
});
|
||||
const result = await callChat();
|
||||
expect(result.stopReason).toBe('tool_calls');
|
||||
expect(result.blocks).toEqual([
|
||||
{ type: 'tool-call', toolCallId: 'tc_1', toolName: 'lookup', input: { q: 'x' } },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3217 — budget recording on the empty-completion throw', () => {
|
||||
let tmp: string;
|
||||
let auditPath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmp = mkdtempSync(join(tmpdir(), 'gbrain-empty-completion-'));
|
||||
auditPath = join(tmp, 'budget.jsonl');
|
||||
_resetBudgetTrackerWarningsForTest();
|
||||
resetGateway();
|
||||
__setGenerateTextTransportForTests(null);
|
||||
configureGateway({
|
||||
chat_model: 'anthropic:claude-sonnet-4-6',
|
||||
env: { ANTHROPIC_API_KEY: 'fake' },
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
__setGenerateTextTransportForTests(null);
|
||||
resetGateway();
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('throw records ACTUAL usage, not the pessimistic maxOutputTokens ceiling', async () => {
|
||||
const usage = { inputTokens: 42, outputTokens: 0 };
|
||||
|
||||
// Control: a successful call with identical usage.
|
||||
installTransport({
|
||||
content: [{ type: 'text', text: 'ok' }],
|
||||
finishReason: 'stop',
|
||||
usage,
|
||||
});
|
||||
const control = new BudgetTracker({ maxCostUsd: 1.0, label: 'control', auditPath });
|
||||
await withBudgetTracker(control, async () => {
|
||||
await callChat();
|
||||
});
|
||||
|
||||
// Under test: an empty completion with the same reported usage.
|
||||
installTransport({
|
||||
content: [],
|
||||
finishReason: 'stop',
|
||||
usage,
|
||||
});
|
||||
const tracker = new BudgetTracker({ maxCostUsd: 1.0, label: 'empty', auditPath });
|
||||
await withBudgetTracker(tracker, async () => {
|
||||
await expect(callChat()).rejects.toThrow(AITransientError);
|
||||
});
|
||||
|
||||
// Same usage → same spend. If the guard threw without carrying usage, the
|
||||
// catch path would charge estimated input + the full 4096-token output
|
||||
// ceiling and this assertion would fail loudly.
|
||||
expect(tracker.totalSpent).toBe(control.totalSpent);
|
||||
expect(tracker.totalSpent).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('omitted usage keeps the pessimistic fallback (never recorded as free)', async () => {
|
||||
// The provider omitted usage entirely. The guard must NOT attach a
|
||||
// synthesized {0,0} to the error — that would bypass
|
||||
// _extractUsageFromError's pessimistic fallback and record the failed
|
||||
// call as costing nothing.
|
||||
installTransport({
|
||||
content: [],
|
||||
finishReason: 'stop',
|
||||
// no `usage` field at all
|
||||
});
|
||||
const tracker = new BudgetTracker({ maxCostUsd: 1.0, label: 'omitted', auditPath });
|
||||
await withBudgetTracker(tracker, async () => {
|
||||
await expect(callChat()).rejects.toThrow(AITransientError);
|
||||
});
|
||||
// Pessimistic fallback = estimated input + full maxOutputTokens ceiling,
|
||||
// which is strictly positive. A {0,0} attach would make this exactly 0.
|
||||
expect(tracker.totalSpent).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -301,3 +301,36 @@ describe('runFactsBackstop — stub guard routing (v0.34.5)', () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3207 — durable facts-absorb submit uses the handler-timeout map', () => {
|
||||
test('defaultTimeoutMsFor stamps facts-absorb with the 10-min budget (matches chronicle_extract)', async () => {
|
||||
const { defaultTimeoutMsFor } = await import('../src/core/minions/handler-timeouts.ts');
|
||||
expect(defaultTimeoutMsFor('facts-absorb')).toBe(10 * 60 * 1000);
|
||||
expect(defaultTimeoutMsFor('facts-absorb')).toBe(defaultTimeoutMsFor('chronicle_extract'));
|
||||
});
|
||||
|
||||
test('short-lived CLI durable submit inherits the map default (no hardcoded 180s override)', async () => {
|
||||
const { markShortLivedCliProcess, __resetShortLivedCliForTests } =
|
||||
await import('../src/core/facts/cli-process-mode.ts');
|
||||
markShortLivedCliProcess();
|
||||
try {
|
||||
const page = meetingPage();
|
||||
const r = await runFactsBackstop(page, makeCtx());
|
||||
expect(r.mode).toBe('queue');
|
||||
if (r.mode === 'queue') expect(r.enqueued).toBe(true);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const rows = await (engine as any).db.query(
|
||||
`SELECT timeout_ms FROM minion_jobs WHERE name = 'facts-absorb' AND data->>'slug' = $1`,
|
||||
[page.slug],
|
||||
);
|
||||
expect(rows.rows.length).toBe(1);
|
||||
// Submit-time stamp comes from HANDLER_DEFAULT_TIMEOUT_MS, not the old
|
||||
// hardcoded 180_000 that overrode any map-only fix (the map's contract:
|
||||
// "An explicit opts.timeout_ms always wins").
|
||||
expect(Number(rows.rows[0].timeout_ms)).toBe(10 * 60 * 1000);
|
||||
} finally {
|
||||
__resetShortLivedCliForTests();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
configureGateway,
|
||||
resetGateway,
|
||||
__setChatTransportForTests,
|
||||
__setGenerateTextTransportForTests,
|
||||
} from '../src/core/ai/gateway.ts';
|
||||
import type { ChatOpts, ChatResult } from '../src/core/ai/gateway.ts';
|
||||
import {
|
||||
@@ -29,6 +30,7 @@ import type { BrainEngine } from '../src/core/engine.ts';
|
||||
beforeEach(() => {
|
||||
resetGateway();
|
||||
__setChatTransportForTests(null);
|
||||
__setGenerateTextTransportForTests(null);
|
||||
configureGateway({
|
||||
chat_model: 'anthropic:claude-sonnet-4-6',
|
||||
env: { ANTHROPIC_API_KEY: 'sk-ant-test' },
|
||||
@@ -129,4 +131,35 @@ describe('extractFactsFromTurn truncation handling (#2113)', () => {
|
||||
expect(calls).toBe(2);
|
||||
expect(facts).toEqual([]);
|
||||
});
|
||||
|
||||
// #3217 follow-through: a completion that spends the WHOLE cap on internal
|
||||
// reasoning (zero text) now throws inside chat() — before this function can
|
||||
// observe stopReason === 'length'. The retry-at-double-cap recovery must
|
||||
// still fire. Uses the generateText transport (NOT the chat transport) so
|
||||
// the REAL empty-completion guard runs.
|
||||
test("contentless 'length' throw from chat() still triggers the double-cap retry", async () => {
|
||||
const seen: Array<{ maxOutputTokens?: number }> = [];
|
||||
__setGenerateTextTransportForTests(async (opts: any) => {
|
||||
seen.push({ maxOutputTokens: opts.maxOutputTokens });
|
||||
return (seen.length === 1
|
||||
? { content: [], finishReason: 'length', usage: { inputTokens: 42, outputTokens: 4000 } }
|
||||
: {
|
||||
content: [{ type: 'text', text: GOOD_JSON }],
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 42, outputTokens: 60 },
|
||||
}) as any;
|
||||
});
|
||||
try {
|
||||
const facts = await extractFactsFromTurn({
|
||||
turnText: 'I gave up alcohol.',
|
||||
source: 'test:truncation',
|
||||
});
|
||||
expect(seen).toHaveLength(2);
|
||||
expect(seen[1]!.maxOutputTokens).toBe(seen[0]!.maxOutputTokens! * 2);
|
||||
expect(facts).toHaveLength(1);
|
||||
expect(facts[0]!.fact).toContain('alcohol');
|
||||
} finally {
|
||||
__setGenerateTextTransportForTests(null);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* #3190 — pack-aware plain link extraction.
|
||||
*
|
||||
* A custom schema pack's `page_types[].path_prefixes`, `link_types[].inference`
|
||||
* and `frontmatter_links` must drive `extractPageLinks` /
|
||||
* `extractFrontmatterLinks` (the plain fs/db extract path + put_page
|
||||
* auto-link), not just the `--ner` path. Three gates pinned here:
|
||||
* 1. entity-dir whitelist unions in pack path_prefixes,
|
||||
* 2. pack verb regexes are consulted before legacy inferLinkType,
|
||||
* 3. pack frontmatter_links extend FRONTMATTER_LINK_MAP (hardcoded map wins
|
||||
* on collision so base-pack-shaped entries can't emit reversed edges).
|
||||
* Plus the guard: codegen'd base packs are ignored (their semantics already
|
||||
* live in the in-code tables).
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
extractEntityRefs,
|
||||
extractPageLinks,
|
||||
extractFrontmatterLinks,
|
||||
type SlugResolver,
|
||||
} from '../src/core/link-extraction.ts';
|
||||
import {
|
||||
packLinkExtractionView,
|
||||
type PackLinkExtraction,
|
||||
} from '../src/core/schema-pack/link-inference.ts';
|
||||
import type { PageType } from '../src/core/types.ts';
|
||||
|
||||
const nullResolver: SlugResolver = { resolve: async () => null };
|
||||
|
||||
/** Minimal custom-pack view matching the issue's genealogy repro. */
|
||||
const packView: PackLinkExtraction = {
|
||||
link_types: [
|
||||
{ name: 'parent_of', inference: { regex: '[Pp]arent of \\[' } },
|
||||
{ name: 'member_of_line', inference: { regex: 'member of the \\[' } },
|
||||
],
|
||||
frontmatter_links: [
|
||||
{ page_type: 'person', fields: ['parents'], link_type: 'parent_of' },
|
||||
// Collides with the hardcoded map (company/key_people is incoming
|
||||
// works_at) — must be skipped, not applied outgoing.
|
||||
{ page_type: 'company', fields: ['key_people'], link_type: 'employs' },
|
||||
],
|
||||
entity_dirs: ['wiki/people', 'lines'],
|
||||
};
|
||||
|
||||
describe('#3190 gate 1 — pack path_prefixes widen the entity-dir whitelist', () => {
|
||||
const content = 'member of the [Pettit](../lines/pettit.md) line and ' +
|
||||
'parent of [Harriet](wiki/people/pettit-harriet-emeline-1828.md).';
|
||||
|
||||
test('without extra dirs, non-whitelisted targets yield zero refs', () => {
|
||||
expect(extractEntityRefs(content)).toEqual([]);
|
||||
});
|
||||
|
||||
test('with pack entity dirs, both targets become candidates', () => {
|
||||
const refs = extractEntityRefs(content, packView.entity_dirs);
|
||||
expect(refs.map(r => r.slug).sort()).toEqual([
|
||||
'lines/pettit',
|
||||
'wiki/people/pettit-harriet-emeline-1828',
|
||||
]);
|
||||
});
|
||||
|
||||
test('wikilinks honor pack dirs too', () => {
|
||||
const refs = extractEntityRefs('[[wiki/people/pettit-james-b-1777|James]]', packView.entity_dirs);
|
||||
expect(refs).toHaveLength(1);
|
||||
expect(refs[0].slug).toBe('wiki/people/pettit-james-b-1777');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3190 gate 2 — pack verb inference runs on the plain extract path', () => {
|
||||
test('pack regex verb wins over legacy mentions', async () => {
|
||||
const { candidates } = await extractPageLinks(
|
||||
'wiki/people/pettit-james-b-1777',
|
||||
'parent of [Harriet](wiki/people/pettit-harriet-emeline-1828.md)',
|
||||
{},
|
||||
'person' as PageType,
|
||||
nullResolver,
|
||||
{ pack: packView },
|
||||
);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].targetSlug).toBe('wiki/people/pettit-harriet-emeline-1828');
|
||||
expect(candidates[0].linkType).toBe('parent_of');
|
||||
});
|
||||
|
||||
test('no pack verb match falls through to legacy inference', async () => {
|
||||
const { candidates } = await extractPageLinks(
|
||||
'wiki/people/a',
|
||||
'see also [Someone](wiki/people/someone)',
|
||||
{},
|
||||
'person' as PageType,
|
||||
nullResolver,
|
||||
{ pack: packView },
|
||||
);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].linkType).toBe('mentions');
|
||||
});
|
||||
|
||||
test('no pack → legacy behavior unchanged (candidate never exists)', async () => {
|
||||
const { candidates } = await extractPageLinks(
|
||||
'wiki/people/pettit-james-b-1777',
|
||||
'parent of [Harriet](wiki/people/pettit-harriet-emeline-1828.md)',
|
||||
{},
|
||||
'person' as PageType,
|
||||
nullResolver,
|
||||
{},
|
||||
);
|
||||
expect(candidates).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3190 gate 3 — pack frontmatter_links extend FRONTMATTER_LINK_MAP', () => {
|
||||
const resolver: SlugResolver = {
|
||||
async resolve(name: string, dirHint?: string | string[]) {
|
||||
const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []);
|
||||
// Simulate step-2 resolution: dir-hint + already-slugified value.
|
||||
if (hints.includes('wiki/people')) return `wiki/people/${name}`;
|
||||
return null;
|
||||
},
|
||||
};
|
||||
|
||||
test('pack-only field produces an outgoing typed edge', async () => {
|
||||
const { candidates, unresolved } = await extractFrontmatterLinks(
|
||||
'wiki/people/pettit-harriet-emeline-1828',
|
||||
'person' as PageType,
|
||||
{ parents: ['pettit-james-b-1777', 'felt-lucy-w-1777'] },
|
||||
resolver,
|
||||
packView,
|
||||
);
|
||||
expect(unresolved).toEqual([]);
|
||||
expect(candidates).toHaveLength(2);
|
||||
for (const c of candidates) {
|
||||
expect(c.linkType).toBe('parent_of');
|
||||
expect(c.fromSlug).toBe('wiki/people/pettit-harriet-emeline-1828');
|
||||
expect(c.linkSource).toBe('frontmatter');
|
||||
}
|
||||
expect(candidates.map(c => c.targetSlug).sort()).toEqual([
|
||||
'wiki/people/felt-lucy-w-1777',
|
||||
'wiki/people/pettit-james-b-1777',
|
||||
]);
|
||||
});
|
||||
|
||||
test('hardcoded map wins on colliding (pageType, field) — no reversed duplicate', async () => {
|
||||
const trackingResolver: SlugResolver = {
|
||||
async resolve(name: string) { return `people/${name}`; },
|
||||
};
|
||||
const { candidates } = await extractFrontmatterLinks(
|
||||
'companies/acme',
|
||||
'company' as PageType,
|
||||
{ key_people: ['alice-example'] },
|
||||
trackingResolver,
|
||||
packView,
|
||||
);
|
||||
// Exactly one edge, from the hardcoded incoming works_at mapping —
|
||||
// the pack's outgoing 'employs' restatement is skipped.
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].linkType).toBe('works_at');
|
||||
expect(candidates[0].fromSlug).toBe('people/alice-example');
|
||||
expect(candidates[0].targetSlug).toBe('companies/acme');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3190 — packLinkExtractionView', () => {
|
||||
const manifest = (over: Record<string, unknown>) => ({
|
||||
manifest: {
|
||||
name: 'family-pack',
|
||||
page_types: [
|
||||
{ name: 'person', path_prefixes: ['wiki/people/', '/lines/'] },
|
||||
{ name: 'weird', path_prefixes: ['../evil/', 'ok_dir/'] },
|
||||
],
|
||||
link_types: [],
|
||||
frontmatter_links: [],
|
||||
...over,
|
||||
} as never,
|
||||
});
|
||||
|
||||
test('derives slug-safe entity dirs from path_prefixes', () => {
|
||||
const view = packLinkExtractionView(manifest({}));
|
||||
expect(view).not.toBeNull();
|
||||
expect(view!.entity_dirs.sort()).toEqual(['lines', 'ok_dir', 'wiki/people']);
|
||||
});
|
||||
|
||||
test('returns null for codegen base packs and missing packs', () => {
|
||||
expect(packLinkExtractionView(null)).toBeNull();
|
||||
expect(packLinkExtractionView(manifest({ name: 'gbrain-base' }))).toBeNull();
|
||||
expect(packLinkExtractionView(manifest({ name: 'gbrain-base-v2' }))).toBeNull();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user