mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-16 01:42:23 +00:00
Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7ff32a8773 | ||
|
|
b3b43d0f91 | ||
|
|
5b9a87f1a3 | ||
|
|
661f1f05cc | ||
|
|
3fec2123d2 | ||
|
|
176836f84d | ||
|
|
539d015cc5 | ||
|
|
91464564cd | ||
|
|
bd049d2969 | ||
|
|
784358f5fd | ||
|
|
d58bb2b0bb | ||
|
|
b252acfce3 | ||
|
|
683b7665f2 |
+14
-7
@@ -55,7 +55,7 @@ export function bigintToStringReplacer(_key: string, value: unknown): unknown {
|
||||
}
|
||||
|
||||
// CLI-only commands that bypass the operation layer
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'maintain', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'reconcile-links', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'retrieval-upgrade', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector']);
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'maintain', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'reconcile-links', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'retrieval-upgrade', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector', 'backfill']);
|
||||
// CLI-only commands whose handlers print their own --help text. These are
|
||||
// excluded from the generic short-circuit so detailed per-command and
|
||||
// per-subcommand usage stays reachable.
|
||||
@@ -872,7 +872,8 @@ export function applyThinClientSourceScope(
|
||||
params.source_id = resolved;
|
||||
}
|
||||
|
||||
async function makeContext(engine: BrainEngine, params: Record<string, unknown>): Promise<OperationContext> {
|
||||
// Exported for tests (same import-safety contract as applyThinClientSourceScope).
|
||||
export async function makeContext(engine: BrainEngine, params: Record<string, unknown>): Promise<OperationContext> {
|
||||
// v0.31.8 (D11): resolve sourceId via the canonical 6-tier chain. Honors
|
||||
// --source / GBRAIN_SOURCE / .gbrain-source / path-match / brain default /
|
||||
// 'default'. Wrapped in try/catch so a doctor / single-source brain that
|
||||
@@ -884,16 +885,21 @@ async function makeContext(engine: BrainEngine, params: Record<string, unknown>)
|
||||
// trusted local boundary) and consumed by federatedSearchScope in
|
||||
// operations.ts, which additionally gates on ctx.remote === false.
|
||||
let localFederated: string[] | undefined;
|
||||
// params.source is set when a CLI flag was parsed for the op (rare; most
|
||||
// CLI ops don't take --source). Falls through to env/dotfile/path-match.
|
||||
const explicit = (params.source as string | undefined) ?? null;
|
||||
try {
|
||||
const { resolveSourceWithTier, localFederatedSourceIds } = await import('./core/source-resolver.ts');
|
||||
// params.source is set when a CLI flag was parsed for the op (rare; most
|
||||
// CLI ops don't take --source). Falls through to env/dotfile/path-match.
|
||||
const explicit = (params.source as string | undefined) ?? null;
|
||||
const resolved = await resolveSourceWithTier(engine, explicit);
|
||||
sourceId = resolved.source_id;
|
||||
localFederated = await localFederatedSourceIds(engine, resolved.source_id, resolved.tier);
|
||||
} catch {
|
||||
// Source resolution failed (e.g. sources table doesn't exist on a fresh
|
||||
} catch (err) {
|
||||
// #1712: an EXPLICIT --source that fails to resolve (invalid id, or a
|
||||
// source that doesn't exist) must error loudly — the blanket swallow
|
||||
// turned `--source __all__` and typos into a silent `default` scope,
|
||||
// which is how three bug reports became debugging sessions.
|
||||
if (explicit) throw err;
|
||||
// Ambient resolution failed (e.g. sources table doesn't exist on a fresh
|
||||
// pre-init brain). Leave sourceId unset; engine read methods fall through
|
||||
// to the cross-source view (D16 back-compat path).
|
||||
sourceId = undefined;
|
||||
@@ -2448,6 +2454,7 @@ TOOLS
|
||||
publish <page.md> [--password] Shareable HTML (strips private data, optional AES-256)
|
||||
check-backlinks <check|fix> [dir] Find/fix missing back-links across brain
|
||||
lint <dir|file> [--fix] Catch LLM artifacts, placeholder dates, bad frontmatter
|
||||
backfill <kind|list> v0.30.1: run a registered backfill (effective-date, ...)
|
||||
orphans [--json] [--count] Find pages with no inbound wikilinks
|
||||
salience [--days N] [--kind P] v0.29: pages ranked by emotional + activity salience
|
||||
anomalies [--since D] [--sigma N] v0.29: cohort-based statistical anomalies (tag, type)
|
||||
|
||||
@@ -8,7 +8,7 @@ import {
|
||||
semverGt,
|
||||
semverLte,
|
||||
} from '../core/semver.ts';
|
||||
import { writeUpdateCache, type UpdateMarker } from '../core/self-upgrade.ts';
|
||||
import { readUpdateCache, writeUpdateCache, type UpdateMarker } from '../core/self-upgrade.ts';
|
||||
|
||||
/** Best-effort cache write — a read-only ~/.gbrain must never make the check throw. */
|
||||
function safeWriteCache(marker: UpdateMarker): void {
|
||||
@@ -45,26 +45,53 @@ function upgradeCommandForMethod(method: string): string {
|
||||
}
|
||||
}
|
||||
|
||||
/** Where the latest version is resolved from. gbrain publishes NO GitHub
|
||||
* releases (the `releases/latest` API is a permanent 404), so the release
|
||||
* train's source of truth is the `VERSION` file on master — same trusted host
|
||||
* `fetchChangelog` already uses. An npm fallback was rejected: the `gbrain`
|
||||
* package on npm is an unrelated GPU library (#505), so it would produce false
|
||||
* upgrade prompts pointing at a stranger's package. */
|
||||
const VERSION_SOURCE_URL = 'https://raw.githubusercontent.com/garrytan/gbrain/master/VERSION';
|
||||
const RELEASE_NOTES_URL = 'https://github.com/garrytan/gbrain/blob/master/CHANGELOG.md';
|
||||
|
||||
/** Extract a version from the raw VERSION file body: first line, optional `v`
|
||||
* prefix, optional `-suffix` channel tag (`0.31.1.1-fixwave` compares as its
|
||||
* numeric base — fail-safe: a suffix-only bump never prompts). Body is bounded
|
||||
* before parsing so a malformed/huge response can't blow up the check. */
|
||||
export function parseVersionFileBody(body: string): string | null {
|
||||
const firstLine = body.slice(0, 256).trim().split('\n')[0].trim();
|
||||
const m = firstLine.match(/^v?(\d+\.\d+\.\d+(?:\.\d+)?)(?:[-+][0-9A-Za-z.-]+)?$/);
|
||||
return m && isValidVersionString(m[1]) ? m[1] : null;
|
||||
}
|
||||
|
||||
export type LatestReleaseResult =
|
||||
| { ok: true; tag: string; published_at: string; url: string }
|
||||
| { ok: false; reason: 'network_error' | 'no_releases' };
|
||||
|
||||
/**
|
||||
* Fetch the latest GitHub release. Exported (v0.42) so the self-upgrade refresh
|
||||
* path and tests can reuse it. 5s timeout (was 10s) — this runs on the detached
|
||||
* refresh, never the hot path, but a tight bound keeps the refresh cheap.
|
||||
* Resolve the latest published gbrain version (from VERSION on master — see
|
||||
* VERSION_SOURCE_URL). Exported (v0.42) so the self-upgrade refresh path and
|
||||
* tests can reuse it. 5s timeout — this runs on the detached refresh, never the
|
||||
* hot path. Failures are discriminated: `network_error` (offline/timeout) vs
|
||||
* `no_releases` (endpoint answered but no usable version).
|
||||
*/
|
||||
export async function fetchLatestRelease(): Promise<{ tag: string; published_at: string; url: string } | null> {
|
||||
export async function fetchLatestRelease(): Promise<LatestReleaseResult> {
|
||||
let res: Response;
|
||||
try {
|
||||
const res = await fetch('https://api.github.com/repos/garrytan/gbrain/releases/latest', {
|
||||
res = await fetch(VERSION_SOURCE_URL, {
|
||||
headers: { 'User-Agent': `gbrain/${VERSION}` },
|
||||
signal: AbortSignal.timeout(5_000),
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
const data = await res.json() as any;
|
||||
return {
|
||||
tag: data.tag_name || '',
|
||||
published_at: data.published_at || '',
|
||||
url: data.html_url || '',
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
return { ok: false, reason: 'network_error' };
|
||||
}
|
||||
try {
|
||||
if (!res.ok) return { ok: false, reason: 'no_releases' };
|
||||
const tag = parseVersionFileBody(await res.text());
|
||||
if (!tag) return { ok: false, reason: 'no_releases' };
|
||||
return { ok: true, tag, published_at: '', url: RELEASE_NOTES_URL };
|
||||
} catch {
|
||||
return { ok: false, reason: 'network_error' };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,17 +145,33 @@ export function extractChangelogBetween(changelog: string, from: string, to: str
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the latest release and write the self-upgrade cache (the marker line
|
||||
* read by the CLI startup hook). Fail-open: on any network failure we cache
|
||||
* `UP_TO_DATE <current>` so the TTL prevents hammering GitHub on every
|
||||
* invocation. Returns the resolved marker for callers that want it. This is the
|
||||
* function the detached single-flight refresh (`gbrain check-update
|
||||
* --refresh-cache`) invokes.
|
||||
* A failed check must NEVER write `up_to_date` — that was #486: the fetch
|
||||
* failed permanently (dead releases API) and every user was told "you're
|
||||
* current" forever. Instead, re-write the last-known-good marker (bumping its
|
||||
* mtime so the cache TTL still throttles retries and a network blip can't
|
||||
* erase a pending upgrade_available notice). No prior marker → write nothing;
|
||||
* the next invocation retries.
|
||||
*/
|
||||
function preserveCacheOnFailedCheck(): void {
|
||||
try {
|
||||
const prior = readUpdateCache();
|
||||
if (prior) safeWriteCache(prior.marker);
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the latest version and write the self-upgrade cache (the marker line
|
||||
* read by the CLI startup hook). On fetch failure the last-known-good marker is
|
||||
* preserved (see preserveCacheOnFailedCheck) — never a fabricated `up_to_date`.
|
||||
* This is the function the detached single-flight refresh (`gbrain
|
||||
* check-update --refresh-cache`) invokes.
|
||||
*/
|
||||
export async function refreshUpdateCache(): Promise<void> {
|
||||
const release = await fetchLatestRelease();
|
||||
if (!release) {
|
||||
safeWriteCache({ kind: 'up_to_date', current: VERSION });
|
||||
if (!release.ok) {
|
||||
preserveCacheOnFailedCheck();
|
||||
return;
|
||||
}
|
||||
const latestVersion = release.tag.replace(/^v/, '');
|
||||
@@ -166,9 +209,8 @@ export async function runCheckUpdate(args: string[]) {
|
||||
|
||||
const release = await fetchLatestRelease();
|
||||
|
||||
if (!release) {
|
||||
// Warm the cache fail-open so the startup hook doesn't re-fetch every call.
|
||||
safeWriteCache({ kind: 'up_to_date', current: VERSION });
|
||||
if (!release.ok) {
|
||||
preserveCacheOnFailedCheck();
|
||||
if (json) {
|
||||
console.log(JSON.stringify({
|
||||
current_version: VERSION,
|
||||
@@ -179,10 +221,12 @@ export async function runCheckUpdate(args: string[]) {
|
||||
release_url: '',
|
||||
changelog_diff: '',
|
||||
published_at: '',
|
||||
error: 'no_releases',
|
||||
error: release.reason,
|
||||
}, null, 2));
|
||||
} else if (release.reason === 'network_error') {
|
||||
console.log(`GBrain ${VERSION} — could not check for updates (network unavailable).`);
|
||||
} else {
|
||||
console.log(`GBrain ${VERSION} — could not check for updates (no releases found or network unavailable).`);
|
||||
console.log(`GBrain ${VERSION} — could not determine the latest published version.`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
+50
-3
@@ -19,6 +19,26 @@ import {
|
||||
} from '../core/pace-mode.ts';
|
||||
import { tryAcquireDbLock, type DbLockHandle } from '../core/db-lock.ts';
|
||||
import { embedBackfillLockId } from '../core/embed-backfill-lock.ts';
|
||||
import { wrapChunkTextsForStoredMode } from '../core/embedding-context.ts';
|
||||
import { titleTierCorpusGeneration } from '../core/contextual-retrieval-service.ts';
|
||||
import type { Page } from '../core/types.ts';
|
||||
|
||||
/**
|
||||
* #3507 — after a plain re-embed fully re-embedded a `per_chunk_synopsis`
|
||||
* page at the title-only tier (see wrapChunkTextsForStoredMode), restamp the
|
||||
* page's CR state to 'title' so `contextual_retrieval_mode` keeps describing
|
||||
* the vectors actually in the column. The reindex sweep restores the synopsis
|
||||
* tier later. No-op for every other mode.
|
||||
*/
|
||||
export async function restampIfDemotedToTitleTier(
|
||||
engine: BrainEngine,
|
||||
page: Pick<Page, 'contextual_retrieval_mode'> | null | undefined,
|
||||
slug: string,
|
||||
sourceId: string,
|
||||
): Promise<void> {
|
||||
if (page?.contextual_retrieval_mode !== 'per_chunk_synopsis') return;
|
||||
await engine.updatePageContextualRetrievalState(slug, sourceId, 'title', titleTierCorpusGeneration());
|
||||
}
|
||||
|
||||
export interface EmbedOpts {
|
||||
/** Embed ALL pages (every chunk). */
|
||||
@@ -599,7 +619,11 @@ async function embedPage(
|
||||
return;
|
||||
}
|
||||
|
||||
const embeddings = await embedBatch(toEmbed.map(c => c.chunk_text), { abortSignal: signal });
|
||||
// #3507: embed with the page's STORED wrapping convention (title-tier
|
||||
// contextual prefix when the page was embedded wrapped), not raw
|
||||
// chunk_text — otherwise a re-embed silently strips the contextual
|
||||
// prefixes the sync path applied. fenced_code chunks stay unwrapped.
|
||||
const embeddings = await embedBatch(wrapChunkTextsForStoredMode(page, toEmbed), { abortSignal: signal });
|
||||
const embeddingMap = new Map<number, Float32Array>();
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
embeddingMap.set(toEmbed[j].chunk_index, embeddings[j]);
|
||||
@@ -622,6 +646,9 @@ async function embedPage(
|
||||
// such a page and then stamps it.
|
||||
if (toEmbed.length === chunks.length) {
|
||||
await engine.setPageEmbeddingSignature(slug, { sourceId, signature: currentEmbeddingSignature() });
|
||||
// #3507: a fully re-embedded per_chunk_synopsis page landed at the
|
||||
// title tier — keep the stamped mode honest.
|
||||
await restampIfDemotedToTitleTier(engine, page, slug, page.source_id);
|
||||
}
|
||||
result.embedded += toEmbed.length;
|
||||
result.pages_processed++;
|
||||
@@ -763,7 +790,8 @@ async function embedAll(
|
||||
}
|
||||
|
||||
try {
|
||||
const embeddings = await embedBatch(toEmbed.map(c => c.chunk_text));
|
||||
// #3507: reproduce the page's stored wrapping convention (see embedPage).
|
||||
const embeddings = await embedBatch(wrapChunkTextsForStoredMode(page, toEmbed));
|
||||
// Build a map of new embeddings by chunk_index
|
||||
const embeddingMap = new Map<number, Float32Array>();
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
@@ -785,6 +813,11 @@ async function embedAll(
|
||||
await observed(pacer, () =>
|
||||
engine.setPageEmbeddingSignature(page.slug, { sourceId: pageSourceId, signature }),
|
||||
);
|
||||
// #3507: --all fully re-embeds; a per_chunk_synopsis page landed at
|
||||
// the title tier — keep the stamped mode honest.
|
||||
await observed(pacer, () =>
|
||||
restampIfDemotedToTitleTier(engine, page, page.slug, pageSourceId),
|
||||
);
|
||||
result.embedded += toEmbed.length;
|
||||
} catch (e: unknown) {
|
||||
serr(`\n Error embedding ${page.slug}: ${e instanceof Error ? e.message : e}`);
|
||||
@@ -1098,7 +1131,13 @@ async function embedAllStale(
|
||||
const keySourceId = stale[0]?.source_id ?? 'default';
|
||||
const slug = stale[0].slug;
|
||||
try {
|
||||
const embeddings = await embedBatchWithBackoff(stale.map(c => c.chunk_text), { abortSignal: effectiveSignal });
|
||||
// #3507: fetch the page row for its title + stored CR mode so the
|
||||
// re-embed reproduces the page's wrapping convention instead of
|
||||
// silently stripping contextual prefixes — `embed --stale` is the
|
||||
// NORMAL post-model-migration path, so raw-text embedding here
|
||||
// quietly converted whole corpora to the unwrapped convention.
|
||||
const pageRow = await observed(pacer, () => engine.getPage(slug, { sourceId: keySourceId }));
|
||||
const embeddings = await embedBatchWithBackoff(wrapChunkTextsForStoredMode(pageRow, stale), { abortSignal: effectiveSignal });
|
||||
// Re-fetch existing chunks and merge to avoid deleting non-stale chunks.
|
||||
const existing = await observed(pacer, () => engine.getChunks(slug, { sourceId: keySourceId }));
|
||||
const staleIdxToEmbedding = new Map<number, Float32Array>();
|
||||
@@ -1126,6 +1165,14 @@ async function embedAllStale(
|
||||
engine.setPageEmbeddingSignature(slug, { sourceId: keySourceId, signature }),
|
||||
);
|
||||
}
|
||||
// #3507: a FULLY re-embedded per_chunk_synopsis page landed at the
|
||||
// title tier — keep the stamped mode honest. Partially-stale pages
|
||||
// stay stamped as-is (mixed provenance; reindex sweeps fix them).
|
||||
if (stale.length === existing.length) {
|
||||
await observed(pacer, () =>
|
||||
restampIfDemotedToTitleTier(engine, pageRow, slug, keySourceId),
|
||||
);
|
||||
}
|
||||
result.embedded += stale.length;
|
||||
} catch (e: unknown) {
|
||||
// Budget/abort-fired cancellations are expected on the way out; don't
|
||||
|
||||
+10
-3
@@ -233,7 +233,7 @@ USAGE
|
||||
gbrain jobs get <id>
|
||||
gbrain jobs cancel <id>
|
||||
gbrain jobs retry <id>
|
||||
gbrain jobs prune [--older-than 30d]
|
||||
gbrain jobs prune [--older-than 30d] [--dry-run]
|
||||
gbrain jobs delete <id>
|
||||
gbrain jobs stats
|
||||
gbrain jobs smoke
|
||||
@@ -633,8 +633,15 @@ HANDLER TYPES (built in)
|
||||
try { await queue.ensureSchema(); }
|
||||
catch (e) { console.error(e instanceof Error ? e.message : String(e)); process.exit(1); }
|
||||
|
||||
const count = await queue.prune({ olderThan: new Date(Date.now() - days * 86400000) });
|
||||
console.log(`Pruned ${count} jobs older than ${days} days.`);
|
||||
// #2712: --dry-run previews the count without deleting. It used to be
|
||||
// silently ignored (the destructive default ran anyway).
|
||||
const dryRun = hasFlag(args, '--dry-run');
|
||||
const count = await queue.prune({ olderThan: new Date(Date.now() - days * 86400000), dryRun });
|
||||
if (dryRun) {
|
||||
console.log(`[dry-run] Would prune ${count} jobs older than ${days} days. Nothing deleted.`);
|
||||
} else {
|
||||
console.log(`Pruned ${count} jobs older than ${days} days.`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@@ -35,7 +35,8 @@ export async function runSelfUpgrade(args: string[]): Promise<void> {
|
||||
const force = args.includes('--force');
|
||||
const json = args.includes('--json');
|
||||
|
||||
const release = await fetchLatestRelease();
|
||||
const result = await fetchLatestRelease();
|
||||
const release = result.ok ? result : null;
|
||||
const latest = release ? release.tag.replace(/^v/, '') : null;
|
||||
const behind = !!latest && isValidVersionString(latest) && isNewerVersion(VERSION, latest);
|
||||
|
||||
|
||||
+15
-8
@@ -3,6 +3,7 @@
|
||||
*
|
||||
* Subcommands:
|
||||
* takes <slug> — list takes for a page
|
||||
* takes list — list all active takes (#2079)
|
||||
* takes search "<query>" [--who h] — keyword search across all takes
|
||||
* takes add <slug> ...flags — append a take (markdown + DB)
|
||||
* takes update <slug> --row N ...flags — update mutable fields
|
||||
@@ -129,11 +130,10 @@ function writeBody(path: string, body: string): void {
|
||||
// --- Subcommands ---
|
||||
|
||||
async function cmdList(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const slug = args[0];
|
||||
if (!slug) {
|
||||
console.error('Usage: gbrain takes <slug> [--json]');
|
||||
process.exit(1);
|
||||
}
|
||||
// #2079: slug is optional. `gbrain takes list` (no slug) lists ALL active
|
||||
// takes — CLI parity with the takes_list operation. A leading flag is not
|
||||
// a slug.
|
||||
const slug = args[0] && !args[0].startsWith('-') ? args[0] : undefined;
|
||||
const json = flagPresent(args, '--json');
|
||||
const holder = flagValue(args, '--who');
|
||||
const kind = flagValue(args, '--kind') as string | undefined;
|
||||
@@ -153,17 +153,19 @@ async function cmdList(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
const scope = slug ?? 'this brain';
|
||||
if (takes.length === 0) {
|
||||
console.log(`No takes on ${slug}.`);
|
||||
console.log(`No takes on ${scope}.`);
|
||||
return;
|
||||
}
|
||||
console.log(`# Takes on ${slug}\n`);
|
||||
console.log(`# Takes on ${scope}\n`);
|
||||
for (const t of takes) {
|
||||
const tag = t.active ? '' : ' [superseded]';
|
||||
const w = Number(t.weight).toFixed(2);
|
||||
const since = t.since_date ?? '';
|
||||
const src = t.source ? ` — ${t.source}` : '';
|
||||
console.log(`#${t.row_num} [${t.kind} • ${t.holder} • w=${w}${since ? ` • ${since}` : ''}]${tag}\n ${t.claim}${src}\n`);
|
||||
const where = slug ? '' : `${t.page_slug} `;
|
||||
console.log(`${where}#${t.row_num} [${t.kind} • ${t.holder} • w=${w}${since ? ` • ${since}` : ''}]${tag}\n ${t.claim}${src}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -555,6 +557,8 @@ export async function runTakes(engine: BrainEngine, args: string[]): Promise<voi
|
||||
Subcommands:
|
||||
takes <slug> [--json] [--who h] [--kind k] [--sort weight|since_date|created_at] [--expired]
|
||||
List takes for a page
|
||||
takes list [--json] [--who h] [--kind k] [--sort ...] [--expired]
|
||||
List all active takes across the brain (#2079)
|
||||
takes search "<query>" [--limit N] [--json]
|
||||
Keyword search across all takes
|
||||
takes add <slug> --claim "..." --kind <fact|take|bet|hunch> --who <holder>
|
||||
@@ -584,6 +588,9 @@ Common flags:
|
||||
const rest = args.slice(1);
|
||||
|
||||
switch (sub) {
|
||||
// #2079: `takes list` used to be parsed as page slug "list" and printed
|
||||
// "No takes on list." — reading exactly like an empty takes table.
|
||||
case 'list': return cmdList(engine, rest);
|
||||
case 'search': return cmdSearch(engine, rest);
|
||||
case 'add': return cmdAdd(engine, rest, await resolveTakesSourceId(engine));
|
||||
case 'update': return cmdUpdate(engine, rest, await resolveTakesSourceId(engine));
|
||||
|
||||
+28
-5
@@ -21,12 +21,35 @@ export const CJK_SLUG_CHARS = '一-鿿-ゟ゠-ヿ가-';
|
||||
export const CJK_RANGES_REGEX = new RegExp(`[${CJK_SLUG_CHARS}]`);
|
||||
|
||||
/**
|
||||
* Page-slug segment grammar (no anchors): alnum-or-CJK lead char, then
|
||||
* alnum/CJK/hyphen continuation. Single source for validatePageSlug
|
||||
* (operations.ts), SlugRegistry's SLUG_RE, and the dream-cycle
|
||||
* SUMMARY_SLUG_RE so every slug validator shares one grammar (#738).
|
||||
* Slug "word" character class (#3417): every script's letters, not just
|
||||
* Latin + CJK. Unicode property escapes — REQUIRES the `u` flag on any
|
||||
* regex composed from this string (without `u`, `\p{Ll}` silently matches
|
||||
* the literal chars `p`, `L`, `l`, `{`, `}`).
|
||||
*
|
||||
* \p{Ll} lowercase letters (a-z, Cyrillic/Greek lowercase, đ, …)
|
||||
* \p{Lm} modifier letters
|
||||
* \p{Lo} caseless-script letters (Hebrew, Arabic, Thai, CJK, Devanagari, …)
|
||||
* \p{M} combining marks that survive the Latin accent-strip pass
|
||||
* (Hebrew niqqud, Arabic harakat, Thai/Devanagari vowel signs)
|
||||
* \p{N} numbers (0-9, Arabic-Indic digits, …)
|
||||
*
|
||||
* Uppercase (\p{Lu}/\p{Lt}) is deliberately excluded: slugifySegment()
|
||||
* lowercases before filtering, so validators stay lowercase-canonical.
|
||||
*
|
||||
* Distinct from CJK_SLUG_CHARS above, which also drives the
|
||||
* countCJKAwareWords density heuristic — do NOT merge the two, or slug
|
||||
* grammar changes silently change chunking behavior.
|
||||
*/
|
||||
export const PAGE_SLUG_SEG = `[a-z0-9${CJK_SLUG_CHARS}][a-z0-9${CJK_SLUG_CHARS}\\-]*`;
|
||||
export const SLUG_WORD_CHARS = '\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}\\p{N}';
|
||||
|
||||
/**
|
||||
* Page-slug segment grammar (no anchors): word-char lead, then word-char or
|
||||
* hyphen continuation. Single source for validatePageSlug (operations.ts),
|
||||
* SlugRegistry's SLUG_RE, and the dream-cycle SUMMARY_SLUG_RE so every slug
|
||||
* validator shares one grammar (#738). Compose with the `u` flag — see
|
||||
* SLUG_WORD_CHARS.
|
||||
*/
|
||||
export const PAGE_SLUG_SEG = `[${SLUG_WORD_CHARS}][${SLUG_WORD_CHARS}\\-]*`;
|
||||
|
||||
export const CJK_SENTENCE_DELIMITERS = ['。', '!', '?']; // 。!?
|
||||
export const CJK_CLAUSE_DELIMITERS = [';', ':', ',', '、']; // ;:,、
|
||||
|
||||
@@ -145,6 +145,19 @@ export function computeCorpusGeneration(args: {
|
||||
return h.digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
/**
|
||||
* #3507 — the corpus_generation a page lands on when a plain re-embed path
|
||||
* (`embed --stale` and friends) re-embeds a `per_chunk_synopsis` page at the
|
||||
* title-only tier (the D14 fallback tier; synopsis re-generation is a paid
|
||||
* backfill concern). Callers restamp
|
||||
* `updatePageContextualRetrievalState(slug, sourceId, 'title', titleTierCorpusGeneration())`
|
||||
* so the stamped mode keeps describing the vectors actually in the column.
|
||||
* Matches what the inline import path writes for its title-tier pages.
|
||||
*/
|
||||
export function titleTierCorpusGeneration(): string {
|
||||
return computeCorpusGeneration({ crMode: 'title', haikuModel: DEFAULT_HAIKU_MODEL });
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute source_text_hash for D27 P1-4 cache key composition. The
|
||||
* synopsis cache invalidates correctly when adjacent text changes (page
|
||||
|
||||
+3
-1
@@ -1179,7 +1179,9 @@ async function runPhaseExtractFacts(
|
||||
summary: `extract_facts skipped: ${result.legacyRowsPending} legacy v0.31 facts pending fence backfill`,
|
||||
details: {
|
||||
legacyRowsPending: result.legacyRowsPending,
|
||||
hint: 'gbrain apply-migrations --yes',
|
||||
// A bare `apply-migrations --yes` no-ops once the v0.32.2 ledger
|
||||
// entry is complete; the retry marker is what re-runs Phase B.
|
||||
hint: 'gbrain apply-migrations --force-retry 0.32.2 && gbrain apply-migrations --yes',
|
||||
warnings: result.warnings,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -26,13 +26,17 @@
|
||||
*
|
||||
* Empty-fence guard (Codex R2-#7; #2484; #2646): the phase refuses to do
|
||||
* its destructive reconciliation pass when genuinely-backfillable legacy
|
||||
* rows still exist — `row_num IS NULL` (never fenced) AND `entity_slug`
|
||||
* resolves to a live page in this source (so the v0_32_2 migration's
|
||||
* Phase B could fence them) AND the row is not soft-expired
|
||||
* (`expired_at IS NULL`). Status returns `warn` with a hint to run
|
||||
* `gbrain apply-migrations --yes`. Without the guard, an interrupted
|
||||
* upgrade where v0_32_2 hasn't run could leave the cycle silently
|
||||
* misreporting "0 facts on people/alice" while legacy rows linger.
|
||||
* rows still exist — in THIS run's source only (`source_id = sourceId`;
|
||||
* a pending row in source A must not jam extraction for source B — the
|
||||
* source-isolation invariant) — `row_num IS NULL` (never fenced) AND
|
||||
* `entity_slug` resolves to a live page in this source (so the v0_32_2
|
||||
* migration's Phase B could fence them) AND the row is not soft-expired
|
||||
* (`expired_at IS NULL`). Status returns `warn` with a hint to re-run
|
||||
* the v0.32.2 fence backfill (`apply-migrations --force-retry 0.32.2`
|
||||
* then `--yes` — a bare `--yes` is a no-op once the ledger says
|
||||
* complete). Without the guard, an interrupted upgrade where v0_32_2
|
||||
* hasn't run could leave the cycle silently misreporting "0 facts on
|
||||
* people/alice" while legacy rows linger.
|
||||
*
|
||||
* The live-page requirement (#2484) is load-bearing: the inline facts
|
||||
* writer keeps producing `row_num IS NULL, entity_slug IS NOT NULL`
|
||||
@@ -225,10 +229,17 @@ export async function runExtractFacts(
|
||||
// soft-expires legacy rows rather than deleting them, so counting
|
||||
// expired rows would leave the guard permanently stuck with no
|
||||
// supported way to drain the backlog.
|
||||
//
|
||||
// Source isolation (#3526): the count is scoped to THIS run's
|
||||
// sourceId. The pre-fix query counted brain-wide, so a single pending
|
||||
// legacy row in any mounted source jammed extract_facts for every
|
||||
// source — a cross-source leak of one source's migration state into
|
||||
// another's cycle (CLAUDE.md source-isolation invariant).
|
||||
const legacy = await engine.executeRaw<{ n: string }>(
|
||||
`SELECT COUNT(*) AS n
|
||||
FROM facts f
|
||||
WHERE f.row_num IS NULL
|
||||
WHERE f.source_id = $1
|
||||
AND f.row_num IS NULL
|
||||
AND f.entity_slug IS NOT NULL
|
||||
AND f.expired_at IS NULL
|
||||
AND EXISTS (
|
||||
@@ -237,15 +248,25 @@ export async function runExtractFacts(
|
||||
AND p.slug = f.entity_slug
|
||||
AND p.deleted_at IS NULL
|
||||
)`,
|
||||
[sourceId],
|
||||
);
|
||||
const legacyCount = parseInt(legacy[0]?.n ?? '0', 10);
|
||||
result.legacyRowsPending = legacyCount;
|
||||
if (legacyCount > 0) {
|
||||
result.guardTriggered = true;
|
||||
// Drain advice must actually work: a bare `apply-migrations --yes`
|
||||
// is a no-op once the v0.32.2 ledger entry says complete (the
|
||||
// runner classifies it as already-applied), so the sanctioned
|
||||
// re-run path is the explicit retry marker first. Phase B is
|
||||
// idempotent — it only touches `row_num IS NULL` rows and de-dupes
|
||||
// against the existing fence — so the re-run is safe. Individual
|
||||
// rows can instead be drained through `forget_fact` (soft-expired
|
||||
// rows stop counting).
|
||||
result.warnings.push(
|
||||
`extract_facts: ${legacyCount} legacy v0.31 fact rows (entity page present, not yet ` +
|
||||
`fenced) pending fence backfill. Run \`gbrain apply-migrations --yes\` to complete ` +
|
||||
`v0_32_2 before this phase can safely reconcile fence → DB.`,
|
||||
`extract_facts: ${legacyCount} legacy v0.31 fact rows in source "${sourceId}" ` +
|
||||
`(entity page present, not yet fenced) pending fence backfill. Re-run the v0.32.2 ` +
|
||||
`fence backfill: \`gbrain apply-migrations --force-retry 0.32.2\` then ` +
|
||||
`\`gbrain apply-migrations --yes\`. Or drain individual rows via \`forget_fact\`.`,
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -48,8 +48,9 @@ import { safeSplitIndex } from '../text-safe.ts';
|
||||
import { PAGE_SLUG_SEG } from '../cjk.ts';
|
||||
|
||||
// Slug grammar from validatePageSlug — shared via PAGE_SLUG_SEG (#738).
|
||||
// Used for the orchestrator-written summary index slug.
|
||||
const SUMMARY_SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`);
|
||||
// Used for the orchestrator-written summary index slug. `u` flag required
|
||||
// by PAGE_SLUG_SEG's \p{...} classes (#3417).
|
||||
const SUMMARY_SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'u');
|
||||
|
||||
// ── Model context budget (D1, D5, D7, D9) ─────────────────────────────
|
||||
|
||||
|
||||
+17
-2
@@ -19,7 +19,8 @@
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import type { ChunkInput } from './types.ts';
|
||||
import { embedBatchWithBackoff } from '../commands/embed.ts';
|
||||
import { embedBatchWithBackoff, restampIfDemotedToTitleTier } from '../commands/embed.ts';
|
||||
import { wrapChunkTextsForStoredMode } from './embedding-context.ts';
|
||||
import { type DbPacer, createNoopPacer, observed } from './db-pacer.ts';
|
||||
import { AbortError } from './abort-check.ts';
|
||||
|
||||
@@ -189,8 +190,15 @@ export async function embedStaleForSource(
|
||||
const keySourceId = stale[0]?.source_id ?? sourceId;
|
||||
const slug = stale[0].slug;
|
||||
try {
|
||||
// #3507: fetch the page row for its title + stored CR mode so the
|
||||
// re-embed reproduces the page's wrapping convention instead of
|
||||
// silently stripping contextual prefixes (mirrors
|
||||
// src/commands/embed.ts:embedAllStale).
|
||||
const pageRow = await observed(pacer, () =>
|
||||
engine.getPage(slug, { sourceId: keySourceId }),
|
||||
);
|
||||
const embeddings = await embedFn(
|
||||
stale.map((c) => c.chunk_text),
|
||||
wrapChunkTextsForStoredMode(pageRow, stale),
|
||||
{ abortSignal: signal },
|
||||
);
|
||||
const existing = await observed(pacer, () =>
|
||||
@@ -233,6 +241,13 @@ export async function embedStaleForSource(
|
||||
engine.setPageEmbeddingSignature(slug, { sourceId: keySourceId, signature }),
|
||||
);
|
||||
}
|
||||
// #3507: a FULLY re-embedded per_chunk_synopsis page landed at the
|
||||
// title tier — keep the stamped mode honest (mixed pages stay as-is).
|
||||
if (stale.length === existing.length) {
|
||||
await observed(pacer, () =>
|
||||
restampIfDemotedToTitleTier(engine, pageRow, slug, keySourceId),
|
||||
);
|
||||
}
|
||||
result.embedded += stale.length;
|
||||
result.pagesProcessed += 1;
|
||||
} catch (e: unknown) {
|
||||
|
||||
@@ -186,3 +186,41 @@ export function modeRequiresHaiku(mode: CRMode): boolean {
|
||||
export function modeRequiresWrapper(mode: CRMode): boolean {
|
||||
return mode !== 'none';
|
||||
}
|
||||
|
||||
/**
|
||||
* #3507 — build the embedding inputs for a re-embed of EXISTING chunk rows,
|
||||
* reproducing the wrapping convention the page's vectors were originally
|
||||
* built under (recorded in `pages.contextual_retrieval_mode`).
|
||||
*
|
||||
* Used by every plain re-embed path (`embed <slug>`, `embed --all`,
|
||||
* `embed --stale`, the embed-backfill Minion loop). Before this helper those
|
||||
* paths embedded raw `chunk_text`, so any re-embed — including the NORMAL
|
||||
* post-model-migration `embed --stale` — silently replaced context-wrapped
|
||||
* vectors with unwrapped ones, degrading retrieval with no signature change
|
||||
* to show for it.
|
||||
*
|
||||
* Convention rules (embed PRESERVES conventions; changing them is
|
||||
* sync/reindex's job):
|
||||
* - mode NULL/undefined/'none' → raw chunk_text (status quo).
|
||||
* - mode 'title' → title-only prefix (pure string concat).
|
||||
* - mode 'per_chunk_synopsis' → title-only prefix. Re-generating Haiku
|
||||
* synopses is a paid backfill concern; title-only is the service's own
|
||||
* documented fallback tier (D14). Callers that fully re-embed a page
|
||||
* this way should restamp the page to 'title' so the column stays
|
||||
* honest (see contextual-retrieval-service.ts:titleTierCorpusGeneration).
|
||||
* - `fenced_code` chunks are NEVER wrapped (D20-T4), same as sync.
|
||||
*/
|
||||
export function wrapChunkTextsForStoredMode(
|
||||
page:
|
||||
| { title?: string | null; contextual_retrieval_mode?: CRMode | null }
|
||||
| null
|
||||
| undefined,
|
||||
chunks: ReadonlyArray<{ chunk_text: string; chunk_source?: string | null }>,
|
||||
): string[] {
|
||||
const mode = page?.contextual_retrieval_mode;
|
||||
if (mode == null || !modeRequiresWrapper(mode)) {
|
||||
return chunks.map((c) => c.chunk_text);
|
||||
}
|
||||
const prefix = buildContextualPrefix(page?.title ?? '', null);
|
||||
return chunks.map((c) => wrapChunkForEmbedding(c.chunk_text, prefix, c.chunk_source));
|
||||
}
|
||||
|
||||
@@ -85,11 +85,21 @@ const RUN_ID_SHORT_LEN = 8;
|
||||
/**
|
||||
* Truncate a run id to the standard 8-char short form used in slug
|
||||
* paths. Idempotent — passing an already-short id returns it unchanged.
|
||||
* Non-hex / non-alphanumeric chars survive (op-checkpoint ids may
|
||||
* include dashes or other separators).
|
||||
* Non-hex / non-alphanumeric chars survive INSIDE the short form
|
||||
* (op-checkpoint ids may include dashes or other separators), but
|
||||
* boundary hyphens are trimmed (#3443): `slugifySegment()` strips
|
||||
* leading/trailing hyphens during repo sync, so a short form like
|
||||
* 'propose-' (from propose-<timestamp> run ids) made the DB receipt
|
||||
* slug and its Git-backed slug disagree — writing the receipt through
|
||||
* to the repo created a normalized sibling instead of materializing
|
||||
* the existing page. Invariant: slugifySegment(shortRunId(x)) ===
|
||||
* shortRunId(x) for slug-safe run ids.
|
||||
*/
|
||||
export function shortRunId(runId: string): string {
|
||||
return runId.slice(0, RUN_ID_SHORT_LEN);
|
||||
// ponytail: truncation-based discrimination is only as good as the run id's
|
||||
// first 8 chars; families that need per-run uniqueness must front-load it.
|
||||
const short = runId.slice(0, RUN_ID_SHORT_LEN).replace(/^-+|-+$/g, '');
|
||||
return short || (runId ? 'run' : '');
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1092,8 +1092,8 @@ export async function importFromFile(
|
||||
chunks: 0,
|
||||
error:
|
||||
`Filename "${relativePath}" produces no usable slug. ` +
|
||||
`Add a "slug:" to the frontmatter, or rename the file to use ` +
|
||||
`ASCII / Chinese / Japanese / Korean characters.`,
|
||||
`Add a "slug:" to the frontmatter, or rename the file to include ` +
|
||||
`at least one letter or number (any script).`,
|
||||
};
|
||||
}
|
||||
} else if (parsed.slug !== expectedSlug) {
|
||||
|
||||
@@ -43,6 +43,11 @@ export const HANDLER_DEFAULT_TIMEOUT_MS: Readonly<Record<string, number>> = {
|
||||
// few writes. Generous 10-min budget (vs the tight null-default) covers a
|
||||
// slow gateway without the 30-min loop budget.
|
||||
chronicle_extract: TEN_MIN_MS,
|
||||
// #3207 — same shape as chronicle_extract: one page = one LLM extraction
|
||||
// call + a few writes. Was missing from this map, so it inherited the tight
|
||||
// null-default and got dead-lettered mid-generation on slow chat providers
|
||||
// (facts silently lost) — exactly the failure this file exists to prevent.
|
||||
'facts-absorb': TEN_MIN_MS,
|
||||
// Per-page contextual reindex jobs process chunks sequentially with one
|
||||
// rate-leased LLM synopsis call per chunk; large transcript pages need more
|
||||
// than the standard 30-min long-job budget.
|
||||
|
||||
@@ -534,10 +534,21 @@ export class MinionQueue {
|
||||
}
|
||||
|
||||
/** Prune old jobs in terminal statuses. Returns count of deleted rows. */
|
||||
async prune(opts?: { olderThan?: Date; status?: MinionJobStatus[] }): Promise<number> {
|
||||
async prune(opts?: { olderThan?: Date; status?: MinionJobStatus[]; dryRun?: boolean }): Promise<number> {
|
||||
const statuses = opts?.status ?? ['completed', 'dead', 'cancelled'];
|
||||
const olderThan = opts?.olderThan ?? new Date(Date.now() - 30 * 86400000);
|
||||
|
||||
// #2712: dryRun counts the would-be-pruned rows without deleting.
|
||||
// Silent-ignoring a safety flag on a delete path is data loss.
|
||||
if (opts?.dryRun) {
|
||||
const rows = await this.engine.executeRaw<{ count: string }>(
|
||||
`SELECT count(*)::text as count FROM minion_jobs
|
||||
WHERE status = ANY($1) AND updated_at < $2`,
|
||||
[statuses, olderThan.toISOString()]
|
||||
);
|
||||
return parseInt(rows[0]?.count ?? '0', 10);
|
||||
}
|
||||
|
||||
const rows = await this.engine.executeRaw<{ count: string }>(
|
||||
`WITH pruned AS (
|
||||
DELETE FROM minion_jobs
|
||||
|
||||
+15
-5
@@ -28,6 +28,7 @@ import { isSearchMode } from './search/mode.ts';
|
||||
import { stampEvidence } from './search/evidence.ts';
|
||||
import type { SearchResult } from './types.ts';
|
||||
import { CJK_SLUG_CHARS, PAGE_SLUG_SEG } from './cjk.ts';
|
||||
import { ALL_SOURCES } from './source-id.ts';
|
||||
import * as db from './db.ts';
|
||||
import { VERSION } from '../version.ts';
|
||||
import {
|
||||
@@ -162,10 +163,11 @@ export function validatePageSlug(slug: string): void {
|
||||
if (slug.length > 255) {
|
||||
throw new OperationError('invalid_params', 'page_slug exceeds 255 characters');
|
||||
}
|
||||
// v0.32.7: CJK ranges (Han / Hiragana / Katakana / Hangul Syllables) allowed
|
||||
// in segments. ASCII shape rules (lead char, hyphen continuation) preserved.
|
||||
if (!new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'i').test(slug)) {
|
||||
throw new OperationError('invalid_params', `Invalid page_slug: ${slug} (allowed: alphanumeric, CJK, hyphens, forward-slash separated segments)`);
|
||||
// #3417: letters/numbers from any script allowed in segments (u flag required
|
||||
// for the \p{...} classes in PAGE_SLUG_SEG). Shape rules (lead char, hyphen
|
||||
// continuation) preserved.
|
||||
if (!new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'iu').test(slug)) {
|
||||
throw new OperationError('invalid_params', `Invalid page_slug: ${slug} (allowed: letters/numbers in any script, hyphens, forward-slash separated segments)`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -486,6 +488,14 @@ export function sourceScopeOpts(ctx: OperationContext): { sourceId?: string; sou
|
||||
// value of `[]` MUST NOT widen scope to "all sources" by being interpreted
|
||||
// as "no filter."
|
||||
if (allowed && allowed.length > 0) return { sourceIds: allowed };
|
||||
// #1712: the __all__ sentinel spans the brain — but ONLY for trusted local
|
||||
// callers (strictly `remote === false`). For remote/untrusted callers the
|
||||
// literal stays as-is: it can never match a real source id (underscores are
|
||||
// rejected at creation), so the read fail-closes to empty rather than
|
||||
// widening past the caller's grant. Do NOT "simplify" this to `{}`.
|
||||
if (ctx.sourceId === ALL_SOURCES) {
|
||||
return ctx.remote === false ? {} : { sourceId: ctx.sourceId };
|
||||
}
|
||||
if (ctx.sourceId) return { sourceId: ctx.sourceId };
|
||||
return {};
|
||||
}
|
||||
@@ -553,7 +563,7 @@ export function resolveRequestedScope(
|
||||
sourceIdParam: string | undefined,
|
||||
allSourcesParam = false,
|
||||
): { sourceId?: string; sourceIds?: string[] } {
|
||||
const wantsAll = allSourcesParam || sourceIdParam === '__all__';
|
||||
const wantsAll = allSourcesParam || sourceIdParam === ALL_SOURCES;
|
||||
if (wantsAll) {
|
||||
return ctx.remote === false ? {} : sourceScopeOpts(ctx);
|
||||
}
|
||||
|
||||
@@ -72,9 +72,10 @@ export class SlugRegistryError extends Error {
|
||||
// SlugRegistry
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Shares the page-slug segment grammar (incl. CJK ranges, #738) with
|
||||
// Shares the page-slug segment grammar (all scripts, #738/#3417) with
|
||||
// validatePageSlug; keeps this site's dir/name shape (>= 2 segments).
|
||||
const SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})+$`);
|
||||
// `u` flag required by PAGE_SLUG_SEG's \p{...} classes.
|
||||
const SLUG_RE = new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})+$`, 'u');
|
||||
|
||||
export class SlugRegistry {
|
||||
constructor(private engine: BrainEngine) {}
|
||||
|
||||
@@ -420,8 +420,10 @@ export class PGLiteEngine implements BrainEngine {
|
||||
let model: string = DEFAULT_EMBEDDING_MODEL;
|
||||
try {
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel() || model;
|
||||
model = gw.getEmbeddingModel();
|
||||
} catch { /* gateway not configured — use defaults */ }
|
||||
|
||||
await this.db.exec(getPGLiteSchema(dims, model));
|
||||
@@ -979,7 +981,8 @@ export class PGLiteEngine implements BrainEngine {
|
||||
const { rows } = await this.db.query(
|
||||
`SELECT id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, deleted_at,
|
||||
effective_date, effective_date_source,
|
||||
source_kind, source_uri, ingested_via, ingested_at
|
||||
source_kind, source_uri, ingested_via, ingested_at,
|
||||
contextual_retrieval_mode
|
||||
FROM pages WHERE ${where.join(' AND ')} LIMIT 1`,
|
||||
params
|
||||
);
|
||||
@@ -2320,15 +2323,26 @@ export class PGLiteEngine implements BrainEngine {
|
||||
|
||||
// Provenance fallback for chunks without an explicit `model`: resolve the
|
||||
// gateway's runtime model, not the compile-time DEFAULT_EMBEDDING_MODEL.
|
||||
// See postgres-engine.ts _upsertChunksOnce for the full rationale — pglite
|
||||
// mirrors it for parity.
|
||||
let resolvedModel: string = DEFAULT_EMBEDDING_MODEL;
|
||||
// #3461: getEmbeddingModel() THROWS when unconfigured (never returns
|
||||
// falsy) — on the throw path fall back to the brain's own
|
||||
// `config.embedding_model` row, then the compile-time default as the
|
||||
// last resort. See postgres-engine.ts _upsertChunksOnce for the full
|
||||
// rationale — pglite mirrors it for parity.
|
||||
let resolvedModel: string | null = null;
|
||||
try {
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
resolvedModel = gw.getEmbeddingModel() || resolvedModel;
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
} catch {
|
||||
// Gateway unconfigured (unit tests / pre-connect): keep the default.
|
||||
try {
|
||||
const cfg = await this.db.query(
|
||||
`SELECT value FROM config WHERE key = 'embedding_model'`,
|
||||
);
|
||||
resolvedModel = ((cfg.rows[0] as { value?: string } | undefined)?.value) ?? null;
|
||||
} catch {
|
||||
// config table unreadable — fall through to the compile-time default.
|
||||
}
|
||||
}
|
||||
if (!resolvedModel) resolvedModel = DEFAULT_EMBEDDING_MODEL;
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const embeddingStr = chunk.embedding
|
||||
@@ -2381,6 +2395,9 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// Code-chunk metadata columns follow the same chunk_text-gated CASE pattern as `embedding`
|
||||
// (#769). Re-chunk trusts EXCLUDED outright; pure re-embed COALESCEs so a caller carrying
|
||||
// only embedding-shaped fields doesn't clobber metadata to NULL.
|
||||
//
|
||||
// #3461: `model` mirrors the `embedding` CASE branch-for-branch so the label always
|
||||
// describes whichever vector wins the upsert. See postgres-engine.ts for rationale.
|
||||
await this.db.query(
|
||||
`INSERT INTO content_chunks ${cols} VALUES ${rowParts.join(', ')}
|
||||
ON CONFLICT (page_id, chunk_index) DO UPDATE SET
|
||||
@@ -2394,7 +2411,14 @@ export class PGLiteEngine implements BrainEngine {
|
||||
THEN EXCLUDED.embedding
|
||||
ELSE content_chunks.embedding
|
||||
END,
|
||||
model = COALESCE(EXCLUDED.model, content_chunks.model),
|
||||
model = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.model
|
||||
WHEN content_chunks.embedding IS NULL THEN EXCLUDED.model
|
||||
WHEN EXCLUDED.embedded_at IS NOT NULL
|
||||
AND (content_chunks.embedded_at IS NULL OR EXCLUDED.embedded_at > content_chunks.embedded_at)
|
||||
THEN EXCLUDED.model
|
||||
ELSE content_chunks.model
|
||||
END,
|
||||
token_count = EXCLUDED.token_count,
|
||||
embedded_at = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text AND EXCLUDED.embedding IS NULL THEN NULL
|
||||
|
||||
@@ -382,8 +382,10 @@ export class PostgresEngine implements BrainEngine {
|
||||
let model: string = DEFAULT_EMBEDDING_MODEL;
|
||||
try {
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
// Both accessors THROW when the gateway is unconfigured (they never
|
||||
// return falsy), so the catch below is the only fallback path (#3461).
|
||||
dims = gw.getEmbeddingDimensions();
|
||||
model = gw.getEmbeddingModel() || model;
|
||||
model = gw.getEmbeddingModel();
|
||||
} catch { /* gateway not yet configured — use defaults */ }
|
||||
|
||||
const sqlText = getPostgresSchema(dims, model);
|
||||
@@ -1031,7 +1033,8 @@ export class PostgresEngine implements BrainEngine {
|
||||
const rows = await tx`
|
||||
SELECT id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, deleted_at,
|
||||
effective_date, effective_date_source,
|
||||
source_kind, source_uri, ingested_via, ingested_at
|
||||
source_kind, source_uri, ingested_via, ingested_at,
|
||||
contextual_retrieval_mode
|
||||
FROM pages
|
||||
WHERE slug = ${slug} ${sourceCondition} ${deletedCondition}
|
||||
LIMIT 1
|
||||
@@ -2437,14 +2440,28 @@ export class PostgresEngine implements BrainEngine {
|
||||
// hardcoded default (e.g. zeroentropyai:zembed-1) onto rows whose vectors
|
||||
// were produced by a different, config-resolved model — corrupting the
|
||||
// provenance that signature-drift staleness + dim-migration logic trust.
|
||||
// Mirrors the resolve-then-fallback pattern used for schema sizing above.
|
||||
let resolvedModel: string = DEFAULT_EMBEDDING_MODEL;
|
||||
//
|
||||
// #3461: getEmbeddingModel() THROWS when the gateway is unconfigured —
|
||||
// it never returns falsy — so an `||` guard here is dead code and the
|
||||
// catch path used to stamp the compile-time default onto rows whose
|
||||
// vectors came from the config-resolved provider. On the throw path we
|
||||
// now fall back to the brain's own `config.embedding_model` row (kept
|
||||
// current by init / migrate / retrieval-upgrade), which names the model
|
||||
// that actually produced this brain's vectors. The compile-time default
|
||||
// is the LAST resort (fresh brain whose config row doesn't exist yet).
|
||||
let resolvedModel: string | null = null;
|
||||
try {
|
||||
const gw = await import('./ai/gateway.ts');
|
||||
resolvedModel = gw.getEmbeddingModel() || resolvedModel;
|
||||
resolvedModel = gw.getEmbeddingModel();
|
||||
} catch {
|
||||
// Gateway unconfigured (unit tests / pre-connect): keep the default.
|
||||
try {
|
||||
const cfg = await sql`SELECT value FROM config WHERE key = 'embedding_model'`;
|
||||
resolvedModel = (cfg[0]?.value as string | undefined) ?? null;
|
||||
} catch {
|
||||
// config table unreadable — fall through to the compile-time default.
|
||||
}
|
||||
}
|
||||
if (!resolvedModel) resolvedModel = DEFAULT_EMBEDDING_MODEL;
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const embeddingStr = chunk.embedding
|
||||
@@ -2508,6 +2525,11 @@ export class PostgresEngine implements BrainEngine {
|
||||
// pure re-embed (chunk_text unchanged) COALESCEs so a caller that only carries embedding
|
||||
// doesn't clobber metadata to NULL. Without this, every embed --stale pass nuked code-def's
|
||||
// primary index for thousands of chunks at once.
|
||||
//
|
||||
// #3461: `model` mirrors the `embedding` CASE branch-for-branch — the label must
|
||||
// describe whichever vector WINS the upsert. The old COALESCE(EXCLUDED.model, …)
|
||||
// relabeled preserved (older-model) vectors with the current gateway model on every
|
||||
// partial re-embed, corrupting provenance without changing the vector.
|
||||
await sql.unsafe(
|
||||
`INSERT INTO content_chunks ${cols} VALUES ${rows.join(', ')}
|
||||
ON CONFLICT (page_id, chunk_index) DO UPDATE SET
|
||||
@@ -2521,7 +2543,14 @@ export class PostgresEngine implements BrainEngine {
|
||||
THEN EXCLUDED.embedding
|
||||
ELSE content_chunks.embedding
|
||||
END,
|
||||
model = COALESCE(EXCLUDED.model, content_chunks.model),
|
||||
model = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text THEN EXCLUDED.model
|
||||
WHEN content_chunks.embedding IS NULL THEN EXCLUDED.model
|
||||
WHEN EXCLUDED.embedded_at IS NOT NULL
|
||||
AND (content_chunks.embedded_at IS NULL OR EXCLUDED.embedded_at > content_chunks.embedded_at)
|
||||
THEN EXCLUDED.model
|
||||
ELSE content_chunks.model
|
||||
END,
|
||||
token_count = EXCLUDED.token_count,
|
||||
embedded_at = CASE
|
||||
WHEN EXCLUDED.chunk_text != content_chunks.chunk_text AND EXCLUDED.embedding IS NULL THEN NULL
|
||||
|
||||
@@ -1777,6 +1777,13 @@ export async function hybridSearchCached(
|
||||
// resolves) into the cache key so a row written under one exclude
|
||||
// policy can't be served to a lookup under another.
|
||||
hardExcludes: resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes),
|
||||
// #3515 — fold the EFFECTIVE detail level into the cache key. detail
|
||||
// gates dedup, chunk-source filtering, and the compiled_truth boost, so
|
||||
// a `--detail low` write (compiled-truth-only result set) must never be
|
||||
// served to a default `medium` lookup. Resolve auto-detect the same way
|
||||
// bare hybridSearch does (opts.detail ?? autoDetectDetail(query)) so an
|
||||
// auto-detected `high` query keys like an explicit `high` one.
|
||||
detail: opts?.detail ?? autoDetectDetail(query),
|
||||
});
|
||||
|
||||
// Cache decision: opts.useCache (explicit) wins over global config; global
|
||||
|
||||
+27
-1
@@ -766,7 +766,17 @@ export function attributeKnob<K extends keyof ModeBundle>(
|
||||
// written between the #3391 stale-fix (which changes which chunks count as
|
||||
// current) and the operator's migration run. Same one-time global cold-miss
|
||||
// pattern as the bumps above.
|
||||
export const KNOBS_HASH_VERSION = 13;
|
||||
//
|
||||
// bump 13→15 (#3515): `detail` folds into the key via ctx.detail (det=).
|
||||
// detail is result-affecting by design — it gates dedup, chunk-source
|
||||
// filtering, and the compiled_truth boost — but was absent from the key, so
|
||||
// a `--detail low` write (compiled-truth-only result set) was served to a
|
||||
// default `medium` lookup for the whole TTL. Same contamination class as
|
||||
// [CDX-4], floor_ratio (v=3), and relationalRetrieval (v=10). v=14 is
|
||||
// claimed by in-flight #3514 (compiled_truth boost scope, #3430), so this
|
||||
// lands as v=15 per the D8 sequencing convention (see the v=4/v=5 note
|
||||
// above). Same one-time global cold-miss pattern as the bumps above.
|
||||
export const KNOBS_HASH_VERSION = 15;
|
||||
|
||||
/**
|
||||
* v0.36 (D8 / CDX-2) — second-arg context for the cache key. The
|
||||
@@ -805,6 +815,17 @@ export interface KnobsHashContext {
|
||||
* 'none' for legacy callers that don't thread excludes.
|
||||
*/
|
||||
hardExcludes?: string[];
|
||||
/**
|
||||
* v=15 (#3515): the EFFECTIVE detail level for this call — per-call
|
||||
* SearchOpts.detail, or the auto-detected level when the caller didn't
|
||||
* specify (hybridSearchCached threads `opts.detail ?? autoDetectDetail(query)`,
|
||||
* matching what bare hybridSearch resolves). detail gates dedup,
|
||||
* chunk-source filtering, and the compiled_truth boost, so a detail=low
|
||||
* write must never be served to a detail=medium lookup. Lives in ctx (not
|
||||
* ResolvedSearchKnobs) because it's per-call, not a mode knob — same path
|
||||
* as col=/prov=. Undefined falls back to 'medium' (the documented default).
|
||||
*/
|
||||
detail?: 'low' | 'medium' | 'high';
|
||||
}
|
||||
|
||||
export function knobsHash(
|
||||
@@ -898,6 +919,11 @@ export function knobsHash(
|
||||
// across processes. Sorted copy so ['a/','b/'] and ['b/','a/'] hash
|
||||
// identically; undefined falls back to 'none' for legacy callers.
|
||||
`hx=${ctx?.hardExcludes ? [...ctx.hardExcludes].sort().join(',') : 'none'}`,
|
||||
// v=15 addition (#3515, append-only): effective detail level. detail
|
||||
// gates dedup, chunk-source filtering, and the compiled_truth boost, so
|
||||
// a low write (compiled-truth-only set) must never be served to a
|
||||
// medium/high lookup. Undefined falls back to 'medium' (the default).
|
||||
`det=${ctx?.detail ?? 'medium'}`,
|
||||
];
|
||||
const h = createHash('sha256');
|
||||
h.update(parts.join('|'));
|
||||
|
||||
@@ -33,6 +33,17 @@
|
||||
|
||||
export const SOURCE_ID_RE = /^[a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?$/;
|
||||
|
||||
/**
|
||||
* Sentinel meaning "span every source" (#1712). Deliberately NOT a valid
|
||||
* source id (underscores are rejected by SOURCE_ID_RE), so it can never
|
||||
* collide with a real source, be created via `sources add`, or leak into
|
||||
* lock ids / path joins. The resolver's explicit/env tiers pass it through
|
||||
* verbatim; `sourceScopeOpts` translates it to an unscoped read for trusted
|
||||
* local callers and keeps it as an unsatisfiable literal for remote callers
|
||||
* (fail-closed).
|
||||
*/
|
||||
export const ALL_SOURCES = '__all__';
|
||||
|
||||
/** Returns true if the string matches the canonical source_id regex. */
|
||||
export function isValidSourceId(s: unknown): s is string {
|
||||
return typeof s === 'string' && SOURCE_ID_RE.test(s);
|
||||
|
||||
@@ -17,9 +17,13 @@ import { readFileSync, lstatSync, type Stats } from 'fs';
|
||||
import { join, dirname, resolve } from 'path';
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import { isSourceFederated } from './sources-load.ts';
|
||||
import { SOURCE_ID_RE, isValidSourceId } from './source-id.ts';
|
||||
import { SOURCE_ID_RE, isValidSourceId, ALL_SOURCES } from './source-id.ts';
|
||||
import { isTrustedDotfile, realpathOrResolve } from './path-confine.ts';
|
||||
|
||||
// Re-export so scope-resolution call sites can import the sentinel from
|
||||
// either module (#1712).
|
||||
export { ALL_SOURCES };
|
||||
|
||||
const DOTFILE = '.gbrain-source';
|
||||
// Canonical SOURCE_ID_RE imported from `source-id.ts` (single source of truth).
|
||||
// Re-exported below as `__testing.SOURCE_ID_RE` for legacy test imports.
|
||||
@@ -83,8 +87,11 @@ export async function resolveSourceId(
|
||||
explicit: string | null | undefined,
|
||||
cwd: string = process.cwd(),
|
||||
): Promise<string> {
|
||||
// 1. Explicit flag wins.
|
||||
// 1. Explicit flag wins. The __all__ sentinel passes through verbatim
|
||||
// (#1712) — it is not a source id, so it skips both the regex and
|
||||
// assertSourceExists; sourceScopeOpts gives it span-everything semantics.
|
||||
if (explicit) {
|
||||
if (explicit === ALL_SOURCES) return ALL_SOURCES;
|
||||
if (!SOURCE_ID_RE.test(explicit)) {
|
||||
throw new Error(`Invalid --source value "${explicit}". Must match [a-z0-9-]{1,32}.`);
|
||||
}
|
||||
@@ -92,9 +99,10 @@ export async function resolveSourceId(
|
||||
return explicit;
|
||||
}
|
||||
|
||||
// 2. Env var.
|
||||
// 2. Env var. Same __all__ pass-through (#2140).
|
||||
const env = process.env.GBRAIN_SOURCE;
|
||||
if (env && env.length > 0) {
|
||||
if (env === ALL_SOURCES) return ALL_SOURCES;
|
||||
if (!SOURCE_ID_RE.test(env)) {
|
||||
throw new Error(`Invalid GBRAIN_SOURCE value "${env}". Must match [a-z0-9-]{1,32}.`);
|
||||
}
|
||||
@@ -173,6 +181,7 @@ export function resolveSourceIdEngineFree(
|
||||
cwd: string = process.cwd(),
|
||||
): string | null {
|
||||
if (explicit) {
|
||||
if (explicit === ALL_SOURCES) return ALL_SOURCES; // #1712 sentinel pass-through
|
||||
if (!SOURCE_ID_RE.test(explicit)) {
|
||||
throw new Error(`Invalid --source value "${explicit}". Must match [a-z0-9-]{1,32}.`);
|
||||
}
|
||||
@@ -180,6 +189,7 @@ export function resolveSourceIdEngineFree(
|
||||
}
|
||||
const env = process.env.GBRAIN_SOURCE;
|
||||
if (env && env.length > 0) {
|
||||
if (env === ALL_SOURCES) return ALL_SOURCES; // #2140 sentinel pass-through
|
||||
if (!SOURCE_ID_RE.test(env)) {
|
||||
throw new Error(`Invalid GBRAIN_SOURCE value "${env}". Must match [a-z0-9-]{1,32}.`);
|
||||
}
|
||||
@@ -315,8 +325,11 @@ export async function resolveSourceWithTier(
|
||||
explicit: string | null | undefined,
|
||||
cwd: string = process.cwd(),
|
||||
): Promise<{ source_id: string; tier: SourceTier; detail?: string }> {
|
||||
// 1. Explicit flag wins.
|
||||
// 1. Explicit flag wins. __all__ sentinel passes through verbatim (#1712).
|
||||
if (explicit) {
|
||||
if (explicit === ALL_SOURCES) {
|
||||
return { source_id: ALL_SOURCES, tier: 'flag', detail: `--source ${ALL_SOURCES} (spans all sources)` };
|
||||
}
|
||||
if (!SOURCE_ID_RE.test(explicit)) {
|
||||
throw new Error(`Invalid --source value "${explicit}". Must match [a-z0-9-]{1,32}.`);
|
||||
}
|
||||
@@ -324,9 +337,12 @@ export async function resolveSourceWithTier(
|
||||
return { source_id: explicit, tier: 'flag', detail: `--source ${explicit}` };
|
||||
}
|
||||
|
||||
// 2. Env var.
|
||||
// 2. Env var. Same __all__ pass-through (#2140).
|
||||
const env = process.env.GBRAIN_SOURCE;
|
||||
if (env && env.length > 0) {
|
||||
if (env === ALL_SOURCES) {
|
||||
return { source_id: ALL_SOURCES, tier: 'env', detail: `GBRAIN_SOURCE=${ALL_SOURCES} (spans all sources)` };
|
||||
}
|
||||
if (!SOURCE_ID_RE.test(env)) {
|
||||
throw new Error(`Invalid GBRAIN_SOURCE value "${env}". Must match [a-z0-9-]{1,32}.`);
|
||||
}
|
||||
|
||||
+13
-8
@@ -11,7 +11,7 @@
|
||||
* pathToSlug() → convert file paths to page slugs
|
||||
*/
|
||||
|
||||
import { CJK_SLUG_CHARS } from './cjk.ts';
|
||||
import { SLUG_WORD_CHARS } from './cjk.ts';
|
||||
// v0.37.7.0 #1169 submodule-detection helpers. Bottom-of-file already
|
||||
// aliases existsSync as `_existsSync` for other purposes; the top-of-file
|
||||
// import keeps the pruneDir helper's deps near its callsite.
|
||||
@@ -396,8 +396,10 @@ export function unsyncableReason(path: string, opts: SyncableOptions = {}): Sync
|
||||
|
||||
/**
|
||||
* Character class for the lowercase-canonical form of a slug segment after
|
||||
* slugifySegment() has run. Lowercase letters, digits, dots, underscores,
|
||||
* hyphens. Exposed so adjacent code (e.g. takes-fence holder validation,
|
||||
* slugifySegment() has run. Letters/numbers in any script (lowercase where
|
||||
* the script has case — #3417), dots, underscores, hyphens. Uses \p{...}
|
||||
* classes, so composed regexes need the `u` flag (this one carries it).
|
||||
* Exposed so adjacent code (e.g. takes-fence holder validation,
|
||||
* v0.32 EXP-4) can reuse the actual repo slug grammar instead of inventing
|
||||
* a stricter parallel one and emitting false-positive warnings on legitimate
|
||||
* `companies/acme.io` / `people/foo_bar` slugs (codex review #3).
|
||||
@@ -405,15 +407,18 @@ export function unsyncableReason(path: string, opts: SyncableOptions = {}): Sync
|
||||
* Pattern is the inner character class only (no anchors); callers wrap it
|
||||
* in `^...$` or compose it with prefixes like `(?:people|companies)/...`.
|
||||
*/
|
||||
export const SLUG_SEGMENT_PATTERN = new RegExp(`[a-z0-9._\\-${CJK_SLUG_CHARS}]+`);
|
||||
export const SLUG_SEGMENT_PATTERN = new RegExp(`[${SLUG_WORD_CHARS}._\\-]+`, 'u');
|
||||
|
||||
/**
|
||||
* Slugify a single path segment: lowercase, strip special chars, spaces → hyphens.
|
||||
* CJK ranges (Han / Hiragana / Katakana / Hangul Syllables) are preserved (v0.32.7).
|
||||
* NFC re-normalize after the NFD-strip-accents pass so Hangul Jamo recomposes back
|
||||
* into precomposed syllables that fall inside the whitelist.
|
||||
* Letters and numbers from EVERY script are preserved (#3417): previously only
|
||||
* Latin + CJK survived, so Hebrew/Arabic/Cyrillic/Greek/Thai/... filenames
|
||||
* collapsed to empty segments and distinct files silently merged onto one slug.
|
||||
* NFC re-normalize after the NFD-strip-accents pass so Hangul Jamo recomposes
|
||||
* back into precomposed syllables, and so NFD filenames (macOS) and NFC
|
||||
* filenames (Linux/git) of the same name produce the SAME slug.
|
||||
*/
|
||||
const SLUGIFY_KEEP_RE = new RegExp(`[^a-z0-9.\\s_\\-${CJK_SLUG_CHARS}]`, 'g');
|
||||
const SLUGIFY_KEEP_RE = new RegExp(`[^${SLUG_WORD_CHARS}.\\s_\\-]`, 'gu');
|
||||
|
||||
export function slugifySegment(segment: string): string {
|
||||
return segment
|
||||
|
||||
@@ -134,6 +134,7 @@ export const TAKES_FENCE_END = '<!--- gbrain:takes:end -->';
|
||||
import { SLUG_SEGMENT_PATTERN } from './sync.ts';
|
||||
export const HOLDER_REGEX = new RegExp(
|
||||
`^(?:world|brain|(?:people|companies)/${SLUG_SEGMENT_PATTERN.source}|${SLUG_SEGMENT_PATTERN.source})$`,
|
||||
'u', // required by SLUG_SEGMENT_PATTERN's \p{...} classes (#3417)
|
||||
);
|
||||
|
||||
/**
|
||||
|
||||
@@ -110,6 +110,12 @@ export function rowToPage(row: Record<string, unknown>): Page {
|
||||
const sourceUri = row.source_uri === undefined ? undefined : (row.source_uri as string | null);
|
||||
const ingestedVia = row.ingested_via === undefined ? undefined : (row.ingested_via as string | null);
|
||||
const ingestedAt = readOptionalDate(row.ingested_at);
|
||||
// #3507: the CR tier the page was last embedded under (three-state, same
|
||||
// pattern as the provenance columns above). Re-embed paths (`embed --stale`
|
||||
// and friends) read this to reproduce the page's stored wrapping convention.
|
||||
const contextualRetrievalMode = row.contextual_retrieval_mode === undefined
|
||||
? undefined
|
||||
: (row.contextual_retrieval_mode as Page['contextual_retrieval_mode']);
|
||||
return {
|
||||
id: row.id as number,
|
||||
slug: row.slug as string,
|
||||
@@ -135,6 +141,7 @@ export function rowToPage(row: Record<string, unknown>): Page {
|
||||
...(sourceUri !== undefined && { source_uri: sourceUri }),
|
||||
...(ingestedVia !== undefined && { ingested_via: ingestedVia }),
|
||||
...(ingestedAt !== undefined && { ingested_at: ingestedAt }),
|
||||
...(contextualRetrievalMode !== undefined && { contextual_retrieval_mode: contextualRetrievalMode }),
|
||||
// v0.31.12: propagate source_id so downstream callers (embed, reconcile-links)
|
||||
// can thread it through getChunks / upsertChunks without defaulting to 'default'.
|
||||
// v0.32.8: Page.source_id is required. Every SELECT feeding rowToPage now
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* #1712 (dupes #2289, #2140) — the `__all__` sentinel must work in EVERY
|
||||
* resolution tier, not just as a per-call `source_id` param.
|
||||
*
|
||||
* The bug: SOURCE_ID_RE forbids underscores, so `--source __all__` and
|
||||
* `GBRAIN_SOURCE=__all__` threw in the resolver; the CLI's makeContext
|
||||
* blanket-caught that and silently fell back to `sourceId: 'default'` —
|
||||
* making the documented span-everything flag STRICTLY NARROWER than passing
|
||||
* no flag at all (the catch also discarded the #2561/#3242 federated
|
||||
* widening). Meanwhile sourceScopeOpts treated a ctx.sourceId of '__all__'
|
||||
* as an unsatisfiable literal.
|
||||
*
|
||||
* Uses the literal '__all__' (not the ALL_SOURCES constant) so these tests
|
||||
* load and run behaviorally against pre-fix trees.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import {
|
||||
resolveSourceId,
|
||||
resolveSourceIdEngineFree,
|
||||
resolveSourceWithTier,
|
||||
} from '../src/core/source-resolver.ts';
|
||||
import {
|
||||
sourceScopeOpts,
|
||||
federatedSearchScope,
|
||||
type OperationContext,
|
||||
} from '../src/core/operations.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
|
||||
// Stub engine: registered sources + no local_path rows + no default config.
|
||||
function makeStub(registeredSources: string[]): BrainEngine {
|
||||
return {
|
||||
kind: 'pglite',
|
||||
executeRaw: async <T>(sql: string, params?: unknown[]): Promise<T[]> => {
|
||||
if (sql.includes('SELECT id FROM sources WHERE id = $1')) {
|
||||
const target = params?.[0];
|
||||
return registeredSources.includes(target as string)
|
||||
? [{ id: target } as unknown as T]
|
||||
: [];
|
||||
}
|
||||
return [];
|
||||
},
|
||||
getConfig: async () => null,
|
||||
} as unknown as BrainEngine;
|
||||
}
|
||||
|
||||
function ctxOf(overrides: Partial<OperationContext> = {}): OperationContext {
|
||||
return {
|
||||
engine: {} as any,
|
||||
config: {} as any,
|
||||
logger: console as any,
|
||||
dryRun: false,
|
||||
remote: true,
|
||||
sourceId: 'default',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Resolver tiers pass the sentinel through verbatim ──────────────────
|
||||
|
||||
describe('source-resolver — __all__ sentinel pass-through', () => {
|
||||
test('resolveSourceId: explicit --source __all__ resolves (no regex throw, no existence check)', async () => {
|
||||
// '__all__' is deliberately NOT in the registered set — the sentinel
|
||||
// must skip assertSourceExists (it is not a source id).
|
||||
const id = await resolveSourceId(makeStub(['default']), '__all__', '/nonexistent');
|
||||
expect(id).toBe('__all__');
|
||||
});
|
||||
|
||||
test('resolveSourceId: GBRAIN_SOURCE=__all__ resolves (#2140)', async () => {
|
||||
await withEnv({ GBRAIN_SOURCE: '__all__' }, async () => {
|
||||
const id = await resolveSourceId(makeStub(['default']), null, '/nonexistent');
|
||||
expect(id).toBe('__all__');
|
||||
});
|
||||
});
|
||||
|
||||
test('resolveSourceIdEngineFree: explicit + env __all__ (thin-client path)', async () => {
|
||||
expect(resolveSourceIdEngineFree('__all__', '/nonexistent')).toBe('__all__');
|
||||
await withEnv({ GBRAIN_SOURCE: '__all__' }, () => {
|
||||
expect(resolveSourceIdEngineFree(null, '/nonexistent')).toBe('__all__');
|
||||
});
|
||||
});
|
||||
|
||||
test('resolveSourceWithTier: flag and env tiers carry the sentinel', async () => {
|
||||
const flag = await resolveSourceWithTier(makeStub(['default']), '__all__', '/nonexistent');
|
||||
expect(flag).toMatchObject({ source_id: '__all__', tier: 'flag' });
|
||||
await withEnv({ GBRAIN_SOURCE: '__all__' }, async () => {
|
||||
const env = await resolveSourceWithTier(makeStub(['default']), null, '/nonexistent');
|
||||
expect(env).toMatchObject({ source_id: '__all__', tier: 'env' });
|
||||
});
|
||||
});
|
||||
|
||||
test('a genuinely invalid --source still throws (SOURCE_ID_RE not loosened)', async () => {
|
||||
await expect(resolveSourceId(makeStub(['default']), 'my_source', '/nonexistent'))
|
||||
.rejects.toThrow(/Invalid --source/);
|
||||
expect(() => resolveSourceIdEngineFree('my_source', '/nonexistent'))
|
||||
.toThrow(/Invalid --source/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── sourceScopeOpts — the single read-scope choke point ─────────────────
|
||||
|
||||
describe('sourceScopeOpts — __all__ sentinel', () => {
|
||||
test('trusted local (remote === false): spans the whole brain (empty scope)', () => {
|
||||
expect(sourceScopeOpts(ctxOf({ remote: false, sourceId: '__all__' }))).toEqual({});
|
||||
});
|
||||
|
||||
test('remote: keeps the unsatisfiable literal — fail-closed, never widens', () => {
|
||||
expect(sourceScopeOpts(ctxOf({ remote: true, sourceId: '__all__' })))
|
||||
.toEqual({ sourceId: '__all__' });
|
||||
});
|
||||
|
||||
test('anything not strictly remote === false is untrusted (fail-closed)', () => {
|
||||
// undefined / missing remote must behave like remote, per the trust rule.
|
||||
const ctx = ctxOf({ sourceId: '__all__' });
|
||||
(ctx as any).remote = undefined;
|
||||
expect(sourceScopeOpts(ctx)).toEqual({ sourceId: '__all__' });
|
||||
});
|
||||
|
||||
test('a federated grant always wins over the sentinel', () => {
|
||||
const ctx = ctxOf({
|
||||
remote: true,
|
||||
sourceId: '__all__',
|
||||
auth: { token: 't', clientId: 'c', scopes: [], allowedSources: ['a', 'b'] } as any,
|
||||
});
|
||||
expect(sourceScopeOpts(ctx)).toEqual({ sourceIds: ['a', 'b'] });
|
||||
});
|
||||
});
|
||||
|
||||
// ── Never narrower than passing no flag (#2561 regression shape) ────────
|
||||
|
||||
describe('__all__ is never narrower than an unqualified read', () => {
|
||||
test('local __all__ spans the brain even when federated widening exists', () => {
|
||||
// Unqualified read on a federated brain widens to the federated array…
|
||||
const unqualified = ctxOf({
|
||||
remote: false,
|
||||
sourceId: 'default',
|
||||
localFederatedSourceIds: ['default', 'src-a', 'src-b'],
|
||||
});
|
||||
expect(federatedSearchScope(unqualified)).toEqual({
|
||||
sourceIds: ['default', 'src-a', 'src-b'],
|
||||
});
|
||||
// …and __all__ must be a superset of that: the whole brain ({}).
|
||||
const all = ctxOf({ remote: false, sourceId: '__all__' });
|
||||
expect(federatedSearchScope(all)).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
// ── makeContext — explicit --source failures error loudly ───────────────
|
||||
|
||||
describe('cli makeContext — no silent default fallback for explicit --source', () => {
|
||||
test('--source __all__ produces ctx.sourceId __all__ (was: silent default)', async () => {
|
||||
const { makeContext } = await import('../src/cli.ts');
|
||||
const ctx = await makeContext(makeStub(['default']), { source: '__all__' });
|
||||
expect(ctx.sourceId).toBe('__all__');
|
||||
expect(ctx.remote).toBe(false);
|
||||
});
|
||||
|
||||
test('an explicit --source that fails to resolve throws instead of becoming default', async () => {
|
||||
const { makeContext } = await import('../src/cli.ts');
|
||||
await expect(makeContext(makeStub(['default']), { source: 'ghost' }))
|
||||
.rejects.toThrow(/not found/);
|
||||
await expect(makeContext(makeStub(['default']), { source: 'my_source' }))
|
||||
.rejects.toThrow(/Invalid --source/);
|
||||
});
|
||||
|
||||
test('ambient resolution failure still falls back silently (pre-init brains)', async () => {
|
||||
const { makeContext } = await import('../src/cli.ts');
|
||||
const broken = {
|
||||
kind: 'pglite',
|
||||
executeRaw: async () => { throw new Error('relation "sources" does not exist'); },
|
||||
getConfig: async () => { throw new Error('relation "config" does not exist'); },
|
||||
} as unknown as BrainEngine;
|
||||
const ctx = await makeContext(broken, {});
|
||||
expect(ctx.sourceId).toBe('default');
|
||||
});
|
||||
});
|
||||
@@ -11,15 +11,34 @@ import { tmpdir } from 'os';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { hardenBrainRepo } from '../src/core/brain-repo-durability.ts';
|
||||
|
||||
// #2943 root cause: `env: process.env` is REQUIRED here. Bun snapshots
|
||||
// process.env at startup, so without it the spawned git — and any post-commit
|
||||
// hook it fires — is blind to beforeEach's HOME/GBRAIN_HOME mutations (the
|
||||
// same Bun quirk as #2747, see resolveGbrainCliPath in brain-repo-durability).
|
||||
// Pre-fix, the hook under test resolved ${GBRAIN_HOME:-$HOME/.gbrain} to the
|
||||
// OPERATOR'S REAL ~/.gbrain: it wrote its log lines there (polluting the real
|
||||
// brain-push.log on every run), the LOCAL-ONLY test never saw them in the
|
||||
// temp log it polls, and the assertion only passed when the scaffolding push
|
||||
// from beforeEach (spawned by hardenBrainRepo WITH explicit env) happened to
|
||||
// still be in flight, lose the ref race, and retry AFTER the test had pointed
|
||||
// origin at the dead path — an accidental, load-dependent signal. That race
|
||||
// is the CI flake.
|
||||
function git(cwd: string, ...args: string[]): string {
|
||||
return execFileSync('git', ['-C', cwd, '-c', 'protocol.file.allow=always', ...args], {
|
||||
stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8', env: process.env,
|
||||
}).trim();
|
||||
}
|
||||
function originHead(bare: string): string {
|
||||
return git(bare, 'rev-parse', 'refs/heads/main');
|
||||
}
|
||||
async function waitForOrigin(bare: string, expectSha: string, ms = 8000): Promise<boolean> {
|
||||
// #2943: 30s poll deadlines (was 8s) for headroom under loaded CI shards —
|
||||
// the unreachable-origin path runs ~6 sequential process spawns after the
|
||||
// hook detaches. Every hook test also passes an explicit 60_000 third-arg
|
||||
// timeout: bun 1.3.14 IGNORES bunfig.toml's `timeout` key, so a bare
|
||||
// `bun test` enforces its 5000ms default and killed these tests before the
|
||||
// internal deadline could even elapse (the runner scripts pass --timeout
|
||||
// explicitly, which is why the inversion only bit direct local runs).
|
||||
async function waitForOrigin(bare: string, expectSha: string, ms = 30_000): Promise<boolean> {
|
||||
const deadline = Date.now() + ms;
|
||||
while (Date.now() < deadline) {
|
||||
try { if (originHead(bare) === expectSha) return true; } catch { /* */ }
|
||||
@@ -28,6 +47,24 @@ async function waitForOrigin(bare: string, expectSha: string, ms = 8000): Promis
|
||||
return false;
|
||||
}
|
||||
|
||||
/** #2943 (index.lock form): hardenBrainRepo installs the post-commit hook
|
||||
* BEFORE committing the scaffolding, so that commit fires the hook and
|
||||
* detaches a background brain_push. If that push loses the ref race against
|
||||
* hardenBrainRepo's own synchronous push, it falls back to `git pull
|
||||
* --rebase`, which takes .git/index.lock — racing the test body's first git
|
||||
* calls ("Unable to create '.../.git/index.lock': File exists"). Wait for the
|
||||
* detached push's terminal log line before handing the repo to the test. */
|
||||
async function waitForHookPushSettled(ms = 30_000): Promise<void> {
|
||||
const log = join(process.env.GBRAIN_HOME!, 'brain-push.log');
|
||||
const terminal = /\[push\] (ok|lock-timeout|LOCAL-ONLY)/;
|
||||
const deadline = Date.now() + ms;
|
||||
while (Date.now() < deadline) {
|
||||
if (existsSync(log) && terminal.test(readFileSync(log, 'utf-8'))) return;
|
||||
await new Promise(r => setTimeout(r, 150));
|
||||
}
|
||||
throw new Error(`detached hook push did not settle within ${ms}ms (${log})`);
|
||||
}
|
||||
|
||||
let root: string, work: string, bare: string;
|
||||
let oldHome: string | undefined, oldGbrainHome: string | undefined;
|
||||
|
||||
@@ -38,14 +75,15 @@ beforeEach(async () => {
|
||||
process.env.GBRAIN_HOME = join(process.env.HOME, '.gbrain');
|
||||
process.env.GBRAIN_GIT_ALLOW_FILE_TRANSPORT = '1';
|
||||
bare = mkdtempSync(join(root, 'origin-')) + '.git';
|
||||
execFileSync('git', ['init', '-q', '--bare', '-b', 'main', bare], { stdio: 'ignore' });
|
||||
execFileSync('git', ['init', '-q', '--bare', '-b', 'main', bare], { stdio: 'ignore', env: process.env });
|
||||
work = mkdtempSync(join(root, 'work-'));
|
||||
execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, work], { stdio: 'ignore' });
|
||||
execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, work], { stdio: 'ignore', env: process.env });
|
||||
git(work, 'config', 'user.email', 't@t.t'); git(work, 'config', 'user.name', 'tester');
|
||||
writeFileSync(join(work, 'README.md'), 'init\n');
|
||||
git(work, 'add', 'README.md'); git(work, 'commit', '-qm', 'init'); git(work, 'push', '-q', 'origin', 'main');
|
||||
git(work, 'remote', 'set-head', 'origin', 'main');
|
||||
await hardenBrainRepo({ repoPath: work, sourceId: 'wiki', pat: 'ghp_x', installCron: false });
|
||||
await waitForHookPushSettled();
|
||||
});
|
||||
afterEach(() => {
|
||||
if (oldHome === undefined) delete process.env.HOME; else process.env.HOME = oldHome;
|
||||
@@ -65,7 +103,7 @@ describe('brain-commit-push.sh (D13 guarantee)', () => {
|
||||
expect(originHead(bare)).toBe(git(work, 'rev-parse', 'HEAD'));
|
||||
// origin actually has the file
|
||||
const verify = mkdtempSync(join(root, 'verify-'));
|
||||
execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, verify], { stdio: 'ignore' });
|
||||
execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, verify], { stdio: 'ignore', env: process.env });
|
||||
expect(existsSync(join(verify, 'people', 'alice.md'))).toBe(true);
|
||||
});
|
||||
|
||||
@@ -102,7 +140,7 @@ describe('brain-commit-push.sh (D13 guarantee)', () => {
|
||||
rmSync(join(work, '.git', 'hooks', 'post-commit'));
|
||||
// Advance the remote from a second clone so a pull is genuinely needed.
|
||||
const other = mkdtempSync(join(root, 'other-'));
|
||||
execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, other], { stdio: 'ignore' });
|
||||
execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, other], { stdio: 'ignore', env: process.env });
|
||||
git(other, 'config', 'user.email', 'o@o.o'); git(other, 'config', 'user.name', 'other');
|
||||
writeFileSync(join(other, 'remote.md'), 'from other\n');
|
||||
git(other, 'add', 'remote.md'); git(other, 'commit', '-qm', 'remote change'); git(other, 'push', '-q', 'origin', 'main');
|
||||
@@ -128,26 +166,26 @@ describe('post-commit hook (D9 local, D7 self-contained)', () => {
|
||||
git(work, 'add', 'note.md'); git(work, 'commit', '-qm', 'note'); // fires .git/hooks/post-commit
|
||||
const head = git(work, 'rev-parse', 'HEAD');
|
||||
expect(await waitForOrigin(bare, head)).toBe(true);
|
||||
});
|
||||
}, 60_000);
|
||||
|
||||
test('the hook works even with the committed helper deleted (self-contained)', async () => {
|
||||
rmSync(join(work, 'scripts', 'brain-commit-push.sh'));
|
||||
git(work, 'add', '-A'); git(work, 'commit', '-qm', 'remove helper');
|
||||
const head = git(work, 'rev-parse', 'HEAD');
|
||||
expect(await waitForOrigin(bare, head)).toBe(true);
|
||||
});
|
||||
}, 60_000);
|
||||
|
||||
test('logs a clear LOCAL-ONLY line when origin is unreachable', async () => {
|
||||
git(work, 'remote', 'set-url', 'origin', join(root, 'gone2.git'));
|
||||
writeFileSync(join(work, 'orphan.md'), 'o\n');
|
||||
git(work, 'add', 'orphan.md'); git(work, 'commit', '-qm', 'orphan');
|
||||
const log = join(process.env.GBRAIN_HOME!, 'brain-push.log');
|
||||
const deadline = Date.now() + 8000;
|
||||
const deadline = Date.now() + 30_000;
|
||||
let found = false;
|
||||
while (Date.now() < deadline) {
|
||||
if (existsSync(log) && readFileSync(log, 'utf-8').includes('NEEDS ATTENTION')) { found = true; break; }
|
||||
await new Promise(r => setTimeout(r, 150));
|
||||
}
|
||||
expect(found).toBe(true);
|
||||
});
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
/**
|
||||
* Serial (stubs globalThis.fetch): exercises the self-upgrade cache REFRESH
|
||||
* orchestration end-to-end — `refreshUpdateCache()` fetches the latest release
|
||||
* and writes the correct marker to the shared cache file that the CLI startup
|
||||
* hook reads. Network is stubbed; the cache write + marker logic are real.
|
||||
* orchestration end-to-end — `refreshUpdateCache()` resolves the latest version
|
||||
* (from the VERSION file on master, #486 — the repo has zero GitHub releases,
|
||||
* so the old `releases/latest` API path could never succeed) and writes the
|
||||
* correct marker to the shared cache file that the CLI startup hook reads.
|
||||
* Network is stubbed; the cache write + marker logic are real.
|
||||
*
|
||||
* Quarantined as *.serial.test.ts because it reassigns the process-global
|
||||
* `fetch` (cross-file-unsafe under the parallel runner).
|
||||
@@ -13,10 +15,11 @@ import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { VERSION } from '../src/version.ts';
|
||||
import { parseSemver } from '../src/core/semver.ts';
|
||||
import { readUpdateCache } from '../src/core/self-upgrade.ts';
|
||||
import { refreshUpdateCache } from '../src/commands/check-update.ts';
|
||||
import { readUpdateCache, writeUpdateCache } from '../src/core/self-upgrade.ts';
|
||||
import { fetchLatestRelease, parseVersionFileBody, refreshUpdateCache, runCheckUpdate } from '../src/commands/check-update.ts';
|
||||
|
||||
const realFetch = globalThis.fetch;
|
||||
const realLog = console.log;
|
||||
let homeDir: string;
|
||||
let priorHome: string | undefined;
|
||||
|
||||
@@ -27,14 +30,13 @@ function bump(kind: 'minor' | 'patch' | 'micro'): string {
|
||||
return `${v[0]}.${v[1]}.${v[2]}.${v[3] + 1}`;
|
||||
}
|
||||
|
||||
function stubReleaseFetch(tag: string | null, ok = true): void {
|
||||
/** Stub the VERSION-file fetch. body === null → network throw. */
|
||||
function stubVersionFetch(body: string | null, status = 200): void {
|
||||
globalThis.fetch = (async (url: any) => {
|
||||
const u = String(url);
|
||||
if (u.includes('/releases/latest')) {
|
||||
if (tag === null) throw new Error('network down');
|
||||
return new Response(JSON.stringify({ tag_name: tag, published_at: '2026-01-01T00:00:00Z', html_url: 'https://x' }), {
|
||||
status: ok ? 200 : 500,
|
||||
});
|
||||
if (u.includes('/gbrain/master/VERSION')) {
|
||||
if (body === null) throw new Error('network down');
|
||||
return new Response(body, { status });
|
||||
}
|
||||
// Changelog fetch (only happens when update available) — return empty.
|
||||
return new Response('', { status: 200 });
|
||||
@@ -49,49 +51,155 @@ beforeEach(() => {
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = realFetch;
|
||||
console.log = realLog;
|
||||
if (priorHome === undefined) delete process.env.GBRAIN_HOME;
|
||||
else process.env.GBRAIN_HOME = priorHome;
|
||||
rmSync(homeDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('fetchLatestRelease — resolves from the VERSION file, discriminates failures', () => {
|
||||
test('bare version body → ok with that tag', async () => {
|
||||
stubVersionFetch('0.99.1.0\n');
|
||||
expect(await fetchLatestRelease()).toMatchObject({ ok: true, tag: '0.99.1.0' });
|
||||
});
|
||||
|
||||
test('network throw → network_error (NOT no_releases — offline users are not told "no releases exist")', async () => {
|
||||
stubVersionFetch(null);
|
||||
expect(await fetchLatestRelease()).toEqual({ ok: false, reason: 'network_error' });
|
||||
});
|
||||
|
||||
test('HTTP 404 → no_releases', async () => {
|
||||
stubVersionFetch('Not Found', 404);
|
||||
expect(await fetchLatestRelease()).toEqual({ ok: false, reason: 'no_releases' });
|
||||
});
|
||||
|
||||
test('garbage body → no_releases', async () => {
|
||||
stubVersionFetch('<html>rate limited</html>');
|
||||
expect(await fetchLatestRelease()).toEqual({ ok: false, reason: 'no_releases' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseVersionFileBody — shape gate over the raw fetch body', () => {
|
||||
test('trailing newline, v prefix, 3-segment legacy, suffix channel', () => {
|
||||
expect(parseVersionFileBody('0.42.67.0\n')).toBe('0.42.67.0');
|
||||
expect(parseVersionFileBody('v0.42.67.0')).toBe('0.42.67.0');
|
||||
expect(parseVersionFileBody('0.31.3\n')).toBe('0.31.3'); // legacy 3-segment
|
||||
expect(parseVersionFileBody('0.31.1.1-fixwave\n')).toBe('0.31.1.1'); // suffix compares as base
|
||||
});
|
||||
|
||||
test('malformed / huge / injected bodies → null', () => {
|
||||
expect(parseVersionFileBody('')).toBeNull();
|
||||
expect(parseVersionFileBody('not a version')).toBeNull();
|
||||
expect(parseVersionFileBody('$(rm -rf /)')).toBeNull();
|
||||
expect(parseVersionFileBody('1.2')).toBeNull(); // 2-segment: not a gbrain version
|
||||
expect(parseVersionFileBody('9'.repeat(10_000_000))).toBeNull(); // bounded, no blowup
|
||||
});
|
||||
});
|
||||
|
||||
describe('refreshUpdateCache — full refresh orchestration (network stubbed)', () => {
|
||||
test('minor-bump release → writes upgrade_available marker', async () => {
|
||||
test('minor-bump VERSION on master → writes upgrade_available marker', async () => {
|
||||
const latest = bump('minor');
|
||||
stubReleaseFetch(`v${latest}`);
|
||||
stubVersionFetch(`${latest}\n`);
|
||||
await refreshUpdateCache();
|
||||
const entry = readUpdateCache();
|
||||
expect(entry?.marker).toEqual({ kind: 'upgrade_available', current: VERSION, latest });
|
||||
});
|
||||
|
||||
test('patch release → writes upgrade_available marker', async () => {
|
||||
test('patch bump → writes upgrade_available marker', async () => {
|
||||
const latest = bump('patch');
|
||||
stubReleaseFetch(`v${latest}`);
|
||||
stubVersionFetch(`${latest}\n`);
|
||||
await refreshUpdateCache();
|
||||
expect(readUpdateCache()?.marker).toEqual({ kind: 'upgrade_available', current: VERSION, latest });
|
||||
});
|
||||
|
||||
test('micro release → writes upgrade_available marker', async () => {
|
||||
test('micro bump → writes upgrade_available marker', async () => {
|
||||
const latest = bump('micro');
|
||||
stubReleaseFetch(`v${latest}`);
|
||||
stubVersionFetch(`${latest}\n`);
|
||||
await refreshUpdateCache();
|
||||
expect(readUpdateCache()?.marker).toEqual({ kind: 'upgrade_available', current: VERSION, latest });
|
||||
});
|
||||
|
||||
test('network failure → writes up_to_date marker (fail-open, TTL prevents hammering)', async () => {
|
||||
stubReleaseFetch(null);
|
||||
test('minor bump published as legacy 3-segment → still detected', async () => {
|
||||
const v = parseSemver(VERSION)!;
|
||||
const latest = `${v[0]}.${v[1] + 1}.0`; // 3-segment, no micro
|
||||
stubVersionFetch(`${latest}\n`);
|
||||
await refreshUpdateCache();
|
||||
expect(readUpdateCache()?.marker).toEqual({ kind: 'upgrade_available', current: VERSION, latest });
|
||||
});
|
||||
|
||||
test('suffix channel release (X.Y.Z.W-fixwave) → compares as numeric base', async () => {
|
||||
const latest = bump('micro');
|
||||
stubVersionFetch(`${latest}-fixwave\n`);
|
||||
await refreshUpdateCache();
|
||||
expect(readUpdateCache()?.marker).toEqual({ kind: 'upgrade_available', current: VERSION, latest });
|
||||
});
|
||||
|
||||
test('same version on master → up_to_date marker', async () => {
|
||||
stubVersionFetch(`${VERSION}\n`);
|
||||
await refreshUpdateCache();
|
||||
expect(readUpdateCache()?.marker).toEqual({ kind: 'up_to_date', current: VERSION });
|
||||
});
|
||||
|
||||
test('non-OK HTTP → fail-open up_to_date', async () => {
|
||||
stubReleaseFetch(`v${bump('minor')}`, false);
|
||||
// The #486 bug class: a failed check must never fabricate "you're current".
|
||||
test('network failure with NO prior cache → writes NOTHING (never a fabricated up_to_date)', async () => {
|
||||
stubVersionFetch(null);
|
||||
await refreshUpdateCache();
|
||||
expect(readUpdateCache()?.marker).toEqual({ kind: 'up_to_date', current: VERSION });
|
||||
expect(readUpdateCache()).toBeNull();
|
||||
});
|
||||
|
||||
test('garbage tag → fail-open up_to_date (forged/invalid version never cached as upgrade)', async () => {
|
||||
stubReleaseFetch('v$(rm -rf /)');
|
||||
test('network failure with prior upgrade_available → pending notice PRESERVED, not erased', async () => {
|
||||
const latest = bump('minor');
|
||||
writeUpdateCache({ kind: 'upgrade_available', current: VERSION, latest });
|
||||
stubVersionFetch(null);
|
||||
await refreshUpdateCache();
|
||||
expect(readUpdateCache()?.marker).toEqual({ kind: 'up_to_date', current: VERSION });
|
||||
expect(readUpdateCache()?.marker).toEqual({ kind: 'upgrade_available', current: VERSION, latest });
|
||||
});
|
||||
|
||||
test('non-OK HTTP → no fabricated up_to_date', async () => {
|
||||
stubVersionFetch('nope', 500);
|
||||
await refreshUpdateCache();
|
||||
expect(readUpdateCache()).toBeNull();
|
||||
});
|
||||
|
||||
test('garbage body → no fabricated marker (forged/invalid version never cached as upgrade)', async () => {
|
||||
stubVersionFetch('$(rm -rf /)');
|
||||
await refreshUpdateCache();
|
||||
expect(readUpdateCache()).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('runCheckUpdate --json — failure discrimination (#486)', () => {
|
||||
function capture(): string[] {
|
||||
const lines: string[] = [];
|
||||
console.log = (...a: unknown[]) => { lines.push(a.join(' ')); };
|
||||
return lines;
|
||||
}
|
||||
|
||||
test('offline → error: network_error (not "no releases exist")', async () => {
|
||||
stubVersionFetch(null);
|
||||
const lines = capture();
|
||||
await runCheckUpdate(['--json']);
|
||||
const out = JSON.parse(lines.join('\n'));
|
||||
expect(out.error).toBe('network_error');
|
||||
expect(out.update_available).toBe(false);
|
||||
expect(readUpdateCache()).toBeNull(); // and no fabricated up_to_date cache
|
||||
});
|
||||
|
||||
test('endpoint answers but no usable version → error: no_releases', async () => {
|
||||
stubVersionFetch('garbage');
|
||||
const lines = capture();
|
||||
await runCheckUpdate(['--json']);
|
||||
expect(JSON.parse(lines.join('\n')).error).toBe('no_releases');
|
||||
});
|
||||
|
||||
test('newer VERSION on master → update_available true with latest_version set', async () => {
|
||||
const latest = bump('minor');
|
||||
stubVersionFetch(`${latest}\n`);
|
||||
const lines = capture();
|
||||
await runCheckUpdate(['--json']);
|
||||
const out = JSON.parse(lines.join('\n'));
|
||||
expect(out.update_available).toBe(true);
|
||||
expect(out.latest_version).toBe(latest);
|
||||
expect(readUpdateCache()?.marker).toEqual({ kind: 'upgrade_available', current: VERSION, latest });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -35,6 +35,13 @@ describe('isNewerVersion', () => {
|
||||
expect(isNewerVersion('0.42.66.0', '0.42.66.1')).toBe(true);
|
||||
});
|
||||
|
||||
test('orders legacy 3-segment against 4-segment: 0.42.67.0 > 0.42.66.1 > 0.42.66', () => {
|
||||
expect(isNewerVersion('0.42.66.1', '0.42.67.0')).toBe(true);
|
||||
expect(isNewerVersion('0.42.66', '0.42.66.1')).toBe(true);
|
||||
expect(isNewerVersion('0.42.66', '0.42.66.0')).toBe(false); // 3-segment == its .0 micro
|
||||
expect(isNewerVersion('0.42.67.0', '0.42.66.1')).toBe(false);
|
||||
});
|
||||
|
||||
test('rejects equal, older, and malformed versions', () => {
|
||||
expect(isNewerVersion('0.42.66.0', '0.42.66.0')).toBe(false);
|
||||
expect(isNewerVersion('0.42.66.1', '0.42.66.0')).toBe(false);
|
||||
|
||||
@@ -136,7 +136,7 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => {
|
||||
return resolveSearchMode({ mode: 'balanced' });
|
||||
}
|
||||
|
||||
test('KNOBS_HASH_VERSION is 13 (cross-modal still appended; 12→13 embedding-provider migration #3390)', () => {
|
||||
test('KNOBS_HASH_VERSION is 15 (cross-modal still appended; 13→15 detail fold #3515)', () => {
|
||||
// v0.35 ladder: 1→2 reranker, 2→3 floor_ratio. v0.36 piggybacks on v=3
|
||||
// with 7 cross-modal knobs + column/provider context. v0.40.4 (salem) +
|
||||
// v0.39 T21 (master) bump to v=4 for graph_signals + schema-pack fields.
|
||||
@@ -146,7 +146,8 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => {
|
||||
// v0.43: 9→10 relational recall arm. #1400: 10→11 query-side input_type
|
||||
// finally reaches asymmetric providers — pre-fix rows were keyed on
|
||||
// document-side query vectors. #2825: 11→12 hard-exclude fold (hx=).
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
// #3515: 13→15 detail fold (det=); v=14 claimed by in-flight #3514.
|
||||
expect(KNOBS_HASH_VERSION).toBe(15);
|
||||
});
|
||||
|
||||
test('flipping unified_multimodal changes the hash', () => {
|
||||
|
||||
@@ -252,6 +252,87 @@ describe('upsertChunks — model provenance uses gateway-resolved model, not com
|
||||
|
||||
resetGateway();
|
||||
});
|
||||
|
||||
// #3461: getEmbeddingModel() THROWS when the gateway is unconfigured — it
|
||||
// never returns falsy — so the reland's `|| resolvedModel` guard was dead
|
||||
// code and the catch path still stamped the compile-time default onto rows
|
||||
// whose vectors came from the config-resolved provider. The engine must
|
||||
// fall back to the brain's own `config.embedding_model` row instead.
|
||||
test('#3461: unconfigured gateway falls back to the brain config model, never the compiled default', async () => {
|
||||
await engine.setConfig('embedding_model', 'voyage:voyage-3-large');
|
||||
// The preload's beforeEach re-configures the gateway before every test,
|
||||
// so the reset must happen INSIDE the test body.
|
||||
resetGateway();
|
||||
|
||||
await engine.putPage('docs/provenance-throw-path', {
|
||||
type: 'concept',
|
||||
title: 'Provenance throw-path page',
|
||||
compiled_truth: 'Chunk written while the gateway is unconfigured.',
|
||||
});
|
||||
await engine.upsertChunks('docs/provenance-throw-path', [
|
||||
{ chunk_index: 0, chunk_text: 'throw-path provenance chunk', chunk_source: 'compiled_truth' },
|
||||
]);
|
||||
|
||||
const rows = await engine.executeRaw<{ model: string }>(
|
||||
`SELECT cc.model FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE p.slug = 'docs/provenance-throw-path'`,
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].model).toBe('voyage:voyage-3-large');
|
||||
|
||||
// Restore the value initSchema wrote for the rest of the file.
|
||||
await engine.setConfig('embedding_model', 'openai:text-embedding-3-large');
|
||||
});
|
||||
|
||||
// #3461 sibling: on a partial re-upsert that carries NO new embedding (the
|
||||
// exact shape `embed --stale` produces for a page's non-stale chunks), the
|
||||
// preserved vector must KEEP its original model label. The old
|
||||
// COALESCE(EXCLUDED.model, …) relabeled it with the current gateway model.
|
||||
test('#3461: preserved vector keeps its original model label on a no-embedding re-upsert', async () => {
|
||||
configureGateway({
|
||||
embedding_model: 'openai:text-embedding-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
env: { OPENAI_API_KEY: 'sk-test' },
|
||||
});
|
||||
|
||||
await engine.putPage('docs/provenance-preserve', {
|
||||
type: 'concept',
|
||||
title: 'Provenance preserve page',
|
||||
compiled_truth: 'Chunk embedded under model A, re-upserted under model B.',
|
||||
});
|
||||
await engine.upsertChunks('docs/provenance-preserve', [
|
||||
{
|
||||
chunk_index: 0,
|
||||
chunk_text: 'stable chunk text',
|
||||
chunk_source: 'compiled_truth',
|
||||
embedding: new Float32Array(VEC1536_A),
|
||||
},
|
||||
]);
|
||||
|
||||
// Model swap: the gateway now resolves a different model, and the
|
||||
// re-upsert (same chunk_text) carries no new embedding.
|
||||
configureGateway({
|
||||
embedding_model: 'voyage:voyage-3-large',
|
||||
embedding_dimensions: 1536,
|
||||
env: { VOYAGE_API_KEY: 'test' },
|
||||
});
|
||||
await engine.upsertChunks('docs/provenance-preserve', [
|
||||
{ chunk_index: 0, chunk_text: 'stable chunk text', chunk_source: 'compiled_truth' },
|
||||
]);
|
||||
|
||||
const rows = await engine.executeRaw<{ model: string; has_embedding: boolean }>(
|
||||
`SELECT cc.model, cc.embedding IS NOT NULL AS has_embedding
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE p.slug = 'docs/provenance-preserve'`,
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].has_embedding).toBe(true); // vector preserved…
|
||||
expect(rows[0].model).toBe('openai:text-embedding-3-large'); // …and its label still describes it
|
||||
|
||||
resetGateway();
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildVectorCastFragment — engine SQL composer (D3)', () => {
|
||||
|
||||
@@ -69,7 +69,7 @@ describeE2E('E2E: JSONB roundtrip — v0.12.1 reliability wave', () => {
|
||||
`;
|
||||
expect(row.t).toBe('object');
|
||||
expect(row.marker).toBe('rawdata-value');
|
||||
});
|
||||
}, 30_000);
|
||||
|
||||
test('logIngest writes pages_updated as array, not double-encoded string', async () => {
|
||||
const engine = getEngine();
|
||||
@@ -91,7 +91,7 @@ describeE2E('E2E: JSONB roundtrip — v0.12.1 reliability wave', () => {
|
||||
expect(row.t).toBe('array');
|
||||
expect(Number(row.n)).toBe(3);
|
||||
expect(row.first).toBe('test/a');
|
||||
});
|
||||
}, 30_000);
|
||||
|
||||
// files.ts:254 (uploadRaw's cloud-upload branch) was changed from
|
||||
// `${JSON.stringify({...})}::jsonb` to `${sql.json({...})}` in v0.12.1.
|
||||
@@ -114,7 +114,7 @@ describeE2E('E2E: JSONB roundtrip — v0.12.1 reliability wave', () => {
|
||||
expect(row.t).toBe('object');
|
||||
expect(row.type).toBe('pdf');
|
||||
expect(row.method).toBe('TUS resumable');
|
||||
});
|
||||
}, 30_000);
|
||||
|
||||
// Source-level tripwire: if anyone re-introduces the old `${JSON.stringify(x)}::jsonb`
|
||||
// pattern for the fixed sites, fail loudly. Greps actual source files per the
|
||||
@@ -129,5 +129,5 @@ describeE2E('E2E: JSONB roundtrip — v0.12.1 reliability wave', () => {
|
||||
const source = await Bun.file(new URL(rel, import.meta.url)).text();
|
||||
expect(source.match(bad)?.[0] ?? null).toBeNull();
|
||||
}
|
||||
});
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
@@ -277,3 +277,79 @@ describe('embedStaleForSource', () => {
|
||||
expect(txtRow.embedded_at).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
// #3507 — re-embed must reproduce the page's STORED contextual-retrieval
|
||||
// wrapping convention. Before the fix, every plain re-embed (including the
|
||||
// normal post-model-migration `embed --stale`) embedded raw chunk_text,
|
||||
// silently replacing context-wrapped vectors with unwrapped ones.
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('contextual-retrieval wrapping on re-embed (#3507)', () => {
|
||||
/** embedFn that records every text it is asked to embed. */
|
||||
function capturingEmbedFn(seen: string[]) {
|
||||
return (texts: string[]): Promise<Float32Array[]> => {
|
||||
seen.push(...texts);
|
||||
return fakeEmbedFn(texts);
|
||||
};
|
||||
}
|
||||
|
||||
async function seedWrappablePage(slug: string, title: string): Promise<void> {
|
||||
await engine.putPage(slug, { type: 'note', title, compiled_truth: 'seeded' });
|
||||
await engine.upsertChunks(slug, [
|
||||
{ chunk_index: 0, chunk_text: 'prose chunk about widgets', chunk_source: 'compiled_truth', token_count: 4 },
|
||||
{ chunk_index: 1, chunk_text: 'const x = 1;', chunk_source: 'fenced_code', token_count: 4 },
|
||||
]);
|
||||
}
|
||||
|
||||
test('title-mode page: stale re-embed sends title-wrapped texts; fenced_code stays raw', async () => {
|
||||
await seedWrappablePage('wrapped-page', 'Widget Notes');
|
||||
await engine.updatePageContextualRetrievalState('wrapped-page', 'default', 'title', 'gen-title');
|
||||
|
||||
const seen: string[] = [];
|
||||
const result = await embedStaleForSource(engine, 'default', { embedFn: capturingEmbedFn(seen) });
|
||||
expect(result.embedded).toBe(2);
|
||||
|
||||
expect(seen).toContain('<context>Widget Notes\n</context>\nprose chunk about widgets');
|
||||
expect(seen).toContain('const x = 1;'); // fenced_code is NEVER wrapped (D20-T4)
|
||||
|
||||
// D20-T1: the canonical chunk_text is NOT rewritten — wrapping is embed-input-only.
|
||||
const chunks = await engine.getChunks('wrapped-page');
|
||||
expect(chunks.map((c) => c.chunk_text).sort()).toEqual(['const x = 1;', 'prose chunk about widgets']);
|
||||
// Mode stamp unchanged for title-tier pages.
|
||||
const rows = await engine.executeRaw<{ contextual_retrieval_mode: string }>(
|
||||
`SELECT contextual_retrieval_mode FROM pages WHERE slug = 'wrapped-page'`,
|
||||
);
|
||||
expect(rows[0].contextual_retrieval_mode).toBe('title');
|
||||
});
|
||||
|
||||
test('per_chunk_synopsis page: re-embed applies the title-tier wrapper and restamps honestly', async () => {
|
||||
await seedWrappablePage('synopsis-page', 'Synopsis Notes');
|
||||
await engine.updatePageContextualRetrievalState('synopsis-page', 'default', 'per_chunk_synopsis', 'gen-synopsis');
|
||||
|
||||
const seen: string[] = [];
|
||||
const result = await embedStaleForSource(engine, 'default', { embedFn: capturingEmbedFn(seen) });
|
||||
expect(result.embedded).toBe(2);
|
||||
|
||||
// Synopsis re-generation is a paid backfill concern; the plain re-embed
|
||||
// lands at the title tier (the service's own D14 fallback tier)…
|
||||
expect(seen).toContain('<context>Synopsis Notes\n</context>\nprose chunk about widgets');
|
||||
// …and the stamped mode is updated so it keeps describing the vectors.
|
||||
const rows = await engine.executeRaw<{ contextual_retrieval_mode: string }>(
|
||||
`SELECT contextual_retrieval_mode FROM pages WHERE slug = 'synopsis-page'`,
|
||||
);
|
||||
expect(rows[0].contextual_retrieval_mode).toBe('title');
|
||||
});
|
||||
|
||||
test('unstamped page (NULL mode) embeds raw chunk_text — convention preserved', async () => {
|
||||
await seedWrappablePage('plain-page', 'Plain Notes');
|
||||
// No updatePageContextualRetrievalState call: pre-CR page.
|
||||
|
||||
const seen: string[] = [];
|
||||
const result = await embedStaleForSource(engine, 'default', { embedFn: capturingEmbedFn(seen) });
|
||||
expect(result.embedded).toBe(2);
|
||||
|
||||
expect(seen).toContain('prose chunk about widgets');
|
||||
expect(seen.some((t) => t.startsWith('<context>'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -907,3 +907,76 @@ describe('runEmbed preserves code-chunk metadata across re-embed (regression for
|
||||
expect(metadataOf(upsertChunkArgs![0])).toEqual(metadataOf(fullCodeChunk));
|
||||
});
|
||||
});
|
||||
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
// #3507 — `embed --stale` must reproduce the page's STORED
|
||||
// contextual-retrieval wrapping convention instead of embedding raw
|
||||
// chunk_text (which silently stripped contextual prefixes on every
|
||||
// re-embed, including the normal post-model-migration path).
|
||||
// ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('embed --stale contextual-retrieval wrapping (#3507)', () => {
|
||||
const wrapChunks = [
|
||||
{ chunk_index: 0, chunk_text: 'prose chunk', chunk_source: 'compiled_truth', embedded_at: null, token_count: 1 },
|
||||
{ chunk_index: 1, chunk_text: 'const x = 1;', chunk_source: 'fenced_code', embedded_at: null, token_count: 1 },
|
||||
];
|
||||
const wrapStale = [
|
||||
{ slug: 'wrapped', chunk_index: 0, chunk_text: 'prose chunk', chunk_source: 'compiled_truth' as const, model: null, token_count: 1, source_id: 'default', page_id: 1 },
|
||||
{ slug: 'wrapped', chunk_index: 1, chunk_text: 'const x = 1;', chunk_source: 'fenced_code' as any, model: null, token_count: 1, source_id: 'default', page_id: 1 },
|
||||
];
|
||||
|
||||
function wrappingHarness(mode: string | null) {
|
||||
const seen: string[] = [];
|
||||
const restamps: any[][] = [];
|
||||
embedBatchBehavior = async (texts: string[]) => {
|
||||
seen.push(...texts);
|
||||
return texts.map(() => new Float32Array(1536));
|
||||
};
|
||||
const engine = mockEngine({
|
||||
countStaleChunks: async () => 2,
|
||||
listStaleChunks: async () => wrapStale,
|
||||
getPage: async () => ({
|
||||
slug: 'wrapped',
|
||||
title: 'Widget Notes',
|
||||
source_id: 'default',
|
||||
compiled_truth: 'x',
|
||||
timeline: '',
|
||||
contextual_retrieval_mode: mode,
|
||||
}),
|
||||
getChunks: async () => wrapChunks,
|
||||
upsertChunks: async () => {},
|
||||
updatePageContextualRetrievalState: async (...args: any[]) => { restamps.push(args); },
|
||||
});
|
||||
return { engine, seen, restamps };
|
||||
}
|
||||
|
||||
test('title-mode page: stale re-embed wraps prose with the title prefix; fenced_code stays raw', async () => {
|
||||
const { engine, seen, restamps } = wrappingHarness('title');
|
||||
const result = await runEmbedCore(engine, { stale: true });
|
||||
expect(result.embedded).toBe(2);
|
||||
expect(seen).toContain('<context>Widget Notes\n</context>\nprose chunk');
|
||||
expect(seen).toContain('const x = 1;');
|
||||
expect(restamps).toHaveLength(0); // title tier: stamp already honest
|
||||
});
|
||||
|
||||
test('per_chunk_synopsis page: fully re-embedded page restamps to the title tier', async () => {
|
||||
const { engine, seen, restamps } = wrappingHarness('per_chunk_synopsis');
|
||||
const result = await runEmbedCore(engine, { stale: true });
|
||||
expect(result.embedded).toBe(2);
|
||||
expect(seen).toContain('<context>Widget Notes\n</context>\nprose chunk');
|
||||
expect(restamps).toHaveLength(1);
|
||||
const [slug, sourceId, newMode] = restamps[0];
|
||||
expect(slug).toBe('wrapped');
|
||||
expect(sourceId).toBe('default');
|
||||
expect(newMode).toBe('title');
|
||||
});
|
||||
|
||||
test('page with no stored CR mode embeds raw chunk_text (convention preserved)', async () => {
|
||||
const { engine, seen, restamps } = wrappingHarness(null);
|
||||
const result = await runEmbedCore(engine, { stale: true });
|
||||
expect(result.embedded).toBe(2);
|
||||
expect(seen).toContain('prose chunk');
|
||||
expect(seen.some((t) => t.startsWith('<context>'))).toBe(false);
|
||||
expect(restamps).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -672,6 +672,51 @@ describe('runExtractFacts — empty-fence guard (Codex R2-#7)', () => {
|
||||
});
|
||||
|
||||
describe('runExtractFacts — multi-source isolation', () => {
|
||||
test('a pending legacy row in source A does NOT jam extraction for source B (#2646 source-scope)', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (engine as any).db.query(
|
||||
`INSERT INTO sources (id, name, config) VALUES ('work', 'work', '{}'::jsonb)
|
||||
ON CONFLICT (id) DO NOTHING`,
|
||||
);
|
||||
|
||||
// Source "work": a genuine pending legacy row (row_num NULL, active,
|
||||
// live backing page) — the exact shape that must gate work's cycle.
|
||||
await engine.putPage('people/alice', {
|
||||
title: 'people/alice', type: 'person',
|
||||
compiled_truth: FACT_FENCE(`| 1 | work fence fact | fact | 1.0 | world | high | 2026-01-01 | | s | |`),
|
||||
frontmatter: {}, timeline: '',
|
||||
}, { sourceId: 'work' });
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (engine as any).db.query(
|
||||
`INSERT INTO facts (source_id, entity_slug, fact, kind, visibility, notability,
|
||||
valid_from, source, confidence)
|
||||
VALUES ('work', 'people/alice', 'work legacy claim', 'fact', 'private', 'medium',
|
||||
now(), 'mcp:put_page', 1.0)`,
|
||||
);
|
||||
|
||||
// Source "default": clean — no legacy rows, one fenced page.
|
||||
await putPage('people/bob', FACT_FENCE(
|
||||
`| 1 | default fact | fact | 1.0 | world | high | 2026-01-01 | | s | |`,
|
||||
));
|
||||
|
||||
// default's run must NOT be jammed by work's pending backlog.
|
||||
const rDefault = await runExtractFacts(engine, { slugs: ['people/bob'], sourceId: 'default' });
|
||||
expect(rDefault.guardTriggered).toBe(false);
|
||||
expect(rDefault.legacyRowsPending).toBe(0);
|
||||
expect(rDefault.factsInserted).toBe(1);
|
||||
|
||||
// work's own run still gates (discriminator stays sharp).
|
||||
const rWork = await runExtractFacts(engine, { slugs: ['people/alice'], sourceId: 'work' });
|
||||
expect(rWork.guardTriggered).toBe(true);
|
||||
expect(rWork.legacyRowsPending).toBe(1);
|
||||
expect(rWork.factsInserted).toBe(0);
|
||||
// The drain advice must be one that actually re-runs Phase B — a bare
|
||||
// `apply-migrations --yes` no-ops once the ledger says complete.
|
||||
expect(rWork.warnings.some(w => w.includes('--force-retry 0.32.2'))).toBe(true);
|
||||
expect(rWork.warnings.some(w => w.includes('forget_fact'))).toBe(true);
|
||||
expect(rWork.warnings.some(w => w.includes('source "work"'))).toBe(true);
|
||||
});
|
||||
|
||||
test('deleteFactsForPage scoping does not affect other sources', async () => {
|
||||
// Seed sources work + home.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
writeReceipt,
|
||||
type ExtractReceiptInput,
|
||||
} from '../../src/core/extract/receipt-writer.ts';
|
||||
import { slugifySegment } from '../../src/core/sync.ts';
|
||||
|
||||
const BASE_INPUT: ExtractReceiptInput = {
|
||||
kind: 'facts.conversation',
|
||||
@@ -81,6 +82,31 @@ describe('shortRunId / dateFromIso — pure helpers', () => {
|
||||
expect(shortRunId('op_check_abc')).toBe('op_check');
|
||||
});
|
||||
|
||||
// #3443 — a short form ending in '-' (e.g. propose-<timestamp> run ids)
|
||||
// desynced the DB receipt slug from its Git-backed slug: slugifySegment()
|
||||
// strips boundary hyphens during repo sync, so the write-through created a
|
||||
// normalized sibling instead of materializing the existing page.
|
||||
test('shortRunId is canonical under slugifySegment for every receipt-producing run-id family (#3443)', () => {
|
||||
const familyRunIds = [
|
||||
'propose-20260724103000-ab12cd34', // cycle/propose-takes.ts
|
||||
`atoms-${Date.now().toString(36)}-pers`, // cycle/extract-atoms.ts
|
||||
`efacts-${Date.now().toString(36)}-pers`, // cycle/extract-facts.ts
|
||||
`concepts-${Date.now().toString(36)}`, // cycle/synthesize-concepts.ts
|
||||
`ecf-${Date.now().toString(36)}-pers`, // extract-conversation-facts.ts
|
||||
];
|
||||
for (const runId of familyRunIds) {
|
||||
const short = shortRunId(runId);
|
||||
expect(slugifySegment(short)).toBe(short);
|
||||
expect(short.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('shortRunId trims boundary hyphens introduced by truncation', () => {
|
||||
expect(shortRunId('propose-20260724103000-ab12cd34')).toBe('propose');
|
||||
// Pathological all-separator prefix still yields a non-empty segment.
|
||||
expect(shortRunId('--------tail')).toBe('run');
|
||||
});
|
||||
|
||||
test('dateFromIso extracts YYYY-MM-DD prefix', () => {
|
||||
expect(dateFromIso('2026-05-27T14:30:00Z')).toBe('2026-05-27');
|
||||
expect(dateFromIso('2026-05-27T14:30:00.123456Z')).toBe('2026-05-27');
|
||||
|
||||
@@ -522,7 +522,7 @@ just content.
|
||||
const result = await importFile(engine, filePath, '🌟🚀.md', { noEmbed: true });
|
||||
expect(result.status).toBe('skipped');
|
||||
expect(result.error).toContain('no usable slug');
|
||||
expect(result.error).toContain('ASCII / Chinese / Japanese / Korean');
|
||||
expect(result.error).toContain('at least one letter or number (any script)');
|
||||
expect((engine as any)._calls.length).toBe(0);
|
||||
});
|
||||
|
||||
|
||||
@@ -354,6 +354,15 @@ describe('MinionQueue: #1737 per-handler default timeout', () => {
|
||||
expect(sub.timeout_ms).toBe(30 * 60 * 1000);
|
||||
});
|
||||
|
||||
// #3207 — facts-absorb is one LLM extraction call per page (same shape as
|
||||
// chronicle_extract) but was missing from HANDLER_DEFAULT_TIMEOUT_MS, so it
|
||||
// inherited the tight null-default wall-clock and was dead-lettered
|
||||
// mid-generation on slow chat providers (facts silently lost).
|
||||
test('facts-absorb gets the 10-min LLM-extraction default (#3207)', async () => {
|
||||
const job = await queue.add('facts-absorb', { slug: 'people/alice-example' });
|
||||
expect(job.timeout_ms).toBe(10 * 60 * 1000);
|
||||
});
|
||||
|
||||
test('contextual per-chunk reindex gets the 60-min default', async () => {
|
||||
const job = await queue.add('contextual_reindex_per_chunk', { page_slug: 'large-transcript' }, undefined, {
|
||||
allowProtectedSubmit: true,
|
||||
@@ -709,6 +718,26 @@ describe('MinionQueue: Prune', () => {
|
||||
const count = await queue.prune({ olderThan: new Date(Date.now() + 86400000) }); // future date = prune everything old enough
|
||||
expect(count).toBe(1); // only the cancelled one
|
||||
});
|
||||
|
||||
// #2712: --dry-run used to be silently ignored — the destructive default
|
||||
// ran and deleted rows while the operator believed they were previewing.
|
||||
test('dryRun counts prunable jobs without deleting', async () => {
|
||||
const job1 = await queue.add('sync', {});
|
||||
await queue.cancelJob(job1.id); // terminal → prunable
|
||||
|
||||
const wouldPrune = await queue.prune({ olderThan: new Date(Date.now() + 86400000), dryRun: true });
|
||||
expect(wouldPrune).toBe(1);
|
||||
|
||||
// The row must still exist after a dry run.
|
||||
const stillThere = await queue.getJob(job1.id);
|
||||
expect(stillThere).not.toBeNull();
|
||||
expect(stillThere!.status).toBe('cancelled');
|
||||
|
||||
// A real prune afterwards actually deletes it.
|
||||
const pruned = await queue.prune({ olderThan: new Date(Date.now() + 86400000) });
|
||||
expect(pruned).toBe(1);
|
||||
expect(await queue.getJob(job1.id)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// --- Stats (1 test) ---
|
||||
|
||||
@@ -277,3 +277,38 @@ describe('hard-exclude cache isolation (#2825)', () => {
|
||||
expect((await cache.lookup(emb, { knobsHash: envExcludeHash })).hit).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('detail cache isolation (#3515)', () => {
|
||||
// Hashes computed the way hybridSearchCached does: same resolved mode, ctx
|
||||
// carrying the effective detail level. A row written by a `--detail low`
|
||||
// call (compiled-truth-only result set) must not be served to a default
|
||||
// `medium` lookup, and vice versa.
|
||||
const lowHash = knobsHash(resolveSearchMode({ mode: 'balanced' }), { detail: 'low' });
|
||||
const mediumHash = knobsHash(resolveSearchMode({ mode: 'balanced' }), { detail: 'medium' });
|
||||
const unsetHash = knobsHash(resolveSearchMode({ mode: 'balanced' }));
|
||||
|
||||
test('detail=low write is NOT served to a default (medium) lookup', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
const emb = makeEmbedding(8);
|
||||
|
||||
// Simulate `query "X" --detail low` populating the cache with the
|
||||
// narrow compiled-truth-only result set.
|
||||
await cache.store('what is the deploy process', emb, makeResults('narrow', 2), {
|
||||
vector_enabled: true, detail_resolved: 'low', expansion_applied: false,
|
||||
}, { knobsHash: lowHash });
|
||||
|
||||
// Default-detail lookup inside the TTL → MISS (falls through to a
|
||||
// fresh, full search) instead of the narrow set.
|
||||
expect((await cache.lookup(emb, { knobsHash: mediumHash })).hit).toBe(false);
|
||||
|
||||
// The low-detail caller still hits its own row.
|
||||
const original = await cache.lookup(emb, { knobsHash: lowHash });
|
||||
expect(original.hit).toBe(true);
|
||||
expect(original.results?.length).toBe(2);
|
||||
});
|
||||
|
||||
test('undefined detail keys like the documented medium default', () => {
|
||||
expect(unsetHash).toBe(mediumHash);
|
||||
expect(unsetHash).not.toBe(lowHash);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -19,3 +19,23 @@ describe('CLI_ONLY command reachability (#2900)', () => {
|
||||
expect(CLI_ONLY.has('reconcile-links')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// #3224 — same drift class: `backfill` has a full `case 'backfill'` handler
|
||||
// (cli.ts, dispatching to commands/backfill.ts) but was missing from CLI_ONLY,
|
||||
// so every invocation hit the generic "Unknown command" branch.
|
||||
describe('CLI_ONLY command reachability (#3224)', () => {
|
||||
test('`backfill` is in CLI_ONLY so dispatch reaches its handler', () => {
|
||||
expect(CLI_ONLY.has('backfill')).toBe(true);
|
||||
});
|
||||
|
||||
test('`gbrain backfill --help` is dispatched, not rejected as unknown', () => {
|
||||
const { spawnSync } = require('node:child_process') as typeof import('node:child_process');
|
||||
const result = spawnSync('bun', ['run', 'src/cli.ts', 'backfill', '--help'], {
|
||||
cwd: process.cwd(),
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, GBRAIN_HOME: '/tmp/gbrain-test-backfill-nonexistent' },
|
||||
});
|
||||
expect(result.stderr ?? '').not.toContain('Unknown command');
|
||||
expect(result.status).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -89,7 +89,7 @@ describe('alias_resolved boost stage', () => {
|
||||
});
|
||||
|
||||
describe('KNOBS_HASH_VERSION', () => {
|
||||
it('is 13 (12→13 embedding-provider migration invalidates rows written against the prior embedding space, #3390)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
it('is 15 (13→15 detail fold makes detail-contaminated rows unreachable, #3515; v=14 claimed by in-flight #3514)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(15);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -413,7 +413,24 @@ describe('knobsHash determinism + cross-mode separation (CDX-4)', () => {
|
||||
// #3390/#3391: bumped 12→13 for the embedding-provider migration wave —
|
||||
// legacy callers hash prov=default before AND after a provider swap, so
|
||||
// pre-migration cache rows must become unreachable on upgrade.
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
// #3515: bumped 13→15 to fold the effective detail level (det=) — a
|
||||
// detail=low write must not be served to a detail=medium lookup. v=14
|
||||
// is claimed by in-flight #3514 (#3430 compiled_truth boost scope).
|
||||
expect(KNOBS_HASH_VERSION).toBe(15);
|
||||
});
|
||||
|
||||
test('#3515: detail set vs unset produces DIFFERENT hashes (cache contamination prevention)', () => {
|
||||
const knobs = resolveSearchMode({ mode: 'balanced' });
|
||||
const low = knobsHash(knobs, { detail: 'low' });
|
||||
const medium = knobsHash(knobs, { detail: 'medium' });
|
||||
const high = knobsHash(knobs, { detail: 'high' });
|
||||
const unset = knobsHash(knobs);
|
||||
expect(low).not.toBe(medium);
|
||||
expect(medium).not.toBe(high);
|
||||
expect(low).not.toBe(high);
|
||||
// Undefined falls back to 'medium' — the documented default — so legacy
|
||||
// callers that don't thread detail share the default-detail rows.
|
||||
expect(unset).toBe(medium);
|
||||
});
|
||||
|
||||
test('T1 (codex): floor_ratio set vs unset produces DIFFERENT hashes (cache contamination prevention)', () => {
|
||||
@@ -578,8 +595,8 @@ describe('v0.40.4 — graph_signals knob', () => {
|
||||
});
|
||||
|
||||
describe('v0.42.3.0 — autocut knobs', () => {
|
||||
test('KNOBS_HASH_VERSION is 13 (12→13 embedding-migration wave, #3390/#3391)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
test('KNOBS_HASH_VERSION is 15 (13→15 detail fold #3515; v=14 claimed by in-flight #3514)', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(15);
|
||||
});
|
||||
|
||||
test('bundle defaults: conservative off, balanced/tokenmax on @0.20', () => {
|
||||
|
||||
@@ -44,7 +44,7 @@ function baseKnobs(): ResolvedSearchKnobs {
|
||||
}
|
||||
|
||||
describe('KNOBS_HASH_VERSION + version invariants', () => {
|
||||
test('version is 13 (…; 10→11 asymmetric input_type #1400; 11→12 hard-excludes #2825; 12→13 embedding-provider migration #3390)', () => {
|
||||
test('version is 15 (…; 11→12 hard-excludes #2825; 12→13 embedding-provider migration #3390; 13→15 detail fold #3515)', () => {
|
||||
// v0.35.0.0: 1→2 to fold reranker fields. v0.35.6.0: 2→3 to fold
|
||||
// floor_ratio. v0.36 wave: piggybacks on v=3 with 7 cross-modal knobs
|
||||
// (D2) PLUS column + provider context (D8/CDX-2 cross-column isolation).
|
||||
@@ -64,7 +64,10 @@ describe('KNOBS_HASH_VERSION + version invariants', () => {
|
||||
// pre-fix document-side query vectors must not be served.
|
||||
// #2825: 11→12 to fold the resolved hard-exclude prefix list (hx=) —
|
||||
// cached rows leaked GBRAIN_SEARCH_EXCLUDE'd slugs across processes.
|
||||
expect(KNOBS_HASH_VERSION).toBe(13);
|
||||
// #3515: 13→15 to fold the effective detail level (det=) — a detail=low
|
||||
// write must not be served to a detail=medium lookup. v=14 claimed by
|
||||
// in-flight #3514 (#3430).
|
||||
expect(KNOBS_HASH_VERSION).toBe(15);
|
||||
});
|
||||
|
||||
test('hash is 16 hex chars regardless of reranker config', () => {
|
||||
|
||||
@@ -31,9 +31,9 @@ function microBump(): string {
|
||||
function stub(tag: string | null, changelog: string): void {
|
||||
globalThis.fetch = (async (url: any) => {
|
||||
const u = String(url);
|
||||
if (u.includes('/releases/latest')) {
|
||||
if (u.includes('/gbrain/master/VERSION')) {
|
||||
if (tag === null) throw new Error('network down');
|
||||
return new Response(JSON.stringify({ tag_name: tag, published_at: '2026-01-01', html_url: 'https://x/rel' }), { status: 200 });
|
||||
return new Response(tag + '\n', { status: 200 });
|
||||
}
|
||||
if (u.includes('CHANGELOG.md')) return new Response(changelog, { status: 200 });
|
||||
return new Response('', { status: 200 });
|
||||
@@ -65,7 +65,7 @@ describe('self-upgrade --check-only surfaces what you get', () => {
|
||||
const out = JSON.parse(captured.join('\n'));
|
||||
expect(out.update_available).toBe(true);
|
||||
expect(out.latest_version).toBe(latest);
|
||||
expect(out.release_url).toBe('https://x/rel');
|
||||
expect(out.release_url).toBe('https://github.com/garrytan/gbrain/blob/master/CHANGELOG.md');
|
||||
expect(out.changelog_diff).toContain('Shiny new thing');
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { slugifySegment, slugifyPath } from '../src/core/sync.ts';
|
||||
import { validatePageSlug } from '../src/core/operations.ts';
|
||||
import { isValidHolder } from '../src/core/takes-fence.ts';
|
||||
|
||||
/**
|
||||
* #3417 — silent data loss for non-Latin, non-CJK scripts.
|
||||
*
|
||||
* Pre-fix, slugifySegment stripped every character outside [a-z0-9._-] + CJK,
|
||||
* so whole filenames in Hebrew / Arabic / Cyrillic / Greek / Thai collapsed to
|
||||
* empty segments. Distinct files then mapped to the SAME slug (their shared
|
||||
* directory prefix) and last-writer-wins overwrote each other with `import`
|
||||
* reporting 0 errors.
|
||||
*
|
||||
* Every assertion here is behavioral (input → output), so this file FAILS on
|
||||
* pre-fix master and passes with the Unicode-property-escape grammar.
|
||||
*/
|
||||
|
||||
describe('#3417: non-Latin scripts survive slugification', () => {
|
||||
// The six script families from the issue, before/after.
|
||||
const cases: Array<[string, string, string]> = [
|
||||
['Hebrew', 'notes/רשימת קניות.md', 'notes/רשימת-קניות'],
|
||||
['Arabic', 'notes/قائمة المهام.md', 'notes/قائمة-المهام'],
|
||||
['Cyrillic', 'notes/Список задач.md', 'notes/список-задач'],
|
||||
// Greek: tonos marks decompose to U+0301 under NFD and are stripped by the
|
||||
// same combining-accent pass that turns café → cafe. Consistent, stable.
|
||||
['Greek', 'notes/Λίστα εργασιών.md', 'notes/λιστα-εργασιων'],
|
||||
['Thai', 'notes/รายการซื้อของ.md', 'notes/รายการซื้อของ'],
|
||||
['Hebrew + digits', 'notes/תוכנית עבודה 2026.md', 'notes/תוכנית-עבודה-2026'],
|
||||
];
|
||||
|
||||
for (const [name, input, expected] of cases) {
|
||||
test(`${name}: ${input} → ${expected}`, () => {
|
||||
expect(slugifyPath(input)).toBe(expected);
|
||||
});
|
||||
}
|
||||
|
||||
test('distinct same-directory files no longer collapse onto one slug', () => {
|
||||
// Pre-fix ALL of these slugified to "notes" — one page, last writer wins.
|
||||
const slugs = [
|
||||
slugifyPath('notes/רשימת קניות.md'),
|
||||
slugifyPath('notes/قائمة المهام.md'),
|
||||
slugifyPath('notes/Список задач.md'),
|
||||
slugifyPath('notes/Λίστα εργασιών.md'),
|
||||
slugifyPath('notes/รายการซื้อของ.md'),
|
||||
];
|
||||
expect(new Set(slugs).size).toBe(slugs.length);
|
||||
for (const s of slugs) expect(s).not.toBe('notes');
|
||||
});
|
||||
|
||||
test('emitted slugs are ACCEPTED by validatePageSlug (three-grammar coherence)', () => {
|
||||
// The trap: fixing only sync.ts makes sync emit slugs put_page rejects.
|
||||
for (const [, input] of cases) {
|
||||
const slug = slugifyPath(input);
|
||||
expect(() => validatePageSlug(slug)).not.toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
test('takes-fence holder grammar accepts non-Latin slugs', () => {
|
||||
expect(isValidHolder('people/גארי-כהן')).toBe(true);
|
||||
expect(isValidHolder('companies/شركة-مثال')).toBe(true);
|
||||
// Uppercase still rejected (lowercase-canonical contract preserved).
|
||||
expect(isValidHolder('people/Garry-Tan')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3417: normalization — NFD (macOS) and NFC (git/Linux) converge', () => {
|
||||
test('Hebrew NFD filename produces the same slug as NFC', () => {
|
||||
const nfc = 'notes/רשימת קניות.md'.normalize('NFC');
|
||||
const nfd = 'notes/רשימת קניות.md'.normalize('NFD');
|
||||
expect(slugifyPath(nfd)).toBe(slugifyPath(nfc));
|
||||
});
|
||||
|
||||
test('Vietnamese NFD filename produces the same slug as NFC', () => {
|
||||
const nfc = 'notes/người dùng.md'.normalize('NFC');
|
||||
const nfd = 'notes/người dùng.md'.normalize('NFD');
|
||||
expect(slugifyPath(nfd)).toBe(slugifyPath(nfc));
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3417: regressions — existing behavior unchanged', () => {
|
||||
test('ASCII kebab-casing, lowercasing, dots, underscores', () => {
|
||||
expect(slugifyPath('notes/Shopping List.md')).toBe('notes/shopping-list');
|
||||
expect(slugifyPath('notes/v1.0.0.md')).toBe('notes/v1.0.0');
|
||||
expect(slugifySegment('my_file_name')).toBe('my_file_name');
|
||||
expect(slugifySegment('notes (march 2024)')).toBe('notes-march-2024');
|
||||
});
|
||||
|
||||
test('Latin accents still strip (café → cafe)', () => {
|
||||
expect(slugifySegment('café résumé')).toBe('cafe-resume');
|
||||
});
|
||||
|
||||
test('CJK still preserved', () => {
|
||||
expect(slugifyPath('notes/购物清单.md')).toBe('notes/购物清单');
|
||||
expect(slugifyPath('inbox/品牌圣经.md')).toBe('inbox/品牌圣经');
|
||||
expect(slugifySegment('한글테스트'.normalize('NFD'))).toBe('한글테스트');
|
||||
});
|
||||
|
||||
test('all-symbol input still collapses to empty (frontmatter-fallback path intact)', () => {
|
||||
expect(slugifySegment('!!!')).toBe('');
|
||||
expect(slugifySegment('🎉🎉')).toBe('');
|
||||
});
|
||||
|
||||
test('control chars, RTL override, punctuation still stripped', () => {
|
||||
expect(slugifySegment('evilgnp')).toBe('evilgnp');
|
||||
expect(slugifySegment('a\u0000b')).toBe('ab');
|
||||
});
|
||||
|
||||
test('validatePageSlug still rejects traversal, backslash, RTL override, uppercase-only weirdness', () => {
|
||||
expect(() => validatePageSlug('../etc/passwd')).toThrow();
|
||||
expect(() => validatePageSlug('notes\\file')).toThrow();
|
||||
expect(() => validatePageSlug('notes/evil')).toThrow();
|
||||
expect(() => validatePageSlug('notes/a\u0007b')).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -259,10 +259,10 @@ describe('SLUG_SEGMENT_PATTERN (v0.32.7)', () => {
|
||||
expect(SLUG_SEGMENT_PATTERN.test('icp-理想客户画像')).toBe(true);
|
||||
});
|
||||
|
||||
test('REGRESSION: rejects non-CJK Unicode (Vietnamese)', () => {
|
||||
// Scope is CJK only; Vietnamese with combining diacritics stays rejected
|
||||
// until we widen to Unicode property escapes in v0.33+.
|
||||
const result = 'người-dùng'.match(new RegExp(`^${SLUG_SEGMENT_PATTERN.source}$`));
|
||||
expect(result).toBeNull();
|
||||
test('accepts non-CJK Unicode (Vietnamese) since the #3417 all-script widening', () => {
|
||||
// Pre-#3417 this was rejected (scope was CJK only). The grammar now uses
|
||||
// Unicode property escapes, so đ/ư/etc. are valid slug characters.
|
||||
const result = 'người-dùng'.match(new RegExp(`^${SLUG_SEGMENT_PATTERN.source}$`, 'u'));
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
/**
|
||||
* #2079 — `gbrain takes list` used to parse "list" as a PAGE SLUG: cmdList
|
||||
* looked up a page named "list" and printed "No takes on list." even when the
|
||||
* brain held many takes — reading exactly like an empty takes table, so
|
||||
* agents concluded there were no takes and moved on.
|
||||
*
|
||||
* Fix: `list` is a real subcommand (CLI parity with the takes_list op).
|
||||
* Bare `takes <slug>` still lists per-page.
|
||||
*/
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { runTakes } from '../src/commands/takes.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
async function captureStdout(fn: () => Promise<void>): Promise<string> {
|
||||
const lines: string[] = [];
|
||||
const orig = console.log;
|
||||
console.log = (...args: unknown[]) => { lines.push(args.join(' ')); };
|
||||
try {
|
||||
await fn();
|
||||
} finally {
|
||||
console.log = orig;
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
await engine.putPage('companies/acme-example', {
|
||||
type: 'company',
|
||||
title: 'Acme Example',
|
||||
compiled_truth: 'Acme Example is a test company.',
|
||||
});
|
||||
const [row] = await engine.executeRaw<{ id: number }>(
|
||||
`SELECT id FROM pages WHERE slug = 'companies/acme-example'`,
|
||||
);
|
||||
await engine.addTakesBatch([{
|
||||
page_id: row.id,
|
||||
row_num: 1,
|
||||
claim: 'Acme will ship the widget by Q3.',
|
||||
kind: 'bet',
|
||||
holder: 'self',
|
||||
weight: 0.7,
|
||||
}]);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
describe('gbrain takes list (#2079)', () => {
|
||||
test('`takes list` lists all takes instead of slug-ifying "list"', async () => {
|
||||
const out = await captureStdout(() => runTakes(engine, ['list']));
|
||||
expect(out).not.toContain('No takes on list.');
|
||||
expect(out).toContain('Acme will ship the widget by Q3.');
|
||||
expect(out).toContain('companies/acme-example');
|
||||
});
|
||||
|
||||
test('`takes list --json` returns the full take rows', async () => {
|
||||
const out = await captureStdout(() => runTakes(engine, ['list', '--json']));
|
||||
const parsed = JSON.parse(out);
|
||||
expect(Array.isArray(parsed)).toBe(true);
|
||||
expect(parsed.length).toBe(1);
|
||||
expect(parsed[0].claim).toContain('Acme will ship');
|
||||
});
|
||||
|
||||
test('per-page form still works: `takes <slug>`', async () => {
|
||||
const out = await captureStdout(() => runTakes(engine, ['companies/acme-example']));
|
||||
expect(out).toContain('# Takes on companies/acme-example');
|
||||
expect(out).toContain('Acme will ship the widget by Q3.');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user