mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 17:02:19 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
582051bbcd | ||
|
|
5e586efad7 | ||
|
|
d23b5a6ed4 |
@@ -65,6 +65,7 @@ on user_asks_about(topic):
|
||||
3. **Don't use hybrid search for known names.** `gbrain query "Pedro Franceschi"` wastes embedding compute. Use `gbrain search "Pedro Franceschi"` or better yet `gbrain get pedro-franceschi` if you know the slug.
|
||||
4. **Token budget awareness.** A full page via `gbrain get` can be large. Read the search chunks first to confirm relevance before pulling the full page. "Did anyone mention the Series A?" -- search results (chunks) are probably enough. "Tell me everything about Pedro" -- get the full page.
|
||||
5. **Hybrid search needs embeddings to have been run.** If `gbrain query` returns nothing but `gbrain search` finds results, the embeddings haven't been generated yet. Run the embedding pipeline first.
|
||||
6. **Slow embedding providers can silently degrade `gbrain query` to keyword-only.** The query-time embed is bounded by a deadline (default 6 seconds, tunable via the `GBRAIN_QUERY_EMBED_TIMEOUT_MS` env var). If the provider misses it, hybrid search falls back to keyword-only — which returns nothing for content keyword FTS can't tokenize (e.g. CJK). The result meta reports `vector_enabled: false` with a `degraded_reason` (`embed_timeout`, `embed_error`, or `no_embedding_provider`), and a once-per-process warning prints to stderr. For a slow local provider (ollama, llama-server), raise `GBRAIN_QUERY_EMBED_TIMEOUT_MS`.
|
||||
|
||||
## How to Verify
|
||||
|
||||
|
||||
@@ -105,9 +105,13 @@ Every ingested item should have its raw source preserved for provenance.
|
||||
|
||||
**Upload command:**
|
||||
```bash
|
||||
gbrain files upload-raw <file> --page <page-slug> --type <type>
|
||||
gbrain files upload-raw <file> --page <page-slug> --type <type> [--source <id>]
|
||||
```
|
||||
Returns JSON: `{storage: "git"}` for small files, `{storage: "supabase", storagePath, reference}` for cloud.
|
||||
Add `--source <id>` when the page slug exists in more than one mounted source — it
|
||||
disambiguates strictly (a slug not found in that source errors rather than silently
|
||||
attaching to another source). The stored path is namespaced per source so the same
|
||||
slug+filename in two sources never collide.
|
||||
|
||||
**The `.redirect.yaml` pointer format:**
|
||||
```yaml
|
||||
|
||||
+12
-8
@@ -140,14 +140,18 @@ export async function runConfig(engine: BrainEngine, args: string[]) {
|
||||
console.error(`[config] Setting it in the DB has no effect on the embed pipeline (silent no-op).`);
|
||||
console.error(`[config]`);
|
||||
if (isPgliteEngine) {
|
||||
console.error(`[config] To switch embedding models/dimensions on PGLite, wipe and re-init:`);
|
||||
console.error(`[config] mv ${dbPath} ${dbPath}.bak`);
|
||||
if (key === 'embedding_model') {
|
||||
console.error(`[config] gbrain init --pglite --embedding-model ${value}`);
|
||||
} else {
|
||||
console.error(`[config] gbrain init --pglite --embedding-dimensions ${value}`);
|
||||
}
|
||||
console.error(`[config] gbrain sync # re-imports your brain repo`);
|
||||
// Point at the canonical one-command recovery (`gbrain reinit-pglite`),
|
||||
// not the stale manual mv+init+sync dance (#2301). reinit-pglite requires
|
||||
// BOTH --embedding-model and --embedding-dimensions (it re-sizes the
|
||||
// pgvector column), so always show the paired flags — the old hint omitted
|
||||
// the dimension, which dead-ended the user.
|
||||
const curModel = loadConfig()?.embedding_model || '<model>';
|
||||
const curDims = loadConfig()?.embedding_dimensions || '<dims>';
|
||||
const model = key === 'embedding_model' ? value : curModel;
|
||||
const dims = key === 'embedding_dimensions' ? value : curDims;
|
||||
console.error(`[config] To switch embedding models/dimensions on PGLite, use the one-command path:`);
|
||||
console.error(`[config] gbrain reinit-pglite --embedding-model ${model} --embedding-dimensions ${dims}`);
|
||||
console.error(`[config] (wipes + re-inits + re-syncs the brain repo; pass --yes to skip the confirm)`);
|
||||
} else {
|
||||
console.error(`[config] To switch embedding models/dimensions on Postgres, see:`);
|
||||
console.error(`[config] docs/embedding-migrations.md`);
|
||||
|
||||
@@ -646,6 +646,40 @@ export async function doctorReportRemote(engine: BrainEngine): Promise<DoctorRep
|
||||
});
|
||||
}
|
||||
|
||||
// 3a. Skill surface honesty (#2330). The focused doctor emits no skill
|
||||
// checks, so computeDoctorReport scored category `skill` as a vacuous 100 —
|
||||
// "structurally dishonest": run_doctor reported skill 100/100 while the skills
|
||||
// directory was unreadable. Emit an EXPLICIT skill check so an unassessed/
|
||||
// unreadable surface can't masquerade as perfect. (We do NOT change
|
||||
// computeDoctorReport's vacuous-truth contract for genuinely-empty categories —
|
||||
// it's pinned; we make the surface emit a real check instead.) ok when the
|
||||
// resolver is reachable, warn when it isn't — never silent.
|
||||
try {
|
||||
const detected = autoDetectSkillsDirReadOnly();
|
||||
if (detected.dir) {
|
||||
const report = checkResolvable(detected.dir);
|
||||
checks.push({
|
||||
name: 'skill_surface',
|
||||
status: report.errors.length > 0 ? 'fail' : report.warnings.length > 0 ? 'warn' : 'ok',
|
||||
message: report.errors.length > 0 || report.warnings.length > 0
|
||||
? `${report.summary.total_skills} skills, ${report.errors.length} error(s)/${report.warnings.length} warning(s)`
|
||||
: `${report.summary.total_skills} skills reachable`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: 'skill_surface',
|
||||
status: 'warn',
|
||||
message: 'Skills directory not found — skill health could not be assessed (run `gbrain doctor` on the host for full skill checks).',
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
checks.push({
|
||||
name: 'skill_surface',
|
||||
status: 'warn',
|
||||
message: `Could not assess skill surface: ${e instanceof Error ? e.message : String(e)}`,
|
||||
});
|
||||
}
|
||||
|
||||
// 3b. Migration wedge hint (v0.31.8 — D14 + D19). The brain server's
|
||||
// filesystem holds the migration ledger; the wedge condition (>=3 consecutive
|
||||
// partials with no later complete) needs the force-retry hint, not plain
|
||||
|
||||
+209
-38
@@ -86,7 +86,7 @@ export async function runFiles(engine: BrainEngine, args: string[]) {
|
||||
console.error(`Usage: gbrain files <command> [args]`);
|
||||
console.error(` list [slug] List files for a page (or all)`);
|
||||
console.error(` upload <file> --page <slug> Upload file linked to page`);
|
||||
console.error(` upload-raw <file> --page <slug> [--type <type>] Smart upload with .redirect.yaml pointer`);
|
||||
console.error(` upload-raw <file> --page <slug> [--source <id>] [--type <type>] Smart upload with .redirect.yaml pointer`);
|
||||
console.error(` signed-url <path> Generate signed URL for stored file`);
|
||||
console.error(` sync <dir> Upload directory to storage`);
|
||||
console.error(` verify Verify all uploads match local`);
|
||||
@@ -169,23 +169,126 @@ async function uploadFile(engine: BrainEngine, args: string[]) {
|
||||
console.log(`Uploaded: ${storagePath} (${humanSize(stat.size)})`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the on-disk brain repo root for a source (#2297). Multi-source: a
|
||||
* source's `local_path` is its checkout; the legacy/default source falls back
|
||||
* to the `sync.repo_path` config anchor written by `gbrain sync`. Returns null
|
||||
* when no repo is resolvable (pure-DB brain) — callers must fail loud rather
|
||||
* than silently claim a git write succeeded.
|
||||
*/
|
||||
export async function resolveRepoRoot(engine: BrainEngine, sourceId: string | null): Promise<string | null> {
|
||||
if (sourceId && sourceId !== 'default') {
|
||||
const rows = await engine.executeRaw<{ local_path: string | null }>(
|
||||
`SELECT local_path FROM sources WHERE id = $1`,
|
||||
[sourceId],
|
||||
);
|
||||
if (rows[0]?.local_path) return rows[0].local_path;
|
||||
}
|
||||
return await engine.getConfig('sync.repo_path');
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up a page by slug to link the file row to a real page + source (#2297).
|
||||
* Slug uniqueness is (source_id, slug); with only `--page <slug>` we take the
|
||||
* first live match. Returns null for a slug with no page (file row still records
|
||||
* the slug + the default source).
|
||||
*/
|
||||
export async function lookupPageBySlug(
|
||||
engine: BrainEngine,
|
||||
pageSlug: string,
|
||||
sourceHint?: string | null,
|
||||
): Promise<{ id: number; source_id: string } | null> {
|
||||
// Slug uniqueness is (source_id, slug), so a bare slug can match multiple
|
||||
// sources (#2297). An explicit --source is honored STRICTLY: a scoped miss
|
||||
// returns null (never falls back to an unscoped match, which could silently
|
||||
// attach the file to the wrong source). Only a bare slug (no hint) takes the
|
||||
// first live match deterministically (id ASC).
|
||||
if (sourceHint) {
|
||||
const scoped = await engine.executeRaw<{ id: number; source_id: string }>(
|
||||
`SELECT id, source_id FROM pages WHERE slug = $1 AND source_id = $2 AND deleted_at IS NULL ORDER BY id LIMIT 1`,
|
||||
[pageSlug, sourceHint],
|
||||
);
|
||||
return scoped[0] ?? null;
|
||||
}
|
||||
const rows = await engine.executeRaw<{ id: number; source_id: string }>(
|
||||
`SELECT id, source_id FROM pages WHERE slug = $1 AND deleted_at IS NULL ORDER BY id LIMIT 1`,
|
||||
[pageSlug],
|
||||
);
|
||||
return rows[0] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the globally-unique `files.storage_path` key (#2297). The schema's
|
||||
* UNIQUE is on storage_path alone, but slugs only disambiguate within a source,
|
||||
* so a bare `<page>/<file>` collides across sources. Prefix non-default sources
|
||||
* with the source id so two sources can hold the same slug+filename without one
|
||||
* ON CONFLICT-clobbering the other. Default-source paths stay byte-identical
|
||||
* (back-compat). NOTE: this is the DB key, not the on-disk path — the physical
|
||||
* copy lives at repoRelPath under the source's own repo root.
|
||||
*/
|
||||
export function namespacedStoragePath(sourceId: string | null, repoRelPath: string): string {
|
||||
return sourceId && sourceId !== 'default' ? `${sourceId}/${repoRelPath}` : repoRelPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Single canonical `files` row writer shared by BOTH the git (small-file) and
|
||||
* cloud (large/media) branches of upload-raw (#2297, DRY). `storage_path` is
|
||||
* UNIQUE in the schema, so it MUST be page-namespaced by the caller to avoid one
|
||||
* page's attachment clobbering another's. metadata goes through executeRawJsonb
|
||||
* (raw object, never JSON.stringify into a ::jsonb cast — the double-encode trap).
|
||||
*/
|
||||
export async function insertFileRow(
|
||||
engine: BrainEngine,
|
||||
row: {
|
||||
sourceId: string | null;
|
||||
pageId: number | null;
|
||||
pageSlug: string | null;
|
||||
filename: string;
|
||||
storagePath: string;
|
||||
mimeType: string | null;
|
||||
size: number;
|
||||
contentHash: string;
|
||||
metadata: Record<string, unknown>;
|
||||
},
|
||||
): Promise<void> {
|
||||
await executeRawJsonb(
|
||||
engine,
|
||||
`INSERT INTO files (source_id, page_id, page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata)
|
||||
VALUES (COALESCE($1, 'default'), $2, $3, $4, $5, $6, $7, $8, $9::jsonb)
|
||||
ON CONFLICT (storage_path) DO UPDATE SET
|
||||
content_hash = EXCLUDED.content_hash,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
mime_type = EXCLUDED.mime_type,
|
||||
page_id = EXCLUDED.page_id,
|
||||
page_slug = EXCLUDED.page_slug`,
|
||||
[row.sourceId, row.pageId, row.pageSlug, row.filename, row.storagePath, row.mimeType, row.size, row.contentHash],
|
||||
[row.metadata],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Smart upload with size routing and .redirect.yaml pointer creation.
|
||||
*
|
||||
* Size routing:
|
||||
* < 100 MB text/PDF → stays in git (brain repo), no cloud upload
|
||||
* < 100 MB text/PDF → copied into the brain repo under <page>/.raw/, recorded
|
||||
* in the files table (storage = 'git')
|
||||
* >= 100 MB OR media → upload to cloud storage, create .redirect.yaml pointer
|
||||
* in the repo, recorded in the files table (storage = cloud)
|
||||
*
|
||||
* The .redirect.yaml pointer stays in the brain repo so git tracks what was stored.
|
||||
* Both branches persist a files row AND write into the brain repo so git tracks
|
||||
* what was stored. Neither silently returns success without persisting (#2297).
|
||||
*/
|
||||
async function uploadRaw(engine: BrainEngine, args: string[]) {
|
||||
const filePath = args.find(a => !a.startsWith('--'));
|
||||
const pageSlug = args.find((a, i) => args[i - 1] === '--page') || null;
|
||||
const fileType = args.find((a, i) => args[i - 1] === '--type') || null;
|
||||
// --source disambiguates a slug that exists in more than one source (#2297);
|
||||
// without it, a bare slug resolves to the first matching source.
|
||||
const sourceHint = args.find((a, i) => args[i - 1] === '--source') || null;
|
||||
const noPointer = args.includes('--no-pointer');
|
||||
|
||||
if (!filePath || !existsSync(filePath)) {
|
||||
console.error('Usage: gbrain files upload-raw <file> --page <slug> [--type <type>] [--no-pointer]');
|
||||
console.error('Usage: gbrain files upload-raw <file> --page <slug> [--source <id>] [--type <type>] [--no-pointer]');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -195,14 +298,71 @@ async function uploadRaw(engine: BrainEngine, args: string[]) {
|
||||
const isMedia = mimeType?.startsWith('video/') || mimeType?.startsWith('audio/') || mimeType?.startsWith('image/');
|
||||
const needsCloud = stat.size >= SIZE_THRESHOLD || isMedia;
|
||||
|
||||
// Resolve the page + source ONCE for both branches (#2297). An explicit
|
||||
// --source + --page pair that resolves to no page errors loudly (the doc
|
||||
// promises strict disambiguation) instead of silently persisting an
|
||||
// unlinked file row under the hinted source.
|
||||
const page = pageSlug ? await lookupPageBySlug(engine, pageSlug, sourceHint) : null;
|
||||
if (pageSlug && sourceHint && !page) {
|
||||
console.error(JSON.stringify({
|
||||
success: false,
|
||||
reason: 'page_not_found_in_source',
|
||||
message: `Page '${pageSlug}' not found in source '${sourceHint}'. `
|
||||
+ 'Check the slug/source pair, or drop --source to attach to the first matching source.',
|
||||
}));
|
||||
process.exit(1);
|
||||
}
|
||||
const sourceId = page?.source_id ?? sourceHint ?? 'default';
|
||||
|
||||
if (!needsCloud) {
|
||||
// Small text/PDF files stay in git
|
||||
// Small text/PDF files are copied INTO the brain repo and recorded in the
|
||||
// files table (#2297). Previously this branch printed success and returned
|
||||
// without copying anything or inserting a row — every small "raw" was
|
||||
// silently lost. The row links to the resolved page + source and the
|
||||
// destination is page-namespaced (storage_path is UNIQUE).
|
||||
const repoRoot = await resolveRepoRoot(engine, sourceId);
|
||||
if (!repoRoot) {
|
||||
console.error(JSON.stringify({
|
||||
success: false,
|
||||
reason: 'no_repo_path',
|
||||
message: 'No brain repo on disk (sources.local_path / sync.repo_path unset). '
|
||||
+ 'Run `gbrain sync` to anchor a repo, or configure cloud storage for raw uploads.',
|
||||
}));
|
||||
process.exit(1);
|
||||
}
|
||||
const content = readFileSync(filePath);
|
||||
const hash = createHash('sha256').update(content).digest('hex');
|
||||
// Physical copy lives at the repo-relative path UNDER the source's own repo
|
||||
// root; the DB storage_path key is additionally source-namespaced so it stays
|
||||
// globally unique across sources (#2297).
|
||||
const repoRelPath = pageSlug
|
||||
? `${pageSlug}/.raw/${filename}`
|
||||
: `unsorted/.raw/${hash.slice(0, 8)}-${filename}`;
|
||||
const storagePath = namespacedStoragePath(sourceId, repoRelPath);
|
||||
const destAbs = join(repoRoot, repoRelPath);
|
||||
mkdirSync(dirname(destAbs), { recursive: true });
|
||||
writeFileSync(destAbs, content);
|
||||
|
||||
await insertFileRow(engine, {
|
||||
sourceId,
|
||||
pageId: page?.id ?? null,
|
||||
pageSlug,
|
||||
filename,
|
||||
storagePath,
|
||||
mimeType,
|
||||
size: stat.size,
|
||||
contentHash: 'sha256:' + hash,
|
||||
metadata: { ...(fileType ? { type: fileType } : {}), upload_method: 'git' },
|
||||
});
|
||||
|
||||
console.log(JSON.stringify({
|
||||
success: true,
|
||||
storage: 'git',
|
||||
path: filePath,
|
||||
path: storagePath,
|
||||
abs_path: destAbs,
|
||||
size: stat.size,
|
||||
size_human: humanSize(stat.size),
|
||||
hash: `sha256:${hash}`,
|
||||
}));
|
||||
return;
|
||||
}
|
||||
@@ -220,48 +380,59 @@ async function uploadRaw(engine: BrainEngine, args: string[]) {
|
||||
const storage = await createStorage(config.storage as any);
|
||||
const content = readFileSync(filePath);
|
||||
const hash = createHash('sha256').update(content).digest('hex');
|
||||
const storagePath = pageSlug ? `${pageSlug}/${filename}` : `unsorted/${hash.slice(0, 8)}-${filename}`;
|
||||
const bucket = (config.storage as any).bucket || 'brain-files';
|
||||
|
||||
// Source-namespace the cloud storage key so the same slug+filename in two
|
||||
// sources doesn't collide on the UNIQUE storage_path (#2297). Default source
|
||||
// keeps its historical path.
|
||||
const cloudRelPath = pageSlug ? `${pageSlug}/${filename}` : `unsorted/${hash.slice(0, 8)}-${filename}`;
|
||||
const storagePath = namespacedStoragePath(sourceId, cloudRelPath);
|
||||
|
||||
const method = content.length >= SIZE_THRESHOLD ? 'TUS resumable' : 'standard';
|
||||
console.error(`Uploading ${humanSize(stat.size)} via ${method}...`);
|
||||
await storage.upload(storagePath, content, mimeType || undefined);
|
||||
|
||||
// Create .redirect.yaml pointer in the brain repo
|
||||
// Create .redirect.yaml pointer IN THE BRAIN REPO, page-namespaced (#2297).
|
||||
// Previously this was written next to the *input* file (filePath +
|
||||
// '.redirect.yaml'), so the pointer never landed in the repo the comment
|
||||
// claimed — git tracked nothing. Resolve the repo root and write it under
|
||||
// the page's .raw/ sidecar so it travels with the page.
|
||||
let pointerPath: string | null = null;
|
||||
if (!noPointer && pageSlug) {
|
||||
const { stringify } = await import('../core/yaml-lite.ts');
|
||||
const pointer = stringify({
|
||||
target: `supabase://${bucket}/${storagePath}`,
|
||||
bucket,
|
||||
storage_path: storagePath,
|
||||
size: stat.size,
|
||||
size_human: humanSize(stat.size),
|
||||
hash: `sha256:${hash}`,
|
||||
mime: mimeType || 'application/octet-stream',
|
||||
uploaded: new Date().toISOString(),
|
||||
...(fileType ? { type: fileType } : {}),
|
||||
});
|
||||
// Write pointer next to the original file
|
||||
pointerPath = filePath + '.redirect.yaml';
|
||||
writeFileSync(pointerPath, pointer);
|
||||
console.error(`Pointer written: ${pointerPath}`);
|
||||
const repoRoot = await resolveRepoRoot(engine, sourceId);
|
||||
if (repoRoot) {
|
||||
const { stringify } = await import('../core/yaml-lite.ts');
|
||||
const pointer = stringify({
|
||||
target: `supabase://${bucket}/${storagePath}`,
|
||||
bucket,
|
||||
storage_path: storagePath,
|
||||
size: stat.size,
|
||||
size_human: humanSize(stat.size),
|
||||
hash: `sha256:${hash}`,
|
||||
mime: mimeType || 'application/octet-stream',
|
||||
uploaded: new Date().toISOString(),
|
||||
...(fileType ? { type: fileType } : {}),
|
||||
});
|
||||
pointerPath = join(repoRoot, pageSlug, '.raw', `${filename}.redirect.yaml`);
|
||||
mkdirSync(dirname(pointerPath), { recursive: true });
|
||||
writeFileSync(pointerPath, pointer);
|
||||
console.error(`Pointer written: ${pointerPath}`);
|
||||
} else {
|
||||
console.error('No brain repo on disk — skipping .redirect.yaml pointer (cloud upload + DB row still recorded).');
|
||||
}
|
||||
}
|
||||
|
||||
// Record in DB. files.metadata is JSONB — pass the object via
|
||||
// executeRawJsonb with an explicit ::jsonb cast so post-v0.31 reads see
|
||||
// an actual object, not a JSON-encoded string (D1 wave).
|
||||
await executeRawJsonb(
|
||||
engine,
|
||||
`INSERT INTO files (page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)
|
||||
ON CONFLICT (storage_path) DO UPDATE SET
|
||||
content_hash = EXCLUDED.content_hash,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
mime_type = EXCLUDED.mime_type`,
|
||||
[pageSlug, filename, storagePath, mimeType, stat.size, 'sha256:' + hash],
|
||||
[{ type: fileType, upload_method: method }],
|
||||
);
|
||||
await insertFileRow(engine, {
|
||||
sourceId,
|
||||
pageId: page?.id ?? null,
|
||||
pageSlug,
|
||||
filename,
|
||||
storagePath,
|
||||
mimeType,
|
||||
size: stat.size,
|
||||
contentHash: 'sha256:' + hash,
|
||||
metadata: { ...(fileType ? { type: fileType } : {}), upload_method: method },
|
||||
});
|
||||
|
||||
// Output JSON for scripting
|
||||
console.log(JSON.stringify({
|
||||
|
||||
@@ -120,6 +120,7 @@ export const SKILL_CHECK_NAMES: ReadonlySet<string> = new Set([
|
||||
'retrieval_reflex_health',
|
||||
'skill_brain_first',
|
||||
'skill_conformance',
|
||||
'skill_surface',
|
||||
'whoknows_health',
|
||||
]);
|
||||
|
||||
|
||||
+45
-11
@@ -1054,7 +1054,11 @@ export class PGLiteEngine implements BrainEngine {
|
||||
source_kind = COALESCE(EXCLUDED.source_kind, pages.source_kind),
|
||||
source_uri = COALESCE(EXCLUDED.source_uri, pages.source_uri),
|
||||
ingested_via = COALESCE(EXCLUDED.ingested_via, pages.ingested_via),
|
||||
ingested_at = COALESCE(EXCLUDED.ingested_at, pages.ingested_at)
|
||||
ingested_at = COALESCE(EXCLUDED.ingested_at, pages.ingested_at),
|
||||
-- #2345: revive on re-put (parity with postgres-engine.ts) — a put on
|
||||
-- a soft-deleted slug lands on the tombstone row via the non-partial
|
||||
-- unique index; without this the write is silently swallowed.
|
||||
deleted_at = NULL
|
||||
RETURNING id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename, source_kind, source_uri, ingested_via, ingested_at`,
|
||||
[sourceId, slug, page.type, pageKind, page.title, page.compiled_truth, page.timeline || '', JSON.stringify(frontmatter), hash, effectiveDate, effectiveDateSource, importFilename, chunkerVersion, sourcePath, sourceKind, sourceUri, ingestedVia, ingestedAt]
|
||||
);
|
||||
@@ -1348,12 +1352,37 @@ export class PGLiteEngine implements BrainEngine {
|
||||
}
|
||||
|
||||
async updateSourceConfig(sourceId: string, patch: Record<string, unknown>): Promise<boolean> {
|
||||
// v0.38: parity with postgres-engine.updateSourceConfig. JSONB `||`
|
||||
// concat operator (overrides same-key, no deep merge). PGLite passes
|
||||
// `JSON.stringify(patch)` as the param; cast to jsonb on the SQL side.
|
||||
// Parity with postgres-engine.updateSourceConfig (#2251): normalize historical
|
||||
// bad shapes (double-encoded JSONB string, or array-of-patch-objects from the
|
||||
// old `|| string` cascade) back to a flat object before the `||` merge, so a
|
||||
// corrupted source self-heals instead of compounding. PGLite runs Postgres
|
||||
// 17.5, so `IS JSON` (PG16+) is available. The array branch wraps each element
|
||||
// in a CASE so jsonb_each never sees a non-object (which would raise
|
||||
// `cannot call jsonb_each on a non-object` and abort the UPDATE). PGLite's
|
||||
// native `db.query` parses the `$1::jsonb` text param itself (not the postgres.js
|
||||
// double-encode path), so `$1::jsonb` is correct here.
|
||||
const result = await this.db.query<{ id: string }>(
|
||||
`UPDATE sources
|
||||
SET config = COALESCE(config, '{}'::jsonb) || $1::jsonb
|
||||
SET config =
|
||||
CASE
|
||||
WHEN jsonb_typeof(config) = 'object' THEN config
|
||||
WHEN jsonb_typeof(config) = 'string'
|
||||
THEN CASE
|
||||
WHEN (config #>> '{}') IS JSON
|
||||
THEN COALESCE(NULLIF((config #>> '{}'), '')::jsonb, '{}'::jsonb)
|
||||
ELSE '{}'::jsonb
|
||||
END
|
||||
WHEN jsonb_typeof(config) = 'array'
|
||||
THEN COALESCE(
|
||||
(SELECT jsonb_object_agg(kv.key, kv.value)
|
||||
FROM jsonb_array_elements(config) elem,
|
||||
jsonb_each(CASE WHEN jsonb_typeof(elem) = 'object'
|
||||
THEN elem ELSE '{}'::jsonb END) kv),
|
||||
'{}'::jsonb
|
||||
)
|
||||
ELSE '{}'::jsonb
|
||||
END
|
||||
|| $1::jsonb
|
||||
WHERE id = $2
|
||||
RETURNING id`,
|
||||
[JSON.stringify(patch), sourceId],
|
||||
@@ -5199,21 +5228,25 @@ export class PGLiteEngine implements BrainEngine {
|
||||
// pages_with_timeline) and v0.10.3 graph layer (link_coverage, timeline_coverage,
|
||||
// most_connected). Both coexist: master's brain_score is the composite
|
||||
// dashboard, v0.10.3 metrics give entity-page-level granularity.
|
||||
// #2330: exclude soft-deleted pages from every page-based metric — parity
|
||||
// with postgres-engine.getHealth and getStats.page_count.
|
||||
const { rows: [h] } = await this.db.query(`
|
||||
WITH entity_pages AS (
|
||||
SELECT id, slug FROM pages WHERE type IN ('person', 'company')
|
||||
SELECT id, slug FROM pages WHERE type IN ('person', 'company') AND deleted_at IS NULL
|
||||
)
|
||||
SELECT
|
||||
(SELECT count(*) FROM pages) as page_count,
|
||||
(SELECT count(*) FROM pages WHERE deleted_at IS NULL) as page_count,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float /
|
||||
GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage,
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE p.updated_at < (SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id)
|
||||
WHERE p.deleted_at IS NULL
|
||||
AND p.updated_at < (SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id)
|
||||
) as stale_pages,
|
||||
-- Bug 11 — orphan = islanded (no inbound AND no outbound).
|
||||
-- See BrainHealth.orphan_pages docstring; docs updated to match this.
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id)
|
||||
WHERE p.deleted_at IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)
|
||||
) as orphan_pages,
|
||||
(SELECT count(*) FROM links l
|
||||
@@ -5221,7 +5254,8 @@ export class PGLiteEngine implements BrainEngine {
|
||||
) as dead_links,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NULL) as missing_embeddings,
|
||||
(SELECT count(*) FROM links) as link_count,
|
||||
(SELECT count(DISTINCT page_id) FROM timeline_entries) as pages_with_timeline,
|
||||
(SELECT count(DISTINCT te.page_id) FROM timeline_entries te
|
||||
JOIN pages p ON p.id = te.page_id WHERE p.deleted_at IS NULL) as pages_with_timeline,
|
||||
(SELECT count(*) FROM entity_pages e
|
||||
WHERE EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = e.id))::float /
|
||||
GREATEST((SELECT count(*) FROM entity_pages), 1)::float as link_coverage,
|
||||
@@ -5235,7 +5269,7 @@ export class PGLiteEngine implements BrainEngine {
|
||||
SELECT p.slug,
|
||||
(SELECT count(*) FROM links l WHERE l.from_page_id = p.id OR l.to_page_id = p.id)::int as link_count
|
||||
FROM pages p
|
||||
WHERE p.type IN ('person', 'company')
|
||||
WHERE p.type IN ('person', 'company') AND p.deleted_at IS NULL
|
||||
ORDER BY link_count DESC
|
||||
LIMIT 5
|
||||
`);
|
||||
|
||||
@@ -1116,7 +1116,11 @@ export class PostgresEngine implements BrainEngine {
|
||||
source_kind = COALESCE(EXCLUDED.source_kind, pages.source_kind),
|
||||
source_uri = COALESCE(EXCLUDED.source_uri, pages.source_uri),
|
||||
ingested_via = COALESCE(EXCLUDED.ingested_via, pages.ingested_via),
|
||||
ingested_at = COALESCE(EXCLUDED.ingested_at, pages.ingested_at)
|
||||
ingested_at = COALESCE(EXCLUDED.ingested_at, pages.ingested_at),
|
||||
-- #2345: revive on re-put. The unique index is non-partial, so a put
|
||||
-- on a soft-deleted slug lands on the tombstone row; without this the
|
||||
-- write is swallowed (page stays invisible but returns success).
|
||||
deleted_at = NULL
|
||||
RETURNING id, source_id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename, source_kind, source_uri, ingested_via, ingested_at
|
||||
`;
|
||||
return rowToPage(rows[0]);
|
||||
@@ -1441,10 +1445,19 @@ export class PostgresEngine implements BrainEngine {
|
||||
ELSE '{}'::jsonb
|
||||
END
|
||||
WHEN jsonb_typeof(config) = 'array'
|
||||
-- Array branch guard (#2251): jsonb_each raises "cannot call
|
||||
-- jsonb_each on a non-object" if ANY array element is a non-object
|
||||
-- scalar. A bare WHERE jsonb_typeof(elem)=object does NOT help --
|
||||
-- the lateral set-returning function is evaluated per elem row
|
||||
-- before the WHERE filters, so it still throws. Wrap each element
|
||||
-- in a CASE so jsonb_each only ever receives an object; non-object
|
||||
-- elements contribute no keys instead of aborting the whole UPDATE
|
||||
-- (which otherwise blocked every cycle last_full_cycle_at write).
|
||||
THEN COALESCE(
|
||||
(SELECT jsonb_object_agg(kv.key, kv.value)
|
||||
FROM jsonb_array_elements(config) elem,
|
||||
jsonb_each(elem) kv),
|
||||
jsonb_each(CASE WHEN jsonb_typeof(elem) = 'object'
|
||||
THEN elem ELSE '{}'::jsonb END) kv),
|
||||
'{}'::jsonb
|
||||
)
|
||||
ELSE '{}'::jsonb
|
||||
@@ -5318,19 +5331,27 @@ export class PostgresEngine implements BrainEngine {
|
||||
// SQL required both — docs now match code so users can trust the
|
||||
// number. A hub page that links out to many but has no back-references
|
||||
// is working as intended, not an orphan.
|
||||
// #2330: exclude soft-deleted pages from EVERY page-based metric, matching
|
||||
// getStats (`page_count`) and the "soft-deleted is hidden everywhere the
|
||||
// user looks" posture. Previously getHealth counted all pages while getStats
|
||||
// excluded deleted, so `get_health.page_count` and `get_stats.page_count`
|
||||
// disagreed (and brain_score ratios were diluted by tombstones). Filtering
|
||||
// numerator AND denominator keeps the ratios well-formed.
|
||||
const [h] = await sql`
|
||||
WITH entity_pages AS (
|
||||
SELECT id, slug FROM pages WHERE type IN ('person', 'company')
|
||||
SELECT id, slug FROM pages WHERE type IN ('person', 'company') AND deleted_at IS NULL
|
||||
)
|
||||
SELECT
|
||||
(SELECT count(*) FROM pages) as page_count,
|
||||
(SELECT count(*) FROM pages WHERE deleted_at IS NULL) as page_count,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float /
|
||||
GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage,
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE p.updated_at < (SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id)
|
||||
WHERE p.deleted_at IS NULL
|
||||
AND p.updated_at < (SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id)
|
||||
) as stale_pages,
|
||||
(SELECT count(*) FROM pages p
|
||||
WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id)
|
||||
WHERE p.deleted_at IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = p.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)
|
||||
) as orphan_pages,
|
||||
(SELECT count(*) FROM links l
|
||||
@@ -5338,7 +5359,8 @@ export class PostgresEngine implements BrainEngine {
|
||||
) as dead_links,
|
||||
(SELECT count(*) FROM content_chunks WHERE embedded_at IS NULL) as missing_embeddings,
|
||||
(SELECT count(*) FROM links) as link_count,
|
||||
(SELECT count(DISTINCT page_id) FROM timeline_entries) as pages_with_timeline,
|
||||
(SELECT count(DISTINCT te.page_id) FROM timeline_entries te
|
||||
JOIN pages p ON p.id = te.page_id WHERE p.deleted_at IS NULL) as pages_with_timeline,
|
||||
(SELECT count(*) FROM entity_pages e
|
||||
WHERE EXISTS (SELECT 1 FROM links l WHERE l.to_page_id = e.id))::float /
|
||||
GREATEST((SELECT count(*) FROM entity_pages), 1)::float as link_coverage,
|
||||
@@ -5351,7 +5373,7 @@ export class PostgresEngine implements BrainEngine {
|
||||
SELECT p.slug,
|
||||
(SELECT count(*) FROM links l WHERE l.from_page_id = p.id OR l.to_page_id = p.id)::int as link_count
|
||||
FROM pages p
|
||||
WHERE p.type IN ('person', 'company')
|
||||
WHERE p.type IN ('person', 'company') AND p.deleted_at IS NULL
|
||||
ORDER BY link_count DESC
|
||||
LIMIT 5
|
||||
`;
|
||||
|
||||
@@ -780,6 +780,11 @@ const QUERY_EMBED_TIMEOUT_MS = (() => {
|
||||
*/
|
||||
const MIN_QUERY_EMBED_BUDGET_MS = 2_000;
|
||||
|
||||
// #2028 — once-per-process guard for the query-embed-failure stderr warning
|
||||
// (bulk keyword-only sessions shouldn't spam one line per query; meta carries
|
||||
// degraded_reason on every affected query instead).
|
||||
let queryEmbedFailureWarned = false;
|
||||
|
||||
export interface QueryEmbedDeadline {
|
||||
/** Aborts the underlying fetch (clean socket close) when the budget elapses. */
|
||||
signal: AbortSignal;
|
||||
@@ -1128,6 +1133,9 @@ export async function hybridSearch(
|
||||
lastRank1Score = noEmbedBudgeted[0] ? (noEmbedBudgeted[0].base_score ?? noEmbedBudgeted[0].score) : undefined;
|
||||
emitMeta({
|
||||
vector_enabled: false,
|
||||
// #2028 — distinguish "no provider configured" from a runtime embed
|
||||
// failure (the timeout path below) so degraded searches are diagnosable.
|
||||
degraded_reason: 'no_embedding_provider',
|
||||
detail_resolved: detailResolved,
|
||||
expansion_applied: false,
|
||||
intent: suggestions.intent,
|
||||
@@ -1213,6 +1221,10 @@ export async function hybridSearch(
|
||||
// - 'image': embedQueryMultimodal + searchVector(embedding_image), skip keyword
|
||||
// - 'both': text + image vector searches in parallel; merged via weighted RRF
|
||||
let vectorLists: SearchResult[][] = [];
|
||||
// #2028 — why the vector arm didn't run, surfaced in the keyword-only
|
||||
// fallback meta so a degraded hybrid search is diagnosable (vector_enabled:
|
||||
// false alone hid a 9-day dead vector arm).
|
||||
let vectorDegradedReason: 'embed_timeout' | 'no_embedding_provider' | 'embed_error' | undefined;
|
||||
let queryEmbedding: Float32Array | null = null;
|
||||
let imageVectorList: SearchResult[] | null = null;
|
||||
let crossModalFellOpen = false;
|
||||
@@ -1323,8 +1335,28 @@ export async function hybridSearch(
|
||||
if (effectiveModality === 'both' && imageVectorList !== null) {
|
||||
vectorLists = [...vectorLists, imageVectorList];
|
||||
}
|
||||
} catch {
|
||||
// Embedding failure is non-fatal, fall back to keyword-only
|
||||
} catch (err) {
|
||||
// Embedding failure is non-fatal, fall back to keyword-only — but never
|
||||
// silently (#2028): a query-embed deadline miss degraded hybrid search
|
||||
// to keyword-only with zero operator signal (empty results for CJK/
|
||||
// vector-dependent content). Classify the reason for meta and warn once
|
||||
// per process on stderr.
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
vectorDegradedReason = /deadline .*exceeded/i.test(msg)
|
||||
? 'embed_timeout'
|
||||
: /not configured|no embedding model/i.test(msg)
|
||||
? 'no_embedding_provider'
|
||||
: 'embed_error';
|
||||
if (!queryEmbedFailureWarned) {
|
||||
queryEmbedFailureWarned = true;
|
||||
console.error(
|
||||
`[search] query embed failed (${msg}); falling back to keyword-only. ` +
|
||||
(vectorDegradedReason === 'embed_timeout'
|
||||
? `The query-embed deadline is ${QUERY_EMBED_TIMEOUT_MS}ms — for a slow local provider, raise GBRAIN_QUERY_EMBED_TIMEOUT_MS. `
|
||||
: '') +
|
||||
`(shown once per process; meta.degraded_reason carries this on every degraded query)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1361,6 +1393,9 @@ export async function hybridSearch(
|
||||
lastRank1Score = kwBudgeted[0] ? (kwBudgeted[0].base_score ?? kwBudgeted[0].score) : undefined;
|
||||
emitMeta({
|
||||
vector_enabled: false,
|
||||
// #2028 — embed_timeout / embed_error / no_embedding_provider, so a
|
||||
// silently-degraded hybrid search names WHY the vector arm didn't run.
|
||||
...(vectorDegradedReason ? { degraded_reason: vectorDegradedReason } : {}),
|
||||
detail_resolved: detailResolved,
|
||||
expansion_applied: expansionApplied,
|
||||
intent: suggestions.intent,
|
||||
|
||||
@@ -1586,6 +1586,14 @@ export interface EvalCaptureFailure {
|
||||
export interface HybridSearchMeta {
|
||||
/** True iff vector search actually ran. False when OPENAI_API_KEY missing or embed failed. */
|
||||
vector_enabled: boolean;
|
||||
/**
|
||||
* #2028 — WHY the vector arm didn't run, set only when vector_enabled is
|
||||
* false. 'embed_timeout' = the query embed missed the GBRAIN_QUERY_EMBED_TIMEOUT_MS
|
||||
* deadline (default 6s); 'no_embedding_provider' = gateway has no reachable
|
||||
* embedding provider for the resolved column; 'embed_error' = any other
|
||||
* embed/vector failure. Omitted when vector ran.
|
||||
*/
|
||||
degraded_reason?: 'embed_timeout' | 'no_embedding_provider' | 'embed_error';
|
||||
/** Post-auto-detect detail level. */
|
||||
detail_resolved: 'low' | 'medium' | 'high' | null;
|
||||
/** True iff multi-query expansion (Haiku) actually fired and produced variants. */
|
||||
|
||||
@@ -248,6 +248,28 @@ describe('getPage / listPages includeDeleted contract (Q3 IRON RULE)', () => {
|
||||
const mia = pages.find((p) => p.slug === 'people/mia')!;
|
||||
expect(mia.deleted_at).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
test('#2345: re-putting a soft-deleted slug revives the page (no silent write-swallow)', async () => {
|
||||
await seedPage(engine, 'people/nadia');
|
||||
await engine.softDeletePage('people/nadia');
|
||||
expect(await engine.getPage('people/nadia')).toBeNull();
|
||||
|
||||
// Re-create via putPage: the upsert lands on the tombstone row (the
|
||||
// unique index is non-partial). Pre-fix the SET list never cleared
|
||||
// deleted_at, so this returned success while the page stayed invisible.
|
||||
await engine.putPage('people/nadia', {
|
||||
type: 'note' as any,
|
||||
title: 'Nadia (recreated)',
|
||||
compiled_truth: 'Recreated content',
|
||||
timeline: '',
|
||||
frontmatter: {},
|
||||
});
|
||||
|
||||
const revived = await engine.getPage('people/nadia');
|
||||
expect(revived).not.toBeNull();
|
||||
expect(revived!.deleted_at).toBeFalsy();
|
||||
expect(revived!.compiled_truth).toBe('Recreated content');
|
||||
});
|
||||
});
|
||||
|
||||
describe('search visibility (soft-deleted pages hidden from searchKeyword)', () => {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
* insufficient against a wedged provider), and that a shared/elapsed deadline
|
||||
* makes a second embed fail FAST (worst case ~one timeout, not two).
|
||||
*/
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
|
||||
import {
|
||||
configureGateway,
|
||||
resetGateway,
|
||||
@@ -83,3 +83,78 @@ describe('embedQueryBounded — query-embed deadline', () => {
|
||||
expect(out.length).toBe(1024);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* #2028 — a degraded hybrid search must NAME why the vector arm didn't run.
|
||||
* Pre-fix the embed catch was bare, so an embed timeout/error emitted only
|
||||
* `vector_enabled: false` — indistinguishable from "no provider configured",
|
||||
* which let a dead vector arm go unnoticed (empty results for CJK content).
|
||||
*/
|
||||
describe('hybridSearch meta.degraded_reason (#2028)', () => {
|
||||
let engine: import('../../src/core/pglite-engine.ts').PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { PGLiteEngine } = await import('../../src/core/pglite-engine.ts');
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
await engine.putPage('notes/hello', {
|
||||
type: 'note' as any,
|
||||
title: 'hello',
|
||||
compiled_truth: 'hello world content',
|
||||
timeline: '',
|
||||
frontmatter: {},
|
||||
});
|
||||
});
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
beforeEach(() => {
|
||||
resetGateway();
|
||||
});
|
||||
afterEach(async () => {
|
||||
__setEmbedTransportForTests(null);
|
||||
resetGateway();
|
||||
});
|
||||
|
||||
test('embed timeout → degraded_reason: embed_timeout', async () => {
|
||||
configureGateway({
|
||||
embedding_model: 'openai:text-embedding-3-small',
|
||||
embedding_dimensions: 1536,
|
||||
env: { OPENAI_API_KEY: 'sk-fake' },
|
||||
});
|
||||
__setEmbedTransportForTests(() => new Promise(() => { /* hang forever */ }));
|
||||
const { hybridSearch } = await import('../../src/core/search/hybrid.ts');
|
||||
let meta: import('../../src/core/types.ts').HybridSearchMeta | undefined;
|
||||
await hybridSearch(engine, 'hello world', {
|
||||
onMeta: (m) => { meta = m; },
|
||||
// Already-elapsed shared deadline → the floored ~2s bound fires fast.
|
||||
_queryEmbedDeadline: { signal: AbortSignal.timeout(1), deadlineAt: Date.now() - 5 },
|
||||
});
|
||||
expect(meta?.vector_enabled).toBe(false);
|
||||
expect(meta?.degraded_reason).toBe('embed_timeout');
|
||||
}, 15000);
|
||||
|
||||
test('embed error → degraded_reason: embed_error', async () => {
|
||||
configureGateway({
|
||||
embedding_model: 'openai:text-embedding-3-small',
|
||||
embedding_dimensions: 1536,
|
||||
env: { OPENAI_API_KEY: 'sk-fake' },
|
||||
});
|
||||
__setEmbedTransportForTests(() => Promise.reject(new Error('provider 500')));
|
||||
const { hybridSearch } = await import('../../src/core/search/hybrid.ts');
|
||||
let meta: import('../../src/core/types.ts').HybridSearchMeta | undefined;
|
||||
await hybridSearch(engine, 'hello world', { onMeta: (m) => { meta = m; } });
|
||||
expect(meta?.vector_enabled).toBe(false);
|
||||
expect(meta?.degraded_reason).toBe('embed_error');
|
||||
}, 15000);
|
||||
|
||||
test('no embedding provider → degraded_reason: no_embedding_provider', async () => {
|
||||
// resetGateway in beforeEach left the gateway unconfigured.
|
||||
const { hybridSearch } = await import('../../src/core/search/hybrid.ts');
|
||||
let meta: import('../../src/core/types.ts').HybridSearchMeta | undefined;
|
||||
await hybridSearch(engine, 'hello world', { onMeta: (m) => { meta = m; } });
|
||||
expect(meta?.vector_enabled).toBe(false);
|
||||
expect(meta?.degraded_reason).toBe('no_embedding_provider');
|
||||
}, 15000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
/**
|
||||
* Regression tests for the silent-failure cleanup wave (post-#2375).
|
||||
*
|
||||
* - #2251 updateSourceConfig array-branch guard (jsonb_each on a non-object)
|
||||
* - #2330 getStats vs getHealth page-count parity (both exclude soft-deleted)
|
||||
* - #2297 files upload-raw persistence helpers (page-namespaced, no clobber)
|
||||
*
|
||||
* Hermetic in-memory PGLite; Postgres parity covered by the e2e suite. The
|
||||
* #2251 + #2330 fixes also land in postgres-engine.ts in lockstep.
|
||||
*/
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
import { resolveRepoRoot, lookupPageBySlug, insertFileRow, namespacedStoragePath } from '../src/commands/files.ts';
|
||||
import { categorizeCheck } from '../src/core/doctor-categories.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
});
|
||||
|
||||
async function seedSource(id: string, localPath: string | null): Promise<void> {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, local_path, config, created_at)
|
||||
VALUES ($1, $2, $3, '{}'::jsonb, NOW())
|
||||
ON CONFLICT (id) DO UPDATE SET local_path = EXCLUDED.local_path`,
|
||||
[id, id, localPath],
|
||||
);
|
||||
}
|
||||
|
||||
async function seedPage(slug: string, opts: { deleted?: boolean; sourceId?: string } = {}): Promise<number> {
|
||||
const rows = await engine.executeRaw<{ id: number }>(
|
||||
`INSERT INTO pages (source_id, slug, type, title, deleted_at)
|
||||
VALUES ($1, $2, 'note', $2, ${opts.deleted ? 'NOW()' : 'NULL'})
|
||||
RETURNING id`,
|
||||
[opts.sourceId ?? 'default', slug],
|
||||
);
|
||||
return rows[0].id;
|
||||
}
|
||||
|
||||
describe('#2251 — updateSourceConfig array-branch guard', () => {
|
||||
test('coerces a corrupted array config (with a non-object element) instead of throwing', async () => {
|
||||
await seedSource('corrupt-arr', '/tmp/corrupt-arr');
|
||||
// Force the historical bad shape: a JSONB ARRAY whose elements include a
|
||||
// non-object scalar. Pre-fix, the array branch ran jsonb_each() on the
|
||||
// string element and the whole UPDATE aborted with
|
||||
// "cannot call jsonb_each on a non-object".
|
||||
await engine.executeRaw(
|
||||
`UPDATE sources SET config = '["stray-string", {"federated": true}]'::jsonb WHERE id = $1`,
|
||||
['corrupt-arr'],
|
||||
);
|
||||
|
||||
// Must NOT throw, and must return true (row matched).
|
||||
const ok = await engine.updateSourceConfig('corrupt-arr', { tracked_branch: 'main' });
|
||||
expect(ok).toBe(true);
|
||||
|
||||
const rows = await engine.executeRaw<{ typ: string; cfg: Record<string, unknown> }>(
|
||||
`SELECT jsonb_typeof(config) AS typ, config AS cfg FROM sources WHERE id = $1`,
|
||||
['corrupt-arr'],
|
||||
);
|
||||
expect(rows[0].typ).toBe('object');
|
||||
// Object element flattened, scalar element skipped, patch merged.
|
||||
expect(rows[0].cfg.federated).toBe(true);
|
||||
expect(rows[0].cfg.tracked_branch).toBe('main');
|
||||
});
|
||||
|
||||
test('still coerces a double-encoded string config', async () => {
|
||||
await seedSource('corrupt-str', '/tmp/corrupt-str');
|
||||
await engine.executeRaw(
|
||||
`UPDATE sources SET config = to_jsonb('{"github_repo":"a/b"}'::text) WHERE id = $1`,
|
||||
['corrupt-str'],
|
||||
);
|
||||
const ok = await engine.updateSourceConfig('corrupt-str', { federated: false });
|
||||
expect(ok).toBe(true);
|
||||
const rows = await engine.executeRaw<{ typ: string; cfg: Record<string, unknown> }>(
|
||||
`SELECT jsonb_typeof(config) AS typ, config AS cfg FROM sources WHERE id = $1`,
|
||||
['corrupt-str'],
|
||||
);
|
||||
expect(rows[0].typ).toBe('object');
|
||||
expect(rows[0].cfg.github_repo).toBe('a/b');
|
||||
expect(rows[0].cfg.federated).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2330 — skill_surface check is categorized as skill (not vacuous meta)', () => {
|
||||
test('the focused-doctor skill_surface check lands in the skill category', () => {
|
||||
// Without this wiring the explicit skill_surface check would fall through to
|
||||
// meta, leaving category_scores.skill a vacuous 100 — the exact dishonesty
|
||||
// #2330 reports. categorizeCheck must map it to skill.
|
||||
expect(categorizeCheck('skill_surface')).toBe('skill');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2330 — getStats and getHealth agree on page_count (both exclude soft-deleted)', () => {
|
||||
test('soft-deleted pages are excluded from BOTH surfaces', async () => {
|
||||
await seedPage('live/one');
|
||||
await seedPage('live/two');
|
||||
await seedPage('gone/three', { deleted: true });
|
||||
|
||||
const stats = await engine.getStats();
|
||||
const health = await engine.getHealth();
|
||||
|
||||
expect(stats.page_count).toBe(2);
|
||||
expect(health.page_count).toBe(2);
|
||||
expect(health.page_count).toBe(stats.page_count);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2297 — upload-raw persistence helpers', () => {
|
||||
test('resolveRepoRoot prefers source.local_path, falls back to sync.repo_path', async () => {
|
||||
await seedSource('teamco', '/repos/teamco');
|
||||
expect(await resolveRepoRoot(engine, 'teamco')).toBe('/repos/teamco');
|
||||
|
||||
await engine.setConfig('sync.repo_path', '/repos/default');
|
||||
expect(await resolveRepoRoot(engine, 'default')).toBe('/repos/default');
|
||||
expect(await resolveRepoRoot(engine, null)).toBe('/repos/default');
|
||||
});
|
||||
|
||||
test('lookupPageBySlug returns the live page id + source, honoring the source hint', async () => {
|
||||
const id = await seedPage('people/alice-example');
|
||||
const found = await lookupPageBySlug(engine, 'people/alice-example');
|
||||
expect(found?.id).toBe(id);
|
||||
expect(found?.source_id).toBe('default');
|
||||
expect(await lookupPageBySlug(engine, 'people/nobody')).toBeNull();
|
||||
|
||||
// Same slug in a second source → the hint disambiguates (#2297).
|
||||
await seedSource('teamco', '/repos/teamco');
|
||||
const teamId = await seedPage('people/alice-example', { sourceId: 'teamco' });
|
||||
const hinted = await lookupPageBySlug(engine, 'people/alice-example', 'teamco');
|
||||
expect(hinted?.id).toBe(teamId);
|
||||
expect(hinted?.source_id).toBe('teamco');
|
||||
|
||||
// Explicit --source is STRICT: a scoped miss returns null even though the
|
||||
// slug exists in `default` — no silent cross-source fallback.
|
||||
expect(await lookupPageBySlug(engine, 'people/alice-example', 'no-such-source')).toBeNull();
|
||||
});
|
||||
|
||||
test('namespacedStoragePath keeps default-source paths but isolates other sources (#2297)', async () => {
|
||||
// Default source: byte-identical to the historical path.
|
||||
expect(namespacedStoragePath('default', 'people/alice/.raw/cv.pdf')).toBe('people/alice/.raw/cv.pdf');
|
||||
expect(namespacedStoragePath(null, 'people/alice/.raw/cv.pdf')).toBe('people/alice/.raw/cv.pdf');
|
||||
// Non-default source: prefixed so the same slug+filename can't collide on
|
||||
// the globally-UNIQUE storage_path.
|
||||
expect(namespacedStoragePath('teamco', 'people/alice/.raw/cv.pdf')).toBe('teamco/people/alice/.raw/cv.pdf');
|
||||
|
||||
// End to end: two sources, same slug+filename → two distinct rows.
|
||||
await seedSource('src-a', '/repos/a');
|
||||
await seedSource('src-b', '/repos/b');
|
||||
const aId = await seedPage('shared/slug', { sourceId: 'src-a' });
|
||||
const bId = await seedPage('shared/slug', { sourceId: 'src-b' });
|
||||
await insertFileRow(engine, {
|
||||
sourceId: 'src-a', pageId: aId, pageSlug: 'shared/slug', filename: 'f.pdf',
|
||||
storagePath: namespacedStoragePath('src-a', 'shared/slug/.raw/f.pdf'),
|
||||
mimeType: 'application/pdf', size: 1, contentHash: 'sha256:a', metadata: {},
|
||||
});
|
||||
await insertFileRow(engine, {
|
||||
sourceId: 'src-b', pageId: bId, pageSlug: 'shared/slug', filename: 'f.pdf',
|
||||
storagePath: namespacedStoragePath('src-b', 'shared/slug/.raw/f.pdf'),
|
||||
mimeType: 'application/pdf', size: 2, contentHash: 'sha256:b', metadata: {},
|
||||
});
|
||||
const rows = await engine.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM files`);
|
||||
expect(rows[0].n).toBe(2);
|
||||
});
|
||||
|
||||
test('insertFileRow persists a row; page-namespaced paths do not clobber across pages', async () => {
|
||||
const aliceId = await seedPage('people/alice-example');
|
||||
const bobId = await seedPage('people/bob-example');
|
||||
|
||||
// Same filename, different page → page-namespaced storage_path keeps both.
|
||||
await insertFileRow(engine, {
|
||||
sourceId: 'default', pageId: aliceId, pageSlug: 'people/alice-example',
|
||||
filename: 'resume.pdf', storagePath: 'people/alice-example/.raw/resume.pdf',
|
||||
mimeType: 'application/pdf', size: 10, contentHash: 'sha256:a', metadata: { upload_method: 'git' },
|
||||
});
|
||||
await insertFileRow(engine, {
|
||||
sourceId: 'default', pageId: bobId, pageSlug: 'people/bob-example',
|
||||
filename: 'resume.pdf', storagePath: 'people/bob-example/.raw/resume.pdf',
|
||||
mimeType: 'application/pdf', size: 20, contentHash: 'sha256:b', metadata: { upload_method: 'git' },
|
||||
});
|
||||
|
||||
const rows = await engine.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM files`);
|
||||
expect(rows[0].n).toBe(2);
|
||||
|
||||
// Re-uploading the SAME storage_path upserts (no duplicate), and metadata
|
||||
// round-trips as a real JSONB object (not a double-encoded string).
|
||||
await insertFileRow(engine, {
|
||||
sourceId: 'default', pageId: aliceId, pageSlug: 'people/alice-example',
|
||||
filename: 'resume.pdf', storagePath: 'people/alice-example/.raw/resume.pdf',
|
||||
mimeType: 'application/pdf', size: 99, contentHash: 'sha256:a2', metadata: { upload_method: 'git', type: 'cv' },
|
||||
});
|
||||
const after = await engine.executeRaw<{ n: number; size_bytes: number; meta_typ: string }>(
|
||||
`SELECT count(*)::int AS n,
|
||||
max(size_bytes) AS size_bytes,
|
||||
jsonb_typeof((SELECT metadata FROM files WHERE storage_path = 'people/alice-example/.raw/resume.pdf')) AS meta_typ
|
||||
FROM files`,
|
||||
);
|
||||
expect(after[0].n).toBe(2); // upsert, not insert
|
||||
expect(Number(after[0].size_bytes)).toBe(99); // updated row
|
||||
expect(after[0].meta_typ).toBe('object'); // metadata is a real object
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user