mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 00:48:18 +00:00
feat(doctor): silent-failure check batch — content-hash duplicates, undeclared DB-only pages, heartbeat staleness, db_only collector collision (#2250 #2784 #2787 #2788) (#3457)
Co-Authored-By: Time Attakc <89218912+time-attack@users.noreply.github.com>
This commit is contained in:
committed by
Sina Matian
co-authored by
Time Attakc
parent
aa5b9e6e2d
commit
7cbb99ffef
File diff suppressed because one or more lines are too long
@@ -69,6 +69,12 @@ health_checks: # typed DSL to verify the integration is working
|
||||
auth_user: "$TWILIO_ACCOUNT_SID"
|
||||
auth_token: "$TWILIO_AUTH_TOKEN"
|
||||
label: "Twilio account"
|
||||
- type: heartbeat_max_age # staleness gate: FAILS `integrations doctor`
|
||||
max_age: 48h # when the newest heartbeat event is older.
|
||||
label: "Data freshness" # The other types are point-in-time and stay
|
||||
# green even when a sense stops producing data.
|
||||
output_paths: # repo-relative dirs the collector writes files to;
|
||||
- daily/voice/ # lets doctor/sync warn if one lands in db_only
|
||||
setup_time: 30 min # estimated time to complete setup
|
||||
---
|
||||
|
||||
@@ -86,7 +92,8 @@ a source install, or the global install copy) are trusted. Recipes discovered at
|
||||
runtime from `$GBRAIN_RECIPES_DIR` or a cwd-local `./recipes/` are marked untrusted:
|
||||
they cannot run `command` health checks, cannot run `http` health checks (SSRF
|
||||
defense), and cannot use the deprecated string health_check form. Untrusted recipes
|
||||
can still use `env_exists` and `any_of` compositions. To ship a recipe that runs
|
||||
can still use `env_exists`, `heartbeat_max_age` (reads only the local heartbeat
|
||||
file — no exec, no network), and `any_of` compositions. To ship a recipe that runs
|
||||
live checks, contribute it upstream so it becomes package-bundled.
|
||||
|
||||
## The Deterministic Collector Pattern
|
||||
|
||||
@@ -51,6 +51,18 @@ When storage configuration is present, `gbrain sync` automatically manages `.git
|
||||
- Skipped when the repo is a git submodule (`.git` is a file, not a directory) — submodule .gitignore changes don't survive parent updates. A warning explains.
|
||||
- Skipped entirely when `GBRAIN_NO_GITIGNORE=1` is set (escape hatch for shared-repo setups where a maintainer wants gbrain to leave .gitignore alone).
|
||||
- Failures (write permission denied, etc.) are caught and logged, never crash sync.
|
||||
- Warns when a configured collector's declared output dir (recipe `output_paths`
|
||||
frontmatter) sits inside a `db_only` path: gitignored files never appear in the
|
||||
git-walking sync diff, and `gbrain import` honors `.gitignore` too — the
|
||||
collector would run green while nothing reaches the DB. The
|
||||
`db_only_collector_collision` doctor check surfaces the same trap.
|
||||
|
||||
Related doctor coverage: `undeclared_db_only_pages` warns about DB pages with no
|
||||
backing file that sit outside every declared `db_only` path. The engine's own
|
||||
derive-phase output prefixes (`life/events/`, `atoms/`, `extracts/`,
|
||||
`dream-cycle-summaries/`) count as implicitly declared for that check, so healthy
|
||||
brains stay quiet without adding them to `gbrain.yml`. They are NOT auto-added to
|
||||
`.gitignore` — only explicitly declared `db_only` dirs are.
|
||||
|
||||
Example `.gitignore` addition:
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
id: calendar-to-brain
|
||||
name: Calendar-to-Brain
|
||||
version: 0.7.0
|
||||
version: 0.8.0
|
||||
description: Google Calendar events become searchable brain pages. Daily files with attendees, locations, and meeting prep context.
|
||||
category: sense
|
||||
requires: [credential-gateway]
|
||||
@@ -28,6 +28,11 @@ health_checks:
|
||||
- type: env_exists
|
||||
name: GOOGLE_CLIENT_ID
|
||||
label: "Google OAuth"
|
||||
- type: heartbeat_max_age
|
||||
max_age: 48h
|
||||
label: "Calendar data freshness"
|
||||
output_paths:
|
||||
- daily/calendar/
|
||||
setup_time: 20 min
|
||||
cost_estimate: "$0 (both options are free)"
|
||||
---
|
||||
|
||||
@@ -52,6 +52,13 @@ import { lagFromContentMs } from '../core/source-health.ts';
|
||||
import { CHUNKER_VERSION } from '../core/chunkers/code.ts';
|
||||
import { LINK_EXTRACTOR_VERSION_TS } from '../core/link-extraction.ts';
|
||||
import { isUndefinedColumnError } from '../core/utils.ts';
|
||||
import {
|
||||
loadStorageConfig,
|
||||
effectiveDbOnlyDirs,
|
||||
DERIVE_PHASE_DB_ONLY_DEFAULTS,
|
||||
findDbOnlyCollisions,
|
||||
} from '../core/storage-config.ts';
|
||||
import { slugifyPath } from '../core/sync.ts';
|
||||
// issue #1777: hidden_by_search_policy — count chunked pages withheld from
|
||||
// default search by the hard-exclude prefix policy. Reuses the canonical
|
||||
// exclude resolver + LIKE escaper + visibility clause so the doctor count can't
|
||||
@@ -3672,6 +3679,198 @@ export async function checkUnverifiedExtractions(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* issue #2250 (reported by @615Works) — content_hash_duplicates.
|
||||
*
|
||||
* `gbrain import` run from the wrong root (one level too deep) drops the
|
||||
* path prefix from every slug, leaving `people/x` and `x` coexisting with
|
||||
* identical content. `dream --phase purge` never removes them (they aren't
|
||||
* file-backed orphans) and nothing surfaced the condition. One GROUP BY —
|
||||
* never an N² hash comparison — flags hash groups that contain BOTH a bare
|
||||
* slug (no '/') and a path-prefixed slug.
|
||||
*/
|
||||
export async function checkContentHashDuplicates(engine: BrainEngine): Promise<Check> {
|
||||
const name = 'content_hash_duplicates';
|
||||
const fix = 'Fix: gbrain pages delete <bare-slug> for each pair, then gbrain pages purge-deleted --older-than 0';
|
||||
try {
|
||||
const rows = await engine.executeRaw<{ source_id: string; content_hash: string; slugs: string }>(
|
||||
`SELECT source_id, content_hash,
|
||||
string_agg(slug, '|' ORDER BY length(slug), slug) AS slugs
|
||||
FROM pages
|
||||
WHERE deleted_at IS NULL AND content_hash IS NOT NULL AND content_hash <> ''
|
||||
GROUP BY source_id, content_hash
|
||||
HAVING count(*) > 1
|
||||
AND count(*) FILTER (WHERE strpos(slug, '/') = 0) > 0
|
||||
AND count(*) FILTER (WHERE strpos(slug, '/') > 0) > 0
|
||||
LIMIT 50`,
|
||||
);
|
||||
if (rows.length === 0) {
|
||||
return { name, status: 'ok', message: 'No content-hash duplicate pairs (bare vs path-prefixed slugs)' };
|
||||
}
|
||||
let pairCount = 0;
|
||||
const samples: string[] = [];
|
||||
for (const r of rows) {
|
||||
const slugs = String(r.slugs).split('|');
|
||||
const prefixed = slugs.filter(s => s.includes('/'));
|
||||
for (const bare of slugs.filter(s => !s.includes('/'))) {
|
||||
const twin = prefixed.find(p => p.endsWith('/' + bare)) ?? prefixed[0];
|
||||
pairCount++;
|
||||
if (samples.length < 5) samples.push(`${bare} <-> ${twin}`);
|
||||
}
|
||||
}
|
||||
return {
|
||||
name,
|
||||
status: 'warn',
|
||||
message: `${pairCount} content-hash duplicate pair(s) detected (same content, differing slug forms — usually an import run from the wrong root, which drops the path prefix). Sample: ${samples.join('; ')}. ${fix}`,
|
||||
details: { pair_count: pairCount, hash_groups: rows.length, sample_pairs: samples },
|
||||
};
|
||||
} catch (e) {
|
||||
return { name, status: 'warn', message: `Could not check content-hash duplicates: ${(e as Error).message}` };
|
||||
}
|
||||
}
|
||||
|
||||
/** Walk a repo for markdown files and return their slugified (lowercased) slugs. */
|
||||
function collectMarkdownSlugs(root: string): Set<string> {
|
||||
const out = new Set<string>();
|
||||
const stack = [''];
|
||||
while (stack.length > 0) {
|
||||
const rel = stack.pop()!;
|
||||
let entries;
|
||||
try {
|
||||
entries = readdirSync(rel ? join(root, rel) : root, { withFileTypes: true });
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const e of entries) {
|
||||
if (e.name.startsWith('.') || e.name === 'node_modules') continue;
|
||||
const childRel = rel ? `${rel}/${e.name}` : e.name;
|
||||
if (e.isDirectory()) stack.push(childRel);
|
||||
else if (/\.mdx?$/i.test(e.name)) out.add(slugifyPath(childRel).toLowerCase());
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* issue #2784 (reported by @alexputici) — undeclared_db_only_pages.
|
||||
*
|
||||
* A markdown page with no backing file that sits outside every declared
|
||||
* db_only path is invisible to any file-lane backup/recovery reasoning: an
|
||||
* operator auditing "what would survive a DB loss" gets a silently wrong
|
||||
* answer. The engine's own derive-phase output prefixes
|
||||
* (DERIVE_PHASE_DB_ONLY_DEFAULTS) count as implicitly declared so the check
|
||||
* stays quiet on healthy brains. Deliberately allowed to stat the source
|
||||
* repo (the one thing the SQL-only check registry could never see).
|
||||
*/
|
||||
export async function checkUndeclaredDbOnlyPages(engine: BrainEngine): Promise<Check> {
|
||||
const name = 'undeclared_db_only_pages';
|
||||
try {
|
||||
const sources = await engine.executeRaw<{ id: string; local_path: string | null }>(
|
||||
`SELECT id, local_path FROM sources WHERE local_path IS NOT NULL`,
|
||||
);
|
||||
const checkable = sources.filter(s => s.local_path && existsSync(s.local_path));
|
||||
if (checkable.length === 0) {
|
||||
return { name, status: 'ok', message: 'Not applicable (no sources with a local repo path on this host)' };
|
||||
}
|
||||
let total = 0;
|
||||
const samples: string[] = [];
|
||||
const perSource: Record<string, number> = {};
|
||||
for (const src of checkable) {
|
||||
let declared: string[] = [];
|
||||
try {
|
||||
declared = loadStorageConfig(src.local_path)?.db_only ?? [];
|
||||
} catch {
|
||||
// invalid gbrain.yml — treated as no declarations; the sync path
|
||||
// already surfaces the config error itself.
|
||||
}
|
||||
const dbOnlyDirs = effectiveDbOnlyDirs(declared);
|
||||
const rows = await engine.executeRaw<{ slug: string }>(
|
||||
`SELECT slug FROM pages WHERE deleted_at IS NULL AND source_id = $1 AND page_kind = 'markdown'`,
|
||||
[src.id],
|
||||
);
|
||||
if (rows.length === 0) continue;
|
||||
const backed = collectMarkdownSlugs(src.local_path!);
|
||||
for (const { slug } of rows) {
|
||||
if (dbOnlyDirs.some(dir => slug.startsWith(dir))) continue;
|
||||
if (backed.has(slug)) continue;
|
||||
total++;
|
||||
perSource[src.id] = (perSource[src.id] ?? 0) + 1;
|
||||
if (samples.length < 5) samples.push(`${slug} (src=${src.id})`);
|
||||
}
|
||||
}
|
||||
if (total === 0) {
|
||||
return {
|
||||
name,
|
||||
status: 'ok',
|
||||
message: `Every DB page is file-backed or under a declared/default db_only path (derive-phase defaults: ${DERIVE_PHASE_DB_ONLY_DEFAULTS.join(' ')})`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
name,
|
||||
status: 'warn',
|
||||
message: `${total} DB page(s) have no backing file and sit outside every declared/default db_only path — invisible to file-lane backup/recovery. Sample: ${samples.join('; ')}. Fix: restore or export the files, or declare their prefixes under storage.db_only in gbrain.yml (derive-phase defaults already cover: ${DERIVE_PHASE_DB_ONLY_DEFAULTS.join(' ')})`,
|
||||
details: { total, per_source: perSource, sample_slugs: samples },
|
||||
};
|
||||
} catch (e) {
|
||||
return { name, status: 'warn', message: `Could not check undeclared db-only pages: ${(e as Error).message}` };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* issue #2788 (reported by @alexputici) — db_only_collector_collision.
|
||||
*
|
||||
* Declaring a collector's output dir in storage.db_only silently kills its
|
||||
* ingestion: manageGitignore auto-gitignores the dir, the git-walking sync
|
||||
* never sees the files, and import honors .gitignore too — everything stays
|
||||
* green while nothing reaches the DB (a 7-week outage in the field). The
|
||||
* recipe's `output_paths` frontmatter is the ground truth; the same warning
|
||||
* also fires at .gitignore-write time inside sync's manageGitignore.
|
||||
*/
|
||||
export async function checkDbOnlyCollectorCollision(
|
||||
engine: BrainEngine,
|
||||
opts?: { collectors?: Array<{ id: string; output_path: string }> },
|
||||
): Promise<Check> {
|
||||
const name = 'db_only_collector_collision';
|
||||
try {
|
||||
let collectors = opts?.collectors;
|
||||
if (!collectors) {
|
||||
const { getConfiguredCollectorOutputs } = await import('./integrations.ts');
|
||||
collectors = getConfiguredCollectorOutputs();
|
||||
}
|
||||
if (collectors.length === 0) {
|
||||
return { name, status: 'ok', message: 'No configured collectors declare output paths' };
|
||||
}
|
||||
const sources = await engine.executeRaw<{ id: string; local_path: string | null }>(
|
||||
`SELECT id, local_path FROM sources WHERE local_path IS NOT NULL`,
|
||||
);
|
||||
const hits: string[] = [];
|
||||
for (const src of sources) {
|
||||
if (!src.local_path || !existsSync(src.local_path)) continue;
|
||||
let dbOnly: string[] = [];
|
||||
try {
|
||||
dbOnly = loadStorageConfig(src.local_path)?.db_only ?? [];
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (dbOnly.length === 0) continue;
|
||||
for (const hit of findDbOnlyCollisions(collectors, dbOnly)) {
|
||||
hits.push(`collector '${hit.id}' writes to '${hit.output_path}' which is inside db_only path '${hit.db_only_dir}' (source ${src.id})`);
|
||||
}
|
||||
}
|
||||
if (hits.length === 0) {
|
||||
return { name, status: 'ok', message: 'No collector output dir falls inside a db_only path' };
|
||||
}
|
||||
return {
|
||||
name,
|
||||
status: 'warn',
|
||||
message: `${hits.length} collector/db_only collision(s): ${hits.join('; ')}. db_only dirs are auto-gitignored, so sync AND import silently skip files there — the collector runs green while nothing reaches the DB. Fix: remove the prefix from storage.db_only in gbrain.yml, or move the collector output.`,
|
||||
details: { collisions: hits },
|
||||
};
|
||||
} catch (e) {
|
||||
return { name, status: 'warn', message: `Could not check collector/db_only collisions: ${(e as Error).message}` };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* issue #1678 — extract_atoms_backlog doctor check.
|
||||
*
|
||||
@@ -7696,6 +7895,14 @@ export async function buildChecks(
|
||||
// per-source dispatch gate sees.
|
||||
progress.heartbeat('cycle_freshness');
|
||||
checks.push(await checkCycleFreshness(engine));
|
||||
// Silent-failure batch (#2250 / #2784 / #2788): wrong-root import
|
||||
// duplicates, undeclared DB-only pages, collector-output-in-db_only.
|
||||
progress.heartbeat('content_hash_duplicates');
|
||||
checks.push(await checkContentHashDuplicates(engine));
|
||||
progress.heartbeat('undeclared_db_only_pages');
|
||||
checks.push(await checkUndeclaredDbOnlyPages(engine));
|
||||
progress.heartbeat('db_only_collector_collision');
|
||||
checks.push(await checkDbOnlyCollectorCollision(engine));
|
||||
}
|
||||
|
||||
// v0.32.3 search-lite — mode + eval_drift surfaces. Status stays 'ok' per
|
||||
|
||||
@@ -55,6 +55,13 @@ interface RecipeFrontmatter {
|
||||
health_checks: HealthCheck[];
|
||||
setup_time: string;
|
||||
cost_estimate?: string;
|
||||
/**
|
||||
* Repo-relative dirs (slug prefixes, trailing '/') this recipe's collector
|
||||
* writes files to. Ground truth for the `db_only_collector_collision`
|
||||
* doctor check (issue #2788): output inside a db_only path is silently
|
||||
* skipped by sync and import (auto-gitignored).
|
||||
*/
|
||||
output_paths: string[];
|
||||
}
|
||||
|
||||
interface ParsedRecipe {
|
||||
@@ -106,7 +113,20 @@ interface AnyOfCheck {
|
||||
checks: HealthCheck[];
|
||||
}
|
||||
|
||||
type HealthCheck = string | HttpCheck | EnvExistsCheck | CommandCheck | AnyOfCheck;
|
||||
/**
|
||||
* Staleness-aware check type (issue #2787, reported by @alexputici). All
|
||||
* other types are point-in-time — a sense whose gateway is up and env vars
|
||||
* are set passes forever even when zero data flows. This one reads the
|
||||
* integration's heartbeat file and FAILS when the newest event is older
|
||||
* than the declared cadence (`max_age`, e.g. "48h", "2d", "90m").
|
||||
*/
|
||||
interface HeartbeatMaxAgeCheck {
|
||||
type: 'heartbeat_max_age';
|
||||
max_age: string;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
type HealthCheck = string | HttpCheck | EnvExistsCheck | CommandCheck | AnyOfCheck | HeartbeatMaxAgeCheck;
|
||||
|
||||
interface CheckResult {
|
||||
integration: string;
|
||||
@@ -141,6 +161,26 @@ export function secretEnv(): Record<string, string | undefined> {
|
||||
return process.env;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a heartbeat_max_age duration string ("30s", "90m", "48h", "2d")
|
||||
* into milliseconds. Returns null on anything unparseable.
|
||||
*/
|
||||
export function parseMaxAge(s: string): number | null {
|
||||
const m = /^(\d+(?:\.\d+)?)\s*(s|m|h|d)$/i.exec(String(s).trim());
|
||||
if (!m) return null;
|
||||
const n = Number(m[1]);
|
||||
if (!Number.isFinite(n) || n <= 0) return null;
|
||||
const unit = { s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 }[m[2].toLowerCase() as 's' | 'm' | 'h' | 'd'];
|
||||
return n * unit;
|
||||
}
|
||||
|
||||
/** Human-readable age for heartbeat_max_age output ("3d", "17h", "42m"). */
|
||||
function formatAge(ms: number): string {
|
||||
if (ms >= 86_400_000) return `${Math.floor(ms / 86_400_000)}d`;
|
||||
if (ms >= 3_600_000) return `${Math.floor(ms / 3_600_000)}h`;
|
||||
return `${Math.max(0, Math.floor(ms / 60_000))}m`;
|
||||
}
|
||||
|
||||
/** Expand $VAR references with gateway-env (config-folded) values */
|
||||
export function expandVars(s: string): string {
|
||||
const env = secretEnv();
|
||||
@@ -299,6 +339,29 @@ export async function executeHealthCheck(
|
||||
}
|
||||
}
|
||||
|
||||
case 'heartbeat_max_age': {
|
||||
// No embedded gate: reads only the local heartbeat file — no exec, no
|
||||
// network. Safe for user-provided recipes.
|
||||
const maxMs = parseMaxAge(check.max_age);
|
||||
if (maxMs === null) {
|
||||
return { ...base, status: 'fail', output: `${check.label || 'heartbeat_max_age'}: invalid max_age '${check.max_age}' (use e.g. 90m, 48h, 2d)` };
|
||||
}
|
||||
const entries = readHeartbeat(integrationId);
|
||||
if (entries.length === 0) {
|
||||
return { ...base, status: 'fail', output: `${check.label || 'heartbeat'}: no heartbeat events in the last 30 days (expected activity within ${check.max_age}) — the sense has stopped producing data` };
|
||||
}
|
||||
let newest = 0;
|
||||
for (const e of entries) {
|
||||
const t = new Date(e.ts).getTime();
|
||||
if (Number.isFinite(t) && t > newest) newest = t;
|
||||
}
|
||||
const ageMs = Date.now() - newest;
|
||||
if (ageMs > maxMs) {
|
||||
return { ...base, status: 'fail', output: `${check.label || 'heartbeat'}: last event ${formatAge(ageMs)} ago exceeds max_age ${check.max_age} — the sense has stopped producing data` };
|
||||
}
|
||||
return { ...base, status: 'ok', output: `${check.label || 'heartbeat'}: last event ${formatAge(ageMs)} ago (within ${check.max_age})` };
|
||||
}
|
||||
|
||||
case 'any_of': {
|
||||
for (const sub of check.checks) {
|
||||
const result = await executeHealthCheck(sub, integrationId, isEmbedded);
|
||||
@@ -340,6 +403,7 @@ export function parseRecipe(content: string, filename: string): ParsedRecipe | n
|
||||
health_checks: (data.health_checks || []) as HealthCheck[],
|
||||
setup_time: data.setup_time || 'unknown',
|
||||
cost_estimate: data.cost_estimate,
|
||||
output_paths: Array.isArray(data.output_paths) ? data.output_paths.map(String) : [],
|
||||
},
|
||||
body: body.trim(),
|
||||
filename,
|
||||
@@ -403,6 +467,25 @@ function loadAllRecipes(): ParsedRecipe[] {
|
||||
return recipes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Output paths of every CONFIGURED recipe (secrets present — the collector
|
||||
* can actually be running). Ground truth for the
|
||||
* `db_only_collector_collision` doctor check and the sync-time warning
|
||||
* (issue #2788). Unconfigured recipes are skipped: a collector that can't
|
||||
* run can't silently die.
|
||||
*/
|
||||
export function getConfiguredCollectorOutputs(): Array<{ id: string; output_path: string }> {
|
||||
const out: Array<{ id: string; output_path: string }> = [];
|
||||
for (const r of loadAllRecipes()) {
|
||||
if (r.frontmatter.output_paths.length === 0) continue;
|
||||
if (getStatus(r) === 'available') continue;
|
||||
for (const p of r.frontmatter.output_paths) {
|
||||
out.push({ id: r.frontmatter.id, output_path: p });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function findRecipe(id: string): ParsedRecipe | null {
|
||||
const recipes = loadAllRecipes();
|
||||
const exact = recipes.find(r => r.frontmatter.id === id);
|
||||
|
||||
+23
-1
@@ -65,7 +65,11 @@ import {
|
||||
slog,
|
||||
serr,
|
||||
} from '../core/console-prefix.ts';
|
||||
import { loadStorageConfig } from '../core/storage-config.ts';
|
||||
import { loadStorageConfig, findDbOnlyCollisions } from '../core/storage-config.ts';
|
||||
// #2788: collector-output vs db_only collision warning at .gitignore-write
|
||||
// time. integrations.ts is side-effect-free at module load (pure recipe I/O
|
||||
// helpers), so a static import is safe here.
|
||||
import { getConfiguredCollectorOutputs } from './integrations.ts';
|
||||
import { getDefaultSourcePath } from '../core/source-resolver.ts';
|
||||
// v0.41.32.0: stamp the durable newest-COMMIT timestamp at sync time so the
|
||||
// remote staleness path reads a column instead of shelling out to git.
|
||||
@@ -5637,6 +5641,24 @@ export function manageGitignore(
|
||||
return;
|
||||
}
|
||||
|
||||
// #2788: a configured collector whose output dir sits inside a db_only
|
||||
// path dies silently — the dir is auto-gitignored below, the git-walking
|
||||
// sync never sees its files, and `gbrain import` honors .gitignore too.
|
||||
// Warn at the moment the config takes effect. Recipe scan failure never
|
||||
// blocks the gitignore housekeeping.
|
||||
try {
|
||||
for (const c of findDbOnlyCollisions(getConfiguredCollectorOutputs(), storageConfig.db_only)) {
|
||||
console.warn(
|
||||
`WARNING: collector '${c.id}' writes to '${c.output_path}', which is inside db_only path ` +
|
||||
`'${c.db_only_dir}'. db_only dirs are auto-gitignored, so gbrain sync and gbrain import ` +
|
||||
`will silently skip its files. Remove the prefix from storage.db_only in gbrain.yml, or ` +
|
||||
`move the collector output.`,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// recipes unavailable in this context — the doctor check still covers it
|
||||
}
|
||||
|
||||
// D4 soft-warn: storage tiering has limited effect on PGLite, but the
|
||||
// .gitignore housekeeping still helps. Warn once per process; proceed.
|
||||
if (engineKind === 'pglite' && !_pgliteTierWarned) {
|
||||
|
||||
@@ -60,6 +60,7 @@ export const BRAIN_CHECK_NAMES: ReadonlySet<string> = new Set([
|
||||
'calibration_freshness',
|
||||
'child_table_orphans',
|
||||
'chronicle_projection_health',
|
||||
'content_hash_duplicates',
|
||||
'content_sanity_audit_recent',
|
||||
'contextual_retrieval_coverage',
|
||||
'contradictions',
|
||||
@@ -111,6 +112,7 @@ export const BRAIN_CHECK_NAMES: ReadonlySet<string> = new Set([
|
||||
'takes_count',
|
||||
'takes_weight_grid',
|
||||
'timeline_coverage',
|
||||
'undeclared_db_only_pages',
|
||||
'unified_multimodal_coverage',
|
||||
'unverified_extractions',
|
||||
'voice_gate_health',
|
||||
@@ -143,6 +145,7 @@ export const OPS_CHECK_NAMES: ReadonlySet<string> = new Set([
|
||||
'batch_retry_health',
|
||||
'brainstorm_health',
|
||||
'connection',
|
||||
'db_only_collector_collision',
|
||||
'federation_health',
|
||||
'home_dir_in_worktree',
|
||||
'index_audit',
|
||||
|
||||
@@ -356,6 +356,52 @@ export function isDbOnly(slug: string, config: StorageConfig): boolean {
|
||||
return config.db_only.some((dir) => matchesTierDir(slug, dir));
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive-phase output prefixes the engine itself writes as DB-only machine
|
||||
* output (issue #2784, reported by @alexputici). These are re-derivable by
|
||||
* design and rarely file-backed, so the `undeclared_db_only_pages` doctor
|
||||
* check treats them as implicitly declared db_only. They are deliberately
|
||||
* NOT merged into `loadStorageConfig` — doing so would auto-gitignore these
|
||||
* dirs via `manageGitignore` and silently kill ingestion for brains that DO
|
||||
* file-back them (the exact #2788 silent-death class).
|
||||
*/
|
||||
export const DERIVE_PHASE_DB_ONLY_DEFAULTS: readonly string[] = [
|
||||
'life/events/',
|
||||
'atoms/',
|
||||
'extracts/',
|
||||
'dream-cycle-summaries/',
|
||||
];
|
||||
|
||||
/** Declared db_only dirs plus the derive-phase defaults, deduped. */
|
||||
export function effectiveDbOnlyDirs(declared: string[]): string[] {
|
||||
return [...new Set([...declared, ...DERIVE_PHASE_DB_ONLY_DEFAULTS])];
|
||||
}
|
||||
|
||||
/**
|
||||
* Collector-output vs db_only collision detection (issue #2788, reported by
|
||||
* @alexputici). A collector output path collides when it equals a db_only
|
||||
* dir or sits anywhere inside one — such dirs are auto-gitignored by sync,
|
||||
* so both the git-walking sync AND `gbrain import` (which honors .gitignore)
|
||||
* silently skip every file the collector writes.
|
||||
*/
|
||||
export function findDbOnlyCollisions(
|
||||
outputs: Array<{ id: string; output_path: string }>,
|
||||
dbOnlyDirs: string[],
|
||||
): Array<{ id: string; output_path: string; db_only_dir: string }> {
|
||||
const hits: Array<{ id: string; output_path: string; db_only_dir: string }> = [];
|
||||
for (const o of outputs) {
|
||||
const out = o.output_path.endsWith('/') ? o.output_path : o.output_path + '/';
|
||||
for (const rawDir of dbOnlyDirs) {
|
||||
const dir = rawDir.endsWith('/') ? rawDir : rawDir + '/';
|
||||
if (out.startsWith(dir)) {
|
||||
hits.push({ id: o.id, output_path: o.output_path, db_only_dir: rawDir });
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
export function getStorageTier(slug: string, config: StorageConfig): StorageTier {
|
||||
if (isDbTracked(slug, config)) return 'db_tracked';
|
||||
if (isDbOnly(slug, config)) return 'db_only';
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
/**
|
||||
* Unit tests for the silent-failure doctor check batch (#2250, #2784, #2788).
|
||||
* Hermetic PGLite; temp dirs stand in for source repos. Postgres parity for
|
||||
* the same checks is pinned by test/e2e/doctor-silent-death-parity.test.ts.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
import {
|
||||
checkContentHashDuplicates,
|
||||
checkUndeclaredDbOnlyPages,
|
||||
checkDbOnlyCollectorCollision,
|
||||
} from '../src/commands/doctor.ts';
|
||||
import {
|
||||
DERIVE_PHASE_DB_ONLY_DEFAULTS,
|
||||
effectiveDbOnlyDirs,
|
||||
findDbOnlyCollisions,
|
||||
} from '../src/core/storage-config.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function makeRepo(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'gbrain-doctor-'));
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (engine) await engine.disconnect();
|
||||
for (const d of tempDirs) rmSync(d, { recursive: true, force: true });
|
||||
}, 60_000);
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
});
|
||||
|
||||
async function addSource(id: string, localPath: string | null): Promise<void> {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, local_path, config) VALUES ($1, $1, $2, '{}'::jsonb)
|
||||
ON CONFLICT (id) DO UPDATE SET local_path = EXCLUDED.local_path`,
|
||||
[id, localPath],
|
||||
);
|
||||
}
|
||||
|
||||
async function addPage(
|
||||
slug: string,
|
||||
opts: { sourceId?: string; hash?: string | null; pageKind?: string; deleted?: boolean } = {},
|
||||
): Promise<void> {
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO pages (slug, source_id, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash, deleted_at)
|
||||
VALUES ($1, $2, 'concept', $3, $1, 'body', '', '{}'::jsonb, $4, $5)`,
|
||||
[
|
||||
slug,
|
||||
opts.sourceId ?? 'default',
|
||||
opts.pageKind ?? 'markdown',
|
||||
opts.hash === undefined ? `h-${slug}` : opts.hash,
|
||||
opts.deleted ? new Date().toISOString() : null,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
describe('content_hash_duplicates (#2250)', () => {
|
||||
test('distinct hashes → ok', async () => {
|
||||
await addPage('people/alice-example');
|
||||
await addPage('projects/widget-co');
|
||||
const c = await checkContentHashDuplicates(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('bare + path-prefixed twins with same hash → warn with pair + remediation', async () => {
|
||||
await addPage('people/alice-example', { hash: 'same' });
|
||||
await addPage('alice-example', { hash: 'same' });
|
||||
const c = await checkContentHashDuplicates(engine);
|
||||
expect(c.status).toBe('warn');
|
||||
expect(c.message).toContain('alice-example <-> people/alice-example');
|
||||
expect(c.message).toContain('gbrain pages delete <bare-slug>');
|
||||
expect(c.message).toContain('gbrain pages purge-deleted --older-than 0');
|
||||
expect((c.details as any).pair_count).toBe(1);
|
||||
});
|
||||
|
||||
test('multiple wrong-root pairs all counted', async () => {
|
||||
await addPage('people/alice-example', { hash: 'h1' });
|
||||
await addPage('alice-example', { hash: 'h1' });
|
||||
await addPage('projects/my-project', { hash: 'h2' });
|
||||
await addPage('my-project', { hash: 'h2' });
|
||||
const c = await checkContentHashDuplicates(engine);
|
||||
expect(c.status).toBe('warn');
|
||||
expect((c.details as any).pair_count).toBe(2);
|
||||
expect(c.message).toContain('my-project <-> projects/my-project');
|
||||
});
|
||||
|
||||
test('two path-prefixed pages with same hash → ok (not the wrong-root pattern)', async () => {
|
||||
await addPage('people/alice-example', { hash: 'same' });
|
||||
await addPage('archive/people/alice-example', { hash: 'same' });
|
||||
const c = await checkContentHashDuplicates(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('soft-deleted twin is ignored', async () => {
|
||||
await addPage('people/alice-example', { hash: 'same' });
|
||||
await addPage('alice-example', { hash: 'same', deleted: true });
|
||||
const c = await checkContentHashDuplicates(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('NULL / empty content_hash never groups', async () => {
|
||||
await addPage('people/alice-example', { hash: null });
|
||||
await addPage('alice-example', { hash: null });
|
||||
await addPage('people/bob-example', { hash: '' });
|
||||
await addPage('bob-example', { hash: '' });
|
||||
const c = await checkContentHashDuplicates(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('same hash across DIFFERENT sources is not flagged (per-source grouping)', async () => {
|
||||
await addSource('other', null);
|
||||
await addPage('people/alice-example', { hash: 'same', sourceId: 'default' });
|
||||
await addPage('alice-example', { hash: 'same', sourceId: 'other' });
|
||||
const c = await checkContentHashDuplicates(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
});
|
||||
});
|
||||
|
||||
describe('undeclared_db_only_pages (#2784)', () => {
|
||||
test('no sources with local_path → ok (not applicable)', async () => {
|
||||
await addPage('floating/page');
|
||||
const c = await checkUndeclaredDbOnlyPages(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
expect(c.message).toContain('Not applicable');
|
||||
});
|
||||
|
||||
test('file-backed page → ok', async () => {
|
||||
const repo = makeRepo();
|
||||
mkdirSync(join(repo, 'people'), { recursive: true });
|
||||
writeFileSync(join(repo, 'people', 'alice-example.md'), '# Alice');
|
||||
await addSource('src-a', repo);
|
||||
await addPage('people/alice-example', { sourceId: 'src-a' });
|
||||
const c = await checkUndeclaredDbOnlyPages(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('derive-phase default prefixes are implicitly declared', async () => {
|
||||
const repo = makeRepo();
|
||||
await addSource('src-a', repo);
|
||||
for (const prefix of DERIVE_PHASE_DB_ONLY_DEFAULTS) {
|
||||
await addPage(`${prefix}page-1`, { sourceId: 'src-a' });
|
||||
}
|
||||
const c = await checkUndeclaredDbOnlyPages(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
expect(c.message).toContain('life/events/');
|
||||
});
|
||||
|
||||
test('declared db_only prefix in gbrain.yml keeps the check quiet', async () => {
|
||||
const repo = makeRepo();
|
||||
writeFileSync(join(repo, 'gbrain.yml'), 'storage:\n db_only:\n - notes/\n');
|
||||
await addSource('src-a', repo);
|
||||
await addPage('notes/db-resident', { sourceId: 'src-a' });
|
||||
const c = await checkUndeclaredDbOnlyPages(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('page with no backing file outside every db_only path → warn with sample + fix', async () => {
|
||||
const repo = makeRepo();
|
||||
await addSource('src-a', repo);
|
||||
await addPage('people/ghost-page', { sourceId: 'src-a' });
|
||||
const c = await checkUndeclaredDbOnlyPages(engine);
|
||||
expect(c.status).toBe('warn');
|
||||
expect(c.message).toContain('people/ghost-page');
|
||||
expect(c.message).toContain('storage.db_only');
|
||||
expect((c.details as any).total).toBe(1);
|
||||
expect((c.details as any).per_source['src-a']).toBe(1);
|
||||
});
|
||||
|
||||
test('code pages are excluded (different slug scheme)', async () => {
|
||||
const repo = makeRepo();
|
||||
await addSource('src-a', repo);
|
||||
await addPage('src-core-thing-ts', { sourceId: 'src-a', pageKind: 'code' });
|
||||
const c = await checkUndeclaredDbOnlyPages(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('source whose local_path is missing on this host is skipped', async () => {
|
||||
await addSource('src-gone', '/nonexistent/gbrain-test-path');
|
||||
await addPage('people/ghost-page', { sourceId: 'src-gone' });
|
||||
const c = await checkUndeclaredDbOnlyPages(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
expect(c.message).toContain('Not applicable');
|
||||
});
|
||||
|
||||
test('effectiveDbOnlyDirs unions declared + defaults, deduped', () => {
|
||||
const dirs = effectiveDbOnlyDirs(['notes/', 'atoms/']);
|
||||
expect(dirs.filter(d => d === 'atoms/').length).toBe(1);
|
||||
expect(dirs).toContain('notes/');
|
||||
for (const d of DERIVE_PHASE_DB_ONLY_DEFAULTS) expect(dirs).toContain(d);
|
||||
});
|
||||
});
|
||||
|
||||
describe('db_only_collector_collision (#2788)', () => {
|
||||
test('no collectors declare output paths → ok', async () => {
|
||||
const c = await checkDbOnlyCollectorCollision(engine, { collectors: [] });
|
||||
expect(c.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('collector output inside a db_only path → warn naming collector, path, and fix', async () => {
|
||||
const repo = makeRepo();
|
||||
writeFileSync(join(repo, 'gbrain.yml'), 'storage:\n db_only:\n - daily/\n');
|
||||
await addSource('src-a', repo);
|
||||
const c = await checkDbOnlyCollectorCollision(engine, {
|
||||
collectors: [{ id: 'calendar-to-brain', output_path: 'daily/calendar/' }],
|
||||
});
|
||||
expect(c.status).toBe('warn');
|
||||
expect(c.message).toContain("collector 'calendar-to-brain'");
|
||||
expect(c.message).toContain("'daily/calendar/'");
|
||||
expect(c.message).toContain("db_only path 'daily/'");
|
||||
expect(c.message).toContain('silently skip');
|
||||
expect(c.message).toContain('storage.db_only');
|
||||
});
|
||||
|
||||
test('exact-match db_only dir also collides', async () => {
|
||||
const repo = makeRepo();
|
||||
writeFileSync(join(repo, 'gbrain.yml'), 'storage:\n db_only:\n - daily/calendar/\n');
|
||||
await addSource('src-a', repo);
|
||||
const c = await checkDbOnlyCollectorCollision(engine, {
|
||||
collectors: [{ id: 'calendar-to-brain', output_path: 'daily/calendar/' }],
|
||||
});
|
||||
expect(c.status).toBe('warn');
|
||||
});
|
||||
|
||||
test('db_only elsewhere → ok', async () => {
|
||||
const repo = makeRepo();
|
||||
writeFileSync(join(repo, 'gbrain.yml'), 'storage:\n db_only:\n - media/x/\n');
|
||||
await addSource('src-a', repo);
|
||||
const c = await checkDbOnlyCollectorCollision(engine, {
|
||||
collectors: [{ id: 'calendar-to-brain', output_path: 'daily/calendar/' }],
|
||||
});
|
||||
expect(c.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('sibling prefix does NOT collide (daily/calendar-x vs daily/calendar/)', () => {
|
||||
const hits = findDbOnlyCollisions(
|
||||
[{ id: 'x', output_path: 'daily/calendar-extra/' }],
|
||||
['daily/calendar/'],
|
||||
);
|
||||
expect(hits.length).toBe(0);
|
||||
});
|
||||
|
||||
test('findDbOnlyCollisions tolerates missing trailing slashes', () => {
|
||||
const hits = findDbOnlyCollisions(
|
||||
[{ id: 'x', output_path: 'daily/calendar' }],
|
||||
['daily'],
|
||||
);
|
||||
expect(hits.length).toBe(1);
|
||||
expect(hits[0].db_only_dir).toBe('daily');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* E2E for the silent-failure doctor batch (#2250 / #2784 / #2788).
|
||||
*
|
||||
* Part 1 (always runs, PGLite): constructs the REAL #2250 failure condition —
|
||||
* the same files imported through the actual import path twice, once with
|
||||
* relative paths computed from the correct brain root and once from a root
|
||||
* one level too deep (which drops the path prefix from every slug) — then
|
||||
* asserts `content_hash_duplicates` fires with the remediation text.
|
||||
*
|
||||
* Part 2 (gated by DATABASE_URL): engine parity. Identical seeds on PGLite
|
||||
* and real Postgres, identical check results — pins the GROUP BY / FILTER /
|
||||
* string_agg SQL shape on both engines.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, relative } from 'node:path';
|
||||
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
|
||||
import type { BrainEngine } from '../../src/core/engine.ts';
|
||||
import { importFromFile } from '../../src/core/import-file.ts';
|
||||
import {
|
||||
checkContentHashDuplicates,
|
||||
checkUndeclaredDbOnlyPages,
|
||||
checkDbOnlyCollectorCollision,
|
||||
} from '../../src/commands/doctor.ts';
|
||||
import { hasDatabase, setupDB, teardownDB, getEngine } from './helpers.ts';
|
||||
|
||||
const SKIP_PG = !hasDatabase();
|
||||
const describePg = SKIP_PG ? describe.skip : describe;
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
function makeDir(prefix: string): string {
|
||||
const d = mkdtempSync(join(tmpdir(), prefix));
|
||||
tempDirs.push(d);
|
||||
return d;
|
||||
}
|
||||
|
||||
afterAll(() => {
|
||||
for (const d of tempDirs) rmSync(d, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('wrong-root import produces content_hash_duplicates (#2250, PGLite)', () => {
|
||||
let engine: PGLiteEngine;
|
||||
let brainRoot: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
|
||||
// A brain with path-prefixed content dirs.
|
||||
brainRoot = makeDir('gbrain-wrongroot-');
|
||||
mkdirSync(join(brainRoot, 'people'), { recursive: true });
|
||||
mkdirSync(join(brainRoot, 'projects'), { recursive: true });
|
||||
// Explicit frontmatter (like real brain files) so the path-based
|
||||
// frontmatter inference doesn't run — the two import roots must produce
|
||||
// byte-identical content, hence identical content hashes.
|
||||
writeFileSync(
|
||||
join(brainRoot, 'people', 'alice-example.md'),
|
||||
'---\ntype: person\ndate: 2026-01-01\n---\n# Alice Example\n\nA founder the brain tracks across meetings and deals.\n',
|
||||
);
|
||||
writeFileSync(
|
||||
join(brainRoot, 'projects', 'widget-co.md'),
|
||||
'---\ntype: project\ndate: 2026-01-01\n---\n# Widget Co\n\nSeed-stage project notes with enough body to chunk.\n',
|
||||
);
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (engine) await engine.disconnect();
|
||||
}, 60_000);
|
||||
|
||||
test('correct-root import alone → check is ok', async () => {
|
||||
for (const rel of ['people/alice-example.md', 'projects/widget-co.md']) {
|
||||
const res = await importFromFile(engine, join(brainRoot, rel), rel, { noEmbed: true });
|
||||
expect(res.status).not.toBe('error');
|
||||
}
|
||||
const c = await checkContentHashDuplicates(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('re-import from a root one level too deep → warn with pairs + purge remediation', async () => {
|
||||
// The wrong-root mistake: import rooted inside people/ and projects/, so
|
||||
// the relative path (and therefore the slug) loses its directory prefix.
|
||||
for (const rel of ['people/alice-example.md', 'projects/widget-co.md']) {
|
||||
const abs = join(brainRoot, rel);
|
||||
const wrongRoot = join(brainRoot, rel.split('/')[0]); // one level too deep
|
||||
const wrongRel = relative(wrongRoot, abs); // "alice-example.md" — prefix dropped
|
||||
const res = await importFromFile(engine, abs, wrongRel, { noEmbed: true });
|
||||
expect(res.status).not.toBe('error');
|
||||
}
|
||||
|
||||
const c = await checkContentHashDuplicates(engine);
|
||||
expect(c.status).toBe('warn');
|
||||
expect(c.message).toContain('alice-example <-> people/alice-example');
|
||||
expect(c.message).toContain('widget-co <-> projects/widget-co');
|
||||
expect(c.message).toContain('gbrain pages delete <bare-slug>');
|
||||
expect(c.message).toContain('gbrain pages purge-deleted --older-than 0');
|
||||
expect((c.details as any).pair_count).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Shared seed + assertions for engine parity. Raw SQL only (both engines
|
||||
* accept the identical statements — that is the point).
|
||||
*/
|
||||
async function seedAndRunAllChecks(engine: BrainEngine, repo: string) {
|
||||
// Shared test DBs can carry leftover sources from other e2e files; blank
|
||||
// their local_path so only the parity source contributes to the checks.
|
||||
await engine.executeRaw(`UPDATE sources SET local_path = NULL WHERE id <> 'parity-src'`);
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, local_path, config) VALUES ('parity-src', 'parity-src', $1, '{}'::jsonb)
|
||||
ON CONFLICT (id) DO UPDATE SET local_path = EXCLUDED.local_path`,
|
||||
[repo],
|
||||
);
|
||||
const addPage = (slug: string, hash: string, sourceId = 'parity-src') =>
|
||||
engine.executeRaw(
|
||||
`INSERT INTO pages (slug, source_id, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash)
|
||||
VALUES ($1, $2, 'concept', 'markdown', $1, 'body', '', '{}'::jsonb, $3)`,
|
||||
[slug, sourceId, hash],
|
||||
);
|
||||
// #2250 shape: one bare/prefixed twin pair + one innocent page.
|
||||
await addPage('people/alice-example', 'dup-hash');
|
||||
await addPage('alice-example', 'dup-hash');
|
||||
await addPage('projects/clean-page', 'clean-hash');
|
||||
// #2784 shape: a ghost page with no backing file, plus a file-backed one
|
||||
// and a derive-phase default one.
|
||||
await addPage('people/ghost-page', 'ghost-hash');
|
||||
await addPage('life/events/derived-1', 'derived-hash');
|
||||
|
||||
const dup = await checkContentHashDuplicates(engine);
|
||||
const undeclared = await checkUndeclaredDbOnlyPages(engine);
|
||||
const collision = await checkDbOnlyCollectorCollision(engine, {
|
||||
collectors: [{ id: 'calendar-to-brain', output_path: 'daily/calendar/' }],
|
||||
});
|
||||
return { dup, undeclared, collision };
|
||||
}
|
||||
|
||||
describePg('engine parity: identical seeds, identical check results (PGLite vs Postgres)', () => {
|
||||
let pglite: PGLiteEngine;
|
||||
let repo: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
repo = makeDir('gbrain-parity-');
|
||||
mkdirSync(join(repo, 'people'), { recursive: true });
|
||||
writeFileSync(join(repo, 'people', 'alice-example.md'), '# Alice');
|
||||
// The bare-slug twin also gets a root-level file so only the deliberate
|
||||
// ghost page (people/ghost-page) counts as undeclared.
|
||||
writeFileSync(join(repo, 'alice-example.md'), '# Alice (bare twin)');
|
||||
mkdirSync(join(repo, 'projects'), { recursive: true });
|
||||
writeFileSync(join(repo, 'projects', 'clean-page.md'), '# Clean');
|
||||
writeFileSync(join(repo, 'gbrain.yml'), 'storage:\n db_only:\n - daily/\n');
|
||||
|
||||
pglite = new PGLiteEngine();
|
||||
await pglite.connect({});
|
||||
await pglite.initSchema();
|
||||
await setupDB();
|
||||
}, 180_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (pglite) await pglite.disconnect();
|
||||
await teardownDB();
|
||||
}, 60_000);
|
||||
|
||||
test('negative: clean engines → content_hash_duplicates ok on both', async () => {
|
||||
for (const engine of [pglite as BrainEngine, getEngine() as BrainEngine]) {
|
||||
const c = await checkContentHashDuplicates(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
test('all three checks agree across engines', async () => {
|
||||
const a = await seedAndRunAllChecks(pglite, repo);
|
||||
const b = await seedAndRunAllChecks(getEngine(), repo);
|
||||
|
||||
for (const r of [a, b]) {
|
||||
expect(r.dup.status).toBe('warn');
|
||||
expect((r.dup.details as any).pair_count).toBe(1);
|
||||
expect(r.dup.message).toContain('alice-example <-> people/alice-example');
|
||||
|
||||
expect(r.undeclared.status).toBe('warn');
|
||||
expect((r.undeclared.details as any).total).toBe(1);
|
||||
expect(r.undeclared.message).toContain('people/ghost-page');
|
||||
|
||||
expect(r.collision.status).toBe('warn');
|
||||
expect(r.collision.message).toContain("db_only path 'daily/'");
|
||||
}
|
||||
|
||||
// Byte-identical verdicts across engines.
|
||||
expect(a.dup.message).toBe(b.dup.message);
|
||||
expect(a.undeclared.details).toEqual(b.undeclared.details);
|
||||
expect(a.collision.message).toBe(b.collision.message);
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -0,0 +1,227 @@
|
||||
/**
|
||||
* Tests for the heartbeat_max_age health-check type (#2787) and the
|
||||
* output_paths recipe frontmatter + configured-collector helper (#2788).
|
||||
* Heartbeat files live under a temp GBRAIN_HOME so nothing touches ~/.gbrain.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import {
|
||||
parseMaxAge,
|
||||
executeHealthCheck,
|
||||
parseRecipe,
|
||||
getConfiguredCollectorOutputs,
|
||||
} from '../src/commands/integrations.ts';
|
||||
|
||||
function tempHome(): string {
|
||||
return mkdtempSync(join(tmpdir(), 'gbrain-hb-'));
|
||||
}
|
||||
|
||||
function writeHeartbeat(home: string, id: string, entries: Array<{ ts: string; event: string; status: string }>): void {
|
||||
const dir = join(home, '.gbrain', 'integrations', id);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(join(dir, 'heartbeat.jsonl'), entries.map(e => JSON.stringify(e)).join('\n') + '\n');
|
||||
}
|
||||
|
||||
describe('parseMaxAge', () => {
|
||||
test('parses h/d/m/s durations', () => {
|
||||
expect(parseMaxAge('48h')).toBe(48 * 3_600_000);
|
||||
expect(parseMaxAge('2d')).toBe(2 * 86_400_000);
|
||||
expect(parseMaxAge('90m')).toBe(90 * 60_000);
|
||||
expect(parseMaxAge('30s')).toBe(30_000);
|
||||
expect(parseMaxAge(' 48H ')).toBe(48 * 3_600_000);
|
||||
});
|
||||
|
||||
test('rejects garbage', () => {
|
||||
expect(parseMaxAge('abc')).toBeNull();
|
||||
expect(parseMaxAge('48')).toBeNull();
|
||||
expect(parseMaxAge('-3h')).toBeNull();
|
||||
expect(parseMaxAge('')).toBeNull();
|
||||
expect(parseMaxAge('0h')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('heartbeat_max_age health check (#2787)', () => {
|
||||
test('fresh heartbeat within max_age → ok', async () => {
|
||||
const home = tempHome();
|
||||
try {
|
||||
writeHeartbeat(home, 'calendar-to-brain', [
|
||||
{ ts: new Date(Date.now() - 3_600_000).toISOString(), event: 'sync', status: 'ok' },
|
||||
]);
|
||||
await withEnv({ GBRAIN_HOME: home }, async () => {
|
||||
const r = await executeHealthCheck(
|
||||
{ type: 'heartbeat_max_age', max_age: '48h', label: 'freshness' } as any,
|
||||
'calendar-to-brain',
|
||||
true,
|
||||
);
|
||||
expect(r.status).toBe('ok');
|
||||
expect(r.output).toContain('within 48h');
|
||||
});
|
||||
} finally {
|
||||
rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('16-day-stale sense FAILS (the #2787 silent-death receipt)', async () => {
|
||||
const home = tempHome();
|
||||
try {
|
||||
writeHeartbeat(home, 'calendar-to-brain', [
|
||||
{ ts: new Date(Date.now() - 16 * 86_400_000).toISOString(), event: 'sync', status: 'ok' },
|
||||
]);
|
||||
await withEnv({ GBRAIN_HOME: home }, async () => {
|
||||
const r = await executeHealthCheck(
|
||||
{ type: 'heartbeat_max_age', max_age: '48h' } as any,
|
||||
'calendar-to-brain',
|
||||
true,
|
||||
);
|
||||
expect(r.status).toBe('fail');
|
||||
expect(r.output).toContain('exceeds max_age 48h');
|
||||
expect(r.output).toContain('stopped producing data');
|
||||
});
|
||||
} finally {
|
||||
rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('no heartbeat data at all → fail', async () => {
|
||||
const home = tempHome();
|
||||
try {
|
||||
await withEnv({ GBRAIN_HOME: home }, async () => {
|
||||
const r = await executeHealthCheck(
|
||||
{ type: 'heartbeat_max_age', max_age: '48h' } as any,
|
||||
'never-ran',
|
||||
true,
|
||||
);
|
||||
expect(r.status).toBe('fail');
|
||||
expect(r.output).toContain('no heartbeat events');
|
||||
});
|
||||
} finally {
|
||||
rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('invalid max_age → fail with guidance, not a crash', async () => {
|
||||
const r = await executeHealthCheck(
|
||||
{ type: 'heartbeat_max_age', max_age: 'soon' } as any,
|
||||
'whatever',
|
||||
true,
|
||||
);
|
||||
expect(r.status).toBe('fail');
|
||||
expect(r.output).toContain("invalid max_age 'soon'");
|
||||
});
|
||||
|
||||
test('not gated on embedded trust (read-only local file)', async () => {
|
||||
const home = tempHome();
|
||||
try {
|
||||
writeHeartbeat(home, 'user-recipe', [
|
||||
{ ts: new Date().toISOString(), event: 'sync', status: 'ok' },
|
||||
]);
|
||||
await withEnv({ GBRAIN_HOME: home }, async () => {
|
||||
const r = await executeHealthCheck(
|
||||
{ type: 'heartbeat_max_age', max_age: '1d' } as any,
|
||||
'user-recipe',
|
||||
false, // NOT embedded
|
||||
);
|
||||
expect(r.status).toBe('ok');
|
||||
});
|
||||
} finally {
|
||||
rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('newest entry wins even when the file is not time-ordered', async () => {
|
||||
const home = tempHome();
|
||||
try {
|
||||
writeHeartbeat(home, 'unordered', [
|
||||
{ ts: new Date(Date.now() - 60_000).toISOString(), event: 'sync', status: 'ok' },
|
||||
{ ts: new Date(Date.now() - 20 * 86_400_000).toISOString(), event: 'sync', status: 'ok' },
|
||||
]);
|
||||
await withEnv({ GBRAIN_HOME: home }, async () => {
|
||||
const r = await executeHealthCheck(
|
||||
{ type: 'heartbeat_max_age', max_age: '48h' } as any,
|
||||
'unordered',
|
||||
true,
|
||||
);
|
||||
expect(r.status).toBe('ok');
|
||||
});
|
||||
} finally {
|
||||
rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('works inside any_of', async () => {
|
||||
const home = tempHome();
|
||||
try {
|
||||
writeHeartbeat(home, 'combo', [
|
||||
{ ts: new Date().toISOString(), event: 'sync', status: 'ok' },
|
||||
]);
|
||||
await withEnv({ GBRAIN_HOME: home }, async () => {
|
||||
const r = await executeHealthCheck(
|
||||
{ type: 'any_of', checks: [{ type: 'heartbeat_max_age', max_age: '1h' }] } as any,
|
||||
'combo',
|
||||
true,
|
||||
);
|
||||
expect(r.status).toBe('ok');
|
||||
});
|
||||
} finally {
|
||||
rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('output_paths frontmatter + configured-collector outputs (#2788)', () => {
|
||||
const RECIPE = `---
|
||||
id: test-collector
|
||||
name: Test Collector
|
||||
version: 0.1.0
|
||||
description: writes files
|
||||
category: sense
|
||||
health_checks: []
|
||||
output_paths:
|
||||
- daily/test-collector/
|
||||
setup_time: 1 min
|
||||
---
|
||||
Body.
|
||||
`;
|
||||
|
||||
test('parseRecipe surfaces output_paths (and defaults to [])', () => {
|
||||
const parsed = parseRecipe(RECIPE, 'test-collector.md');
|
||||
expect(parsed).not.toBeNull();
|
||||
expect(parsed!.frontmatter.output_paths).toEqual(['daily/test-collector/']);
|
||||
const bare = parseRecipe('---\nid: bare\n---\nBody.', 'bare.md');
|
||||
expect(bare!.frontmatter.output_paths).toEqual([]);
|
||||
});
|
||||
|
||||
test('the shipped calendar-to-brain recipe declares heartbeat_max_age + output_paths', () => {
|
||||
const content = require('node:fs').readFileSync(
|
||||
join(import.meta.dir, '..', 'recipes', 'calendar-to-brain.md'),
|
||||
'utf-8',
|
||||
);
|
||||
const parsed = parseRecipe(content, 'calendar-to-brain.md');
|
||||
expect(parsed).not.toBeNull();
|
||||
expect(parsed!.frontmatter.output_paths).toEqual(['daily/calendar/']);
|
||||
const hb = parsed!.frontmatter.health_checks.find(
|
||||
(c: any) => typeof c === 'object' && c.type === 'heartbeat_max_age',
|
||||
) as any;
|
||||
expect(hb).toBeDefined();
|
||||
expect(hb.max_age).toBe('48h');
|
||||
});
|
||||
|
||||
test('getConfiguredCollectorOutputs includes secretless recipes with output_paths', async () => {
|
||||
const home = tempHome();
|
||||
const recipesDir = mkdtempSync(join(tmpdir(), 'gbrain-recipes-'));
|
||||
try {
|
||||
writeFileSync(join(recipesDir, 'test-collector.md'), RECIPE);
|
||||
await withEnv({ GBRAIN_HOME: home, GBRAIN_RECIPES_DIR: recipesDir }, async () => {
|
||||
const outputs = getConfiguredCollectorOutputs();
|
||||
expect(outputs).toContainEqual({ id: 'test-collector', output_path: 'daily/test-collector/' });
|
||||
});
|
||||
} finally {
|
||||
rmSync(home, { recursive: true, force: true });
|
||||
rmSync(recipesDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -343,7 +343,7 @@ describe('all recipes', () => {
|
||||
expect(typeof check).toBe('string');
|
||||
} else {
|
||||
// Typed checks must have a valid type
|
||||
expect(['http', 'env_exists', 'command', 'any_of']).toContain((check as any).type);
|
||||
expect(['http', 'env_exists', 'command', 'any_of', 'heartbeat_max_age']).toContain((check as any).type);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,3 +188,53 @@ describe('manageGitignore', () => {
|
||||
expect(warnings.filter((w) => /submodule/.test(w))).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// #2788: collector-output vs db_only collision warning at .gitignore-write time.
|
||||
describe('manageGitignore collector/db_only collision warning (#2788)', () => {
|
||||
let recipesDir: string;
|
||||
const SECRET_ENV_KEYS = ['CLAWVISOR_URL', 'CLAWVISOR_AGENT_TOKEN', 'GOOGLE_CLIENT_ID', 'GOOGLE_CLIENT_SECRET'];
|
||||
let savedEnv: Record<string, string | undefined>;
|
||||
|
||||
beforeEach(() => {
|
||||
recipesDir = mkdtempSync(join(tmpdir(), 'gbrain-recipes-'));
|
||||
savedEnv = {};
|
||||
// Make embedded recipes (calendar-to-brain) deterministically unconfigured
|
||||
// and point recipe discovery at our temp dir.
|
||||
for (const k of [...SECRET_ENV_KEYS, 'GBRAIN_RECIPES_DIR', 'GBRAIN_HOME']) {
|
||||
savedEnv[k] = process.env[k];
|
||||
}
|
||||
for (const k of SECRET_ENV_KEYS) delete process.env[k];
|
||||
process.env.GBRAIN_RECIPES_DIR = recipesDir;
|
||||
process.env.GBRAIN_HOME = recipesDir; // heartbeat reads stay hermetic
|
||||
writeFileSync(
|
||||
join(recipesDir, 'test-collector.md'),
|
||||
'---\nid: test-collector\nname: Test Collector\noutput_paths:\n - media/x/inbox/\n---\nBody.\n',
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const [k, v] of Object.entries(savedEnv)) {
|
||||
if (v === undefined) delete process.env[k];
|
||||
else process.env[k] = v;
|
||||
}
|
||||
rmSync(recipesDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('warns when a configured collector output dir sits inside a db_only path', () => {
|
||||
writeStorageConfig(); // db_only includes media/x/
|
||||
manageGitignore(tmp);
|
||||
const hit = warnings.find((w) => /collector 'test-collector'/.test(w));
|
||||
expect(hit).toBeDefined();
|
||||
expect(hit).toContain("'media/x/inbox/'");
|
||||
expect(hit).toContain("db_only path 'media/x/'");
|
||||
expect(hit).toContain('silently skip');
|
||||
// .gitignore management still happens — the warning never blocks it.
|
||||
expect(existsSync(join(tmp, '.gitignore'))).toBe(true);
|
||||
});
|
||||
|
||||
test('no warning when the collector writes outside every db_only path', () => {
|
||||
writeFileSync(join(tmp, 'gbrain.yml'), 'storage:\n db_only:\n - archive/\n');
|
||||
manageGitignore(tmp);
|
||||
expect(warnings.filter((w) => /collector 'test-collector'/.test(w))).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user