diff --git a/docs/architecture/KEY_FILES.md b/docs/architecture/KEY_FILES.md index f8402bafc..6766a21f2 100644 --- a/docs/architecture/KEY_FILES.md +++ b/docs/architecture/KEY_FILES.md @@ -23,12 +23,13 @@ per-release `**vX.Y.Z:**` narration — CI enforces this - `src/core/context-engine.ts` + `src/openclaw-context-engine.ts` — the deterministic context engine OpenClaw loads on every turn (`assemble()` injects the Live Context block, zero-LLM). `createGBrainContextEngine({workspaceDir, resolveEntities?})` accepts an OPTIONAL host-injected resolver (`ENGINE_API_VERSION` 0.2.0, additive — older hosts work unchanged; the plugin entry maps `ctx.resolveEntities`/`ctx.brainQuery` onto it). `assemble()` runs the Retrieval Reflex after the Live Context block: extracts the current turn's user text, builds prior-context text (every message EXCEPT the current turn — suppression must not see the triggering mention), passes the rolling window (`getWindowTurns`, last 12 user/assistant turns; the reflex slices to its configured `retrieval_reflex_window_turns`), and appends the pointer block. `warmReflex()` fires at construction. - `src/core/context/` — Retrieval Reflex (Layer 1, issue #1981). `entity-salience.ts`: pure, zero-LLM, precision-biased `extractCandidates(text)` (capitalized runs + `@handles`, STOPWORDS + soft COMMON_WORDS + sentence-start guard, deterministic, capped) + `extractCandidatesFromWindow(turns)` (#2095: merges per-turn extraction across the last N turns by normalizeAlias form with occurrence/newest-turn/user-mention metadata; salience-ordered — recency > frequency > user-role — so the cap drops stale assistant chatter first). `retrieval-reflex.ts`: `resolveEntitiesToPointers(engine, sourceId, candidates, opts)` — alias arm (`resolveAliases`, caught per-arm for pre-v110 brains) + exact title/slug-suffix arm (the recall fix: real slugs are namespaced `people/x` but `slugify` drops the prefix); pointers carry `source_id`/`arm`/`confidence`/`matchedNorm` (#2095 — `ARM_CONFIDENCE` alias 0.9 / title 0.8 / slug-suffix 0.6 lives next to the arm definitions; arm-2 provenance classified in JS since the combined OR can't report which predicate matched); opts: `sourceIds?` federated scope (alias arm loops per source, arm 2 uses `source_id = ANY`), `suppression?` ('slug-and-title' legacy default; 'slug-only' REQUIRED under windowing — the title rule would suppress every entity merely mentioned in a prior window turn), ambient-channel event logging is DELIVERY-side, not in-resolver — `logDeliveredReflexPointers(engine, pointers)` fires only once a block is actually handed to the consumer (serve's resolve-IPC `onDelivered` hook post-write; `buildReflexAddition` post-timeout on the direct rung), so abandoned/timed-out blocks never pollute the volunteered-vs-used stats; synopsis runs through `stripTakesFence`/`stripFactsFence` (the same privacy boundary `get_page` applies) so private facts never reach the prompt; capped at `MAX_POINTERS`. `reflex.ts`: the orchestrator + engine-aware resolver ladder (host `resolveEntities` → PGLite serve IPC → Postgres cached process-singleton → disabled), zero-candidate fast path, fail-open + timeout, heartbeat write for the doctor check, `reflexEnabled(cfg)` (file/env gate, default ON; DB-plane does NOT gate — `assemble()` is sync); windowed extraction when `windowTurns` present and `retrieval_reflex_window_turns` (default 4; 1 = exact legacy behavior) > 1 — switches suppression to slug-only; accept-side reflex-channel logging fires after the per-turn timeout admits the block (direct-Postgres rung only — IPC logs server-side at delivery; host-injected resolvers are a documented gap). `resolve-ipc.ts`: local unix-socket resolve protocol (client + server) so PGLite resolves through the single connection `gbrain serve` holds (a second opener would hit the exclusive lock; a subprocess would force-steal it past the 5-min staleness window and crash). Wired into `src/mcp/server.ts` (serve binds `/.gbrain-resolve.sock` on PGLite, cleaned up on shutdown). Doctor surface: `retrieval_reflex_health` in `src/commands/doctor.ts` (reads the heartbeat for truthful runtime status; categorized in `doctor-categories.ts`). Config: `retrieval_reflex` + `retrieval_reflex_max_pointers` + `retrieval_reflex_window_turns` in `src/core/config.ts` (env `GBRAIN_RETRIEVAL_REFLEX_WINDOW_TURNS`). `volunteer.ts` (#2095): `parseWindow` (lenient `user:`/`assistant:` prefixes, unprefixed → one user turn), `volunteerContext` (extract → resolve → +0.05 multi-turn/newest-turn boost → `min_confidence` 0.7 gate → cap 3/5; deterministic rationale strings, never raw conversation text; slug-only suppression), `volunteerUsageStats` (per-arm/channel precision from the `pages.last_retrieved_at > volunteered_at` join — APPROXIMATE: the 5-min last-retrieved throttle causes false negatives, unrelated reads false positives). `volunteer-events.ts` (#2095): `insertVolunteerEvents` (ONE multi-row parameterized INSERT), `logVolunteerEventsFireAndForget` + bounded drain registered as the `volunteer-events` background-work sink (order 4), `purgeStaleVolunteerEvents` (90-day GC, called from the dream cycle's purge phase). Policy layer ships as the `retrieval-reflex` recipe (`recipes/retrieval-reflex/`). Pinned by `test/context/entity-salience.test.ts`, `test/retrieval-reflex.test.ts`, `test/context/resolve-ipc.test.ts`, `test/doctor-retrieval-reflex.test.ts`, `test/volunteer-context.test.ts`, `test/e2e/volunteer-context-postgres.test.ts`. - `src/commands/watch.ts` — `gbrain watch` (#2095): the push transport. Reads turns from stdin as they arrive (`user:`/`assistant:` prefixes; unprefixed = user turn), keeps a rolling in-process window (`--window-turns`, default 4), calls `volunteerContext` per turn, streams pointers to stdout (`--json` for JSONL with turn attribution), logs `channel: 'watch'` events with a per-session id. Session dedupe feeds already-pushed slugs back as priorContext so the core's slug-only suppression dedupes. Blocks in the stdin iteration (interactive alive until Ctrl-C/Ctrl-D; piped ends at EOF) — deliberately NOT in DAEMON_COMMANDS; SIGINT closes the stream so teardown flows through finishCliTeardown. Per-turn resolution failures are fail-open. Registered in CLI_ONLY + CLI_ONLY_SELF_HELP + THIN_CLIENT_REFUSED_COMMANDS (thin clients use the `volunteer_context` MCP op). Pinned by `test/watch-command.test.ts`. -- `src/commands/integrations.ts` — recipe install. The resolver-row install fence is keyed by `manifest.recipe` (`gbrain::resolver-rows`), so a second `copy-into-host-repo` recipe no longer writes a block mislabeled with the first recipe's name. Pinned by `test/integrations-install.test.ts`. +- `src/commands/integrations.ts` — recipe install. The resolver-row install fence is keyed by `manifest.recipe` (`gbrain::resolver-rows`), so a second `copy-into-host-repo` recipe no longer writes a block mislabeled with the first recipe's name. Pinned by `test/integrations-install.test.ts`. Health-check DSL includes the staleness-aware `heartbeat_max_age` type (#2787): declares the sense's expected cadence (`max_age: 48h`), and `integrations doctor` FAILS when the newest heartbeat event is older — the only check type that catches a green-but-dead sense (all others are point-in-time). Not embedded-gated (reads only the local heartbeat file). Recipe frontmatter carries `output_paths` (repo-relative dirs the collector writes, e.g. calendar-to-brain → `daily/calendar/`); `getConfiguredCollectorOutputs()` surfaces them for the #2788 db_only-collision check/warning. Pinned by `test/integrations-heartbeat-max-age.test.ts`. - `src/core/audit/audit-writer.ts` — shared JSONL audit primitive consolidating the hand-rolled audit modules. Exports `createAuditWriter({kind, recordSchema})` returning `{log, readRecent}` plus shared helpers `computeIsoWeekFilename(kind, now?)` and `resolveAuditDir()` (honors `GBRAIN_AUDIT_DIR`). ISO-week file rotation; best-effort writes (stderr warn on failure, never throws); read-path scans current-week + previous-week files for boundary spans. Refactored onto it for parity: `src/core/rerank-audit.ts`, `src/core/audit-slug-fallback.ts`, `src/core/minions/handlers/shell-audit.ts`, `src/core/minions/handlers/supervisor-audit.ts`, `src/core/facts/phantom-audit.ts` (each module's public API preserved bit-for-bit). The `graph-signals-failures` audit (`logGraphSignalsFailure`) uses the same primitive. One hand-rolled audit remains at `src/core/skillpack/audit.ts`. Pinned by `test/audit/audit-writer.test.ts`. - `src/core/cli-force-exit.ts` (#2084) — single owner of one-shot CLI exit + teardown, designed as a PAIR with the `import.meta.main` seam at the bottom of `src/cli.ts`. `finishCliTeardown({engine, drainTimeoutMs?})` is teardown-ONLY (never exits on the clean path): arms a REF'D backstop (unref'd would let a hung teardown exit naturally, skipping the flush and exiting with whatever PGLite scribbled into `process.exitCode`) whose deadline is COMPUTED from the bounds it guards (`computeTeardownDeadlineMs` = sinks × drainTimeoutMs + facts-abort grace + 2 × pool-end bound + slack, floor 10s; `GBRAIN_TEARDOWN_DEADLINE_MS` env override is the incident escape hatch), drains every background-work sink, disconnects the engine (a throw is warned + swallowed — the exit code reports the OPERATION, not the cleanup), then returns. The exit VERDICT lives in a gbrain-owned channel (`setCliExitVerdict`/`currentExitCode`; mirror-writes `process.exitCode` but NEVER reads it back) because PGLite's Emscripten runtime scribbles its own status into `process.exitCode` at arbitrary points mid-run — every writer that means to set the CLI exit code (op-dispatch catch, reindex, frontmatter, transcripts, brainstorm, autopilot, doctor's FAIL verdict, extract, and cli.ts's swept inner exits — friction, claw-test, smoke-test, the no-DB eval runners, status/status-thin, whoknows-thin) calls `setCliExitVerdict`; `test/cli-exit-verdict-pin.test.ts` greps src/ so the next raw `process.exitCode =` write fails CI instead of silently reporting success on failure. The deadline arms at TEARDOWN start, never before the op handler (the pre-#2084 placement measured handler + teardown combined, so PgBouncer deployments paid a flat 10s force-exit tax on every query and any >10s op was killed mid-run with exit 0). All nine cli.ts disconnect sites route through it; the ONE process exit happens in cli.ts's `main().then/catch` via `flushThenExit(currentExitCode())`, gated by `shouldForceExitAfterMain()` (daemon list: `serve`) — the CLI never waits for Bun's event loop to drain, because `endPoolBounded` deliberately races past stuck PgBouncer sockets that would keep it alive. `flushThenExit(code)` fences stdout+stderr (`write('', cb)` raced with an unref'd guard, EPIPE-safe both sync and async) then holds a REF'D aliveness grace for non-TTY stdio before `process.exit` — Bun delivers queued pipe writes only while the process is alive (no flush API reaches `process.stdout`'s native queue; write callbacks fire on accept, not delivery), so the grace IS the flush (#1959 truncation class). Scope claim is deliberately cli.ts-only: command modules' mid-run engine lifecycles stay local (process-exit semantics inside them would be wrong) and are absorbed by the final explicit exit. Pinned by `test/cli-finish-teardown.test.ts`, `test/flush-then-exit-harness.test.ts` (real spawned-Bun pipe semantics), `test/cli-should-force-exit.test.ts`, `test/cli-pipe-truncation.test.ts` (real-CLI piped --tools-json byte-stable), `test/cli-exit-verdict-pin.test.ts`, the `#2084` describes in `test/fix-wave-structural.test.ts` + `test/e2e/pglite-cli-exit.serial.test.ts`, and `test/e2e/pgbouncer-teardown.test.ts` (CI transaction-mode pooler — the #1972/#2015/#2084 class, finally reproducible in CI). - `src/core/cli-options.ts` extension — `CliOptions` gains `explain: boolean`. `parseGlobalFlags` recognizes `--explain` anywhere in argv (stripped before command dispatch). `src/cli.ts` `formatResult` for `search` + `query` cases routes to `formatResultsExplain` from `src/core/search/explain-formatter.ts` when `CliOptions.explain` is set; falls through to the existing JSON / human formatters otherwise. - `src/commands/search.ts:gbrain search stats` extension — `graph_signals` section (enabled/source/failures_count/failures_by_reason). JSON envelope adds a `graph_signals` sibling property; `_meta.metric_glossary` adds `graph_signals.enabled` + `graph_signals.failures_by_reason`. Human output prints the section after the existing block. Reads `search.graph_signals` config first, falls back to the mode default. Pinned by `test/search/search-stats-graph-signals.test.ts`. +- `src/commands/doctor.ts` extension — silent-failure batch (#2250/#2784/#2788): `content_hash_duplicates` (single GROUP BY over `(source_id, content_hash)` with FILTER aggregates — never N² — flagging hash groups that hold BOTH a bare and a path-prefixed slug, the wrong-import-root pattern; warn carries sample pairs + the `pages delete` → `purge-deleted --older-than 0` remediation); `undeclared_db_only_pages` (per source with a local repo: markdown pages with no backing file outside every declared + derive-phase-default db_only prefix — the one check deliberately allowed to stat the repo); `db_only_collector_collision` (configured recipe `output_paths` inside a declared db_only dir — auto-gitignore means sync AND import silently skip the collector's files; same warning fires in sync's `manageGitignore` at config-write time). All warn-level, engine-parity pinned by `test/e2e/doctor-silent-death-parity.test.ts`; units in `test/doctor-silent-death-checks.test.ts`. - `src/commands/doctor.ts` extension — `graph_signals_coverage` check wired into both `runDoctor` (local) and `doctorReportRemote` (HTTP/JSON thin-client path). Reads `search.graph_signals` config first, falls back to mode default; silent `ok` when disabled. Computes inbound link coverage on the page set; warns at <10% with `gbrain extract all` fix hint; `ok` at ≥30% ("fire on most queries") and 10-29% ("fire occasionally"), each with the percentage embedded. Pinned by cases in `test/doctor.test.ts`. - `src/core/engine-factory.ts` — Engine factory with dynamic imports (`'pglite'` | `'postgres'`). - `src/core/pglite-engine.ts` — PGLite (embedded Postgres 17.5 via WASM) implementation, all BrainEngine methods. `listLinkSources({sourceId?, sourceIds?})` returns distinct `link_source` provenances + counts (`ORDER BY count DESC, link_source ASC NULLS LAST`; scalar + federated scoped; parity with postgres-engine.ts) powering `gbrain link-sources`. `addLinksBatch`/`addTimelineEntriesBatch`/`addTakesBatch` pass the whole batch as one JSONB document via `jsonb_to_recordset(($1::jsonb)->'rows')` (bound through `executeRawJsonb` with a `{ rows }` wrapper; rows built by the shared `src/core/batch-rows.ts` helpers, NUL-stripped), and are `batchRetry`-wrapped. `connect()` wraps `PGlite.create()` in a try/catch that emits an actionable error (macOS 26.3 WASM bug #223, points at `gbrain doctor`); the lock is released on failure so the next process can retry cleanly. `searchKeyword`/`searchKeywordChunks` multiply `ts_rank` by the source-factor CASE at chunk grain; `searchVector` is a two-stage CTE — inner CTE keeps `ORDER BY cc.embedding <=> vec` so HNSW stays usable, outer SELECT re-ranks by `raw_score * source_factor`, inner LIMIT scales with offset to preserve pagination. `searchTakes`/`searchTakesVector` take full `SearchOpts` and apply the standard source-scope predicates (federated `sourceIds[]` wins over scalar `sourceId`, via the joined page's `source_id`) alongside the holder allow-list — parity SQL in postgres-engine.ts; pinned by `test/e2e/think-source-isolation-pglite.test.ts`. `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL — probes for forward-referenced state the embedded blob needs (`pages.source_id`, `links.link_source`, `links.origin_page_id`, `content_chunks.symbol_name`, `content_chunks.language`, `sources` FK target, plus `files.source_id`, `files.page_id`, `oauth_clients.source_id`, `oauth_clients.federated_read`, `sources.archived`, `sources.archived_at`, `sources.archive_expires_at`, `timeline_entries.event_page_id` — column-only, migration v121 stays the source of truth for its FK + indexes) and adds only what's missing; threads the DDL connection from `initSchema` so probes run inside the advisory-lock scope; no-op on fresh installs and modern brains (closes the upgrade-wedge bug class #239/#243/#266/#357/#366/#374/#375/#378/#395/#396/#1018/#974/#820). `getBrainScore` returns 100/100 with full breakdown (35/25/15/15/10) when `pageCount === 0` (vacuous truth — empty brain has no coverage problem); Pinned by `test/brain-score-breakdown.test.ts` empty-brain assertion + `test/doctor-report-remote.serial.test.ts`. `disconnect()` uses snapshot+early-null (snapshot `_db`/`_lock`, null instance fields BEFORE any `await` so a concurrent `connect()` can't see a partial mid-close state) wrapped in try/finally guaranteeing lock-release even if `db.close()` throws; KEEPS close-then-release order (release-then-close was rejected: it would widen the window where a sibling process connects to a still-closing brain); Pinned by `test/pglite-engine-disconnect.serial.test.ts`. `PGlite.create()` runs inside `preservingProcessExitCode` (#2084): PGLite's Emscripten runtime writes its own status into `process.exitCode` (99 at create; in-memory brains run initdb whose status lands on a later tick; the exit status at close — and assigning `undefined` cannot clear a prior value), which would otherwise silently clobber an errored op's exit 1 back to 0. The wrapper keeps the global tidy for external readers; `db.close()` stays unwrapped (its 0-write is baseline behavior test runners depend on). The CLI's exit verdict is immune either way — it lives in the gbrain-owned channel in `cli-force-exit.ts` and never reads `process.exitCode` back. Exports `classifyPgliteInitError(message): 'bunfs' | 'macos-26-3' | 'unknown'` + `buildPgliteInitErrorMessage(verdict, original)` routing the catch-block hint by failure shape (`bunfs` matches literal `$$bunfs` OR `ENOENT[\s\S]*pglite\.data` co-occurrence, surfaces a paste-ready `bun upgrade` + Node fallback; `macos-26-3` keeps the #223 link; `unknown` falls through); Pinned by `test/pglite-init-classifier.test.ts`. Implements `deletePages(slugs, {sourceId})` + `resolveSlugsByPaths(paths, {sourceId})` via `slug = ANY($1::text[])` array-param binding, caller-chunking primitive throwing when input exceeds `DELETE_BATCH_SIZE`, `deletePages` returns `RETURNING slug` rows so callers filter `pagesAffected` to confirmed deletes. Implements the embedding-signature stale-detection quartet — `sumStaleChunkChars({sourceId?, signature?})`, `setPageEmbeddingSignature(slug, {sourceId?, signature})`, `invalidateStaleSignatureEmbeddings({signature, sourceId?})`, widened `countStaleChunks({sourceId?, signature?})` (the `signature` opt widens via `JOIN pages p ... WHERE cc.embedding IS NULL OR (p.embedding_signature IS NOT NULL AND p.embedding_signature <> $signature)`, NULL grandfathered); parity SQL with postgres-engine.ts. PGLite-specific DDL (pgvector, pg_trgm, triggers). Engine-path helper dependencies (`retry`, ontology, recency decay) avoid dynamic `import()`; the only lazy dynamic imports are `ai/gateway.ts` in `initSchema` and `_upsertChunksOnce`, line-marked because their local catches preserve compiled-default and stored-config fallbacks that eager module evaluation would bypass. @@ -51,7 +52,7 @@ per-release `**vX.Y.Z:**` narration — CI enforces this - `src/core/sync.ts` — Pure sync functions (manifest parsing, filtering, slug conversion). Exported `pruneDir(name: string): boolean` is the single source of truth for descent-time directory exclusion across walkers — blocks `node_modules` (no leading dot, so naive walkers slipped through and inflated MISSING_OPEN counts via vendor packages), `vendor`/`dist`/`build`/`venv`, dot-prefix dirs, and `*.raw` sidecars — NOT `ops/`, which is ordinary user content (#2404; the bundled daily-task-manager stores `ops/tasks` there); `isSyncable` applies it per path segment, and `walkMarkdownFiles` in `src/commands/extract.ts` + `listTextFiles` in `src/core/cycle/transcript-discovery.ts` consult it BEFORE recursing to save the IO of walking thousands of vendor files (closes #923 + #202). `manageGitignore` worktree discriminator matches the gitdir path segment (`/modules/` = submodule, `/worktrees/` = worktree, per Git's documented layout) so Conductor worktrees (first-class repos) get `.gitignore` management for storage-tiering (closes #889). The sync-failure ledger (failure store, error classifier, the shared bookmark gate, and the doctor severity rule) lives in `src/core/sync-failure-ledger.ts`; `sync.ts` re-exports `classifyErrorCode`, `summarizeFailuresByCode`, `loadSyncFailures`, `unacknowledgedSyncFailures`, `acknowledgeSyncFailures`, `recordSyncFailures`, `decideSyncFailureSeverity`, `applySyncFailureGate`, and the `SyncFailure` type for backward-compatible imports — see its entry below. - `src/core/sync-failure-ledger.ts` — the bounded auto-skip sync failure ledger (issue #1939; formerly inline "Bug 9" in `sync.ts`). A LEAF module (imports only fs/path/crypto/config) so `sync.ts` can re-export it without a circular dependency. State lives in `~/.gbrain/sync-failures.jsonl`, one JSON object per line, keyed by `(source_id, path)` with a per-key `attempts` count and a 3-state machine: `open` (fresh/blocking) → `acknowledged` (human resolved via `gbrain sync --skip-failed`) or `auto_skipped` (chronic). `classifyErrorCode(errorMsg)` regex classifier with 12 codes (`SLUG_MISMATCH`, `YAML_PARSE`, `YAML_DUPLICATE_KEY`, `MISSING_OPEN`, `MISSING_CLOSE`, `NESTED_QUOTES`, `EMPTY_FRONTMATTER`, `NULL_BYTES`, `INVALID_UTF8`, `STATEMENT_TIMEOUT`, `FILE_TOO_LARGE`, `SYMLINK_NOT_ALLOWED`) plus `UNKNOWN` (also recognizes `PAGE_JUNK_PATTERN` from the content-sanity gate); `summarizeFailuresByCode(failures)` returns sorted `[{code, count}]`; `MISSING_OPEN`/`MISSING_CLOSE`/`EMPTY_FRONTMATTER` regexes match the `markdown.ts` validator strings, `FILE_TOO_LARGE` covers `import-file.ts:199, 352, 401`, `SYMLINK_NOT_ALLOWED` covers `:347`. All mutations run under `withLedgerLock` (cross-process file lock) with an atomic rename write. The auto-skip threshold resolves via `resolveAutoSkipThreshold()` from `GBRAIN_SYNC_AUTOSKIP_AFTER` (default `DEFAULT_AUTOSKIP_AFTER = 3`; `0` disables the valve = pure fail-closed). Two pure decision functions are the unit-test surface: `decideGateAction({fileFailures, sentinels, attemptsByPath, threshold, skipFailed})` returns `hard_block | block | advance | advance_then_autoskip` (sentinels like `` ALWAYS hard-block, even with `--skip-failed`, so a history rewrite can't auto-skip; any FRESH failure with `attempts < threshold` blocks fail-closed; only when ALL failures are chronic does it `advance_then_autoskip`), and `decideSyncFailureSeverity({entries, nowMs, failHours})` returns the `sync_failures` doctor status (`ok` when zero unresolved; `fail` when ≥10 OPEN-blocking or the oldest OPEN failure has blocked the bookmark past `failHours`; otherwise `warn` — `auto_skipped`-only rows stay WARN-visible regardless of count because the bookmark already advanced). `applySyncFailureGate(input)` is the one orchestrator BOTH sync paths (incremental + full/`runImport`) call: it records/clears ledger rows, runs `decideGateAction`, then executes effects in the crash-safe order (advance the bookmark FIRST via the injected `advance()` callback, THEN auto-skip the chronic set) so a crash can never mark a file skipped while leaving sync wedged. `isSkippablePath` rejects `<…>` sentinels. Pinned by `test/sync-failure-ledger.serial.test.ts` + `test/sync-failures.test.ts`. - `src/core/storage.ts` — Pluggable storage interface (S3, Supabase Storage, local). -- `src/core/storage-config.ts` — Storage tiering: `loadStorageConfig` reads `gbrain.yml`, normalizes deprecated keys (`git_tracked`/`supabase_only`) to canonical (`db_tracked`/`db_only`) with once-per-process deprecation warning, and runs `normalizeAndValidateStorageConfig` (auto-fixes missing trailing `/`, throws `StorageConfigError` on tier overlap). Path-segment matcher: `media/x/` does NOT match `media/xerox/foo`. Uses a dedicated parser for the `gbrain.yml` shape rather than gray-matter (broken on delimiter-less YAML). +- `src/core/storage-config.ts` — Storage tiering: `loadStorageConfig` reads `gbrain.yml`, normalizes deprecated keys (`git_tracked`/`supabase_only`) to canonical (`db_tracked`/`db_only`) with once-per-process deprecation warning, and runs `normalizeAndValidateStorageConfig` (auto-fixes missing trailing `/`, throws `StorageConfigError` on tier overlap). Path-segment matcher: `media/x/` does NOT match `media/xerox/foo`. Uses a dedicated parser for the `gbrain.yml` shape rather than gray-matter (broken on delimiter-less YAML). Also carries `DERIVE_PHASE_DB_ONLY_DEFAULTS` (`life/events/`, `atoms/`, `extracts/`, `dream-cycle-summaries/`) + `effectiveDbOnlyDirs` — the engine's derive-phase output prefixes treated as implicitly-declared db_only by the `undeclared_db_only_pages` doctor check but deliberately NOT merged into `loadStorageConfig` (a global merge would auto-gitignore those dirs and silently kill ingestion for brains that file-back them, the #2788 class) — and `findDbOnlyCollisions` (pure collector-output vs db_only overlap detector shared by the `db_only_collector_collision` doctor check and sync's `manageGitignore` warning). Pinned by `test/storage-config.test.ts` + `test/doctor-silent-death-checks.test.ts`. - `src/core/disk-walk.ts` — `walkBrainRepo(repoPath)` returns `Map` from one recursive `readdirSync`. Skips dot-dirs, `node_modules`, non-`.md` files. Used by `gbrain storage status` to replace per-page `existsSync + statSync` (~400K syscalls on 200K-page brains → tens). - `src/core/git-head.ts` — local git HEAD freshness probe for `gbrain doctor`. `isSourceUnchangedSinceSync(localPath, lastCommit, opts?)` returns true iff `localPath` is a git repo whose current HEAD matches `lastCommit`; when `opts.requireCleanWorkingTree` is true also requires a clean working tree (mirrors `gbrain sync`'s force-walk gate at `sync.ts:1075` so doctor and sync agree on "is there work to do?"). `requireCleanWorkingTree` is `boolean | 'ignore-untracked'` — in `'ignore-untracked'` mode the clean probe runs `git status --porcelain --untracked-files=no` so a quiet repo with stray untracked dirs (`?? companies/`, `?? media/`) is still "unchanged" (sync's incremental path keys off the commit diff and never imports untracked files); `GitCleanProbe` gains an `ignoreUntracked?` second arg. Two probe seams (`_setGitHeadProbeForTests`, `_setGitCleanProbeForTests`) keep unit tests R2-compliant (no `mock.module`). Uses `execFileSync` with array args so shell metachars in `local_path` cannot escape to a shell (the regression test runs real `execFileSync` against `'/nonexistent/$(touch )/repo'` and asserts the sentinel is never created). Fail-open on every error (missing path, not a git repo, git not installed, timeout, NULL inputs, dirty-probe errored → false) preserving the caller's prior time-based behavior. The chunker-version-match check lives in the caller (doctor.ts) because it depends on engine state (`sources.chunker_version` vs `CHUNKER_VERSION` from `src/core/chunkers/code.ts`). Pinned by `test/core/git-head.test.ts` (incl. the shell-injection regression guard). - `src/core/source-health.ts` — per-source health metrics for `gbrain sources status` + doctor's `federation_health`. Commit-relative staleness: `newestCommitMs(localPath)` = HEAD committer time via `git log -1 --format=%ct` (fail-open null; NO working-tree mtime parsing — committed content only, robust against the porcelain-mtime bug farm); pure `lagFromContentMs(contentMs|null, lastSyncMs|null, nowMs)` = remote/column comparator (null lastSync → null; negative wall-clock → skew passthrough; `contentMs <= lastSync` → 0; else/null-content → wall-clock). `computeAllSourceMetrics(engine, sources, {probeContent?})`: LOCAL (`probeContent:true`, `gbrain sources status`) → `isSourceUnchangedSinceSync(..., {requireCleanWorkingTree:'ignore-untracked'}) ? 0 : wall-clock` (live commit-hash catches HEAD moving to an old-dated commit a timestamp compare would miss); REMOTE (default, `federation_health` on the HTTP MCP path) → `lagFromContentMs(row.newest_content_at, ...)`, NO git subprocess (trust boundary). `commitTimeMs(localPath, sha)` is the `newestCommitMs` sibling pinned to an arbitrary commit (committer time via `git show -s --format=%ct `, fail-open null, execFileSync array args) — the resumable sync stamps `newest_content_at` against its pinned target commit, not whatever HEAD raced to. Pinned by `test/source-health.test.ts`. diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 3d65cb311..31c85dcd5 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -69,6 +69,12 @@ health_checks: # typed DSL to verify the integration is working auth_user: "$TWILIO_ACCOUNT_SID" auth_token: "$TWILIO_AUTH_TOKEN" label: "Twilio account" + - type: heartbeat_max_age # staleness gate: FAILS `integrations doctor` + max_age: 48h # when the newest heartbeat event is older. + label: "Data freshness" # The other types are point-in-time and stay + # green even when a sense stops producing data. +output_paths: # repo-relative dirs the collector writes files to; + - daily/voice/ # lets doctor/sync warn if one lands in db_only setup_time: 30 min # estimated time to complete setup --- @@ -86,7 +92,8 @@ a source install, or the global install copy) are trusted. Recipes discovered at runtime from `$GBRAIN_RECIPES_DIR` or a cwd-local `./recipes/` are marked untrusted: they cannot run `command` health checks, cannot run `http` health checks (SSRF defense), and cannot use the deprecated string health_check form. Untrusted recipes -can still use `env_exists` and `any_of` compositions. To ship a recipe that runs +can still use `env_exists`, `heartbeat_max_age` (reads only the local heartbeat +file — no exec, no network), and `any_of` compositions. To ship a recipe that runs live checks, contribute it upstream so it becomes package-bundled. ## The Deterministic Collector Pattern diff --git a/docs/storage-tiering.md b/docs/storage-tiering.md index 0c853d3a3..e5da8be7d 100644 --- a/docs/storage-tiering.md +++ b/docs/storage-tiering.md @@ -51,6 +51,18 @@ When storage configuration is present, `gbrain sync` automatically manages `.git - Skipped when the repo is a git submodule (`.git` is a file, not a directory) — submodule .gitignore changes don't survive parent updates. A warning explains. - Skipped entirely when `GBRAIN_NO_GITIGNORE=1` is set (escape hatch for shared-repo setups where a maintainer wants gbrain to leave .gitignore alone). - Failures (write permission denied, etc.) are caught and logged, never crash sync. +- Warns when a configured collector's declared output dir (recipe `output_paths` + frontmatter) sits inside a `db_only` path: gitignored files never appear in the + git-walking sync diff, and `gbrain import` honors `.gitignore` too — the + collector would run green while nothing reaches the DB. The + `db_only_collector_collision` doctor check surfaces the same trap. + +Related doctor coverage: `undeclared_db_only_pages` warns about DB pages with no +backing file that sit outside every declared `db_only` path. The engine's own +derive-phase output prefixes (`life/events/`, `atoms/`, `extracts/`, +`dream-cycle-summaries/`) count as implicitly declared for that check, so healthy +brains stay quiet without adding them to `gbrain.yml`. They are NOT auto-added to +`.gitignore` — only explicitly declared `db_only` dirs are. Example `.gitignore` addition: diff --git a/recipes/calendar-to-brain.md b/recipes/calendar-to-brain.md index afcd87116..558fd27e3 100644 --- a/recipes/calendar-to-brain.md +++ b/recipes/calendar-to-brain.md @@ -1,7 +1,7 @@ --- id: calendar-to-brain name: Calendar-to-Brain -version: 0.7.0 +version: 0.8.0 description: Google Calendar events become searchable brain pages. Daily files with attendees, locations, and meeting prep context. category: sense requires: [credential-gateway] @@ -28,6 +28,11 @@ health_checks: - type: env_exists name: GOOGLE_CLIENT_ID label: "Google OAuth" + - type: heartbeat_max_age + max_age: 48h + label: "Calendar data freshness" +output_paths: + - daily/calendar/ setup_time: 20 min cost_estimate: "$0 (both options are free)" --- diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index c10c7242d..b918ed8ef 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -52,6 +52,13 @@ import { lagFromContentMs } from '../core/source-health.ts'; import { CHUNKER_VERSION } from '../core/chunkers/code.ts'; import { LINK_EXTRACTOR_VERSION_TS } from '../core/link-extraction.ts'; import { isUndefinedColumnError } from '../core/utils.ts'; +import { + loadStorageConfig, + effectiveDbOnlyDirs, + DERIVE_PHASE_DB_ONLY_DEFAULTS, + findDbOnlyCollisions, +} from '../core/storage-config.ts'; +import { slugifyPath } from '../core/sync.ts'; // issue #1777: hidden_by_search_policy — count chunked pages withheld from // default search by the hard-exclude prefix policy. Reuses the canonical // exclude resolver + LIKE escaper + visibility clause so the doctor count can't @@ -3672,6 +3679,198 @@ export async function checkUnverifiedExtractions( } } +/** + * issue #2250 (reported by @615Works) — content_hash_duplicates. + * + * `gbrain import` run from the wrong root (one level too deep) drops the + * path prefix from every slug, leaving `people/x` and `x` coexisting with + * identical content. `dream --phase purge` never removes them (they aren't + * file-backed orphans) and nothing surfaced the condition. One GROUP BY — + * never an N² hash comparison — flags hash groups that contain BOTH a bare + * slug (no '/') and a path-prefixed slug. + */ +export async function checkContentHashDuplicates(engine: BrainEngine): Promise { + const name = 'content_hash_duplicates'; + const fix = 'Fix: gbrain pages delete for each pair, then gbrain pages purge-deleted --older-than 0'; + try { + const rows = await engine.executeRaw<{ source_id: string; content_hash: string; slugs: string }>( + `SELECT source_id, content_hash, + string_agg(slug, '|' ORDER BY length(slug), slug) AS slugs + FROM pages + WHERE deleted_at IS NULL AND content_hash IS NOT NULL AND content_hash <> '' + GROUP BY source_id, content_hash + HAVING count(*) > 1 + AND count(*) FILTER (WHERE strpos(slug, '/') = 0) > 0 + AND count(*) FILTER (WHERE strpos(slug, '/') > 0) > 0 + LIMIT 50`, + ); + if (rows.length === 0) { + return { name, status: 'ok', message: 'No content-hash duplicate pairs (bare vs path-prefixed slugs)' }; + } + let pairCount = 0; + const samples: string[] = []; + for (const r of rows) { + const slugs = String(r.slugs).split('|'); + const prefixed = slugs.filter(s => s.includes('/')); + for (const bare of slugs.filter(s => !s.includes('/'))) { + const twin = prefixed.find(p => p.endsWith('/' + bare)) ?? prefixed[0]; + pairCount++; + if (samples.length < 5) samples.push(`${bare} <-> ${twin}`); + } + } + return { + name, + status: 'warn', + message: `${pairCount} content-hash duplicate pair(s) detected (same content, differing slug forms — usually an import run from the wrong root, which drops the path prefix). Sample: ${samples.join('; ')}. ${fix}`, + details: { pair_count: pairCount, hash_groups: rows.length, sample_pairs: samples }, + }; + } catch (e) { + return { name, status: 'warn', message: `Could not check content-hash duplicates: ${(e as Error).message}` }; + } +} + +/** Walk a repo for markdown files and return their slugified (lowercased) slugs. */ +function collectMarkdownSlugs(root: string): Set { + const out = new Set(); + const stack = ['']; + while (stack.length > 0) { + const rel = stack.pop()!; + let entries; + try { + entries = readdirSync(rel ? join(root, rel) : root, { withFileTypes: true }); + } catch { + continue; + } + for (const e of entries) { + if (e.name.startsWith('.') || e.name === 'node_modules') continue; + const childRel = rel ? `${rel}/${e.name}` : e.name; + if (e.isDirectory()) stack.push(childRel); + else if (/\.mdx?$/i.test(e.name)) out.add(slugifyPath(childRel).toLowerCase()); + } + } + return out; +} + +/** + * issue #2784 (reported by @alexputici) — undeclared_db_only_pages. + * + * A markdown page with no backing file that sits outside every declared + * db_only path is invisible to any file-lane backup/recovery reasoning: an + * operator auditing "what would survive a DB loss" gets a silently wrong + * answer. The engine's own derive-phase output prefixes + * (DERIVE_PHASE_DB_ONLY_DEFAULTS) count as implicitly declared so the check + * stays quiet on healthy brains. Deliberately allowed to stat the source + * repo (the one thing the SQL-only check registry could never see). + */ +export async function checkUndeclaredDbOnlyPages(engine: BrainEngine): Promise { + const name = 'undeclared_db_only_pages'; + try { + const sources = await engine.executeRaw<{ id: string; local_path: string | null }>( + `SELECT id, local_path FROM sources WHERE local_path IS NOT NULL`, + ); + const checkable = sources.filter(s => s.local_path && existsSync(s.local_path)); + if (checkable.length === 0) { + return { name, status: 'ok', message: 'Not applicable (no sources with a local repo path on this host)' }; + } + let total = 0; + const samples: string[] = []; + const perSource: Record = {}; + for (const src of checkable) { + let declared: string[] = []; + try { + declared = loadStorageConfig(src.local_path)?.db_only ?? []; + } catch { + // invalid gbrain.yml — treated as no declarations; the sync path + // already surfaces the config error itself. + } + const dbOnlyDirs = effectiveDbOnlyDirs(declared); + const rows = await engine.executeRaw<{ slug: string }>( + `SELECT slug FROM pages WHERE deleted_at IS NULL AND source_id = $1 AND page_kind = 'markdown'`, + [src.id], + ); + if (rows.length === 0) continue; + const backed = collectMarkdownSlugs(src.local_path!); + for (const { slug } of rows) { + if (dbOnlyDirs.some(dir => slug.startsWith(dir))) continue; + if (backed.has(slug)) continue; + total++; + perSource[src.id] = (perSource[src.id] ?? 0) + 1; + if (samples.length < 5) samples.push(`${slug} (src=${src.id})`); + } + } + if (total === 0) { + return { + name, + status: 'ok', + message: `Every DB page is file-backed or under a declared/default db_only path (derive-phase defaults: ${DERIVE_PHASE_DB_ONLY_DEFAULTS.join(' ')})`, + }; + } + return { + name, + status: 'warn', + message: `${total} DB page(s) have no backing file and sit outside every declared/default db_only path — invisible to file-lane backup/recovery. Sample: ${samples.join('; ')}. Fix: restore or export the files, or declare their prefixes under storage.db_only in gbrain.yml (derive-phase defaults already cover: ${DERIVE_PHASE_DB_ONLY_DEFAULTS.join(' ')})`, + details: { total, per_source: perSource, sample_slugs: samples }, + }; + } catch (e) { + return { name, status: 'warn', message: `Could not check undeclared db-only pages: ${(e as Error).message}` }; + } +} + +/** + * issue #2788 (reported by @alexputici) — db_only_collector_collision. + * + * Declaring a collector's output dir in storage.db_only silently kills its + * ingestion: manageGitignore auto-gitignores the dir, the git-walking sync + * never sees the files, and import honors .gitignore too — everything stays + * green while nothing reaches the DB (a 7-week outage in the field). The + * recipe's `output_paths` frontmatter is the ground truth; the same warning + * also fires at .gitignore-write time inside sync's manageGitignore. + */ +export async function checkDbOnlyCollectorCollision( + engine: BrainEngine, + opts?: { collectors?: Array<{ id: string; output_path: string }> }, +): Promise { + const name = 'db_only_collector_collision'; + try { + let collectors = opts?.collectors; + if (!collectors) { + const { getConfiguredCollectorOutputs } = await import('./integrations.ts'); + collectors = getConfiguredCollectorOutputs(); + } + if (collectors.length === 0) { + return { name, status: 'ok', message: 'No configured collectors declare output paths' }; + } + const sources = await engine.executeRaw<{ id: string; local_path: string | null }>( + `SELECT id, local_path FROM sources WHERE local_path IS NOT NULL`, + ); + const hits: string[] = []; + for (const src of sources) { + if (!src.local_path || !existsSync(src.local_path)) continue; + let dbOnly: string[] = []; + try { + dbOnly = loadStorageConfig(src.local_path)?.db_only ?? []; + } catch { + continue; + } + if (dbOnly.length === 0) continue; + for (const hit of findDbOnlyCollisions(collectors, dbOnly)) { + hits.push(`collector '${hit.id}' writes to '${hit.output_path}' which is inside db_only path '${hit.db_only_dir}' (source ${src.id})`); + } + } + if (hits.length === 0) { + return { name, status: 'ok', message: 'No collector output dir falls inside a db_only path' }; + } + return { + name, + status: 'warn', + message: `${hits.length} collector/db_only collision(s): ${hits.join('; ')}. db_only dirs are auto-gitignored, so sync AND import silently skip files there — the collector runs green while nothing reaches the DB. Fix: remove the prefix from storage.db_only in gbrain.yml, or move the collector output.`, + details: { collisions: hits }, + }; + } catch (e) { + return { name, status: 'warn', message: `Could not check collector/db_only collisions: ${(e as Error).message}` }; + } +} + /** * issue #1678 — extract_atoms_backlog doctor check. * @@ -7696,6 +7895,14 @@ export async function buildChecks( // per-source dispatch gate sees. progress.heartbeat('cycle_freshness'); checks.push(await checkCycleFreshness(engine)); + // Silent-failure batch (#2250 / #2784 / #2788): wrong-root import + // duplicates, undeclared DB-only pages, collector-output-in-db_only. + progress.heartbeat('content_hash_duplicates'); + checks.push(await checkContentHashDuplicates(engine)); + progress.heartbeat('undeclared_db_only_pages'); + checks.push(await checkUndeclaredDbOnlyPages(engine)); + progress.heartbeat('db_only_collector_collision'); + checks.push(await checkDbOnlyCollectorCollision(engine)); } // v0.32.3 search-lite — mode + eval_drift surfaces. Status stays 'ok' per diff --git a/src/commands/integrations.ts b/src/commands/integrations.ts index bf68cf2a2..e6d576394 100644 --- a/src/commands/integrations.ts +++ b/src/commands/integrations.ts @@ -55,6 +55,13 @@ interface RecipeFrontmatter { health_checks: HealthCheck[]; setup_time: string; cost_estimate?: string; + /** + * Repo-relative dirs (slug prefixes, trailing '/') this recipe's collector + * writes files to. Ground truth for the `db_only_collector_collision` + * doctor check (issue #2788): output inside a db_only path is silently + * skipped by sync and import (auto-gitignored). + */ + output_paths: string[]; } interface ParsedRecipe { @@ -106,7 +113,20 @@ interface AnyOfCheck { checks: HealthCheck[]; } -type HealthCheck = string | HttpCheck | EnvExistsCheck | CommandCheck | AnyOfCheck; +/** + * Staleness-aware check type (issue #2787, reported by @alexputici). All + * other types are point-in-time — a sense whose gateway is up and env vars + * are set passes forever even when zero data flows. This one reads the + * integration's heartbeat file and FAILS when the newest event is older + * than the declared cadence (`max_age`, e.g. "48h", "2d", "90m"). + */ +interface HeartbeatMaxAgeCheck { + type: 'heartbeat_max_age'; + max_age: string; + label?: string; +} + +type HealthCheck = string | HttpCheck | EnvExistsCheck | CommandCheck | AnyOfCheck | HeartbeatMaxAgeCheck; interface CheckResult { integration: string; @@ -141,6 +161,26 @@ export function secretEnv(): Record { return process.env; } +/** + * Parse a heartbeat_max_age duration string ("30s", "90m", "48h", "2d") + * into milliseconds. Returns null on anything unparseable. + */ +export function parseMaxAge(s: string): number | null { + const m = /^(\d+(?:\.\d+)?)\s*(s|m|h|d)$/i.exec(String(s).trim()); + if (!m) return null; + const n = Number(m[1]); + if (!Number.isFinite(n) || n <= 0) return null; + const unit = { s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 }[m[2].toLowerCase() as 's' | 'm' | 'h' | 'd']; + return n * unit; +} + +/** Human-readable age for heartbeat_max_age output ("3d", "17h", "42m"). */ +function formatAge(ms: number): string { + if (ms >= 86_400_000) return `${Math.floor(ms / 86_400_000)}d`; + if (ms >= 3_600_000) return `${Math.floor(ms / 3_600_000)}h`; + return `${Math.max(0, Math.floor(ms / 60_000))}m`; +} + /** Expand $VAR references with gateway-env (config-folded) values */ export function expandVars(s: string): string { const env = secretEnv(); @@ -299,6 +339,29 @@ export async function executeHealthCheck( } } + case 'heartbeat_max_age': { + // No embedded gate: reads only the local heartbeat file — no exec, no + // network. Safe for user-provided recipes. + const maxMs = parseMaxAge(check.max_age); + if (maxMs === null) { + return { ...base, status: 'fail', output: `${check.label || 'heartbeat_max_age'}: invalid max_age '${check.max_age}' (use e.g. 90m, 48h, 2d)` }; + } + const entries = readHeartbeat(integrationId); + if (entries.length === 0) { + return { ...base, status: 'fail', output: `${check.label || 'heartbeat'}: no heartbeat events in the last 30 days (expected activity within ${check.max_age}) — the sense has stopped producing data` }; + } + let newest = 0; + for (const e of entries) { + const t = new Date(e.ts).getTime(); + if (Number.isFinite(t) && t > newest) newest = t; + } + const ageMs = Date.now() - newest; + if (ageMs > maxMs) { + return { ...base, status: 'fail', output: `${check.label || 'heartbeat'}: last event ${formatAge(ageMs)} ago exceeds max_age ${check.max_age} — the sense has stopped producing data` }; + } + return { ...base, status: 'ok', output: `${check.label || 'heartbeat'}: last event ${formatAge(ageMs)} ago (within ${check.max_age})` }; + } + case 'any_of': { for (const sub of check.checks) { const result = await executeHealthCheck(sub, integrationId, isEmbedded); @@ -340,6 +403,7 @@ export function parseRecipe(content: string, filename: string): ParsedRecipe | n health_checks: (data.health_checks || []) as HealthCheck[], setup_time: data.setup_time || 'unknown', cost_estimate: data.cost_estimate, + output_paths: Array.isArray(data.output_paths) ? data.output_paths.map(String) : [], }, body: body.trim(), filename, @@ -403,6 +467,25 @@ function loadAllRecipes(): ParsedRecipe[] { return recipes; } +/** + * Output paths of every CONFIGURED recipe (secrets present — the collector + * can actually be running). Ground truth for the + * `db_only_collector_collision` doctor check and the sync-time warning + * (issue #2788). Unconfigured recipes are skipped: a collector that can't + * run can't silently die. + */ +export function getConfiguredCollectorOutputs(): Array<{ id: string; output_path: string }> { + const out: Array<{ id: string; output_path: string }> = []; + for (const r of loadAllRecipes()) { + if (r.frontmatter.output_paths.length === 0) continue; + if (getStatus(r) === 'available') continue; + for (const p of r.frontmatter.output_paths) { + out.push({ id: r.frontmatter.id, output_path: p }); + } + } + return out; +} + function findRecipe(id: string): ParsedRecipe | null { const recipes = loadAllRecipes(); const exact = recipes.find(r => r.frontmatter.id === id); diff --git a/src/commands/sync.ts b/src/commands/sync.ts index e857ec96d..de214ce52 100644 --- a/src/commands/sync.ts +++ b/src/commands/sync.ts @@ -65,7 +65,11 @@ import { slog, serr, } from '../core/console-prefix.ts'; -import { loadStorageConfig } from '../core/storage-config.ts'; +import { loadStorageConfig, findDbOnlyCollisions } from '../core/storage-config.ts'; +// #2788: collector-output vs db_only collision warning at .gitignore-write +// time. integrations.ts is side-effect-free at module load (pure recipe I/O +// helpers), so a static import is safe here. +import { getConfiguredCollectorOutputs } from './integrations.ts'; import { getDefaultSourcePath } from '../core/source-resolver.ts'; // v0.41.32.0: stamp the durable newest-COMMIT timestamp at sync time so the // remote staleness path reads a column instead of shelling out to git. @@ -5637,6 +5641,24 @@ export function manageGitignore( return; } + // #2788: a configured collector whose output dir sits inside a db_only + // path dies silently — the dir is auto-gitignored below, the git-walking + // sync never sees its files, and `gbrain import` honors .gitignore too. + // Warn at the moment the config takes effect. Recipe scan failure never + // blocks the gitignore housekeeping. + try { + for (const c of findDbOnlyCollisions(getConfiguredCollectorOutputs(), storageConfig.db_only)) { + console.warn( + `WARNING: collector '${c.id}' writes to '${c.output_path}', which is inside db_only path ` + + `'${c.db_only_dir}'. db_only dirs are auto-gitignored, so gbrain sync and gbrain import ` + + `will silently skip its files. Remove the prefix from storage.db_only in gbrain.yml, or ` + + `move the collector output.`, + ); + } + } catch { + // recipes unavailable in this context — the doctor check still covers it + } + // D4 soft-warn: storage tiering has limited effect on PGLite, but the // .gitignore housekeeping still helps. Warn once per process; proceed. if (engineKind === 'pglite' && !_pgliteTierWarned) { diff --git a/src/core/doctor-categories.ts b/src/core/doctor-categories.ts index c101f6154..e041b1780 100644 --- a/src/core/doctor-categories.ts +++ b/src/core/doctor-categories.ts @@ -60,6 +60,7 @@ export const BRAIN_CHECK_NAMES: ReadonlySet = new Set([ 'calibration_freshness', 'child_table_orphans', 'chronicle_projection_health', + 'content_hash_duplicates', 'content_sanity_audit_recent', 'contextual_retrieval_coverage', 'contradictions', @@ -111,6 +112,7 @@ export const BRAIN_CHECK_NAMES: ReadonlySet = new Set([ 'takes_count', 'takes_weight_grid', 'timeline_coverage', + 'undeclared_db_only_pages', 'unified_multimodal_coverage', 'unverified_extractions', 'voice_gate_health', @@ -143,6 +145,7 @@ export const OPS_CHECK_NAMES: ReadonlySet = new Set([ 'batch_retry_health', 'brainstorm_health', 'connection', + 'db_only_collector_collision', 'federation_health', 'home_dir_in_worktree', 'index_audit', diff --git a/src/core/storage-config.ts b/src/core/storage-config.ts index 46c1835fd..164dfc8b2 100644 --- a/src/core/storage-config.ts +++ b/src/core/storage-config.ts @@ -356,6 +356,52 @@ export function isDbOnly(slug: string, config: StorageConfig): boolean { return config.db_only.some((dir) => matchesTierDir(slug, dir)); } +/** + * Derive-phase output prefixes the engine itself writes as DB-only machine + * output (issue #2784, reported by @alexputici). These are re-derivable by + * design and rarely file-backed, so the `undeclared_db_only_pages` doctor + * check treats them as implicitly declared db_only. They are deliberately + * NOT merged into `loadStorageConfig` — doing so would auto-gitignore these + * dirs via `manageGitignore` and silently kill ingestion for brains that DO + * file-back them (the exact #2788 silent-death class). + */ +export const DERIVE_PHASE_DB_ONLY_DEFAULTS: readonly string[] = [ + 'life/events/', + 'atoms/', + 'extracts/', + 'dream-cycle-summaries/', +]; + +/** Declared db_only dirs plus the derive-phase defaults, deduped. */ +export function effectiveDbOnlyDirs(declared: string[]): string[] { + return [...new Set([...declared, ...DERIVE_PHASE_DB_ONLY_DEFAULTS])]; +} + +/** + * Collector-output vs db_only collision detection (issue #2788, reported by + * @alexputici). A collector output path collides when it equals a db_only + * dir or sits anywhere inside one — such dirs are auto-gitignored by sync, + * so both the git-walking sync AND `gbrain import` (which honors .gitignore) + * silently skip every file the collector writes. + */ +export function findDbOnlyCollisions( + outputs: Array<{ id: string; output_path: string }>, + dbOnlyDirs: string[], +): Array<{ id: string; output_path: string; db_only_dir: string }> { + const hits: Array<{ id: string; output_path: string; db_only_dir: string }> = []; + for (const o of outputs) { + const out = o.output_path.endsWith('/') ? o.output_path : o.output_path + '/'; + for (const rawDir of dbOnlyDirs) { + const dir = rawDir.endsWith('/') ? rawDir : rawDir + '/'; + if (out.startsWith(dir)) { + hits.push({ id: o.id, output_path: o.output_path, db_only_dir: rawDir }); + break; + } + } + } + return hits; +} + export function getStorageTier(slug: string, config: StorageConfig): StorageTier { if (isDbTracked(slug, config)) return 'db_tracked'; if (isDbOnly(slug, config)) return 'db_only'; diff --git a/test/doctor-silent-death-checks.test.ts b/test/doctor-silent-death-checks.test.ts new file mode 100644 index 000000000..d26e5b140 --- /dev/null +++ b/test/doctor-silent-death-checks.test.ts @@ -0,0 +1,266 @@ +/** + * Unit tests for the silent-failure doctor check batch (#2250, #2784, #2788). + * Hermetic PGLite; temp dirs stand in for source repos. Postgres parity for + * the same checks is pinned by test/e2e/doctor-silent-death-parity.test.ts. + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { resetPgliteState } from './helpers/reset-pglite.ts'; +import { + checkContentHashDuplicates, + checkUndeclaredDbOnlyPages, + checkDbOnlyCollectorCollision, +} from '../src/commands/doctor.ts'; +import { + DERIVE_PHASE_DB_ONLY_DEFAULTS, + effectiveDbOnlyDirs, + findDbOnlyCollisions, +} from '../src/core/storage-config.ts'; + +let engine: PGLiteEngine; +const tempDirs: string[] = []; + +function makeRepo(): string { + const dir = mkdtempSync(join(tmpdir(), 'gbrain-doctor-')); + tempDirs.push(dir); + return dir; +} + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}, 60_000); + +afterAll(async () => { + if (engine) await engine.disconnect(); + for (const d of tempDirs) rmSync(d, { recursive: true, force: true }); +}, 60_000); + +beforeEach(async () => { + await resetPgliteState(engine); +}); + +async function addSource(id: string, localPath: string | null): Promise { + await engine.executeRaw( + `INSERT INTO sources (id, name, local_path, config) VALUES ($1, $1, $2, '{}'::jsonb) + ON CONFLICT (id) DO UPDATE SET local_path = EXCLUDED.local_path`, + [id, localPath], + ); +} + +async function addPage( + slug: string, + opts: { sourceId?: string; hash?: string | null; pageKind?: string; deleted?: boolean } = {}, +): Promise { + await engine.executeRaw( + `INSERT INTO pages (slug, source_id, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash, deleted_at) + VALUES ($1, $2, 'concept', $3, $1, 'body', '', '{}'::jsonb, $4, $5)`, + [ + slug, + opts.sourceId ?? 'default', + opts.pageKind ?? 'markdown', + opts.hash === undefined ? `h-${slug}` : opts.hash, + opts.deleted ? new Date().toISOString() : null, + ], + ); +} + +describe('content_hash_duplicates (#2250)', () => { + test('distinct hashes → ok', async () => { + await addPage('people/alice-example'); + await addPage('projects/widget-co'); + const c = await checkContentHashDuplicates(engine); + expect(c.status).toBe('ok'); + }); + + test('bare + path-prefixed twins with same hash → warn with pair + remediation', async () => { + await addPage('people/alice-example', { hash: 'same' }); + await addPage('alice-example', { hash: 'same' }); + const c = await checkContentHashDuplicates(engine); + expect(c.status).toBe('warn'); + expect(c.message).toContain('alice-example <-> people/alice-example'); + expect(c.message).toContain('gbrain pages delete '); + expect(c.message).toContain('gbrain pages purge-deleted --older-than 0'); + expect((c.details as any).pair_count).toBe(1); + }); + + test('multiple wrong-root pairs all counted', async () => { + await addPage('people/alice-example', { hash: 'h1' }); + await addPage('alice-example', { hash: 'h1' }); + await addPage('projects/my-project', { hash: 'h2' }); + await addPage('my-project', { hash: 'h2' }); + const c = await checkContentHashDuplicates(engine); + expect(c.status).toBe('warn'); + expect((c.details as any).pair_count).toBe(2); + expect(c.message).toContain('my-project <-> projects/my-project'); + }); + + test('two path-prefixed pages with same hash → ok (not the wrong-root pattern)', async () => { + await addPage('people/alice-example', { hash: 'same' }); + await addPage('archive/people/alice-example', { hash: 'same' }); + const c = await checkContentHashDuplicates(engine); + expect(c.status).toBe('ok'); + }); + + test('soft-deleted twin is ignored', async () => { + await addPage('people/alice-example', { hash: 'same' }); + await addPage('alice-example', { hash: 'same', deleted: true }); + const c = await checkContentHashDuplicates(engine); + expect(c.status).toBe('ok'); + }); + + test('NULL / empty content_hash never groups', async () => { + await addPage('people/alice-example', { hash: null }); + await addPage('alice-example', { hash: null }); + await addPage('people/bob-example', { hash: '' }); + await addPage('bob-example', { hash: '' }); + const c = await checkContentHashDuplicates(engine); + expect(c.status).toBe('ok'); + }); + + test('same hash across DIFFERENT sources is not flagged (per-source grouping)', async () => { + await addSource('other', null); + await addPage('people/alice-example', { hash: 'same', sourceId: 'default' }); + await addPage('alice-example', { hash: 'same', sourceId: 'other' }); + const c = await checkContentHashDuplicates(engine); + expect(c.status).toBe('ok'); + }); +}); + +describe('undeclared_db_only_pages (#2784)', () => { + test('no sources with local_path → ok (not applicable)', async () => { + await addPage('floating/page'); + const c = await checkUndeclaredDbOnlyPages(engine); + expect(c.status).toBe('ok'); + expect(c.message).toContain('Not applicable'); + }); + + test('file-backed page → ok', async () => { + const repo = makeRepo(); + mkdirSync(join(repo, 'people'), { recursive: true }); + writeFileSync(join(repo, 'people', 'alice-example.md'), '# Alice'); + await addSource('src-a', repo); + await addPage('people/alice-example', { sourceId: 'src-a' }); + const c = await checkUndeclaredDbOnlyPages(engine); + expect(c.status).toBe('ok'); + }); + + test('derive-phase default prefixes are implicitly declared', async () => { + const repo = makeRepo(); + await addSource('src-a', repo); + for (const prefix of DERIVE_PHASE_DB_ONLY_DEFAULTS) { + await addPage(`${prefix}page-1`, { sourceId: 'src-a' }); + } + const c = await checkUndeclaredDbOnlyPages(engine); + expect(c.status).toBe('ok'); + expect(c.message).toContain('life/events/'); + }); + + test('declared db_only prefix in gbrain.yml keeps the check quiet', async () => { + const repo = makeRepo(); + writeFileSync(join(repo, 'gbrain.yml'), 'storage:\n db_only:\n - notes/\n'); + await addSource('src-a', repo); + await addPage('notes/db-resident', { sourceId: 'src-a' }); + const c = await checkUndeclaredDbOnlyPages(engine); + expect(c.status).toBe('ok'); + }); + + test('page with no backing file outside every db_only path → warn with sample + fix', async () => { + const repo = makeRepo(); + await addSource('src-a', repo); + await addPage('people/ghost-page', { sourceId: 'src-a' }); + const c = await checkUndeclaredDbOnlyPages(engine); + expect(c.status).toBe('warn'); + expect(c.message).toContain('people/ghost-page'); + expect(c.message).toContain('storage.db_only'); + expect((c.details as any).total).toBe(1); + expect((c.details as any).per_source['src-a']).toBe(1); + }); + + test('code pages are excluded (different slug scheme)', async () => { + const repo = makeRepo(); + await addSource('src-a', repo); + await addPage('src-core-thing-ts', { sourceId: 'src-a', pageKind: 'code' }); + const c = await checkUndeclaredDbOnlyPages(engine); + expect(c.status).toBe('ok'); + }); + + test('source whose local_path is missing on this host is skipped', async () => { + await addSource('src-gone', '/nonexistent/gbrain-test-path'); + await addPage('people/ghost-page', { sourceId: 'src-gone' }); + const c = await checkUndeclaredDbOnlyPages(engine); + expect(c.status).toBe('ok'); + expect(c.message).toContain('Not applicable'); + }); + + test('effectiveDbOnlyDirs unions declared + defaults, deduped', () => { + const dirs = effectiveDbOnlyDirs(['notes/', 'atoms/']); + expect(dirs.filter(d => d === 'atoms/').length).toBe(1); + expect(dirs).toContain('notes/'); + for (const d of DERIVE_PHASE_DB_ONLY_DEFAULTS) expect(dirs).toContain(d); + }); +}); + +describe('db_only_collector_collision (#2788)', () => { + test('no collectors declare output paths → ok', async () => { + const c = await checkDbOnlyCollectorCollision(engine, { collectors: [] }); + expect(c.status).toBe('ok'); + }); + + test('collector output inside a db_only path → warn naming collector, path, and fix', async () => { + const repo = makeRepo(); + writeFileSync(join(repo, 'gbrain.yml'), 'storage:\n db_only:\n - daily/\n'); + await addSource('src-a', repo); + const c = await checkDbOnlyCollectorCollision(engine, { + collectors: [{ id: 'calendar-to-brain', output_path: 'daily/calendar/' }], + }); + expect(c.status).toBe('warn'); + expect(c.message).toContain("collector 'calendar-to-brain'"); + expect(c.message).toContain("'daily/calendar/'"); + expect(c.message).toContain("db_only path 'daily/'"); + expect(c.message).toContain('silently skip'); + expect(c.message).toContain('storage.db_only'); + }); + + test('exact-match db_only dir also collides', async () => { + const repo = makeRepo(); + writeFileSync(join(repo, 'gbrain.yml'), 'storage:\n db_only:\n - daily/calendar/\n'); + await addSource('src-a', repo); + const c = await checkDbOnlyCollectorCollision(engine, { + collectors: [{ id: 'calendar-to-brain', output_path: 'daily/calendar/' }], + }); + expect(c.status).toBe('warn'); + }); + + test('db_only elsewhere → ok', async () => { + const repo = makeRepo(); + writeFileSync(join(repo, 'gbrain.yml'), 'storage:\n db_only:\n - media/x/\n'); + await addSource('src-a', repo); + const c = await checkDbOnlyCollectorCollision(engine, { + collectors: [{ id: 'calendar-to-brain', output_path: 'daily/calendar/' }], + }); + expect(c.status).toBe('ok'); + }); + + test('sibling prefix does NOT collide (daily/calendar-x vs daily/calendar/)', () => { + const hits = findDbOnlyCollisions( + [{ id: 'x', output_path: 'daily/calendar-extra/' }], + ['daily/calendar/'], + ); + expect(hits.length).toBe(0); + }); + + test('findDbOnlyCollisions tolerates missing trailing slashes', () => { + const hits = findDbOnlyCollisions( + [{ id: 'x', output_path: 'daily/calendar' }], + ['daily'], + ); + expect(hits.length).toBe(1); + expect(hits[0].db_only_dir).toBe('daily'); + }); +}); diff --git a/test/e2e/doctor-silent-death-parity.test.ts b/test/e2e/doctor-silent-death-parity.test.ts new file mode 100644 index 000000000..7c4b1bd28 --- /dev/null +++ b/test/e2e/doctor-silent-death-parity.test.ts @@ -0,0 +1,194 @@ +/** + * E2E for the silent-failure doctor batch (#2250 / #2784 / #2788). + * + * Part 1 (always runs, PGLite): constructs the REAL #2250 failure condition — + * the same files imported through the actual import path twice, once with + * relative paths computed from the correct brain root and once from a root + * one level too deep (which drops the path prefix from every slug) — then + * asserts `content_hash_duplicates` fires with the remediation text. + * + * Part 2 (gated by DATABASE_URL): engine parity. Identical seeds on PGLite + * and real Postgres, identical check results — pins the GROUP BY / FILTER / + * string_agg SQL shape on both engines. + */ + +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, relative } from 'node:path'; +import { PGLiteEngine } from '../../src/core/pglite-engine.ts'; +import type { BrainEngine } from '../../src/core/engine.ts'; +import { importFromFile } from '../../src/core/import-file.ts'; +import { + checkContentHashDuplicates, + checkUndeclaredDbOnlyPages, + checkDbOnlyCollectorCollision, +} from '../../src/commands/doctor.ts'; +import { hasDatabase, setupDB, teardownDB, getEngine } from './helpers.ts'; + +const SKIP_PG = !hasDatabase(); +const describePg = SKIP_PG ? describe.skip : describe; + +const tempDirs: string[] = []; +function makeDir(prefix: string): string { + const d = mkdtempSync(join(tmpdir(), prefix)); + tempDirs.push(d); + return d; +} + +afterAll(() => { + for (const d of tempDirs) rmSync(d, { recursive: true, force: true }); +}); + +describe('wrong-root import produces content_hash_duplicates (#2250, PGLite)', () => { + let engine: PGLiteEngine; + let brainRoot: string; + + beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); + + // A brain with path-prefixed content dirs. + brainRoot = makeDir('gbrain-wrongroot-'); + mkdirSync(join(brainRoot, 'people'), { recursive: true }); + mkdirSync(join(brainRoot, 'projects'), { recursive: true }); + // Explicit frontmatter (like real brain files) so the path-based + // frontmatter inference doesn't run — the two import roots must produce + // byte-identical content, hence identical content hashes. + writeFileSync( + join(brainRoot, 'people', 'alice-example.md'), + '---\ntype: person\ndate: 2026-01-01\n---\n# Alice Example\n\nA founder the brain tracks across meetings and deals.\n', + ); + writeFileSync( + join(brainRoot, 'projects', 'widget-co.md'), + '---\ntype: project\ndate: 2026-01-01\n---\n# Widget Co\n\nSeed-stage project notes with enough body to chunk.\n', + ); + }, 120_000); + + afterAll(async () => { + if (engine) await engine.disconnect(); + }, 60_000); + + test('correct-root import alone → check is ok', async () => { + for (const rel of ['people/alice-example.md', 'projects/widget-co.md']) { + const res = await importFromFile(engine, join(brainRoot, rel), rel, { noEmbed: true }); + expect(res.status).not.toBe('error'); + } + const c = await checkContentHashDuplicates(engine); + expect(c.status).toBe('ok'); + }); + + test('re-import from a root one level too deep → warn with pairs + purge remediation', async () => { + // The wrong-root mistake: import rooted inside people/ and projects/, so + // the relative path (and therefore the slug) loses its directory prefix. + for (const rel of ['people/alice-example.md', 'projects/widget-co.md']) { + const abs = join(brainRoot, rel); + const wrongRoot = join(brainRoot, rel.split('/')[0]); // one level too deep + const wrongRel = relative(wrongRoot, abs); // "alice-example.md" — prefix dropped + const res = await importFromFile(engine, abs, wrongRel, { noEmbed: true }); + expect(res.status).not.toBe('error'); + } + + const c = await checkContentHashDuplicates(engine); + expect(c.status).toBe('warn'); + expect(c.message).toContain('alice-example <-> people/alice-example'); + expect(c.message).toContain('widget-co <-> projects/widget-co'); + expect(c.message).toContain('gbrain pages delete '); + expect(c.message).toContain('gbrain pages purge-deleted --older-than 0'); + expect((c.details as any).pair_count).toBe(2); + }); +}); + +/** + * Shared seed + assertions for engine parity. Raw SQL only (both engines + * accept the identical statements — that is the point). + */ +async function seedAndRunAllChecks(engine: BrainEngine, repo: string) { + // Shared test DBs can carry leftover sources from other e2e files; blank + // their local_path so only the parity source contributes to the checks. + await engine.executeRaw(`UPDATE sources SET local_path = NULL WHERE id <> 'parity-src'`); + await engine.executeRaw( + `INSERT INTO sources (id, name, local_path, config) VALUES ('parity-src', 'parity-src', $1, '{}'::jsonb) + ON CONFLICT (id) DO UPDATE SET local_path = EXCLUDED.local_path`, + [repo], + ); + const addPage = (slug: string, hash: string, sourceId = 'parity-src') => + engine.executeRaw( + `INSERT INTO pages (slug, source_id, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash) + VALUES ($1, $2, 'concept', 'markdown', $1, 'body', '', '{}'::jsonb, $3)`, + [slug, sourceId, hash], + ); + // #2250 shape: one bare/prefixed twin pair + one innocent page. + await addPage('people/alice-example', 'dup-hash'); + await addPage('alice-example', 'dup-hash'); + await addPage('projects/clean-page', 'clean-hash'); + // #2784 shape: a ghost page with no backing file, plus a file-backed one + // and a derive-phase default one. + await addPage('people/ghost-page', 'ghost-hash'); + await addPage('life/events/derived-1', 'derived-hash'); + + const dup = await checkContentHashDuplicates(engine); + const undeclared = await checkUndeclaredDbOnlyPages(engine); + const collision = await checkDbOnlyCollectorCollision(engine, { + collectors: [{ id: 'calendar-to-brain', output_path: 'daily/calendar/' }], + }); + return { dup, undeclared, collision }; +} + +describePg('engine parity: identical seeds, identical check results (PGLite vs Postgres)', () => { + let pglite: PGLiteEngine; + let repo: string; + + beforeAll(async () => { + repo = makeDir('gbrain-parity-'); + mkdirSync(join(repo, 'people'), { recursive: true }); + writeFileSync(join(repo, 'people', 'alice-example.md'), '# Alice'); + // The bare-slug twin also gets a root-level file so only the deliberate + // ghost page (people/ghost-page) counts as undeclared. + writeFileSync(join(repo, 'alice-example.md'), '# Alice (bare twin)'); + mkdirSync(join(repo, 'projects'), { recursive: true }); + writeFileSync(join(repo, 'projects', 'clean-page.md'), '# Clean'); + writeFileSync(join(repo, 'gbrain.yml'), 'storage:\n db_only:\n - daily/\n'); + + pglite = new PGLiteEngine(); + await pglite.connect({}); + await pglite.initSchema(); + await setupDB(); + }, 180_000); + + afterAll(async () => { + if (pglite) await pglite.disconnect(); + await teardownDB(); + }, 60_000); + + test('negative: clean engines → content_hash_duplicates ok on both', async () => { + for (const engine of [pglite as BrainEngine, getEngine() as BrainEngine]) { + const c = await checkContentHashDuplicates(engine); + expect(c.status).toBe('ok'); + } + }, 60_000); + + test('all three checks agree across engines', async () => { + const a = await seedAndRunAllChecks(pglite, repo); + const b = await seedAndRunAllChecks(getEngine(), repo); + + for (const r of [a, b]) { + expect(r.dup.status).toBe('warn'); + expect((r.dup.details as any).pair_count).toBe(1); + expect(r.dup.message).toContain('alice-example <-> people/alice-example'); + + expect(r.undeclared.status).toBe('warn'); + expect((r.undeclared.details as any).total).toBe(1); + expect(r.undeclared.message).toContain('people/ghost-page'); + + expect(r.collision.status).toBe('warn'); + expect(r.collision.message).toContain("db_only path 'daily/'"); + } + + // Byte-identical verdicts across engines. + expect(a.dup.message).toBe(b.dup.message); + expect(a.undeclared.details).toEqual(b.undeclared.details); + expect(a.collision.message).toBe(b.collision.message); + }, 120_000); +}); diff --git a/test/integrations-heartbeat-max-age.test.ts b/test/integrations-heartbeat-max-age.test.ts new file mode 100644 index 000000000..2c6c1f739 --- /dev/null +++ b/test/integrations-heartbeat-max-age.test.ts @@ -0,0 +1,227 @@ +/** + * Tests for the heartbeat_max_age health-check type (#2787) and the + * output_paths recipe frontmatter + configured-collector helper (#2788). + * Heartbeat files live under a temp GBRAIN_HOME so nothing touches ~/.gbrain. + */ + +import { describe, test, expect } from 'bun:test'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { withEnv } from './helpers/with-env.ts'; +import { + parseMaxAge, + executeHealthCheck, + parseRecipe, + getConfiguredCollectorOutputs, +} from '../src/commands/integrations.ts'; + +function tempHome(): string { + return mkdtempSync(join(tmpdir(), 'gbrain-hb-')); +} + +function writeHeartbeat(home: string, id: string, entries: Array<{ ts: string; event: string; status: string }>): void { + const dir = join(home, '.gbrain', 'integrations', id); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, 'heartbeat.jsonl'), entries.map(e => JSON.stringify(e)).join('\n') + '\n'); +} + +describe('parseMaxAge', () => { + test('parses h/d/m/s durations', () => { + expect(parseMaxAge('48h')).toBe(48 * 3_600_000); + expect(parseMaxAge('2d')).toBe(2 * 86_400_000); + expect(parseMaxAge('90m')).toBe(90 * 60_000); + expect(parseMaxAge('30s')).toBe(30_000); + expect(parseMaxAge(' 48H ')).toBe(48 * 3_600_000); + }); + + test('rejects garbage', () => { + expect(parseMaxAge('abc')).toBeNull(); + expect(parseMaxAge('48')).toBeNull(); + expect(parseMaxAge('-3h')).toBeNull(); + expect(parseMaxAge('')).toBeNull(); + expect(parseMaxAge('0h')).toBeNull(); + }); +}); + +describe('heartbeat_max_age health check (#2787)', () => { + test('fresh heartbeat within max_age → ok', async () => { + const home = tempHome(); + try { + writeHeartbeat(home, 'calendar-to-brain', [ + { ts: new Date(Date.now() - 3_600_000).toISOString(), event: 'sync', status: 'ok' }, + ]); + await withEnv({ GBRAIN_HOME: home }, async () => { + const r = await executeHealthCheck( + { type: 'heartbeat_max_age', max_age: '48h', label: 'freshness' } as any, + 'calendar-to-brain', + true, + ); + expect(r.status).toBe('ok'); + expect(r.output).toContain('within 48h'); + }); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); + + test('16-day-stale sense FAILS (the #2787 silent-death receipt)', async () => { + const home = tempHome(); + try { + writeHeartbeat(home, 'calendar-to-brain', [ + { ts: new Date(Date.now() - 16 * 86_400_000).toISOString(), event: 'sync', status: 'ok' }, + ]); + await withEnv({ GBRAIN_HOME: home }, async () => { + const r = await executeHealthCheck( + { type: 'heartbeat_max_age', max_age: '48h' } as any, + 'calendar-to-brain', + true, + ); + expect(r.status).toBe('fail'); + expect(r.output).toContain('exceeds max_age 48h'); + expect(r.output).toContain('stopped producing data'); + }); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); + + test('no heartbeat data at all → fail', async () => { + const home = tempHome(); + try { + await withEnv({ GBRAIN_HOME: home }, async () => { + const r = await executeHealthCheck( + { type: 'heartbeat_max_age', max_age: '48h' } as any, + 'never-ran', + true, + ); + expect(r.status).toBe('fail'); + expect(r.output).toContain('no heartbeat events'); + }); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); + + test('invalid max_age → fail with guidance, not a crash', async () => { + const r = await executeHealthCheck( + { type: 'heartbeat_max_age', max_age: 'soon' } as any, + 'whatever', + true, + ); + expect(r.status).toBe('fail'); + expect(r.output).toContain("invalid max_age 'soon'"); + }); + + test('not gated on embedded trust (read-only local file)', async () => { + const home = tempHome(); + try { + writeHeartbeat(home, 'user-recipe', [ + { ts: new Date().toISOString(), event: 'sync', status: 'ok' }, + ]); + await withEnv({ GBRAIN_HOME: home }, async () => { + const r = await executeHealthCheck( + { type: 'heartbeat_max_age', max_age: '1d' } as any, + 'user-recipe', + false, // NOT embedded + ); + expect(r.status).toBe('ok'); + }); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); + + test('newest entry wins even when the file is not time-ordered', async () => { + const home = tempHome(); + try { + writeHeartbeat(home, 'unordered', [ + { ts: new Date(Date.now() - 60_000).toISOString(), event: 'sync', status: 'ok' }, + { ts: new Date(Date.now() - 20 * 86_400_000).toISOString(), event: 'sync', status: 'ok' }, + ]); + await withEnv({ GBRAIN_HOME: home }, async () => { + const r = await executeHealthCheck( + { type: 'heartbeat_max_age', max_age: '48h' } as any, + 'unordered', + true, + ); + expect(r.status).toBe('ok'); + }); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); + + test('works inside any_of', async () => { + const home = tempHome(); + try { + writeHeartbeat(home, 'combo', [ + { ts: new Date().toISOString(), event: 'sync', status: 'ok' }, + ]); + await withEnv({ GBRAIN_HOME: home }, async () => { + const r = await executeHealthCheck( + { type: 'any_of', checks: [{ type: 'heartbeat_max_age', max_age: '1h' }] } as any, + 'combo', + true, + ); + expect(r.status).toBe('ok'); + }); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); +}); + +describe('output_paths frontmatter + configured-collector outputs (#2788)', () => { + const RECIPE = `--- +id: test-collector +name: Test Collector +version: 0.1.0 +description: writes files +category: sense +health_checks: [] +output_paths: + - daily/test-collector/ +setup_time: 1 min +--- +Body. +`; + + test('parseRecipe surfaces output_paths (and defaults to [])', () => { + const parsed = parseRecipe(RECIPE, 'test-collector.md'); + expect(parsed).not.toBeNull(); + expect(parsed!.frontmatter.output_paths).toEqual(['daily/test-collector/']); + const bare = parseRecipe('---\nid: bare\n---\nBody.', 'bare.md'); + expect(bare!.frontmatter.output_paths).toEqual([]); + }); + + test('the shipped calendar-to-brain recipe declares heartbeat_max_age + output_paths', () => { + const content = require('node:fs').readFileSync( + join(import.meta.dir, '..', 'recipes', 'calendar-to-brain.md'), + 'utf-8', + ); + const parsed = parseRecipe(content, 'calendar-to-brain.md'); + expect(parsed).not.toBeNull(); + expect(parsed!.frontmatter.output_paths).toEqual(['daily/calendar/']); + const hb = parsed!.frontmatter.health_checks.find( + (c: any) => typeof c === 'object' && c.type === 'heartbeat_max_age', + ) as any; + expect(hb).toBeDefined(); + expect(hb.max_age).toBe('48h'); + }); + + test('getConfiguredCollectorOutputs includes secretless recipes with output_paths', async () => { + const home = tempHome(); + const recipesDir = mkdtempSync(join(tmpdir(), 'gbrain-recipes-')); + try { + writeFileSync(join(recipesDir, 'test-collector.md'), RECIPE); + await withEnv({ GBRAIN_HOME: home, GBRAIN_RECIPES_DIR: recipesDir }, async () => { + const outputs = getConfiguredCollectorOutputs(); + expect(outputs).toContainEqual({ id: 'test-collector', output_path: 'daily/test-collector/' }); + }); + } finally { + rmSync(home, { recursive: true, force: true }); + rmSync(recipesDir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/integrations.test.ts b/test/integrations.test.ts index 4e2a90f7e..9b806889f 100644 --- a/test/integrations.test.ts +++ b/test/integrations.test.ts @@ -343,7 +343,7 @@ describe('all recipes', () => { expect(typeof check).toBe('string'); } else { // Typed checks must have a valid type - expect(['http', 'env_exists', 'command', 'any_of']).toContain((check as any).type); + expect(['http', 'env_exists', 'command', 'any_of', 'heartbeat_max_age']).toContain((check as any).type); } } } diff --git a/test/storage-sync.test.ts b/test/storage-sync.test.ts index c386717b5..406a3ae00 100644 --- a/test/storage-sync.test.ts +++ b/test/storage-sync.test.ts @@ -188,3 +188,53 @@ describe('manageGitignore', () => { expect(warnings.filter((w) => /submodule/.test(w))).toEqual([]); }); }); + +// #2788: collector-output vs db_only collision warning at .gitignore-write time. +describe('manageGitignore collector/db_only collision warning (#2788)', () => { + let recipesDir: string; + const SECRET_ENV_KEYS = ['CLAWVISOR_URL', 'CLAWVISOR_AGENT_TOKEN', 'GOOGLE_CLIENT_ID', 'GOOGLE_CLIENT_SECRET']; + let savedEnv: Record; + + beforeEach(() => { + recipesDir = mkdtempSync(join(tmpdir(), 'gbrain-recipes-')); + savedEnv = {}; + // Make embedded recipes (calendar-to-brain) deterministically unconfigured + // and point recipe discovery at our temp dir. + for (const k of [...SECRET_ENV_KEYS, 'GBRAIN_RECIPES_DIR', 'GBRAIN_HOME']) { + savedEnv[k] = process.env[k]; + } + for (const k of SECRET_ENV_KEYS) delete process.env[k]; + process.env.GBRAIN_RECIPES_DIR = recipesDir; + process.env.GBRAIN_HOME = recipesDir; // heartbeat reads stay hermetic + writeFileSync( + join(recipesDir, 'test-collector.md'), + '---\nid: test-collector\nname: Test Collector\noutput_paths:\n - media/x/inbox/\n---\nBody.\n', + ); + }); + + afterEach(() => { + for (const [k, v] of Object.entries(savedEnv)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + rmSync(recipesDir, { recursive: true, force: true }); + }); + + test('warns when a configured collector output dir sits inside a db_only path', () => { + writeStorageConfig(); // db_only includes media/x/ + manageGitignore(tmp); + const hit = warnings.find((w) => /collector 'test-collector'/.test(w)); + expect(hit).toBeDefined(); + expect(hit).toContain("'media/x/inbox/'"); + expect(hit).toContain("db_only path 'media/x/'"); + expect(hit).toContain('silently skip'); + // .gitignore management still happens — the warning never blocks it. + expect(existsSync(join(tmp, '.gitignore'))).toBe(true); + }); + + test('no warning when the collector writes outside every db_only path', () => { + writeFileSync(join(tmp, 'gbrain.yml'), 'storage:\n db_only:\n - archive/\n'); + manageGitignore(tmp); + expect(warnings.filter((w) => /collector 'test-collector'/.test(w))).toEqual([]); + }); +});