mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-15 09:22:18 +00:00
Compare commits
20
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f60f245512 | ||
|
|
b325f28239 | ||
|
|
564ffae186 | ||
|
|
428bdc9cd1 | ||
|
|
1d98298a5c | ||
|
|
74f1ba20f1 | ||
|
|
af209a6c61 | ||
|
|
9a59748bb7 | ||
|
|
1d78013c07 | ||
|
|
a1a2671c21 | ||
|
|
e744eda66c | ||
|
|
2ea5b71177 | ||
|
|
8b40678e46 | ||
|
|
ee9ceb327a | ||
|
|
cb02932388 | ||
|
|
9c2dc4cd54 | ||
|
|
058fe69575 | ||
|
|
9e2093fc9b | ||
|
|
0de9eb68ba | ||
|
|
d97f159793 |
@@ -37,6 +37,6 @@ jobs:
|
||||
- run: bun install
|
||||
- name: Pre-test gates (shard 1 only — they're not test files)
|
||||
if: matrix.shard == 1
|
||||
run: scripts/check-jsonb-pattern.sh && scripts/check-progress-to-stdout.sh && scripts/check-wasm-embedded.sh && bun run typecheck
|
||||
run: bun run verify
|
||||
- name: Run test shard ${{ matrix.shard }}/4
|
||||
run: scripts/test-shard.sh ${{ matrix.shard }} 4
|
||||
|
||||
@@ -29,6 +29,12 @@ test/.cache/
|
||||
.claude/
|
||||
export/
|
||||
|
||||
# Conductor workspace-local agent artifacts: plans, todos, run-unit-parallel
|
||||
# failure logs and per-shard test output. v0.26.4 (run-unit-parallel.sh)
|
||||
# writes .context/test-failures.log + .context/test-summary.txt +
|
||||
# .context/test-shards/. Workspace-local by design — never committed.
|
||||
.context/
|
||||
|
||||
# Tier 3 PGLite snapshot fixture (built on demand by build:pglite-snapshot)
|
||||
test/fixtures/pglite-snapshot.tar
|
||||
test/fixtures/pglite-snapshot.version
|
||||
|
||||
+1337
-2
File diff suppressed because it is too large
Load Diff
@@ -40,13 +40,13 @@ strict behavior when unset.
|
||||
|
||||
## Key files
|
||||
|
||||
- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Also exports upload validators: `validateUploadPath`, `validatePageSlug`, `validateFilename`, plus `matchesSlugAllowList(slug, prefixes)` (v0.23 glob matcher: `<prefix>/*` matches recursive children; bare `<prefix>` matches exact only). `OperationContext.remote` flags untrusted callers; `OperationContext.allowedSlugPrefixes` (v0.23) is the trusted-workspace allow-list set by the dream cycle. `put_page` enforces: when `viaSubagent` and `allowedSlugPrefixes` is set, slug must match the allow-list; else the legacy `wiki/agents/<id>/...` namespace check applies. Auto-link enabled for trusted-workspace writes (skipped only when `remote=true && !trustedWorkspace`). As of v0.26.0, every `Operation` also carries `scope?: 'read' | 'write' | 'admin'` + `localOnly?: boolean`. All ops are annotated; `sync_brain`, `file_upload`, `file_list`, and `file_url` are `admin + localOnly` (rejected over HTTP). `OperationContext.auth?: AuthInfo` is threaded through HTTP dispatch for scope enforcement in `serve-http.ts` before the op runs.
|
||||
- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Also exports upload validators: `validateUploadPath`, `validatePageSlug`, `validateFilename`, plus `matchesSlugAllowList(slug, prefixes)` (v0.23 glob matcher: `<prefix>/*` matches recursive children; bare `<prefix>` matches exact only). `OperationContext.remote` flags untrusted callers; `OperationContext.allowedSlugPrefixes` (v0.23) is the trusted-workspace allow-list set by the dream cycle. `put_page` enforces: when `viaSubagent` and `allowedSlugPrefixes` is set, slug must match the allow-list; else the legacy `wiki/agents/<id>/...` namespace check applies. Auto-link enabled for trusted-workspace writes (skipped only when `remote=true && !trustedWorkspace`). As of v0.26.0, every `Operation` also carries `scope?: 'read' | 'write' | 'admin'` + `localOnly?: boolean`. All ops are annotated; `sync_brain`, `file_upload`, `file_list`, and `file_url` are `admin + localOnly` (rejected over HTTP). `OperationContext.auth?: AuthInfo` is threaded through HTTP dispatch for scope enforcement in `serve-http.ts` before the op runs. **v0.26.9 (D12 + F7b):** `OperationContext.remote` is now a REQUIRED field in the TypeScript type — the compiler is the first defense against transports that forget to set it. Four trust-boundary call sites (`put_page` allowlist, file_upload trust-narrowing, submit_job protected-name guard, auto-link skip) flipped from falsy-default (`!ctx.remote`) to fail-closed semantics (`ctx.remote === false` for "trusted-only" sites and `ctx.remote !== false` for "untrust unless explicit-false"). Anything that isn't strictly `false` is now treated as remote. Closed an HTTP MCP shell-job RCE: a `read+write`-scoped OAuth token could submit `shell` jobs because the HTTP request handler's literal context skipped `remote: true` and `submit_job`'s protected-name guard saw a falsy undefined. Stdio MCP set the field correctly via dispatch.ts; HTTP inlined a parallel context-builder for several releases and lost it.
|
||||
- `src/core/engine.ts` — Pluggable engine interface (BrainEngine). `clampSearchLimit(limit, default, cap)` takes an explicit cap so per-operation caps can be tighter than `MAX_SEARCH_LIMIT`. Exports `LinkBatchInput` / `TimelineBatchInput` for the v0.12.1 bulk-insert API (`addLinksBatch` / `addTimelineEntriesBatch`). As of v0.13.1, `BrainEngine` has a `readonly kind: 'postgres' | 'pglite'` discriminator so migrations (`src/core/migrate.ts`) and other consumers can branch on engine without `instanceof` + dynamic imports.
|
||||
- `src/core/engine-factory.ts` — Engine factory with dynamic imports (`'pglite'` | `'postgres'`)
|
||||
- `src/core/pglite-engine.ts` — PGLite (embedded Postgres 17.5 via WASM) implementation, all 40 BrainEngine methods. `addLinksBatch` / `addTimelineEntriesBatch` use multi-row `unnest()` with manual `$N` placeholders. As of v0.13.1, `connect()` wraps `PGlite.create()` in a try/catch that emits an actionable error naming the macOS 26.3 WASM bug (#223) and pointing at `gbrain doctor`; the lock is released on failure so the next process can retry cleanly. v0.22.0: `searchKeyword` and `searchKeywordChunks` multiply `ts_rank` by the source-factor CASE expression at the chunk-grain level; `searchVector` becomes a two-stage CTE — inner CTE keeps `ORDER BY cc.embedding <=> vec` so HNSW stays usable, outer SELECT re-ranks by `raw_score * source_factor`. Inner LIMIT scales with offset to preserve pagination contract. As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL — probes for the specific forward-referenced state the embedded schema blob needs (`pages.source_id`, `links.link_source`, `links.origin_page_id`, `content_chunks.symbol_name`, `content_chunks.language`, `sources` FK target table) and adds only what's missing. Closes the upgrade-wedge bug class that bit users 10+ times across 6 schema versions over 2 years (#239/#243/#266/#357/#366/#374/#375/#378/#395/#396). No-op on fresh installs and modern brains.
|
||||
- `src/core/pglite-schema.ts` — PGLite-specific DDL (pgvector, pg_trgm, triggers)
|
||||
- `src/core/postgres-engine.ts` — Postgres + pgvector implementation (Supabase / self-hosted). `addLinksBatch` / `addTimelineEntriesBatch` use `INSERT ... SELECT FROM unnest($1::text[], ...) JOIN pages ON CONFLICT DO NOTHING RETURNING 1` — 4-5 array params regardless of batch size, sidesteps the 65535-parameter cap. As of v0.12.3, `searchKeyword` / `searchVector` scope `statement_timeout` via `sql.begin` + `SET LOCAL` so the GUC dies with the transaction instead of leaking across the pooled postgres.js connection (contributed by @garagon). `getEmbeddingsByChunkIds` uses `tryParseEmbedding` so one corrupt row skips+warns instead of killing the query. v0.22.0: `searchKeyword`, `searchKeywordChunks`, and `searchVector` apply source-aware ranking by inlining the source-factor CASE and `NOT (col LIKE …)` hard-exclude clause from `src/core/search/sql-ranking.ts`. `searchVector` switches to a two-stage CTE (HNSW-safe inner ORDER BY, source-boost re-rank in the outer SELECT) and carries `p.source_id` through inner→outer for v0.18 multi-source callers. v0.22.1 (#406): `_savedConfig` retains the connect config; `reconnect()` tears down + recreates the pool from saved config (called by supervisor watchdog after 3 consecutive health-check failures). `executeRaw` is a single-statement passthrough — no per-call retry (D3 dropped that as unsound for non-idempotent statements; recovery is supervisor-driven). v0.22.1 (#363, contributed by @orendi84): `connect()` applies `resolveSessionTimeouts()` from `db.ts` as connection-time startup parameters (`statement_timeout`, `idle_in_transaction_session_timeout`) so orphan pgbouncer backends can't hold locks for hours. v0.22.1 (#409, contributed by @atrevino47): `countStaleChunks()` + `listStaleChunks()` server-side-filter on `embedding IS NULL` for `embed --stale`, eliminating ~76 MB/call client-side pull on a fully-embedded brain; `upsertChunks()` resets both `embedding` AND `embedded_at` to NULL when chunk_text changes without a new embedding (consistency). As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL on the same forward-reference probe set as the PGLite engine, so old Postgres brains pinned at v0.13/v0.18/v0.19 walk forward cleanly instead of wedging on `column "..." does not exist`.
|
||||
- `src/core/utils.ts` — Shared SQL utilities extracted from postgres-engine.ts. Exports `parseEmbedding(value)` (throws on unknown input, used by migration + ingest paths where data integrity matters) and as of v0.12.3 `tryParseEmbedding(value)` (returns `null` + warns once per process, used by search/rescore paths where availability matters more than strictness).
|
||||
- `src/core/postgres-engine.ts` — Postgres + pgvector implementation (Supabase / self-hosted). `addLinksBatch` / `addTimelineEntriesBatch` use `INSERT ... SELECT FROM unnest($1::text[], ...) JOIN pages ON CONFLICT DO NOTHING RETURNING 1` — 4-5 array params regardless of batch size, sidesteps the 65535-parameter cap. As of v0.12.3, `searchKeyword` / `searchVector` scope `statement_timeout` via `sql.begin` + `SET LOCAL` so the GUC dies with the transaction instead of leaking across the pooled postgres.js connection (contributed by @garagon). `getEmbeddingsByChunkIds` uses `tryParseEmbedding` so one corrupt row skips+warns instead of killing the query. v0.22.0: `searchKeyword`, `searchKeywordChunks`, and `searchVector` apply source-aware ranking by inlining the source-factor CASE and `NOT (col LIKE …)` hard-exclude clause from `src/core/search/sql-ranking.ts`. `searchVector` switches to a two-stage CTE (HNSW-safe inner ORDER BY, source-boost re-rank in the outer SELECT) and carries `p.source_id` through inner→outer for v0.18 multi-source callers. v0.22.1 (#406): `_savedConfig` retains the connect config; `reconnect()` tears down + recreates the pool from saved config (called by supervisor watchdog after 3 consecutive health-check failures). `executeRaw` is a single-statement passthrough — no per-call retry (D3 dropped that as unsound for non-idempotent statements; recovery is supervisor-driven). v0.22.1 (#363, contributed by @orendi84): `connect()` applies `resolveSessionTimeouts()` from `db.ts` as connection-time startup parameters (`statement_timeout`, `idle_in_transaction_session_timeout`) so orphan pgbouncer backends can't hold locks for hours. v0.22.1 (#409, contributed by @atrevino47): `countStaleChunks()` + `listStaleChunks()` server-side-filter on `embedding IS NULL` for `embed --stale`, eliminating ~76 MB/call client-side pull on a fully-embedded brain; `upsertChunks()` resets both `embedding` AND `embedded_at` to NULL when chunk_text changes without a new embedding (consistency). As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL on the same forward-reference probe set as the PGLite engine, so old Postgres brains pinned at v0.13/v0.18/v0.19 walk forward cleanly instead of wedging on `column "..." does not exist`. **v0.28.1:** `disconnect()` is now idempotent. New `_connectionStyle` instance field tracks whether the engine owns its pool (worker engines) or shares the module-level singleton; second call on an instance-pool engine is a no-op rather than falling through to `db.disconnect()` and clobbering the singleton. Pinned by `test/e2e/postgres-engine-disconnect-idempotency.test.ts` (2 cases). Closes the bug class where any test sharing an engine across multiple `worker.start()` / `worker.stop()` cycles silently broke its own DB connectivity.
|
||||
- `src/core/utils.ts` — Shared SQL utilities extracted from postgres-engine.ts. Exports `parseEmbedding(value)` (throws on unknown input, used by migration + ingest paths where data integrity matters) and as of v0.12.3 `tryParseEmbedding(value)` (returns `null` + warns once per process, used by search/rescore paths where availability matters more than strictness). **v0.26.9 (D14):** adds `isUndefinedColumnError(err)` predicate — pattern-matches Postgres SQLSTATE 42703 / "column ... does not exist" with engine-driver shape variation tolerated. Replaces bare `catch {}` blocks in `oauth-provider.ts` so genuine errors (lock timeout, network blip, permission denied) propagate while column-missing falls through to the legacy fallback path. Reusable from any future code that needs the same column-existence probe semantics.
|
||||
- `src/core/db.ts` — Connection management, schema initialization. v0.22.1 (#363, contributed by @orendi84): `resolveSessionTimeouts()` returns `statement_timeout` + `idle_in_transaction_session_timeout` (defaults: 5min each, env-overridable via `GBRAIN_STATEMENT_TIMEOUT` / `GBRAIN_IDLE_TX_TIMEOUT` / `GBRAIN_CLIENT_CHECK_INTERVAL`). Both `connect()` (module singleton) and `PostgresEngine.connect()` (worker pool) consume the result via postgres.js's `connection` option, sending GUCs as startup parameters that survive PgBouncer transaction mode (unlike the prior `setSessionDefaults` post-pool SET, kept as a back-compat no-op shim).
|
||||
- `src/commands/migrate-engine.ts` — Bidirectional engine migration (`gbrain migrate --to supabase/pglite`)
|
||||
- `src/core/import-file.ts` — importFromFile + importFromContent (chunk + embed + tags)
|
||||
@@ -67,7 +67,13 @@ strict behavior when unset.
|
||||
- `src/core/search/eval.ts` — Retrieval eval harness: P@k, R@k, MRR, nDCG@k metrics + runEval() orchestrator
|
||||
- `src/core/search/source-boost.ts` (v0.22.0) — Source-type boost map keyed by slug prefix. `DEFAULT_SOURCE_BOOSTS` (originals/ 1.5, concepts/ 1.3, writing/ 1.4, people/companies/deals/ 1.2, daily/ 0.8, media/x/ 0.7, wintermute/chat/ 0.5) and `DEFAULT_HARD_EXCLUDES` (test/, archive/, attachments/, .raw/). `parseSourceBoostEnv` / `parseHardExcludesEnv` parse comma-separated `prefix:factor` pairs from `GBRAIN_SOURCE_BOOST` / `GBRAIN_SEARCH_EXCLUDE` env vars. `resolveBoostMap` and `resolveHardExcludes` merge defaults + env + caller `SearchOpts.exclude_slug_prefixes`/`include_slug_prefixes`.
|
||||
- `src/core/search/sql-ranking.ts` (v0.22.0) — Pure SQL string builders. `buildSourceFactorCase(slugColumn, boostMap, detail)` emits a CASE expression with longest-prefix-match wins (returns literal `'1.0'` when `detail === 'high'` for temporal-bypass parity with COMPILED_TRUTH_BOOST). `buildHardExcludeClause(slugColumn, prefixes)` emits `NOT (col LIKE 'p1%' OR col LIKE 'p2%')` — OR-chain wrapped in NOT, NOT `NOT LIKE ALL/ANY` (those quantifiers don't express set-exclusion). LIKE meta-character escape covers all three of `%`, `_`, AND `\` (backslash matters because it's Postgres LIKE's default escape char). Single-quote doubling on SQL string literals so injection-style inputs are inert text.
|
||||
- `src/commands/eval.ts` — `gbrain eval` command: single-run table + A/B config comparison. v0.25.0 adds sub-subcommand dispatch on `args[0]` so `gbrain eval export` + `gbrain eval prune` + `gbrain eval replay` route into session-capture handlers; bare `gbrain eval --qrels …` fall-through preserves the legacy IR-metrics flow.
|
||||
- `src/commands/eval.ts` — `gbrain eval` command: single-run table + A/B config comparison. v0.25.0 adds sub-subcommand dispatch on `args[0]` so `gbrain eval export` + `gbrain eval prune` + `gbrain eval replay` route into session-capture handlers; bare `gbrain eval --qrels …` fall-through preserves the legacy IR-metrics flow. v0.27.x adds `gbrain eval cross-modal` to the dispatch (the user-facing path is the cli.ts no-DB branch — `src/commands/eval.ts:cross-modal` only fires when callers re-enter with an existing engine).
|
||||
- `src/commands/eval-cross-modal.ts` (v0.27.x) — multi-model quality gate. Three different-provider frontier models score the OUTPUT against the TASK on a 5-dim list. Verdict `pass` (exit 0) / `fail` (exit 1) / `inconclusive` (exit 2; <2/3 model successes per Q3=A in plans/radiant-napping-lerdorf.md). Reuses `src/core/ai/gateway.ts:chat()` so config/auth/aliasing comes from the gateway recipe registry — no parallel provider stack. Self-configures the gateway (`configureGateway(loadConfig() + process.env)`) since the cli.ts dispatch bypasses `connectEngine()`. Default cycles 3 in TTY, 1 in non-TTY (T11=B partial cost guardrail). Receipts land at `gbrainPath('eval-receipts')/<slug>-<sha8-of-output>.json`. The full `--budget-usd` cap is a v0.27.x follow-up TODO.
|
||||
- `src/core/cross-modal-eval/json-repair.ts` (v0.27.x) — `parseModelJSON(raw)` named export with a 4-strategy fallback chain (direct parse → fence-strip → trailing-comma + single-quote + embedded-newline repair → regex nuclear option). Adversarial input throws rather than fabricating scores — the aggregator treats a throw as "this model contributed nothing this cycle" so the gate stays correct at >=2/3 successes.
|
||||
- `src/core/cross-modal-eval/aggregate.ts` (v0.27.x) — pure verdict logic. Pass criterion: `(successes >= 2) AND (every dim mean >= 7) AND (every dim min across models >= 5)` (Q2=A floor). Inconclusive when <2/3 models returned parseable scores (Q3=A regression guard for the v1 .mjs `Object.values({}).every(...) === true` empty-array PASS bug).
|
||||
- `src/core/cross-modal-eval/runner.ts` (v0.27.x) — orchestrator. Each cycle runs `Promise.allSettled([gwChat(slotA), gwChat(slotB), gwChat(slotC)])` (T4=A — bare allSettled, no rate-leases for the CLI path; minion-integration TODO recovers cross-process concurrency). Stops early on PASS or INCONCLUSIVE; runs up to 3 cycles. Default slots: `openai:gpt-4o` / `anthropic:claude-opus-4-7` / `google:gemini-1.5-pro`. `estimateCost()` exports a small per-model pricing table (drifts; refresh alongside model-family bumps).
|
||||
- `src/core/cross-modal-eval/receipt-name.ts` (v0.27.x) — receipt filename binds (slug, SKILL.md sha-8). `findReceiptForSkill(skillPath, receiptDir)` returns `'found' | 'stale' | 'missing'` (T10=A). Skillify-check item 11 surfaces the status as informational (T7=C); the audit does NOT fail on missing/stale receipts.
|
||||
- `src/core/cross-modal-eval/receipt-write.ts` (v0.27.x) — wraps `fs.writeFileSync` with `mkdirSync({recursive:true})` ahead of every write (T5 correction; `gbrainPath()` does NOT auto-mkdir).
|
||||
- `src/commands/eval-export.ts` (v0.25.0) — streams `eval_candidates` rows as NDJSON to stdout with `schema_version: 1` prefix on every line. EPIPE-safe, progress heartbeats on stderr, stable id-desc tiebreaker so `--since` windows never dupe/miss rows.
|
||||
- `src/commands/eval-prune.ts` (v0.25.0) — explicit retention cleanup. Requires `--older-than DUR`. `--dry-run` reports would-delete count.
|
||||
- `src/commands/eval-replay.ts` (v0.25.0) — contributor-facing replay tool. Reads NDJSON from `gbrain eval export`, re-runs each captured `query` / `search` op against the current brain, computes set-Jaccard@k between captured + current `retrieved_slugs`, top-1 stability rate, and latency Δ. Stable JSON shape (`schema_version: 1`) for CI gating; human mode prints a regression table. Pure Bun, zero new deps. The dev-loop half of BrainBench-Real that closes the gap between "data captured" and "data used to gate a PR." See `docs/eval-bench.md` for the workflow.
|
||||
@@ -77,7 +83,10 @@ strict behavior when unset.
|
||||
- `src/core/search/hybrid.ts` — Cathedral II `Promise<SearchResult[]>` return shape unchanged in v0.25.0. Adds `onMeta?: (m: HybridSearchMeta) => void` callback so op-layer capture can record what hybridSearch actually did. Existing callers leave it undefined.
|
||||
- `docs/eval-capture.md` (v0.25.0) — stable NDJSON schema reference for gbrain-evals consumers.
|
||||
- `test/public-exports.test.ts` (v0.25.0 / R2) — runtime contract test. Imports each of the 17 public subpaths via package name and pins a canary symbol per module. Paired with `scripts/check-exports-count.sh`.
|
||||
- `src/core/embedding.ts` — OpenAI text-embedding-3-large, batch, retry, backoff
|
||||
- `src/core/embedding.ts` — OpenAI text-embedding-3-large, batch, retry, backoff. **v0.28.7:** `BATCH_SIZE` reverted 50→100 — the original Voyage safety guard halved OpenAI throughput on every page. Per-recipe pre-split + recursive halving + adaptive shrink-on-miss now live in the gateway, so the outer paginator goes back to its original purpose: progress-callback granularity, not batch protection.
|
||||
- `src/core/ai/types.ts` — provider/recipe types. **v0.28.7 (#680):** `EmbeddingTouchpoint` extended with optional `chars_per_token` (default 4 chars/token, matching OpenAI tiktoken on English) and `safety_factor` (default 0.8, budget-utilization ceiling). Both consulted only when `max_batch_tokens` is also set. Voyage declares `chars_per_token=1` + `safety_factor=0.5` to handle dense payloads (CJK/JSON/base64) that overshoot tiktoken. The pre-split budget is `max_batch_tokens × safety_factor / chars_per_token`.
|
||||
- `src/core/ai/gateway.ts` — unified seam for every AI call. **v0.28.7 (#680):** module-scoped `_embedTransport` defaulting to AI SDK `embedMany`, with `__setEmbedTransportForTests(fn)` test seam so tests drive the public `embed()` function with a stubbed transport instead of probing private helpers. `splitByTokenBudget` and `isTokenLimitError` are now exported `@internal` — pure functions reused directly by the test file. Module-level `_shrinkState: Map<recipeId, {factor, consecutiveSuccesses}>` halves the recipe's effective `safety_factor` on token-limit miss (floor 0.05) and heals back ×1.5 toward the ceiling after `SHRINK_HEAL_AFTER=10` consecutive successes. `configureGateway()` walks every registered recipe at construction time and emits a once-per-process stderr warning for any embedding touchpoint missing `max_batch_tokens` (excluding the canonical OpenAI fast-path recipe). `resetGateway()` clears `_shrinkState`, the warned-set, and restores the real transport. ASCII flow diagram embedded in the `embed()` JSDoc covers the routing decision, recursion + halving, and shrinkState lifecycle.
|
||||
- `src/core/ai/recipes/voyage.ts` — Voyage AI openai-compatible recipe. **v0.28.7 (#680):** declares `chars_per_token=1` + `safety_factor=0.5` so the gateway pre-splits Voyage batches at a 60K-character budget (50% of 120K-token cap with the dense-tokenizer ratio). Closes the v0.27 backfill loop where ~26% of the corpus stayed un-embedded because tiktoken-grounded budgeting silently undercounted Voyage's actual token usage.
|
||||
- `src/core/check-resolvable.ts` — Resolver validation: reachability, MECE overlap, DRY checks, structured fix objects. v0.14.1: `CROSS_CUTTING_PATTERNS.conventions` is an array (notability gate accepts both `conventions/quality.md` and `_brain-filing-rules.md`). New `extractDelegationTargets()` parses `> **Convention:**`, `> **Filing rule:**`, and inline backtick references. DRY suppression is proximity-based via `DRY_PROXIMITY_LINES = 40`.
|
||||
- `src/core/repo-root.ts` — Shared `findRepoRoot(startDir?)` (v0.16.4): walks up from `startDir` (default `process.cwd()`) looking for `skills/RESOLVER.md`. Zero-dependency module imported by both `doctor.ts` and `check-resolvable.ts`. Parameterized `startDir` makes tests hermetic.
|
||||
- `src/commands/check-resolvable.ts` — Standalone CLI wrapper (v0.16.4) over `checkResolvable()`. Exports `parseFlags`, `resolveSkillsDir`, `DEFERRED`, `runCheckResolvable`. Exit rule: **1 on any issue (warnings OR errors)**, stricter than doctor's `ok` flag — honors README:259. Stable JSON envelope `{ok, skillsDir, report, autoFix, deferred, error, message}` — same shape on success and error paths. `--fix` path runs `autoFixDryViolations` BEFORE `checkResolvable` (same ordering as doctor). `scripts/skillify-check.ts` subprocess-calls `gbrain check-resolvable --json` (cached per process) and fails loud on binary-missing — no silent false-pass. **v0.19:** AGENTS.md workspaces now resolve natively (see `src/core/resolver-filenames.ts`) — gbrain inspects the 107-skill OpenClaw deployment whether the routing file is `RESOLVER.md` or `AGENTS.md`. `DEFERRED[]` is empty — Checks 5 + 6 shipped as real code, not issue URLs.
|
||||
@@ -101,10 +110,12 @@ strict behavior when unset.
|
||||
- `src/commands/extract.ts` — `gbrain extract links|timeline|all [--source fs|db]`: batch link/timeline extraction. fs walks markdown files, db walks pages from the engine (mutation-immune snapshot iteration; use this for live brains with no local checkout). As of v0.12.1 there is no in-memory dedup pre-load — candidates are buffered 100 at a time and flushed via `addLinksBatch` / `addTimelineEntriesBatch`; `ON CONFLICT DO NOTHING` enforces uniqueness at the DB layer, and the `created` counter returns real rows inserted (truthful on re-runs). v0.22.1 (#417): `ExtractOpts.slugs?: string[]` enables incremental extract — when set, `extractForSlugs()` reads ONLY those slugs' files (single combined links+timeline pass) instead of the full directory walk. CLI `gbrain extract` keeps full-walk behavior; the cycle path threads sync's `pagesAffected` through. `walkMarkdownFiles(brainDir)` still runs at line 455 to build `allSlugs` for link resolution — see `TODOS.md` for replacing it with `engine.getAllSlugs()`.
|
||||
- `src/commands/graph-query.ts` — `gbrain graph-query <slug> [--type T] [--depth N] [--direction in|out|both]`: typed-edge relationship traversal (renders indented tree)
|
||||
- `src/core/link-extraction.ts` — shared library for the v0.12.0 graph layer. extractEntityRefs (canonical, replaces backlinks.ts duplicate) matches both `[Name](people/slug)` markdown links and Obsidian `[[people/slug|Name]]` wikilinks as of v0.12.3. extractPageLinks, inferLinkType heuristics (attended/works_at/invested_in/founded/advises/source/mentions), parseTimelineEntries, isAutoLinkEnabled config helper. `DIR_PATTERN` covers `people`, `companies`, `deals`, `topics`, `concepts`, `projects`, `entities`, `tech`, `finance`, `personal`, `openclaw`. Used by extract.ts, operations.ts auto-link post-hook, and backlinks.ts.
|
||||
- `src/core/zombie-reap.ts` (v0.28.1) — idempotent `installSigchldHandler()` so JS-spawned children get reaped via Bun's internal `waitpid()`. Bun (like Node) only auto-reaps when a SIGCHLD listener is registered; without it, every child the worker spawns (shell jobs, embed batches, sub-agents) becomes a zombie on exit and holds connection slots. Called once at module load from `src/cli.ts` (with Windows platform guard — SIGCHLD doesn't exist on Windows). Cross-file leak guard via `_uninstallSigchldHandlerForTests()` for tests. Layer 1 of the three-layer zombie defense; Layer 2 is tini-as-PID-1 wrapping the worker subtree (via `src/core/minions/spawn-helpers.ts`); Layer 3 is the container's own tini for hard Bun crashes.
|
||||
- `src/core/minions/` — Minions job queue: BullMQ-inspired, Postgres-native (queue, worker, backoff, types, protected-names, quiet-hours, stagger, handlers/shell).
|
||||
- `src/core/minions/queue.ts` — MinionQueue class (submit, claim, complete, fail, stall detection, parent-child, depth/child-cap, per-job timeouts, cascade-kill, attachments, idempotency keys, child_done inbox, removeOnComplete/Fail). `add()` takes a 4th `trusted` arg (separate from `opts` to prevent spread leakage); protected names in `PROTECTED_JOB_NAMES` require `{allowProtectedSubmit: true}` and the check runs trim-normalized (whitespace-bypass safe). v0.14.1 #219: `add()` plumbs `max_stalled` through with a `[1, 100]` clamp; omitted values let the schema DEFAULT (5) kick in. v0.19.0: `handleWallClockTimeouts(lockDurationMs)` is Layer 3 kill shot for jobs where `FOR UPDATE SKIP LOCKED` stall detection and the timeout sweep both fail to evict (wedged worker holding a row lock via a pending transaction). v0.19.1: `maxWaiting` coalesce path now uses `pg_advisory_xact_lock` keyed on `(name, queue)` to serialize concurrent submits for the same key, and filters on `queue` in addition to `name` so cross-queue same-name jobs don't suppress each other.
|
||||
- `src/core/minions/worker.ts` — MinionWorker class (handler registry, lock renewal, graceful shutdown, timeout safety net). v0.14.0 abort-path fix: aborted jobs now call `failJob` with reason (`timeout`/`cancel`/`lock-lost`/`shutdown`) instead of returning silently. `shutdownAbort` (instance field) fires on process SIGTERM/SIGINT and propagates to `ctx.shutdownSignal` — shell handler listens to it; non-shell handlers don't. v0.22.1 (#403): per-job timeout fires `abort.abort(new Error('timeout'))` then a 30-second grace-then-evict safety net force-evicts the job from `inFlight` and marks it dead in DB if the handler ignores the abort signal — frees the slot even when a handler wedges (the 98-waiting-0-active prod incident driver).
|
||||
- `src/core/minions/supervisor.ts` — MinionSupervisor process manager. Spawns `gbrain jobs work` as a child, restarts on crash with exponential backoff, periodic health check. v0.22.1 (#406): `consecutiveHealthFailures` counter; on 3 consecutive failures emits `health_warn` with `reason: 'db_connection_degraded'` and calls `engine.reconnect()` to swap in a fresh pool, then resets the counter. Worker exit classifier emits `likely_cause` field on `worker_exited` events: `oom_or_external_kill` (SIGKILL), `graceful_shutdown` (SIGTERM), `runtime_error` (code 1), `clean_exit` (code 0), `unknown`.
|
||||
- `src/core/minions/worker.ts` — MinionWorker class (handler registry, lock renewal, graceful shutdown, timeout safety net). v0.14.0 abort-path fix: aborted jobs now call `failJob` with reason (`timeout`/`cancel`/`lock-lost`/`shutdown`) instead of returning silently. `shutdownAbort` (instance field) fires on process SIGTERM/SIGINT and propagates to `ctx.shutdownSignal` — shell handler listens to it; non-shell handlers don't. v0.22.1 (#403): per-job timeout fires `abort.abort(new Error('timeout'))` then a 30-second grace-then-evict safety net force-evicts the job from `inFlight` and marks it dead in DB if the handler ignores the abort signal — frees the slot even when a handler wedges (the 98-waiting-0-active prod incident driver). **v0.28.1 engine-ownership invariant:** `start()` no longer calls `engine.disconnect()` on shutdown — that was a leaky abstraction (the worker disconnected an engine it didn't own). The CLI handler in `src/commands/jobs.ts case 'work'` now owns engine lifecycle via try/finally with loud error logging on disconnect failure. Pinned by `test/worker-shutdown-disconnect.test.ts` asserting the inverse (`disconnectSpy).not.toHaveBeenCalled()`).
|
||||
- `src/core/minions/supervisor.ts` — MinionSupervisor process manager. Spawns `gbrain jobs work` as a child, restarts on crash with exponential backoff, periodic health check. v0.22.1 (#406): `consecutiveHealthFailures` counter; on 3 consecutive failures emits `health_warn` with `reason: 'db_connection_degraded'` and calls `engine.reconnect()` to swap in a fresh pool, then resets the counter. Worker exit classifier emits `likely_cause` field on `worker_exited` events: `oom_or_external_kill` (SIGKILL), `graceful_shutdown` (SIGTERM), `runtime_error` (code 1), `clean_exit` (code 0), `unknown`. **v0.28.1:** consumes `detectTini()` + `buildSpawnInvocation()` from `src/core/minions/spawn-helpers.ts` to wrap the worker subtree in tini-as-PID-1 when tini is on `PATH` (handles native-addon zombie reaping that the in-process SIGCHLD reaper can't reach). Exposes `isTiniDetected` read-only accessor for tests.
|
||||
- `src/core/minions/spawn-helpers.ts` (v0.28.1) — pure `detectTini()` + `buildSpawnInvocation()` helpers consumed by both `supervisor.ts` and `autopilot.ts`. Resolves the DRY violation between the two spawn sites and makes the tini wrapping testable without `mock.module()` (rule R2 of `scripts/check-test-isolation.sh`). `detectTini()` calls `execFileSync('which', ['tini'])` with explicit `env: process.env` so Bun sees runtime PATH mutations (the env-snapshot bug fix). `buildSpawnInvocation(tiniPath, cmd, args)` returns `{cmd, args}` with tini prepended when present, or the bare invocation otherwise. Pinned by `test/spawn-helpers.test.ts` (5 cases) and `test/supervisor-tini.test.ts` (4 cases).
|
||||
- `src/core/minions/types.ts` — `MinionJobInput` + `MinionJobStatus` + handler context types. `MinionJobInput.max_stalled` (new in v0.14.1) is optional; omitted values let the schema DEFAULT (5) kick in, provided values are clamped to `[1, 100]`.
|
||||
- `src/core/minions/protected-names.ts` — side-effect-free constant module exporting `PROTECTED_JOB_NAMES` + `isProtectedJobName()`. Kept pure so queue core can import without loading handler modules.
|
||||
- `src/core/minions/handlers/shell.ts` — `shell` job handler. Spawns `/bin/sh -c cmd` (absolute path, PATH-override-safe) or `argv[0] argv[1..]` (no shell). Env allowlist: `PATH, HOME, USER, LANG, TZ, NODE_ENV` + caller `env:` overrides. UTF-8-safe stdout/stderr tail via `string_decoder.StringDecoder`. Abort (either `ctx.signal` or `ctx.shutdownSignal`) fires SIGTERM → 5s grace → SIGKILL on child. Requires `GBRAIN_ALLOW_SHELL_JOBS=1` on worker (gated by `registerBuiltinHandlers`).
|
||||
@@ -122,14 +133,14 @@ strict behavior when unset.
|
||||
- `src/core/minions/attachments.ts` — Attachment validation (path traversal, null byte, oversize, base64, duplicate detection)
|
||||
- `src/commands/agent.ts` (v0.16) — `gbrain agent run <prompt> [flags]` CLI. Submits `subagent` (or N children + 1 aggregator) under `{allowProtectedSubmit: true}`. Single-entry `--fanout-manifest` short-circuits. Children get `on_child_fail: 'continue'` + `max_stalled: 3`. `--follow` is the default on TTY; streams logs + polls `waitForCompletion` in parallel. Ctrl-C detaches, does not cancel.
|
||||
- `src/commands/agent-logs.ts` (v0.16) — `gbrain agent logs <job> [--follow] [--since]`. Merges JSONL heartbeat audit + `subagent_messages` into a chronological timeline. `parseSince` accepts ISO-8601 or relative (`5m`, `1h`, `2d`). Transcript tail renders only for terminal jobs.
|
||||
- `src/commands/jobs.ts` — `gbrain jobs` CLI subcommands + `gbrain jobs work` daemon. v0.13.1 surfaces the full `MinionJobInput` retry/backoff/timeout/idempotency surface as first-class CLI flags on `jobs submit`: `--max-stalled`, `--backoff-type fixed|exponential`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key`. `jobs smoke --sigkill-rescue` is the opt-in regression guard for #219. v0.16 wires `registerBuiltinHandlers` to always register `subagent` + `subagent_aggregator` (no env flag — `ANTHROPIC_API_KEY` is the natural cost gate, trust is via `PROTECTED_JOB_NAMES`) and loads `GBRAIN_PLUGIN_PATH` plugins at worker startup with a loud startup-line per plugin. `shell` handler still gated by `GBRAIN_ALLOW_SHELL_JOBS=1` (RCE surface, separate concern). v0.22.10 (#521): the `autopilot-cycle` handler now forwards `job.data.phases` to `runCycle` (was previously discarded — caller-supplied phase selection silently became a full cycle). Phases are validated against `ALL_PHASES` from `src/core/cycle.ts`; invalid names are filtered out and an empty/missing array falls back to the default 6-phase cycle. v0.22.13 (PR #490 CODEX-1+CODEX-4): `sync` handler now resolves `sourceId` at entry by looking up `sources.local_path` (mirrors `cycle.ts:480`'s autopilot fix from PR #475) so multi-source brains read the per-source `last_commit` anchor instead of the global config key. Concurrency routed through the shared `autoConcurrency()` policy in `src/core/sync-concurrency.ts` instead of the prior hardcoded `4`; PGLite stays serial. `noEmbed` default is `true` (embed is a separate job — submit `gbrain embed --stale` after sync, or rely on the autopilot cycle's embed phase).
|
||||
- `src/commands/jobs.ts` — `gbrain jobs` CLI subcommands + `gbrain jobs work` daemon. **v0.28.1:** `case 'work'` now wraps `worker.start()` in try/finally and owns engine lifecycle — calls `engine.disconnect()` on shutdown with loud error logging on failure. Replaces the prior call inside `MinionWorker.start()` (which violated engine ownership: the worker disconnected an engine it didn't own, and clobbered the module-level singleton on PostgresEngine via the now-fixed idempotency bug). Pool slots now free immediately on shutdown instead of waiting for TCP keepalive (~minutes). v0.13.1 surfaces the full `MinionJobInput` retry/backoff/timeout/idempotency surface as first-class CLI flags on `jobs submit`: `--max-stalled`, `--backoff-type fixed|exponential`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key`. `jobs smoke --sigkill-rescue` is the opt-in regression guard for #219. v0.16 wires `registerBuiltinHandlers` to always register `subagent` + `subagent_aggregator` (no env flag — `ANTHROPIC_API_KEY` is the natural cost gate, trust is via `PROTECTED_JOB_NAMES`) and loads `GBRAIN_PLUGIN_PATH` plugins at worker startup with a loud startup-line per plugin. `shell` handler still gated by `GBRAIN_ALLOW_SHELL_JOBS=1` (RCE surface, separate concern). v0.22.10 (#521): the `autopilot-cycle` handler now forwards `job.data.phases` to `runCycle` (was previously discarded — caller-supplied phase selection silently became a full cycle). Phases are validated against `ALL_PHASES` from `src/core/cycle.ts`; invalid names are filtered out and an empty/missing array falls back to the default 6-phase cycle. v0.22.13 (PR #490 CODEX-1+CODEX-4): `sync` handler now resolves `sourceId` at entry by looking up `sources.local_path` (mirrors `cycle.ts:480`'s autopilot fix from PR #475) so multi-source brains read the per-source `last_commit` anchor instead of the global config key. Concurrency routed through the shared `autoConcurrency()` policy in `src/core/sync-concurrency.ts` instead of the prior hardcoded `4`; PGLite stays serial. `noEmbed` default is `true` (embed is a separate job — submit `gbrain embed --stale` after sync, or rely on the autopilot cycle's embed phase).
|
||||
- `src/commands/features.ts` — `gbrain features --json --auto-fix`: usage scan + feature adoption salesman
|
||||
- `src/commands/autopilot.ts` — `gbrain autopilot --install`: self-maintaining brain daemon (sync+extract+embed)
|
||||
- `src/commands/autopilot.ts` — `gbrain autopilot --install`: self-maintaining brain daemon (sync+extract+embed). **v0.28.1:** consumes `detectTini()` from `src/core/minions/spawn-helpers.ts` and resolves it once at startup instead of per worker respawn (was paying an `execFileSync` cost on every restart).
|
||||
- `src/mcp/server.ts` — MCP stdio server (generated from operations). v0.22.7: tool-call handler delegates to `dispatchToolCall` from `src/mcp/dispatch.ts` so stdio + HTTP transports share one validation, context-build, and error-format path.
|
||||
- `src/mcp/dispatch.ts` (v0.22.7) — Shared tool-call dispatch consumed by both stdio (`server.ts`) and HTTP transports. Exports `dispatchToolCall(engine, name, params, opts)`, `buildOperationContext(engine, params, opts)`, and `validateParams(op, params)`. Single source of truth for `(ctx, params)` handler arg order and the 5-field `OperationContext` shape (engine + config + logger + dryRun + remote). Defaults to `remote: true` (untrusted); local CLI callers pass `remote: false`. Closed F1/F2/F3 drift bugs in the original v0.22.5 HTTP transport.
|
||||
- `src/mcp/dispatch.ts` (v0.22.7) — Shared tool-call dispatch consumed by both stdio (`server.ts`) and HTTP transports. Exports `dispatchToolCall(engine, name, params, opts)`, `buildOperationContext(engine, params, opts)`, and `validateParams(op, params)`. Single source of truth for `(ctx, params)` handler arg order and the 5-field `OperationContext` shape (engine + config + logger + dryRun + remote). Defaults to `remote: true` (untrusted); local CLI callers pass `remote: false`. Closed F1/F2/F3 drift bugs in the original v0.22.5 HTTP transport. **v0.26.9 (F8):** adds `summarizeMcpParams(opName, params)` — privacy-preserving redactor for `mcp_request_log` and the admin SSE feed. Returns `{redacted, kind, declared_keys, unknown_key_count, approx_bytes}`. Intersects submitted top-level keys against the operation's declared `params` allow-list (declared keys preserved as a sorted array for debug visibility; unknown keys counted but never named, closing the attacker-controlled-key-name leak). Byte counts bucketed up to nearest 1KB so an attacker can't binary-search secret-content sizes via repeated probes. Operators on a personal laptop who want raw payload visibility opt back in with `gbrain serve --http --log-full-params` (loud stderr warning at startup). Canonical helper — new logging code paths route through it rather than `JSON.stringify(params)`.
|
||||
- `src/mcp/rate-limit.ts` (v0.22.7) — Bounded-LRU token-bucket limiter. `buildDefaultLimiters()` returns the two-bucket pipeline: pre-auth IP (30/60s, fires BEFORE the DB lookup so brute-force load against `access_tokens` is actually capped) + post-auth token-id (60/60s). Tracks `lastTouchedMs` separately from `lastRefillMs` so an exhausted key can't be reset by hammering past the TTL. LRU cap bounds memory under attacker-controlled key growth.
|
||||
- `src/commands/serve-http.ts` (v0.26.0) — Express 5 HTTP MCP server with OAuth 2.1, admin dashboard, and SSE live activity feed. Started via `gbrain serve --http [--port N] [--token-ttl N] [--enable-dcr] [--public-url URL]`. Supersedes the v0.22.7 `src/mcp/http-transport.ts` simple bearer-auth path. Combines MCP SDK's `mcpAuthRouter` (authorize / token / register / revoke endpoints), a custom `client_credentials` handler (SDK's token endpoint throws `UnsupportedGrantTypeError` for CC; the custom handler runs BEFORE the router and falls through for `auth_code` / `refresh_token`), `requireBearerAuth` middleware for `/mcp` with scope enforcement before op dispatch, `localOnly` rejection, and `express-rate-limit` at 50 req / 15 min on `/token`. Serves the built admin SPA from `admin/dist/` with SPA fallback. `/admin/events` SSE endpoint broadcasts every MCP request to connected admin browsers. `cookie-parser` middleware wired (Express 5 has no built-in). Startup logging prints port, engine, configured issuer URL (honors `--public-url`), registered-client count, DCR status, and admin bootstrap token.
|
||||
- `src/core/oauth-provider.ts` (v0.26.0) — `GBrainOAuthProvider` implementing the MCP SDK's `OAuthServerProvider` + `OAuthRegisteredClientsStore` interfaces. Backed by raw SQL (works on both PGLite and Postgres — OAuth is infrastructure, not a BrainEngine concern). Full OAuth 2.1 spec: `authorize` + `exchangeAuthorizationCode` with PKCE (for ChatGPT), `client_credentials` (for Perplexity / Claude), `refresh_token` with rotation, `revokeToken`, `registerClient` (DCR path validates redirect_uri must be `https://` or loopback per RFC 6749 §3.1.2.1). All tokens + client secrets SHA-256 hashed before storage. Auth codes single-use with 10-minute TTL via atomic `DELETE...RETURNING` (closes RFC 6749 §10.5 TOCTOU race). Refresh rotation also `DELETE...RETURNING` (closes §10.4 stolen-token detection bypass). `pgArray()` escapes commas/quotes/braces in elements so a comma-bearing redirect_uri can't smuggle a second array element. Legacy `access_tokens` fallback in `verifyAccessToken` grandfathers pre-v0.26 bearer tokens as `read+write+admin`. `sweepExpiredTokens()` runs on startup wrapped in try/catch. **v0.26.2:** module-private `coerceTimestamp()` boundary helper at the top of the file normalizes postgres-driver-as-string BIGINT columns to JS numbers at every read site (5 call sites: `getClient` L112+L113 for DCR `/register` RFC 7591 §3.2.1 numeric timestamps, `exchangeRefreshToken` L274 + `verifyAccessToken` L296+L303 for the SDK's `typeof === 'number'` bearerAuth check). Throws on non-finite input (NaN/Infinity) so corrupt rows fail loud at the boundary instead of riding through as `expiresAt: NaN`; returns undefined for SQL NULL so callers decide NULL semantics explicitly (refresh + access token paths treat NULL as expired). Helper intentionally NOT promoted to `src/core/utils.ts` — codex review flagged repo-wide BIGINT precision-loss risk for a generic helper.
|
||||
- `src/commands/serve-http.ts` (v0.26.0) — Express 5 HTTP MCP server with OAuth 2.1, admin dashboard, and SSE live activity feed. Started via `gbrain serve --http [--port N] [--token-ttl N] [--enable-dcr] [--public-url URL] [--log-full-params]`. Supersedes the v0.22.7 `src/mcp/http-transport.ts` simple bearer-auth path. Combines MCP SDK's `mcpAuthRouter` (authorize / token / register / revoke endpoints), a custom `client_credentials` handler (SDK's token endpoint throws `UnsupportedGrantTypeError` for CC; the custom handler runs BEFORE the router and falls through for `auth_code` / `refresh_token`), `requireBearerAuth` middleware for `/mcp` with scope enforcement before op dispatch, `localOnly` rejection, and `express-rate-limit` at 50 req / 15 min on `/token`. Serves the built admin SPA from `admin/dist/` with SPA fallback. `/admin/events` SSE endpoint broadcasts every MCP request to connected admin browsers. `cookie-parser` middleware wired (Express 5 has no built-in). Startup logging prints port, engine, configured issuer URL (honors `--public-url`), registered-client count, DCR status, and admin bootstrap token. **v0.26.9 hardening pass:** F7 sets `remote: true` explicitly on the `/mcp` request handler's OperationContext literal (closes the HTTP shell-job RCE — without this, `submit_job`'s protected-name guard at `operations.ts:1391` saw a falsy undefined and skipped, letting a `read+write`-scoped OAuth token submit `shell` jobs). F8 wires `summarizeMcpParams` from `src/mcp/dispatch.ts` into both `mcp_request_log` writes and the admin SSE feed by default (raw payloads opt-in via `--log-full-params` with stderr warning). F9 sets cookie `Secure` flag when behind HTTPS or a public-URL proxy. F10 caps the magic-link nonce store with an LRU bound. F12 routes DCR disable through the `GBrainOAuthProvider` constructor's `dcrDisabled` option instead of the prior monkey-patch on the express router. F14 wraps `transport.handleRequest` in try/catch so SDK throws return a JSON-RPC 500 envelope instead of express's default HTML error page. F15 unifies OperationError + unexpected exceptions through `buildError` / `serializeError` so `/mcp` always returns the same envelope shape. **v0.28.1:** `/health` endpoint extracted into pure `probeHealth(engine)` async function with `HEALTH_TIMEOUT_MS = 3000` exported constant — drops the timeout from 5s to 3s so Fly.io's 5s health-check deadline gets 2s of headroom for TCP, response framing, and clock skew. Races `engine.getStats()` against the timeout via `Promise.race`; saturated pool returns 503 with `Health check timed out (database pool may be saturated)` instead of hanging. `clearTimeout` in finally block prevents pending-timer pile-up under high probe rates (race-leak fix from adversarial review).
|
||||
- `src/core/oauth-provider.ts` (v0.26.0) — `GBrainOAuthProvider` implementing the MCP SDK's `OAuthServerProvider` + `OAuthRegisteredClientsStore` interfaces. Backed by raw SQL (works on both PGLite and Postgres — OAuth is infrastructure, not a BrainEngine concern). Full OAuth 2.1 spec: `authorize` + `exchangeAuthorizationCode` with PKCE (for ChatGPT), `client_credentials` (for Perplexity / Claude), `refresh_token` with rotation, `revokeToken`, `registerClient` (DCR path validates redirect_uri must be `https://` or loopback per RFC 6749 §3.1.2.1). All tokens + client secrets SHA-256 hashed before storage. Auth codes single-use with 10-minute TTL via atomic `DELETE...RETURNING` (closes RFC 6749 §10.5 TOCTOU race). Refresh rotation also `DELETE...RETURNING` (closes §10.4 stolen-token detection bypass). `pgArray()` escapes commas/quotes/braces in elements so a comma-bearing redirect_uri can't smuggle a second array element. Legacy `access_tokens` fallback in `verifyAccessToken` grandfathers pre-v0.26 bearer tokens as `read+write+admin`. `sweepExpiredTokens()` runs on startup wrapped in try/catch. **v0.26.9 RFC 6749/7009 hardening pass:** F1+F2 fold `client_id` atomically into the `DELETE WHERE` clauses for both auth-code exchange and refresh rotation — pre-fix the post-hoc client compare burned the row on wrong-client paths so the legitimate client couldn't retry. F3 enforces refresh-scope-subset against the original grant on the row (RFC 6749 §6), not the client's currently-allowed scopes — fixes the case where revoking a scope from a client wouldn't shrink the agent's existing refresh tokens. F4 binds `client_id` on `revokeToken` so a client can only revoke its own tokens (RFC 7009 §2.1). F7c validates the `/token` request's `redirect_uri` against the value stored at `/authorize` (RFC 6749 §4.1.3) — empty-string treated as missing rather than wildcard match (adversarial-review fix). F5 swaps bare `catch {}` blocks in `verifyAccessToken` and `getClient` for `isUndefinedColumnError` from `src/core/utils.ts` — only SQLSTATE 42703 falls through to legacy fallback; lock timeouts and network blips throw and surface. F6 makes `sweepExpiredTokens()` actually return the count via `RETURNING 1` + array length, not a fire-and-forget zero. F12 adds `dcrDisabled` constructor option so `serve-http.ts` can disable the `/register` endpoint without monkey-patching the router. **v0.26.2:** module-private `coerceTimestamp()` boundary helper at the top of the file normalizes postgres-driver-as-string BIGINT columns to JS numbers at every read site (5 call sites: `getClient` L112+L113 for DCR `/register` RFC 7591 §3.2.1 numeric timestamps, `exchangeRefreshToken` L274 + `verifyAccessToken` L296+L303 for the SDK's `typeof === 'number'` bearerAuth check). Throws on non-finite input (NaN/Infinity) so corrupt rows fail loud at the boundary instead of riding through as `expiresAt: NaN`; returns undefined for SQL NULL so callers decide NULL semantics explicitly (refresh + access token paths treat NULL as expired). Helper intentionally NOT promoted to `src/core/utils.ts` — codex review flagged repo-wide BIGINT precision-loss risk for a generic helper.
|
||||
- `admin/` (v0.26.0) — React 19 + Vite + TypeScript admin SPA embedded in the binary via `admin/dist/` served by `serve-http.ts`. 7 screens: Login (bootstrap token → session cookie), Dashboard (metrics + SSE feed + token health), Agents (sortable table + sparklines + Register button), Register (modal with scope checkboxes + grant type selector), Credentials reveal (full-screen modal with Copy + Download JSON + yellow one-time-only warning), Request Log (filterable paginated), Agent Detail drawer (Details / Activity / Config Export tabs + Revoke). Design tokens: `#0a0a0f` bg, Inter for UI, JetBrains Mono for data, 4-32px spacing scale, rounded pill badges. HTTP-only SameSite=Strict cookie auth. 65KB gzip. Build: `cd admin && bun install && bun run build`; output at `admin/dist/` is committed for self-contained binaries.
|
||||
- `src/commands/auth.ts` — Token management. `gbrain auth create/list/revoke/test` for legacy bearer tokens (v0.22.7 wired as a first-class CLI subcommand) plus `gbrain auth register-client` (v0.26.0) and `gbrain auth revoke-client <client_id>` (v0.26.2) for OAuth 2.1 client lifecycle. `revoke-client` runs an atomic `DELETE...RETURNING` on `oauth_clients`; FK `ON DELETE CASCADE` on `oauth_tokens.client_id` and `oauth_codes.client_id` purges every active token + authorization code in a single transaction. `process.exit(1)` on no-such-client (idempotent — re-running on the same id produces the same exit-1 message). Legacy tokens stored as SHA-256 hashes in `access_tokens`; OAuth clients in `oauth_clients`. As of v0.26.0, legacy tokens grandfather to `read+write+admin` scopes on the OAuth HTTP server, so pre-v0.26 deployments keep working with no migration.
|
||||
- `src/commands/upgrade.ts` — Self-update CLI. `runPostUpgrade()` enumerates migrations from the TS registry (src/commands/migrations/index.ts) and tail-calls `runApplyMigrations(['--yes', '--non-interactive'])` so the mechanical side of every outstanding migration runs unconditionally.
|
||||
@@ -137,8 +148,8 @@ strict behavior when unset.
|
||||
- `src/commands/repair-jsonb.ts` — `gbrain repair-jsonb [--dry-run] [--json]`: rewrites `jsonb_typeof='string'` rows in place across 5 affected columns (pages.frontmatter, raw_data.data, ingest_log.pages_updated, files.metadata, page_versions.frontmatter). Fixes v0.12.0 double-encode bug on Postgres; PGLite no-ops. Idempotent.
|
||||
- `src/commands/orphans.ts` — `gbrain orphans [--json] [--count] [--include-pseudo]`: surfaces pages with zero inbound wikilinks, grouped by domain. Auto-generated/raw/pseudo pages filtered by default. Also exposed as `find_orphans` MCP operation. Shipped in v0.12.3 (contributed by @knee5).
|
||||
- `src/commands/integrity.ts` — `gbrain integrity check|auto|review|extract`: bare-tweet detection, dead-link detection, three-bucket repair (auto-repair / review-queue / skip). `scanIntegrity()` is the shared library function called from `gbrain doctor` (sampled at limit=500) and `cmdCheck` (full scan). v0.22.8: batch-load fast path on Postgres uses `SELECT DISTINCT ON (slug)` in a single SQL query to fix the PgBouncer round-trip timeout (60s → ~6s) while preserving `engine.getAllSlugs()`'s `Set<string>` semantics on multi-source brains. Gated by `engine.kind === 'postgres'` at the call site so PGLite never enters batch; fallback `catch` logs at `GBRAIN_DEBUG=1` so real Postgres errors are diagnosable.
|
||||
- `src/commands/doctor.ts` — `gbrain doctor [--json] [--fast] [--fix] [--dry-run] [--index-audit]`: health checks. v0.12.3 added `jsonb_integrity` + `markdown_body_completeness` reliability checks. v0.14.1: `--fix` delegates inlined cross-cutting rules to `> **Convention:** see [path](path).` callouts (pipes DRY violations into `src/core/dry-fix.ts`); `--fix --dry-run` previews without writing. v0.14.2: `schema_version` check fails loudly when `version=0` (migrations never ran — the #218 `bun install -g` signature) and routes users to `gbrain apply-migrations --yes`; new opt-in `--index-audit` flag (Postgres-only) reports zero-scan indexes from `pg_stat_user_indexes` (informational only, no auto-drop). v0.15.2: every DB check is wrapped in a progress phase; `markdown_body_completeness` runs under a 1s heartbeat timer so 10+ min scans are observable on 50K-page brains. v0.19.1 added `queue_health` (Postgres-only) with two subchecks: stalled-forever active jobs (started_at > 1h) and waiting-depth-per-name > threshold (default 10, override via `GBRAIN_QUEUE_WAITING_THRESHOLD`). Worker-heartbeat subcheck intentionally deferred to follow-up B7 because it needs a `minion_workers` table to produce ground-truth signal. Fix hints point at `gbrain repair-jsonb`, `gbrain sync --force`, `gbrain apply-migrations`, and `gbrain jobs get/cancel <id>`. v0.22.12 (#500): `sync_failures` check shows `[CODE=N, ...]` breakdown for both unacked entries (warn) and acked-historical entries (ok), surfacing systemic failure modes (`SLUG_MISMATCH=2685`) instead of a bare count.
|
||||
- `src/core/migrate.ts` — schema-migration runner. Owns the `MIGRATIONS` array (source of truth for schema DDL). v0.14.2 extended the `Migration` interface with `sqlFor?: { postgres?, pglite? }` (engine-specific SQL overrides `sql`) and `transaction?: boolean` (set to false for `CREATE INDEX CONCURRENTLY`, which Postgres refuses inside a transaction; ignored on PGLite since it has no concurrent writers). Migration v14 (fix wave) uses a handler branching on `engine.kind` to run CONCURRENTLY on Postgres (with a pre-drop of any invalid remnant via `pg_index.indisvalid`) and plain `CREATE INDEX` on PGLite. v15 bumps `minion_jobs.max_stalled` default 1→5 and backfills existing non-terminal rows. v0.22.6.1: migration v24 (`rls_backfill_missing_tables`) uses `sqlFor: { pglite: '' }` to no-op on PGLite — PGLite has no RLS engine and is single-tenant by definition, and the v24 ALTERs target subagent tables that don't exist in pglite-schema.ts. Closes #395 (contributed by @jdcastro2). **v30 (v0.23):** creates `dream_verdicts (file_path TEXT, content_hash TEXT, worth_processing BOOL, reasons JSONB, judged_at TIMESTAMPTZ, PK(file_path, content_hash))`. RLS-enabled when running as a BYPASSRLS role. The synthesize phase reads/writes this table to avoid re-judging on backfill re-runs.
|
||||
- `src/commands/doctor.ts` — `gbrain doctor [--json] [--fast] [--fix] [--dry-run] [--index-audit]`: health checks. v0.12.3 added `jsonb_integrity` + `markdown_body_completeness` reliability checks. v0.14.1: `--fix` delegates inlined cross-cutting rules to `> **Convention:** see [path](path).` callouts (pipes DRY violations into `src/core/dry-fix.ts`); `--fix --dry-run` previews without writing. v0.14.2: `schema_version` check fails loudly when `version=0` (migrations never ran — the #218 `bun install -g` signature) and routes users to `gbrain apply-migrations --yes`; new opt-in `--index-audit` flag (Postgres-only) reports zero-scan indexes from `pg_stat_user_indexes` (informational only, no auto-drop). v0.15.2: every DB check is wrapped in a progress phase; `markdown_body_completeness` runs under a 1s heartbeat timer so 10+ min scans are observable on 50K-page brains. v0.19.1 added `queue_health` (Postgres-only) with two subchecks: stalled-forever active jobs (started_at > 1h) and waiting-depth-per-name > threshold (default 10, override via `GBRAIN_QUEUE_WAITING_THRESHOLD`). Worker-heartbeat subcheck intentionally deferred to follow-up B7 because it needs a `minion_workers` table to produce ground-truth signal. Fix hints point at `gbrain repair-jsonb`, `gbrain sync --force`, `gbrain apply-migrations`, and `gbrain jobs get/cancel <id>`. v0.22.12 (#500): `sync_failures` check shows `[CODE=N, ...]` breakdown for both unacked entries (warn) and acked-historical entries (ok), surfacing systemic failure modes (`SLUG_MISMATCH=2685`) instead of a bare count. v0.26.7 (#612): `rls_event_trigger` check (post-install drift detector for migration v35's auto-RLS event trigger). Lives outside the `// 5. RLS` slice that the structural doctor.test.ts guards anchor on, so the existing test guards stay intact. Healthy `evtenabled` set is `('O','A')` only — `R` is replica-only and would not fire in normal sessions; `D` is disabled. Fix hint is `gbrain apply-migrations --force-retry 35`.
|
||||
- `src/core/migrate.ts` — schema-migration runner. Owns the `MIGRATIONS` array (source of truth for schema DDL). v0.14.2 extended the `Migration` interface with `sqlFor?: { postgres?, pglite? }` (engine-specific SQL overrides `sql`) and `transaction?: boolean` (set to false for `CREATE INDEX CONCURRENTLY`, which Postgres refuses inside a transaction; ignored on PGLite since it has no concurrent writers). Migration v14 (fix wave) uses a handler branching on `engine.kind` to run CONCURRENTLY on Postgres (with a pre-drop of any invalid remnant via `pg_index.indisvalid`) and plain `CREATE INDEX` on PGLite. v15 bumps `minion_jobs.max_stalled` default 1→5 and backfills existing non-terminal rows. v0.22.6.1: migration v24 (`rls_backfill_missing_tables`) uses `sqlFor: { pglite: '' }` to no-op on PGLite — PGLite has no RLS engine and is single-tenant by definition, and the v24 ALTERs target subagent tables that don't exist in pglite-schema.ts. Closes #395 (contributed by @jdcastro2). **v30 (v0.23):** creates `dream_verdicts (file_path TEXT, content_hash TEXT, worth_processing BOOL, reasons JSONB, judged_at TIMESTAMPTZ, PK(file_path, content_hash))`. RLS-enabled when running as a BYPASSRLS role. The synthesize phase reads/writes this table to avoid re-judging on backfill re-runs. **v35 (v0.26.7):** auto-RLS event trigger + one-time backfill. `auto_rls_on_create_table` fires on `ddl_command_end` for `WHEN TAG IN ('CREATE TABLE','CREATE TABLE AS','SELECT INTO')` and runs `ALTER TABLE … ENABLE ROW LEVEL SECURITY` on every new `public.*` table — no FORCE (matches v24/v29/schema.sql posture so non-BYPASSRLS apps can still read their own tables). The same migration backfills RLS on every existing `public.*` base table whose comment doesn't match the doctor regex (`^GBRAIN:RLS_EXEMPT\s+reason=\S.{3,}`). Per-table failure aborts the offending CREATE TABLE (event triggers fire inside the DDL transaction); no EXCEPTION wrap — that would convert loud rollback into silent permissive default. PGLite no-op via `sqlFor.pglite: ''`. Breaking change: operators with intentionally-RLS-off public tables must add the GBRAIN:RLS_EXEMPT comment BEFORE upgrade or the backfill will flip them on.
|
||||
- `src/core/progress.ts` — Shared bulk-action progress reporter. Writes to stderr. Modes: `auto` (TTY: `\r`-rewriting; non-TTY: plain lines), `human`, `json` (JSONL), `quiet`. Rate-gated by `minIntervalMs` and `minItems`. `startHeartbeat(reporter, note)` helper for single long queries. `child()` composes phase paths. Singleton SIGINT/SIGTERM coordinator emits `abort` events for every live phase. EPIPE defense on both sync throws and stream `'error'` events. Zero dependencies. Introduced in v0.15.2.
|
||||
- `src/core/cli-options.ts` — Global CLI flag parser. `parseGlobalFlags(argv)` returns `{cliOpts, rest}` with `--quiet` / `--progress-json` / `--progress-interval=<ms>` stripped. `getCliOptions()` / `setCliOptions()` expose a module-level singleton so commands reach the resolved flags without parameter threading. `cliOptsToProgressOptions()` maps to reporter options. `childGlobalFlags()` returns the flag suffix to append to `execSync('gbrain ...')` calls in migration orchestrators. `OperationContext.cliOpts` extends shared-op dispatch for MCP callers.
|
||||
- `src/core/db-lock.ts` (v0.22.13) — generic `tryAcquireDbLock(engine, lockId, ttlMinutes)` over the existing `gbrain_cycle_locks` table. Parameterized lock id so different scopes can nest cleanly: `gbrain-cycle` for the broad cycle (held by `cycle.ts`) and `gbrain-sync` (`SYNC_LOCK_ID` constant) for `performSync`'s narrower writer window. Same UPSERT-with-TTL semantics as the prior cycle-only helper, just generalized. Survives PgBouncer transaction pooling (unlike session-scoped `pg_try_advisory_lock`); crashed holders auto-release once their TTL expires.
|
||||
@@ -209,6 +220,8 @@ strict behavior when unset.
|
||||
- `src/commands/backlinks.ts` — Back-link checker and fixer (enforces Iron Law)
|
||||
- `src/commands/lint.ts` — Page quality linter (catches LLM artifacts, placeholder dates)
|
||||
- `src/commands/report.ts` — Structured report saver (audit trail for maintenance/enrichment)
|
||||
- `src/core/destructive-guard.ts` (v0.26.5) — three-layer protection against accidental data loss in gbrain. `assessDestructiveImpact(engine, sourceId)` counts pages/chunks/embeddings/files for a source. `checkDestructiveConfirmation(impact, opts)` is the fail-closed gate (`--confirm-destructive` required when data is present; `--yes` alone is rejected). `softDeleteSource` / `restoreSource` / `listArchivedSources` / `purgeExpiredSources` drive the source-level archive lifecycle via the column shape introduced in migration v34 (`sources.archived BOOLEAN`, `archived_at TIMESTAMPTZ`, `archive_expires_at TIMESTAMPTZ`). v0.26.5 added the page-level analog through `BrainEngine.softDeletePage` / `restorePage` / `purgeDeletedPages` plus `pages.deleted_at TIMESTAMPTZ` and a partial purge index. The MCP `delete_page` op rewires to `softDeletePage`; new ops `restore_page` (`scope: write`) and `purge_deleted_pages` (`scope: admin`, `localOnly: true`) round out the surface. Search visibility (`buildVisibilityClause` in `src/core/search/sql-ranking.ts`) hides soft-deleted pages and archived sources from `searchKeyword` / `searchKeywordChunks` / `searchVector` in both engines. The autopilot cycle's new 9th `purge` phase calls `purgeExpiredSources` + `engine.purgeDeletedPages(72)` so the 72h TTL is real, not honor-system.
|
||||
- `src/commands/pages.ts` (v0.26.5) — `gbrain pages purge-deleted [--older-than HOURS|Nd] [--dry-run] [--json]` operator escape hatch. Mirror of `gbrain sources purge` for the page-level lifecycle. Hard-deletes pages whose `deleted_at` is older than the cutoff; cascades to content_chunks/page_links/chunk_relations.
|
||||
- `openclaw.plugin.json` — ClawHub bundle plugin manifest
|
||||
|
||||
### BrainBench — in a sibling repo (v0.20+)
|
||||
@@ -244,10 +257,24 @@ Key commands added for Minions (job queue):
|
||||
- `gbrain jobs smoke [--sigkill-rescue]` — health smoke test. `--sigkill-rescue` is the v0.13.1 regression guard for #219: simulates a killed worker and asserts the stalled job is requeued instead of dead-lettered on first stall.
|
||||
- `gbrain jobs work [--queue Q] [--concurrency N]` — start worker daemon (Postgres only)
|
||||
|
||||
Key commands added in v0.26.5 (destructive-guard, end-to-end):
|
||||
- `gbrain sources archive <id>` — soft-delete a source. Hides from search via the new `sources.archived` column + cascading visibility filter. Preserves data for 72h. (PR #595 cherry-pick.)
|
||||
- `gbrain sources restore <id> [--no-federate]` — un-archive a soft-deleted source. Re-federates by default.
|
||||
- `gbrain sources archived [--json]` — list soft-deleted sources with their TTL.
|
||||
- `gbrain sources purge [<id>] [--confirm-destructive]` — permanent delete; with no id, purges all sources whose TTL expired.
|
||||
- `gbrain sources remove <id> [--confirm-destructive] [--dry-run]` — `--yes` alone no longer enough on populated sources. Boxed impact preview before destruction.
|
||||
- `gbrain pages purge-deleted [--older-than HOURS|Nd] [--dry-run] [--json]` — operator escape hatch for page-level soft-delete cleanup. Mirror of `gbrain sources purge`. The autopilot cycle's new `purge` phase calls the same library function automatically every run.
|
||||
- MCP `delete_page` op semantically shifts from hard-delete to soft-delete. New ops: `restore_page` (`scope: write`), `purge_deleted_pages` (`scope: admin`, `localOnly: true`).
|
||||
- `get_page` and `list_pages` extended with `include_deleted: boolean` (default false).
|
||||
- New autopilot cycle phase `purge` (9th, runs after `orphans`). `gbrain dream --phase purge` runs only the purge sweep.
|
||||
- Index strategy note: the partial index `pages_deleted_at_purge_idx ON pages (deleted_at) WHERE deleted_at IS NOT NULL` supports the autopilot purge query. Search filters (`WHERE deleted_at IS NULL`) do NOT need their own index — soft-deleted cardinality stays low and Postgres won't use the partial index for the negative predicate. Don't add a regular `(deleted_at)` index without measuring.
|
||||
- Schema migration v34 (`destructive_guard_columns`) adds `pages.deleted_at` + the partial purge index; promotes `archived` from `sources.config` JSONB to real columns; backfills any pre-v0.26.5 JSONB shape.
|
||||
|
||||
Key commands added in v0.25.0:
|
||||
- `gbrain eval export [--since DUR] [--limit N] [--tool query|search]` — stream captured `eval_candidates` rows as NDJSON to stdout. Every line starts with `"schema_version": 1` per the stable contract in `docs/eval-capture.md`. EPIPE-safe, progress heartbeats on stderr, deterministic ordering. Primary consumer is the sibling `gbrain-evals` repo for BrainBench-Real replay.
|
||||
- `gbrain eval prune --older-than DUR [--dry-run]` — explicit retention cleanup for `eval_candidates`. Requires `--older-than` (never deletes without a window). Duration strings: 30d, 7d, 1h, 90m, 3600s.
|
||||
- `gbrain eval replay --against FILE.ndjson [--limit N] [--top-regressions K] [--json] [--verbose]` — contributor-facing dev loop. Reads a captured NDJSON snapshot, re-runs each `query` / `search` op against the current brain, computes mean set-Jaccard@k between captured + current `retrieved_slugs`, top-1 stability rate, and latency Δ. JSON mode (`schema_version: 1`) for CI gating; human mode prints a regression table sorted worst-first. Closes the gap between "data captured" and "data used to gate a PR." See `docs/eval-bench.md` for the workflow.
|
||||
- `gbrain eval cross-modal --task "..." --output <path> [--cycles N] [--slot-a-model ID] [--slot-b-model ID] [--slot-c-model ID] [--receipt-dir DIR] [--json]` (v0.27.x) — multi-model quality gate. Three different-provider frontier models score the OUTPUT against the TASK on 5 documented dimensions. Pass criterion: every dim mean >=7 AND no model scored any dim <5. Exit codes: 0 PASS, 1 FAIL, 2 INCONCLUSIVE (<2/3 models returned parseable scores). Default cycles=3 in TTY, **cycles=1 in non-TTY** (limits accidental scripted bulk spend). Default slots: `openai:gpt-4o` / `anthropic:claude-opus-4-7` / `google:gemini-1.5-pro` — refresh alongside model-family bumps. Receipts land at `~/.gbrain/.gbrain/eval-receipts/<slug>-<sha8-of-output>.json` (gbrainPath honors GBRAIN_HOME). Bypasses `connectEngine()` via the cli.ts no-DB branch — runs cleanly before `gbrain init`. Reuses `src/core/ai/gateway.ts:chat()` for config/auth (no parallel provider stack). Cost-estimate prints to stderr before each cycle (T11=B partial cost guardrail; full `--budget-usd N` is a follow-up TODO).
|
||||
- `gbrain doctor` gains an `eval_capture` check: reads `eval_capture_failures` for the last 24h, groups by reason, warns when non-zero. Cross-process visibility (doctor runs in a separate process from MCP). Pre-v31 brains get `Skipped (table unavailable)` — non-fatal.
|
||||
- Config addition: `eval: { capture?: boolean, scrub_pii?: boolean }` in `~/.gbrain/config.json`. **File-plane only** — `gbrain config set` writes the DB plane and does NOT control capture.
|
||||
- **`GBRAIN_CONTRIBUTOR_MODE=1` env var** is the contributor-facing toggle. Capture is **off by default** as of v0.25.0; production users get a quiet brain. Resolution order: explicit `eval.capture` config wins both directions, then env var, then off. Documented in README.md, CONTRIBUTING.md, and `docs/eval-bench.md`.
|
||||
@@ -267,7 +294,7 @@ Key commands added in v0.14.2:
|
||||
- `gbrain doctor` gains two new checks: `sync_failures` (surfaces unacknowledged parse failures with exact paths + fix hints) and `brain_score` (renders the 5-component breakdown when score < 100: embed coverage / 35, link density / 25, timeline coverage / 15, orphans / 15, dead links / 10 — sum equals total).
|
||||
|
||||
Key commands added in v0.26.0 (OAuth 2.1 + HTTP server + admin dashboard):
|
||||
- `gbrain serve --http [--port 3131] [--token-ttl 3600] [--enable-dcr]` — HTTP MCP server with OAuth 2.1, admin dashboard at `/admin`, SSE activity feed at `/admin/events`, health check at `/health`. Prints admin bootstrap token on first start. Alongside (not replacing) stdio `gbrain serve`.
|
||||
- `gbrain serve --http [--port 3131] [--token-ttl 3600] [--enable-dcr] [--log-full-params]` — HTTP MCP server with OAuth 2.1, admin dashboard at `/admin`, SSE activity feed at `/admin/events`, health check at `/health`. Prints admin bootstrap token on first start. Alongside (not replacing) stdio `gbrain serve`. As of v0.26.9, `mcp_request_log.params` and the SSE feed default to a redacted summary (`{redacted, kind, declared_keys, unknown_key_count, approx_bytes}`); pass `--log-full-params` to log raw payloads on a personal laptop with a startup warning.
|
||||
- **OAuth client registration** — three paths:
|
||||
1. CLI: `gbrain auth register-client <name> --grant-types <types> --scopes <scopes>` (wired into `src/commands/auth.ts` as a thin wrapper over `GBrainOAuthProvider.registerClientManual`). Default grant types: `client_credentials`. Default scopes: `read`.
|
||||
2. Admin dashboard: Register client modal → credential reveal with Copy + Download JSON.
|
||||
@@ -297,6 +324,118 @@ Key commands added in v0.22.16 (claw-test friction loop):
|
||||
|
||||
## Testing
|
||||
|
||||
### Test command tiers (v0.26.4 — parallel fast loop)
|
||||
|
||||
Five tiers of test commands, each with a clear scope:
|
||||
|
||||
| Command | What it runs | Wallclock | When to use |
|
||||
|---|---|---|---|
|
||||
| `bun run test` | Parallel unit-test fast loop. 8-shard fan-out via `scripts/run-unit-parallel.sh`, then a serial pass over `*.serial.test.ts`. Excludes `*.slow.test.ts` and `test/e2e/*`. No pre-checks, no typecheck. | ~85s on a Mac dev box (3650+ tests) | Inner edit loop. Default. |
|
||||
| `bun run verify` | CI's authoritative pre-test gate set: `check:privacy && check:jsonb && check:progress && check:wasm && bun run typecheck`. The 4 checks `.github/workflows/test.yml` runs on shard 1 + typecheck. Single source of truth — CI literally calls `bun run verify`. | ~12s (wasm-compile dominates) | Before pushing; before `/ship`. |
|
||||
| `bun run test:full` | `verify && bun run test && bun run test:slow && [smart e2e]`. The local equivalent of "everything CI runs." Smart e2e: runs e2e only when `DATABASE_URL` is set; else loud skip notice to stderr. | ~3-5min depending on slow + e2e | Pre-merge sanity, before opening a PR. |
|
||||
| `bun run test:slow` | Just the `*.slow.test.ts` set (intentional cold-path correctness checks). | seconds-to-minutes | When touching slow-path code. |
|
||||
| `bun run test:serial` | Just the `*.serial.test.ts` set (cross-file-contention quarantine; runs at `--max-concurrency=1`). | ~1s per quarantined file | Debugging a specific quarantined file. |
|
||||
| `bun run test:e2e` | Real Postgres E2E. Requires Docker + `DATABASE_URL`. Sequential (template-DB parallelization is a v0.27+ TODO). | ~5-10min | Pre-ship; nightly. |
|
||||
| `bun run check:all` | All 7 historical pre-checks (privacy + jsonb + progress + no-legacy-getconnection + trailing-newline + wasm + exports-count). Superset of `verify`. | ~10s | Local-only sweep. The 4 not in `verify` are nice-to-haves. |
|
||||
|
||||
### CI vs local: intentionally divergent file sets
|
||||
|
||||
- **CI matrix** (`.github/workflows/test.yml`) runs `scripts/test-shard.sh` 4-way, which uses FNV-1a hash bucketing and INCLUDES `*.slow.test.ts`. CI is the ground truth for "did everything pass."
|
||||
- **Local fast loop** (`scripts/run-unit-shard.sh` via the parallel wrapper) uses round-robin-by-index sharding and EXCLUDES `*.slow.test.ts` AND `*.serial.test.ts`. Local trades coverage for inner-loop speed; CI catches what local skips.
|
||||
|
||||
This divergence is intentional. Don't try to make them equal — the two scripts deliberately solve different problems. The regression test at `test/scripts/run-unit-shard.test.ts` pins what the local fast loop should and shouldn't include.
|
||||
|
||||
### Failure-first logging
|
||||
|
||||
When `bun run test` finds any failure, the wrapper:
|
||||
|
||||
1. Writes failure blocks (each prefixed with `--- shard N: <test name> ---`) to `.context/test-failures.log` (workspace-local, gitignored). On systems without a writable `.context/`, falls back to `/tmp/gbrain-test-failures.log`.
|
||||
2. Prints a loud stderr banner with the absolute log path, plus the last 30 lines of the failure log inlined. Banner survives `| head` / `| tail` / agent-side log truncation.
|
||||
3. Writes a one-line-per-shard summary to `.context/test-summary.txt` (`shard N/M: pass=X fail=Y skip=Z rc=W`).
|
||||
4. Exits non-zero. Empty failure log + non-zero exit = infrastructure problem (wedged shard, killed child); the banner says so.
|
||||
|
||||
If a shard wedges (per-shard `GBRAIN_TEST_SHARD_TIMEOUT` cap, default 600s), the wrapper writes `--- shard N: WEDGED after ${SHARD_TIMEOUT}s ---` to the failure log, includes the last 50 lines of the shard log, and proceeds with other shards' results.
|
||||
|
||||
### File taxonomy
|
||||
|
||||
- `*.test.ts` → fast loop (parallel 8-shard fan-out).
|
||||
- `*.slow.test.ts` → run via `bun run test:slow` only (intentional cold-path tests; would dominate the fast loop's wallclock).
|
||||
- `*.serial.test.ts` → run via `bun run test:serial` after the parallel pass completes; uses `--max-concurrency=1`. Quarantine for tests that share file-wide state and race when run alongside other files in the same `bun test` process. Currently: `test/brain-registry.serial.test.ts`, `test/reconcile-links.serial.test.ts`, `test/core/cycle.serial.test.ts`, `test/embed.serial.test.ts` (the latter two added in v0.26.7 — they use `mock.module(...)` which leaks across files in the shard process). **Do not put the parallelism back on a serial file unless you've fixed the contention root cause** (it just re-introduces the flake).
|
||||
- `test/e2e/*.test.ts` → real-Postgres E2E. Skipped when `DATABASE_URL` is unset.
|
||||
|
||||
The intra-file parallelism project (turn `bun test` into `bun test --concurrent` after sweeping shared-state contention sites) is sliced across v0.26.7 (foundation), v0.26.8 (env-mutation sweep), and v0.26.9 (PGLite sweep + codemod + measurement). v0.26.4 ships file-level parallelism only.
|
||||
|
||||
### Test-isolation lint and helpers (v0.26.7)
|
||||
|
||||
The cross-file flake class is enforced statically by `scripts/check-test-isolation.sh`, wired into `bun run verify` and `bun run check:all`. Rules (non-serial unit files only; `*.serial.test.ts` and `test/e2e/*` are skipped):
|
||||
|
||||
| Rule | What it bans | Fix |
|
||||
|---|---|---|
|
||||
| **R1** | `process.env.X = ...`, bracket assignment, `delete process.env.X`, `Object.assign(process.env, ...)`, `Reflect.set(process.env, ...)` | Use `withEnv()` from `test/helpers/with-env.ts`, OR rename file to `*.serial.test.ts` |
|
||||
| **R2** | `mock.module(...)` anywhere in the file | Rename file to `*.serial.test.ts` (no DI on production code for testability) |
|
||||
| **R3** | `new PGLiteEngine(` outside ~50 lines after a `beforeAll(` line | Use the canonical block (below) inside `beforeAll(` |
|
||||
| **R4** | Files creating `new PGLiteEngine(` without `engine.disconnect(` inside an `afterAll(` block | Add `afterAll(() => engine.disconnect())` |
|
||||
|
||||
Files that violated these rules at the v0.26.7 baseline are listed in `scripts/check-test-isolation.allowlist`. **The allow-list MUST shrink over time** — never add new entries. v0.26.8 (env sweep) and v0.26.9 (PGLite sweep) remove entries as files get fixed.
|
||||
|
||||
#### Canonical PGLite block (R3 + R4 compliant)
|
||||
|
||||
Every test file that needs a PGLite engine should use this exact pattern:
|
||||
|
||||
```ts
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
});
|
||||
```
|
||||
|
||||
Why this exact shape: `beforeAll` creates a single engine per file (PGLite WASM cold-start + initSchema is ~20s); `beforeEach` truncates user data via `resetPgliteState` ("two orders of magnitude faster" than fresh-engine-per-test); `afterAll` disconnects so the engine doesn't leak across file boundaries within a shard process.
|
||||
|
||||
#### `withEnv` pattern (R1 fix)
|
||||
|
||||
```ts
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
|
||||
test('reads OPENAI_API_KEY', async () => {
|
||||
await withEnv({ OPENAI_API_KEY: 'sk-test' }, async () => {
|
||||
expect(loadConfig().openai_key).toBe('sk-test');
|
||||
});
|
||||
});
|
||||
|
||||
// Delete a var (override is undefined):
|
||||
await withEnv({ GBRAIN_HOME: undefined }, fn);
|
||||
|
||||
// Multiple keys:
|
||||
await withEnv({ A: '1', B: '2', C: undefined }, fn);
|
||||
```
|
||||
|
||||
`withEnv` saves the prior value of every key it touches and restores via try/finally — including when the callback throws. **It is cross-test safe but NOT intra-file concurrent-safe.** `process.env` is process-global; two `test.concurrent()` calls in the same file both touching the same key will race. Files using `withEnv` stay outside the future `test.concurrent()` codemod's eligibility filter.
|
||||
|
||||
#### When to quarantine instead of fix
|
||||
|
||||
Rename to `*.serial.test.ts` when:
|
||||
- The file uses `mock.module(...)` (R2 — there's no clean fix without changing production code).
|
||||
- The file is genuinely env-coupled (e.g. `gbrain-home-isolation.test.ts`, `claw-test-cli.test.ts`) — module-load env readers + ESM caching defeat dynamic-import-after-env tricks.
|
||||
- The file's tests intentionally share state across `it()` boundaries.
|
||||
|
||||
Quarantine count cap: 10 (informational). Beyond that, push back on the design.
|
||||
|
||||
### Inventory (legacy)
|
||||
|
||||
`bun test` runs all tests. After the v0.12.1 release: ~75 unit test files + 8 E2E test files (1412 unit pass, 119 E2E when `DATABASE_URL` is set — skip gracefully otherwise). Unit tests run
|
||||
without a database. E2E tests skip gracefully when `DATABASE_URL` is not set.
|
||||
|
||||
@@ -309,6 +448,7 @@ parity), `test/cli.test.ts` (CLI structure), `test/config.test.ts` (config redac
|
||||
`test/import-resume.test.ts` (import checkpoints), `test/migrate.test.ts` (migration; v8/v9 helper-btree-index SQL structural assertions + 1000-row wall-clock fixtures that guard the O(n²)→O(n log n) fix + v0.13.1 assertions on v12/v13 SQL shape, `sqlFor` + `transaction:false` runner semantics, the `max_stalled DEFAULT 1` regression guard, and v0.22.6.1 v24 `sqlFor.pglite: ''` no-op assertion),
|
||||
`test/bootstrap.test.ts` (v0.22.6.1 — bootstrap contract: no-op on fresh install, idempotent across two `initSchema()` calls, no-op on modern brain that already has every probed column, full bootstrap path on simulated pre-v0.18 brain, fresh-install regression guard, pre-v0.13 `links` shape coverage),
|
||||
`test/schema-bootstrap-coverage.test.ts` (v0.22.6.1 CI guard — `REQUIRED_BOOTSTRAP_COVERAGE` lists every forward reference in PGLITE_SCHEMA_SQL; the test fails loudly if `applyForwardReferenceBootstrap` skips one. When you add a column-with-index to the embedded schema blob, you extend both arrays or this guard fails. The pattern that broke gbrain ten times in two years is now structurally prevented.),
|
||||
`test/helpers/schema-diff.ts` + `test/helpers/schema-diff.test.ts` + `test/e2e/schema-drift.test.ts` (v0.26.6 #588 — cross-engine schema parity gate. Helper exports pure `snapshotSchema(query)` / `diffSnapshots(pg, pglite, opts)` / `formatDiffForFailure(diff)` / `isCleanDiff(diff)` over a four-tuple per column (`data_type`, `udt_name`, `is_nullable`, `column_default`). E2E test spins up fresh PGLite + Postgres, runs `engine.initSchema()` on each (bootstrap + schema replay + migrations), snapshots `information_schema.columns`, then diffs. 2-table allowlist (`files`, `file_migration_ledger`) — every other Postgres table must reach PGLite via PGLITE_SCHEMA_SQL or a migration's `sqlFor.pglite` branch. Sentinels for `oauth_clients`, `mcp_request_log`, `access_tokens`, `eval_candidates` give tighter blame messages. Skip-gracefully without `DATABASE_URL`. Wired into `scripts/e2e-test-map.ts` so changes to `src/schema.sql`, `src/core/pglite-schema.ts`, or `src/core/migrate.ts` trigger it. The failure message names every drift with a paste-ready hint pointing at `src/core/pglite-schema.ts`.),
|
||||
`test/setup-branching.test.ts` (setup flow), `test/slug-validation.test.ts` (slug validation),
|
||||
`test/storage.test.ts` (storage backends), `test/supabase-admin.test.ts` (Supabase admin),
|
||||
`test/yaml-lite.test.ts` (YAML parsing), `test/check-update.test.ts` (version check + update CLI),
|
||||
@@ -356,7 +496,9 @@ parity), `test/cli.test.ts` (CLI structure), `test/config.test.ts` (config redac
|
||||
`test/doctor.test.ts` (doctor command + v0.12.3 assertions that `jsonb_integrity` scans the four v0.12.0 write sites and `markdown_body_completeness` is present),
|
||||
`test/utils.test.ts` (shared SQL utilities + `tryParseEmbedding` null-return and single-warn semantics),
|
||||
`test/build-llms.test.ts` (llms.txt/llms-full.txt generator: path resolution, idempotence, spec shape, regen-drift guard, content contract, AGENTS.md install-path mirror, size-budget enforcement — 7 cases),
|
||||
`test/oauth.test.ts` (v0.26.0 OAuth 2.1 provider — 27 cases: register, getClient, `client_credentials` grant exchange, `authorization_code` flow with PKCE challenge / verifier, refresh token rotation, `verifyAccessToken` with both OAuth + legacy `access_tokens` fallback, `revokeToken`, `sweepExpiredTokens`, and a contract test asserting `scope` + `localOnly` annotations are set correctly on all 30 operations; **v0.26.2** adds 5 `coerceTimestamp` unit cases (null/undefined/string/number/throw-on-NaN), NULL-`expires_at`-as-expired contract tests for both refresh + access token paths, and a cascade-delete contract test asserting `revoke-client` purges `oauth_tokens` + `oauth_codes` rows via FK CASCADE),
|
||||
`test/oauth.test.ts` (v0.26.0 OAuth 2.1 provider — 27 cases: register, getClient, `client_credentials` grant exchange, `authorization_code` flow with PKCE challenge / verifier, refresh token rotation, `verifyAccessToken` with both OAuth + legacy `access_tokens` fallback, `revokeToken`, `sweepExpiredTokens`, and a contract test asserting `scope` + `localOnly` annotations are set correctly on all 30 operations; **v0.26.2** adds 5 `coerceTimestamp` unit cases (null/undefined/string/number/throw-on-NaN), NULL-`expires_at`-as-expired contract tests for both refresh + access token paths, and a cascade-delete contract test asserting `revoke-client` purges `oauth_tokens` + `oauth_codes` rows via FK CASCADE; **v0.26.9** adds 14 cases pinning the F1/F2/F3/F4/F5/F6/F7c/F12 invariants, including the F1/F4 cross-client isolation pattern (wrong-client attempt MUST reject AND rightful owner MUST still succeed atomically afterward) and the empty-string `redirect_uri` bypass guard surfaced during adversarial review),
|
||||
`test/mcp-dispatch-summarize.test.ts` (v0.26.9 — 7 cases pinning F8 `summarizeMcpParams` invariants: declared-keys allow-list intersection, attacker-key-name leak guard (unknown keys counted not named), 1KB byte bucketing for size-probe defense, missing op falls through to fully-redacted shape, declared-keys sorted for deterministic output),
|
||||
`test/trust-boundary-contract.test.ts` (v0.26.9 — 4 cases pinning F7b fail-closed semantics under cast bypass: `ctx.remote === undefined` treated as remote/untrusted at every flipped call site, `as any` and `Partial<>` spreads can't downgrade trust by accident),
|
||||
`test/check-resolvable-cli.test.ts` (v0.19 CLI wrapper: exit codes, JSON envelope shape, AGENTS.md fallback chain),
|
||||
`test/regression-v0_16_4.test.ts` (findRepoRoot regression guard — hermetic startDir parameterization),
|
||||
`test/filing-audit.test.ts` (v0.19 Check 6: `writes_pages` / `writes_to` frontmatter, filing-rules JSON validation),
|
||||
@@ -365,7 +507,8 @@ parity), `test/cli.test.ts` (CLI structure), `test/config.test.ts` (config redac
|
||||
`test/skillify-scaffold.test.ts` (v0.19 `gbrain skillify scaffold` stubs: SKILL.md, script, tests, routing-eval fixtures),
|
||||
`test/skillpack-install.test.ts` (v0.19 `gbrain skillpack install` managed-block install / update / no-clobber semantics),
|
||||
`test/skillpack-sync-guard.test.ts` (v0.19 sync-guard: bundled skills stay byte-identical to `skills/` source),
|
||||
`test/http-transport.test.ts` (v0.22.7 HTTP transport: 23 unit cases covering bearer auth + missing/no-Bearer/unknown/revoked + `/health` bypass, F1+F2 round-trip via dispatch.ts, F3 invalid_params, application/json response shape (not SSE), CORS default-deny + allowlist, body cap on Content-Length AND chunked, two-bucket rate limit (refill, exhaust+Retry-After, LRU eviction, TTL prune, pre-auth IP fires before DB), and `mcp_request_log` audit on success + auth_failed).
|
||||
`test/http-transport.test.ts` (v0.22.7 HTTP transport: 23 unit cases covering bearer auth + missing/no-Bearer/unknown/revoked + `/health` bypass, F1+F2 round-trip via dispatch.ts, F3 invalid_params, application/json response shape (not SSE), CORS default-deny + allowlist, body cap on Content-Length AND chunked, two-bucket rate limit (refill, exhaust+Retry-After, LRU eviction, TTL prune, pre-auth IP fires before DB), and `mcp_request_log` audit on success + auth_failed),
|
||||
`test/restart-sweep.test.ts` (v0.28.3 — 27 bun:test cases for the `recipes/restart-sweep.md` inlined script: sentinel-anchored fenced-block extraction with salted tmp filenames to bypass ESM cache; constructor-time env reads (proves no module-load snapshot); idempotency layer load/save/atomic-tmp-rename/corrupt-JSON-recovery/30-day-prune; `(sessionKey, lastAlertedAt)` cooldown gate with 6h threshold (the C1 fix that survives synthesized restartTime); AGGRESSIVE-gate two-state tests; execFile argv shape proving shell metachars in `OPENCLAW_TELEGRAM_GROUP` cannot reach `/bin/sh`; real-`\n`-not-literal alert formatting; `GBRAIN_HOME` state path override).
|
||||
|
||||
E2E tests (`test/e2e/`): Run against real Postgres+pgvector. Require `DATABASE_URL`.
|
||||
- `bun run test:e2e` runs Tier 1 (mechanical, all operations, no API keys). Includes 9 dedicated cases for the postgres-engine `addLinksBatch` / `addTimelineEntriesBatch` bind path — postgres-js's `unnest()` binding is structurally different from PGLite's and gets its own coverage.
|
||||
@@ -383,7 +526,7 @@ E2E tests (`test/e2e/`): Run against real Postgres+pgvector. Require `DATABASE_U
|
||||
- `test/e2e/engine-parity.test.ts` (v0.22.0) — Postgres ↔ PGLite top-result and result-set parity for `searchKeyword` + `searchVector`. Codex flagged that Postgres ranks pages then picks best chunk while PGLite returns chunks directly — without parity coverage the source-boost fix could pass on PGLite and fail on Postgres. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/postgres-bootstrap.test.ts` (v0.22.6.1) — exercises `PostgresEngine.initSchema()` directly against a fresh real Postgres database. Asserts the bootstrap path is no-op on fresh installs and that SCHEMA_SQL replays cleanly through the engine path (not via the standalone `db.initSchema` from `src/core/db.ts`, which would have produced false-positive coverage). Codex caught the E2E-shape gap during plan review.
|
||||
- `test/e2e/http-transport.test.ts` (v0.22.7) — 8 cases against real Postgres covering `gbrain serve --http` end-to-end: bearer auth round-trip, `last_used_at` SQL-level debounce semantics, `mcp_request_log` row insertion on success and auth_failed paths, `/health` DB-down → 503 (DB-probing health check), and the F1+F2+F3 dispatch round-trip with a real operation. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/serve-http-oauth.test.ts` (v0.26.0, expanded v0.26.2) — real-Postgres E2E against `gbrain serve --http` with full OAuth 2.1. Spawns a subprocess server, registers a client via the CLI, mints `client_credentials` tokens, exercises the `/mcp` JSON-RPC pipeline. **v0.26.2 adds:** real DCR `/register` HTTP-level response-shape test (asserts `typeof body.client_id_issued_at === 'number'` over the wire — RFC 7591 §3.2.1 spec compliance, not just internal-store shape); real CLI subprocess test for `revoke-client` (registers → mints token → revokes via `execSync` → asserts token rejected at `/mcp` → asserts re-run exits 1); server fixture flips on `--enable-dcr` so `/register` is reachable. **bun execSync env-inheritance fix:** bun's `execSync` does NOT inherit env mutations done via `process.env.X = ...`, only OS-level env from before bun started. helpers.ts loads `.env.testing` and sets `DATABASE_URL` via `process.env` mutation, which is invisible to subprocesses unless `env: { ...process.env }` is passed explicitly — every subprocess call in this file passes `env: { ...process.env }` for that reason. Reference fix for the next maintainer hitting the same failure mode in sibling sync/cycle/dream/claw-test E2Es. `afterAll` cleanup is guarded on `clientId` (won't throw if `beforeAll` failed before registration); cleanup errors surface to stderr without throwing so real test failures aren't masked. Tracks DCR-registered clients alongside the manual one. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/serve-http-oauth.test.ts` (v0.26.0, expanded v0.26.2, expanded v0.26.9) — real-Postgres E2E against `gbrain serve --http` with full OAuth 2.1. Spawns a subprocess server, registers a client via the CLI, mints `client_credentials` tokens, exercises the `/mcp` JSON-RPC pipeline. **v0.26.2 adds:** real DCR `/register` HTTP-level response-shape test (asserts `typeof body.client_id_issued_at === 'number'` over the wire — RFC 7591 §3.2.1 spec compliance, not just internal-store shape); real CLI subprocess test for `revoke-client` (registers → mints token → revokes via `execSync` → asserts token rejected at `/mcp` → asserts re-run exits 1); server fixture flips on `--enable-dcr` so `/register` is reachable. **bun execSync env-inheritance fix:** bun's `execSync` does NOT inherit env mutations done via `process.env.X = ...`, only OS-level env from before bun started. helpers.ts loads `.env.testing` and sets `DATABASE_URL` via `process.env` mutation, which is invisible to subprocesses unless `env: { ...process.env }` is passed explicitly — every subprocess call in this file passes `env: { ...process.env }` for that reason. Reference fix for the next maintainer hitting the same failure mode in sibling sync/cycle/dream/claw-test E2Es. `afterAll` cleanup is guarded on `clientId` (won't throw if `beforeAll` failed before registration); cleanup errors surface to stderr without throwing so real test failures aren't masked. Tracks DCR-registered clients alongside the manual one. **v0.26.9** adds 2 regressions for the F7 trust-boundary fix: an HTTP MCP `submit_job` for `name: "shell"` MUST reject with a permission error (proving the request handler now sets `remote: true` and `submit_job`'s protected-name guard fires), and the same guard rejects subagent submission. Closes the OAuth-token-to-RCE escalation path. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/sync-parallel.test.ts` (v0.22.13 PR #490) — DATABASE_URL-gated. T2: 60-file Postgres sync at concurrency=4 imports all + no connection leak (probes `pg_stat_activity` before/after to confirm worker engines disconnected). P4: 120-file serial-vs-parallel benchmark prints `SYNC_PARALLEL_BENCH N files | serial=Xms | parallel(4)=Yms | speedup=Zx` for CHANGELOG quoting. Asserts parallel ≤ serial × 1.5 (CI-noise tolerant; not a strict speedup gate).
|
||||
- Tier 2 (`skills.test.ts`) requires OpenClaw + API keys, runs nightly in CI
|
||||
- If `.env.testing` doesn't exist in this directory, check sibling worktrees for one:
|
||||
|
||||
+80
-12
@@ -52,14 +52,22 @@ docs/ Architecture docs
|
||||
## Running tests
|
||||
|
||||
```bash
|
||||
# Recommended: full CI guard chain + tests (matches what CI runs)
|
||||
bun run test # privacy + jsonb + progress + wasm + typecheck + bun test
|
||||
|
||||
# Just the test runner (skips CI guards)
|
||||
bun test # all tests (unit + E2E skipped without DB)
|
||||
# Inner edit loop (~85s on a Mac dev box, 3700+ unit tests)
|
||||
bun run test # parallel 8-shard fan-out + serial post-pass
|
||||
bun test test/markdown.test.ts # specific unit test
|
||||
|
||||
# E2E tests (requires Postgres with pgvector)
|
||||
# Pre-push gate (matches what CI runs on shard 1 + typecheck)
|
||||
bun run verify # privacy + jsonb + progress + test-isolation + wasm + admin-build + typecheck
|
||||
|
||||
# Pre-merge sanity (everything CI runs)
|
||||
bun run test:full # verify + parallel unit + slow + smart e2e
|
||||
|
||||
# Slow / serial / e2e in isolation
|
||||
bun run test:slow # *.slow.test.ts only (cold-path correctness)
|
||||
bun run test:serial # *.serial.test.ts only (--max-concurrency=1)
|
||||
bun run test:e2e # real-Postgres E2E (requires DATABASE_URL)
|
||||
|
||||
# E2E setup (Postgres with pgvector)
|
||||
docker compose -f docker-compose.test.yml up -d
|
||||
DATABASE_URL=postgresql://postgres:postgres@localhost:5434/gbrain_test bun run test:e2e
|
||||
|
||||
@@ -67,12 +75,72 @@ DATABASE_URL=postgresql://postgres:postgres@localhost:5434/gbrain_test bun run t
|
||||
DATABASE_URL=postgresql://... bun run test:e2e
|
||||
```
|
||||
|
||||
Use `bun run test` before pushing. The guard chain catches: banned fork-name leaks
|
||||
(`scripts/check-privacy.sh`), `JSON.stringify(x)::jsonb` interpolation patterns
|
||||
(`scripts/check-jsonb-pattern.sh`), `\r` progress bleed to stdout
|
||||
(`scripts/check-progress-to-stdout.sh`), trailing-newline drift across tracked
|
||||
files (`scripts/check-trailing-newline.sh`), and silent fallback to recursive
|
||||
chunking in the compiled binary (`scripts/check-wasm-embedded.sh`).
|
||||
Use `bun run verify` before pushing. The guard chain catches: banned fork-name
|
||||
leaks (`scripts/check-privacy.sh`), `JSON.stringify(x)::jsonb` interpolation
|
||||
patterns (`scripts/check-jsonb-pattern.sh`), `\r` progress bleed to stdout
|
||||
(`scripts/check-progress-to-stdout.sh`), test-isolation rule violations
|
||||
(`scripts/check-test-isolation.sh` — see "Writing tests that survive the parallel
|
||||
loop" below), silent fallback to recursive chunking in the compiled binary
|
||||
(`scripts/check-wasm-embedded.sh`), and stale admin-dashboard build artifacts
|
||||
(`scripts/check-admin-build.sh`). `bun run check:all` runs the full historical
|
||||
sweep including the trailing-newline and exports-count checks.
|
||||
|
||||
### Writing tests that survive the parallel loop
|
||||
|
||||
`bun run test` shards 92+ unit-test files across 8 worker processes. Files in the
|
||||
same shard share a process, so process-global state leaks between them. Four
|
||||
lint rules (`scripts/check-test-isolation.sh`, R1-R4) enforce isolation:
|
||||
|
||||
| Rule | What it bans | Fix |
|
||||
|---|---|---|
|
||||
| **R1** | Direct `process.env.X = ...` mutation | Use `withEnv()` from `test/helpers/with-env.ts`, or rename to `*.serial.test.ts` |
|
||||
| **R2** | `mock.module(...)` anywhere in the file | Rename to `*.serial.test.ts` |
|
||||
| **R3** | `new PGLiteEngine(` outside ~50 lines after `beforeAll(` | Use the canonical PGLite block (see below) |
|
||||
| **R4** | `new PGLiteEngine(` without paired `afterAll(disconnect)` | Add the `afterAll(() => engine.disconnect())` |
|
||||
|
||||
Canonical PGLite block (R3 + R4 compliant — paste this verbatim):
|
||||
|
||||
```ts
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
afterAll(async () => { await engine.disconnect(); });
|
||||
beforeEach(async () => { await resetPgliteState(engine); });
|
||||
```
|
||||
|
||||
Env-touching tests:
|
||||
|
||||
```ts
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
|
||||
test('reads OPENAI_API_KEY', async () => {
|
||||
await withEnv({ OPENAI_API_KEY: 'sk-test' }, async () => {
|
||||
expect(loadConfig().openai_key).toBe('sk-test');
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
`withEnv` saves and restores keys via try/finally including when the callback
|
||||
throws. Cross-test safe; **NOT** intra-file concurrent-safe (`process.env` is
|
||||
process-global). Files using `withEnv` stay outside the future
|
||||
`test.concurrent()` codemod's eligibility filter.
|
||||
|
||||
When to quarantine instead of fix: rename to `*.serial.test.ts` if the file
|
||||
uses `mock.module(...)`, is genuinely env-coupled (module-load env readers +
|
||||
ESM caching defeat dynamic-import-after-env tricks), or intentionally shares
|
||||
state across `it()` boundaries. Quarantine count cap: 10 (informational).
|
||||
|
||||
Files that violated these rules at the v0.26.7 baseline are listed in
|
||||
`scripts/check-test-isolation.allowlist`. **The allow-list MUST shrink over
|
||||
time** ... never add new entries. v0.26.8 (env sweep) and v0.26.9 (PGLite sweep
|
||||
+ codemod) remove entries as files get fixed.
|
||||
|
||||
### Local CI gate (recommended before pushing, v0.23.1+)
|
||||
|
||||
|
||||
@@ -51,6 +51,14 @@ postinstall hook on global installs, so schema migrations never run and the CLI
|
||||
aborts with `Aborted()` the first time it opens PGLite. Use `git clone + bun install
|
||||
&& bun link` as shown above. See [#218](https://github.com/garrytan/gbrain/issues/218).
|
||||
|
||||
**Do NOT use `bun add -g gbrain` or `npm install -g gbrain`.** The npm registry
|
||||
has an unrelated package squatting that name (`gbrain@1.3.x`) — you'd silently
|
||||
install the wrong binary and overwrite the canonical one. v0.28.5+ detects this
|
||||
and prints a recovery message on `gbrain upgrade`, but the `git clone + bun link`
|
||||
path above is the only reliable install method until we publish under
|
||||
`@garrytan/gbrain` (tracked v0.29 follow-up). See
|
||||
[#658](https://github.com/garrytan/gbrain/issues/658).
|
||||
|
||||
```
|
||||
3 results (hybrid search, 0.12s):
|
||||
|
||||
@@ -439,6 +447,7 @@ GBrain ships integration recipes that your agent sets up for you. Each recipe te
|
||||
| [X-to-Brain](recipes/x-to-brain.md) | — | Twitter timeline + mentions + deletions |
|
||||
| [Calendar-to-Brain](recipes/calendar-to-brain.md) | credential-gateway | Google Calendar to searchable daily pages |
|
||||
| [Meeting Sync](recipes/meeting-sync.md) | — | Circleback transcripts to brain pages with attendees |
|
||||
| [Restart Sweep](recipes/restart-sweep.md) | OpenClaw + Telegram | Detect dropped Telegram messages after OpenClaw gateway restarts |
|
||||
|
||||
**Data research recipes** extract structured data from email into tracked brain pages. Built-in recipes for investor updates (MRR, ARR, runway, headcount), expense tracking, and company metrics. Create your own with `gbrain research init`.
|
||||
|
||||
@@ -729,7 +738,7 @@ ADMIN
|
||||
gbrain serve MCP server (stdio)
|
||||
gbrain serve --http [--port 3131] HTTP MCP server with OAuth 2.1 + admin dashboard
|
||||
[--token-ttl 3600] [--enable-dcr]
|
||||
[--public-url URL]
|
||||
[--public-url URL] [--log-full-params]
|
||||
gbrain auth create|list|revoke|test Legacy bearer token management
|
||||
gbrain auth register-client <name> Register an OAuth 2.1 client
|
||||
--grant-types client_credentials,authorization_code
|
||||
@@ -740,6 +749,11 @@ ADMIN
|
||||
# programmatically via oauthProvider.registerClientManual() for host-repo wrappers.
|
||||
gbrain integrations Integration recipe dashboard
|
||||
gbrain sources list|add|remove|... Multi-source brain management (v0.18)
|
||||
v0.28.2: --url <https://...> registers a federated
|
||||
remote git repo; clone is auto-managed under
|
||||
$GBRAIN_HOME/clones/<id>/ and re-cloned on sync if
|
||||
it goes missing. Also exposed via MCP for remote
|
||||
agent setup (whoami + sources_{add,list,remove,status}).
|
||||
gbrain dream [--dry-run] [--phase N] 8-phase maintenance cycle (lint→backlinks→sync→synthesize
|
||||
→extract→patterns→embed→orphans). v0.23 added synthesize +
|
||||
patterns: transcripts → reflections + cross-session themes.
|
||||
@@ -787,7 +801,7 @@ The skills in this repo are those patterns, generalized. What took 11 days to bu
|
||||
|
||||
## Contributing
|
||||
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md). Run `bun test` for unit tests. For the full local CI gate (gitleaks + unit + all 29 E2E files in Docker, the same checks GH Actions runs), use `bun run ci:local` ... or `bun run ci:local:diff` for the diff-aware subset during fast iteration.
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md). Run `bun run test` for the parallel unit-test fast loop (~85s on a Mac dev box, 3700+ tests) or `bun run verify` for the pre-push gate (privacy + jsonb + progress + test-isolation + wasm + admin-build + typecheck). For the full local CI gate (gitleaks + unit + all 29 E2E files in Docker, the same checks GH Actions runs), use `bun run ci:local` ... or `bun run ci:local:diff` for the diff-aware subset during fast iteration.
|
||||
|
||||
If you're working on retrieval or any of the search/embedding/ranking surface, set `GBRAIN_CONTRIBUTOR_MODE=1` in your shell rc and use `gbrain eval replay` to gate your changes against a snapshot of real captured queries — the dev loop is documented in [`docs/eval-bench.md`](docs/eval-bench.md). Capture is **off by default** for production users (no surprise data accumulation); the env var is the contributor opt-in.
|
||||
|
||||
|
||||
+11
@@ -166,3 +166,14 @@ psql "$DATABASE_URL" -c \
|
||||
`body_too_large`, `parse_error`, `unknown_method`. Failed-auth rows have
|
||||
`token_name = NULL`. Inserts are fire-and-forget so audit failures
|
||||
never block requests.
|
||||
|
||||
**v0.26.9 redaction default.** The `params` column now stores
|
||||
`{redacted, kind, declared_keys, unknown_key_count, approx_bytes}` instead
|
||||
of raw JSON-RPC payloads. Declared keys (intersected against the operation's
|
||||
spec) preserve for debug visibility; unknown keys are counted but never
|
||||
named so attackers can't probe key existence; byte sizes bucket to 1KB so
|
||||
content sizes can't be binary-searched. The same shape is broadcast on the
|
||||
admin SSE feed at `/admin/events`. Operators on a personal laptop who want
|
||||
raw payloads back can pass `gbrain serve --http --log-full-params` (loud
|
||||
stderr warning at startup). Multi-tenant deployments should leave it
|
||||
on the redacted default.
|
||||
|
||||
@@ -1,5 +1,296 @@
|
||||
# TODOS
|
||||
|
||||
## cross-modal-eval (v0.27.x follow-ups from PR #674 plan)
|
||||
|
||||
### `--budget-usd` hard cap + per-call cost telemetry (T11=B follow-up)
|
||||
**Priority:** P2
|
||||
|
||||
**What:** `gbrain eval cross-modal` ships in v0.27.x with a partial cost guardrail: default `--cycles 1` in non-TTY plus a stderr cost-estimate printed before each run. The full `--budget-usd N` hard cap (refuse to start the next cycle if estimated spend would exceed) and per-call actual-cost telemetry written into the receipt are intentionally deferred.
|
||||
|
||||
**Why:** Codex pushback on the original P2=B "defer everything" decision was right — even with `>=2/3` success required for a verdict (Q3=A), 3 cycles × 3 calls = 9 frontier calls per run, repeated across N skills if anyone scripts a bulk audit. The TTY/non-TTY cycle default catches the worst case; the hard cap catches the next class of mistakes.
|
||||
|
||||
**Pros:** Deterministic spend ceiling. Real per-call cost in the receipt drives a feedback loop that lets us refine the price-table constant in `src/core/cross-modal-eval/runner.ts:estimateCost`. Future bulk-audit integrations get a safety net by default.
|
||||
**Cons:** ~80 lines of pricing-table + parsing + threading. Pricing values drift; the file becomes a small maintenance burden between model-family bumps.
|
||||
**Context:** Pricing table lives at `src/core/cross-modal-eval/runner.ts:estimateCost`. Once we have real telemetry from a few weeks of usage, we can switch the table to "last observed" instead of "list price" and get more accurate caps. v0.27.x candidate.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
### Subagent integration (recovers cross-process rate-leases — T4 deferred)
|
||||
**Priority:** P2
|
||||
|
||||
**What:** Wire `gbrain eval cross-modal` to be invokable as a `gbrain agent run` child job. Today the CLI runs synchronously and bypasses `src/core/minions/rate-leases.ts` because the lease helper requires a `minion_jobs.id` that the CLI path doesn't have (T4=A in plans/radiant-napping-lerdorf.md).
|
||||
|
||||
**Why:** Cross-process concurrency cap. A user running `gbrain eval cross-modal` in one terminal alongside `gbrain agent run` in another can hit Anthropic 429s due to combined load. As a minion job, the eval gets the rate-lease behavior for free, plus stagger / quiet-hours / retry surface from the existing Minions queue.
|
||||
|
||||
**Pros:** No new helper API; reuses what's already there. Closes the cross-process gap that today's `Promise.allSettled` design intentionally leaves open.
|
||||
**Cons:** Requires a job handler registration + receipt-path threading through job context. Probably ~150 lines plus tests. Behavior parity (verdict / receipt shape) needs to be pinned with a parametrized test.
|
||||
**Context:** Pattern is the same as `src/core/minions/handlers/subagent.ts`. v0.27.x candidate.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
### Skill adoption telemetry (revisit T7=C with data)
|
||||
**Priority:** P3
|
||||
|
||||
**What:** Track how many skills land cross-modal eval receipts. If adoption stalls at, say, <30% of skills after 30 days, consider flipping the 11th item from `required:false` (T7=C, current) to `required:true` (T7=A) in v0.28.x.
|
||||
|
||||
**Why:** T7=C ships the gate as informational so existing audits don't regress. The forcing function is documentation alone. We don't yet know if that's enough.
|
||||
|
||||
**Pros:** Data-driven decision instead of guessing. Lightweight: count receipt files in `gbrainPath('eval-receipts')` against the count of skills under `skills/*/SKILL.md`.
|
||||
**Cons:** "Adoption stalled" is a judgment call without a baseline. Could become a debate.
|
||||
**Context:** New check in `gbrain doctor` would surface the count. v0.28.x candidate.
|
||||
**Depends on:** None.
|
||||
|
||||
### `docs/cross-modal-eval.md` user guide
|
||||
**Priority:** P3
|
||||
|
||||
**What:** Add a user-facing guide. Cover the gateway-config flow, receipt forensics, the `<slug>-<sha8>.json` filename convention, default models + how to override them, the relationship to `skills/cross-modal-review/SKILL.md`, and worked examples on a real skill.
|
||||
|
||||
**Why:** SKILL.md teaches the workflow but lives under `skills/skillify/`. CLAUDE.md "Key files" entries are agent-facing, not human-facing. A `docs/cross-modal-eval.md` is the natural home for "I'm a user, how do I use this command?" answers.
|
||||
|
||||
**Pros:** Discoverable from CLAUDE.md "Key files" reference. Mirrors `docs/eval-bench.md` precedent.
|
||||
**Cons:** Doc-write task; ~250 lines of prose.
|
||||
**Context:** v0.27.x candidate.
|
||||
**Depends on:** None.
|
||||
|
||||
## /health endpoint hardening (v0.28.1 follow-up)
|
||||
|
||||
### Cancel `engine.getStats()` when /health times out
|
||||
**Priority:** P2
|
||||
|
||||
**What:** `probeHealth()` in `src/commands/serve-http.ts` races `engine.getStats()` against a 3s timeout. When the timeout wins, the original `getStats()` keeps running on a saturated pool. Under sustained probe traffic with a slow DB, timed-out probes pile up expensive `count(*)` queries that turn a partial slowdown into a total outage.
|
||||
|
||||
**Why:** Both adversarial reviewers (Claude + Codex) flagged this independently during the v0.28.1 ship. Deferred because cancellation requires `AbortController` plumbing through `BrainEngine.getStats()` which doesn't exist yet — wider blast radius than v0.28.1's zombie-reaping scope justified.
|
||||
|
||||
**Pros:** Closes the self-DoS path. /health returning 503 stops contributing to pool saturation.
|
||||
**Cons:** Touches the BrainEngine interface (PostgresEngine + PGLiteEngine implementations). Needs postgres.js or PgBouncer-level query cancellation. Wider blast radius.
|
||||
**Context:** Drop-in replacement for `Promise.race([getStats(), timeout])` is `getStats({ signal })` consumed via AbortController. Reviewer findings: see PR #637 (v0.28.1) adversarial review section.
|
||||
**Depends on:** AbortController plumbing in BrainEngine interface.
|
||||
|
||||
### Replace `/health` with a lighter liveness probe
|
||||
**Priority:** P3
|
||||
|
||||
**What:** `engine.getStats()` does `count(*) FROM pages, content_chunks, links, tags, timeline_entries` plus `GROUP BY type`. On a large but otherwise healthy brain, this can normally exceed 3s and cause false-positive 503s + orchestrator restart loops.
|
||||
|
||||
**Why:** Codex flagged that the new 3s timeout is aggressive for the cost of the probe. Pre-existing behavior (the /health endpoint was already doing full stats in v0.27 with no timeout). Worth splitting probe purpose: `/health` for liveness (`SELECT 1`), `/stats` for the full counts.
|
||||
|
||||
**Pros:** Liveness probe stays under 100ms even on saturated pools. Operators get a separate `/stats` for the count breakdown when they actually want it.
|
||||
**Cons:** Behavior change for orchestrator setups that scrape /health as both liveness AND count source.
|
||||
**Context:** PR #637 (v0.28.1) adversarial review. Pair with the AbortController follow-up above.
|
||||
## Remote-source MCP follow-ups (v0.28.2)
|
||||
|
||||
### Token rotation: `gbrain auth rotate <name>` + `rotate_token` MCP op
|
||||
**Priority:** P2
|
||||
|
||||
**What:** Atomic rotate for legacy + OAuth tokens. Issue a new token in the same TX as the revocation of the old, no overlap window. Refresh-token rotation already exists for OAuth; this is the unified user-facing surface (CLI + MCP).
|
||||
|
||||
**Why:** Today rotation is `revoke + create`, with a window where neither token works. For long-lived bearer keys handed to agents, that's a reload outage every time the key gets rotated.
|
||||
|
||||
**Pros:** Single command does the right thing. Atomic cutover. Operators stop scripting around the gap.
|
||||
**Cons:** Needs careful testing of the legacy `access_tokens` UPDATE path (returns single-use new token before the row mutates) plus an MCP op that grants a new token bound to the original client_id without requiring a new authorize round trip.
|
||||
**Context:** Item 4 from the gstack /setup-gbrain v1.28.1.0 enhancement request. v0.28.x candidate.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
### Migration introspection in `get_health`
|
||||
**Priority:** P3
|
||||
|
||||
**What:** Extend `BrainEngine.getHealth()` return shape with `migrations: { pending: [...], wedged: [...] }`. `gbrain doctor` already shows this; expose it via the MCP op so remote agents can detect partial-migration state without invoking `doctor` separately.
|
||||
|
||||
**Why:** Closes a remote-diagnostic gap. gstack /setup-gbrain Path 4 hit a wedged-migration brain mid-session; the only readback was SSH + `gbrain doctor`. With this, the same diagnostic flows through MCP.
|
||||
|
||||
**Pros:** Pure additive change to the `get_health` op shape. No new op surface. Consumers ignore the new field if they don't care.
|
||||
**Cons:** Wedged detection logic lives in `gbrain doctor`'s code today; need to extract or duplicate. Care needed not to leak migration internals to non-admin scopes (current op is admin-only — fine).
|
||||
**Context:** Item 5 from the gstack /setup-gbrain v1.28.1.0 enhancement request.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
### Accept-header friendliness on `/mcp`
|
||||
**Priority:** P3
|
||||
|
||||
**What:** MCP SDK rejects requests missing `text/event-stream` in the Accept header with a generic 406 Not Acceptable. Pre-check the header at the express middleware layer and return a 400 with a descriptive hint pointing at the spec.
|
||||
|
||||
**Why:** Other MCP clients (curl scripts, custom integrations) hit the SDK's 406 and get no diagnostic. gstack's verify-helper sets both headers correctly so the headline path works.
|
||||
|
||||
**Pros:** Operator UX improvement. Faster debugging when clients fail discovery.
|
||||
**Cons:** Tight coupling to the SDK behavior — if it later loosens, the pre-check becomes redundant.
|
||||
**Context:** Item 6 from the gstack /setup-gbrain v1.28.1.0 enhancement request.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
### `gbrain sources rebase-clone <id>`
|
||||
**Priority:** P3
|
||||
|
||||
**What:** Recover from `url-drift` (config.remote_url updated but the on-disk clone still points at the old origin). Currently `sync` refuses with a structured error pointing at this command — but the command itself doesn't exist yet. Implement: prompt for confirmation (rm-rf the clone is destructive), then re-clone via the same temp-dir + rename atomicity contract as `sources add --url`.
|
||||
|
||||
**Why:** Closes the loop on the URL-drift code path the v0.28.2 sync added. Without it, operators have to `sources remove --confirm-destructive` + `sources add --url` (loses page count, history).
|
||||
|
||||
**Pros:** Cleaner UX for URL changes. Preserves the source row + history.
|
||||
**Cons:** Destructive on-disk; needs `--confirm-destructive` gate. Edge case: what if sync is mid-run when rebase fires? The existing sync-lock guards this, but worth pinning in tests.
|
||||
**Context:** v0.28.2 plan filed this explicitly as a follow-up.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
### `--filter=blob:none` partial-clone option for federated sources
|
||||
**Priority:** P3
|
||||
|
||||
**What:** v0.28.2 defaults `gbrain sources add --url` to `--depth=1` (no history). For users who want commit-aware features later (page-state-at-commit-X, blame, who-edited-what), expose `--filter=blob:none` as an opt-in: keeps full graph metadata, lazy-fetches blobs.
|
||||
|
||||
**Why:** `--depth=1` is a one-way door — once cloned, you can't reconstruct history without re-cloning the whole repo. Partial clones preserve history while staying small.
|
||||
|
||||
**Pros:** Forward-compat for commit-aware brain features. Negligible cost on first clone for typical brain repos. Better than the alternative (full clones for everyone).
|
||||
**Cons:** First-clone latency is higher on long-history repos. Adds one more flag to the `add` surface.
|
||||
**Context:** Eng review A5 — the boring choice for v0.28.2 was `--depth=1`. This is the unboring follow-up.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
### DNS rebinding defense for `parseRemoteUrl`
|
||||
**Priority:** P3
|
||||
|
||||
**What:** `isInternalUrl` (`src/core/url-safety.ts`) does lexical/string-based classification only — no DNS resolution. An attacker who controls a public hostname's A/AAAA records can resolve to internal IPs (`127.0.0.1`, `169.254.169.254`, RFC 1918) and bypass the SSRF gate. The gate catches direct IP literals + metadata hostnames; it doesn't catch `https://attacker-controlled.example/repo.git` where DNS points internal.
|
||||
|
||||
**Why:** Defense in depth. The current gate is sufficient for naive abuse (typing `192.168.1.1` directly), but a deliberate attacker with DNS control can bypass it. Adding async DNS resolution + revalidation closes the hole.
|
||||
|
||||
**Pros:** Closes the cleanest remaining SSRF bypass. Mirrors the redirect-revalidation pattern at `integrations.ts:289`. Pinned by a future test using a mock resolver.
|
||||
**Cons:** Async DNS makes `parseRemoteUrl` `async`. Every caller (CLI, MCP op, test) needs to update. ~50-line change.
|
||||
**Context:** Codex finding from v0.28.2 ship adversarial review. The IPv6 ULA + link-local portion of the same finding shipped in v0.28.2; DNS rebinding deferred.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
### `sources.chunker_version` PGLite-schema parity
|
||||
**Priority:** P3
|
||||
|
||||
**What:** `src/schema.sql:33` declares `sources.chunker_version` and `src/commands/sync.ts:253` reads/writes it, but `src/core/pglite-schema.ts:28` omits the column. PGLite users hit a schema-mismatch error on the sync write path.
|
||||
|
||||
**Why:** Pre-existing bug surfaced during the v0.28.2 codex review. Not introduced by remote-source work, but adjacent to source-sync code. Worth fixing as a small parity PR before more source-local state lands.
|
||||
|
||||
**Pros:** Closes a quiet schema drift between the two engine implementations. ~10 lines.
|
||||
**Cons:** Needs a migration entry to add the column to existing PGLite brains. Migration version bump.
|
||||
**Context:** Codex D5 from v0.28.2 plan review.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
## OAuth/MCP hardening (v0.26.7 follow-up)
|
||||
|
||||
### F11 — `auth register-client --redirect-uri` flag
|
||||
**Priority:** P3
|
||||
|
||||
**What:** `gbrain auth register-client` always passes `[]` for redirect URIs; there is no CLI flag to set them. Operators who want to register an `authorization_code` client without DCR have to hand-edit the database.
|
||||
|
||||
**Why:** Operator UX gap, not a trust-boundary issue. Codex C11 correctly flagged it as scope creep on the v0.26.7 hardening pass — kept out of that PR but worth doing.
|
||||
|
||||
**Pros:** Closes the operator-experience gap. Validates `https://` or loopback per RFC 6749 §3.1.2.1 at registration time. Repeatable flag.
|
||||
**Cons:** ~30 lines of argv parsing + URL validation. Adds one more flag to the `auth register-client` surface. Low value relative to the OAuth provider hardening that already shipped.
|
||||
**Context:** Eva-brain has the implementation under `src/commands/auth.ts:registerClient`. Lift verbatim — the `localhost`/`127.0.0.1`/`::1` exact-match validation is correct; codex spot-check confirmed it does NOT match `localhost.evil.com`. v0.27 candidate.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
### F13 — `gbrain serve --http` argv positive-int validator
|
||||
**Priority:** P3
|
||||
|
||||
**What:** `parseInt(args[idx + 1])` on `--port` and `--token-ttl` accepts the next flag as the value if the argument is missing (e.g., `--port --token-ttl 100` parses port as NaN → fallback 3131). Negative integers like `--port -1` parse to -1, server fails to bind with a confusing error.
|
||||
|
||||
**Why:** Hygiene, not security. Codex C11 flagged as scope creep. Cheap to do later.
|
||||
|
||||
**Pros:** Replaces `parseInt(...) || fallback` with a `parsePositiveIntOption(args, flag, fallback, {max?})` helper that validates the next arg isn't a flag, matches `^[1-9]\d*$`, and clamps to a max. Exits 2 with a clear error.
|
||||
**Cons:** ~20 lines of helper + threading through `serve.ts`. Behavior change: previously-silent bad input now exits loud. Probably fine; no consumer relies on the silent fallback.
|
||||
**Context:** Eva-brain has the helper at `src/commands/serve.ts`. v0.27 candidate.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
## destructive-guard (v0.26.5 follow-up)
|
||||
|
||||
### Adjacent 2 — Storage objects orphan on hard purge
|
||||
**Priority:** P2
|
||||
|
||||
**What:** When `purgeExpiredSources` (sources cascade) or `purgeDeletedPages` (page-level) deletes rows, the underlying object-storage payloads referenced by `files.storage_uri` (S3 / Supabase Storage) are NOT torn down. The cascade FK on `files.source_id` removes the DB row that points at the object; the object itself stays.
|
||||
|
||||
**Why:** Bound today by most brains carrying `Files: 0` (operator preview boxes confirm this in the wild). The leak compounds the moment attachments / images / audio start landing — every soft-delete + 72h TTL purge silently abandons object-storage bytes.
|
||||
|
||||
**Pros:** Closes a real data-leak path. Operators stop paying for orphaned bytes. Aligns sources/pages purge with the file lifecycle.
|
||||
**Cons:** Storage backend code is non-trivial (S3 vs Supabase vs local-fs paths each have different cleanup APIs). Single-flight delete + retries on 5xx; needs an audit log.
|
||||
**Context:** Plan calls this out explicitly in v0.26.5 CEO review (`~/.claude/plans/take-a-look-and-gentle-pine.md` Adjacent 2). Targets: `src/core/storage.ts` for the object-storage interface, `src/core/destructive-guard.ts` `purgeExpiredSources` for the call site, plus a new sweep in the cycle's purge phase. v0.26.6 candidate.
|
||||
**Depends on:** Schema is fine (already has `files.storage_uri`). Just needs the storage delete plumbing.
|
||||
|
||||
### Adjacent 3 — sources remove + sources purge race against gbrain sync
|
||||
**Priority:** P3
|
||||
|
||||
**What:** `gbrain sources remove <id>` and the new `gbrain sources purge <id>` paths don't acquire `SYNC_LOCK_ID` (the `gbrain-sync` writer lock from PR #490). If `gbrain sync` is mid-import for the same source, the parent row can DELETE while sync is INSERTing children, surfacing as a loud FK violation.
|
||||
|
||||
**Why:** Failure mode is loud (FK violation, not data corruption), and the race window is narrow. Worth closing while the destructive surface is touched, not before.
|
||||
|
||||
**Pros:** Single line at the top of `runRemove` and `runPurge`. Reuses `tryAcquireDbLock(engine, SYNC_LOCK_ID, 5)`. No design surface.
|
||||
**Cons:** Adds an extra "couldn't acquire lock" exit path the operator has to recognize and retry.
|
||||
**Context:** Plan calls this out in CEO review Adjacent 3. Targets: `src/commands/sources.ts` `runRemove` and `runPurge`. v0.26.6 candidate. Pattern: `try { await fn() } finally { await release() }` mirrors the cycle.ts use of the same primitive.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
### Auth revoke-client gets the destructive-guard pattern
|
||||
**Priority:** P3
|
||||
|
||||
**What:** `gbrain auth revoke-client <client_id>` (v0.26.2) lands without an impact preview or `--confirm-destructive` gate. CASCADE-purges every active token + auth code in one transaction; one stray client_id wipes a production integration.
|
||||
|
||||
**Why:** Lower urgency than sources/pages because operators run this explicitly with a known client_id, not reflexively. But if the v0.26.5 posture is "every destructive surface gets the same gate," this surface should adopt it.
|
||||
|
||||
**Pros:** Posture consistency — every destructive verb in the gbrain CLI follows one pattern. Operators get the impact preview before nuking a production OAuth client.
|
||||
**Cons:** Marginal — single-row delete with cascade. The CASCADE is the blast radius, not the verb itself.
|
||||
**Context:** Plan flags this in CEO review. Targets: `src/commands/auth.ts` `runRevokeClient` (current shape: atomic DELETE...RETURNING with CASCADE on `oauth_tokens` + `oauth_codes`). Add an impact preview that counts `oauth_tokens` and `oauth_codes` for the client, then gate behind `--confirm-destructive`.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
## test infra (v0.26.4 follow-up — intra-file parallelism)
|
||||
|
||||
### Sweep cross-file shared-state contention; enable `bun test --concurrent` for another 2-3x speedup
|
||||
**Priority:** P0
|
||||
**Status:** v0.26.7 shipped foundation slice (helpers + lint + mock.module quarantine). v0.26.8 (env sweep) and v0.26.9 (PGLite sweep + codemod + measurement) carry the rest.
|
||||
|
||||
**What:** v0.26.4 shipped file-level parallel fan-out (8 shards) and got `bun run test` from 18 minutes to ~85s — a 12x speedup. The next layer is **intra-file** parallelism via Bun's `--concurrent` flag (or per-test `test.concurrent()` markers). This requires every test file to be safe under concurrent execution within the same `bun test` process.
|
||||
|
||||
The constraint: when multiple test files load into the same bun process (which is what `bun test foo.test.ts bar.test.ts ...` does inside a shard), they share module-level state. Three contention surfaces today:
|
||||
|
||||
- **~58 PGLiteEngine instantiations** across `test/` (per codex's grep). Many use module-level `let engine: PGLiteEngine` patterns. Race when multiple test files load and each invokes `new PGLiteEngine().connect({})`. **(carrying to v0.26.9)**
|
||||
- **~40 process.env mutations** without restore. `process.env.X = '...'` not paired with `afterEach` cleanup leaks across files in the same process. **(carrying to v0.26.8 — `withEnv` helper shipped in v0.26.7)**
|
||||
- ~~**2 top-level `mock.module(...)` calls** in `test/core/cycle.test.ts:26` and `test/embed.test.ts`. Top-level mocks affect every other test file in the same process.~~ **(quarantined as `*.serial.test.ts` in v0.26.7)**
|
||||
|
||||
The repo already has the right helper: `test/helpers/reset-pglite.ts` exports `resetPgliteState(engine)` which is "two orders of magnitude faster" than fresh-engine-per-test (per the helper's own comment). Sweep all PGLite sites to use one shared engine + this reset in `beforeEach`. Do NOT introduce a `freshPglite()` allocator — codex correctly flagged that the repo already rejected that direction.
|
||||
|
||||
Two flakes already known and quarantined as `*.serial.test.ts` (run after parallel pass at `--max-concurrency=1`):
|
||||
- `test/brain-registry.serial.test.ts` (was `brain-registry.test.ts`)
|
||||
- `test/reconcile-links.serial.test.ts` (was `reconcile-links.test.ts`)
|
||||
|
||||
After the sweep, both should be fixable and renameable back to plain `*.test.ts`.
|
||||
|
||||
**Why:**
|
||||
- 2-3x additional speedup on top of v0.26.4's 12x. Target: `bun run test` < 30s on a Mac dev box.
|
||||
- Forces the test architecture to be principled (no shared mutable state across files in the same process).
|
||||
- The empirical proof point: when `bun run test` was first measured at v0.26.4, two flakes surfaced under cross-file pressure that pass cleanly in isolation. That same pattern WILL surface more flakes if the suite grows. Better to sweep proactively than to keep growing the `*.serial.test.ts` quarantine.
|
||||
|
||||
**Pros:**
|
||||
- Real architectural win, not just speed: tests become composable.
|
||||
- Existing helper (`test/helpers/reset-pglite.ts`) already validates the pattern.
|
||||
- Quarantined flakes auto-resolve: rename back to `*.test.ts` after the sweep.
|
||||
|
||||
**Cons:**
|
||||
- 1-2 weeks of careful refactoring across ~100 test files.
|
||||
- Some tests genuinely need shared file-wide state (top-level mocks for module-replacement tests). Those stay quarantined as `*.serial.test.ts` permanently — but the count should shrink to a known small set, not grow.
|
||||
|
||||
**Context:** v0.26.4 plan considered doing this in scope (Codex Tension #2 = C). After empirical measurement showed `--max-concurrency=4` does nothing on tests not marked `test.concurrent()`, the user chose to ship v0.26.4 as file-level-only and file this as the v0.27+ project. Plan file: `~/.claude/plans/system-instruction-you-are-working-tranquil-ladybug.md`. Codex critical findings #2, #3, #6 are all relevant.
|
||||
|
||||
**Acceptance criteria:**
|
||||
1. All ~58 PGLiteEngine sites use shared-engine + `resetPgliteState()` in `beforeEach`. **(v0.26.9)**
|
||||
2. All ~40 `process.env` mutations use a `withEnv(...)` helper that saves + restores. **(v0.26.8 — helper shipped v0.26.7)**
|
||||
3. ~~The 2 top-level `mock.module()` calls scoped to `beforeEach`/`afterEach`, OR the file moves to `*.serial.test.ts`.~~ **DONE in v0.26.7 (quarantined)**
|
||||
4. Wrapper passes `--concurrent` (or every test marked `.concurrent()`). **(v0.26.9 — codemod with `find` recursive per Codex F3)**
|
||||
5. `bun run test` runs 5 times consecutively without flakes. **(v0.26.9)**
|
||||
6. Quarantine count `≤10` after the sweep (raised from 5 per D15; v0.26.7 added 2, currently 4: brain-registry, reconcile-links, cycle, embed).
|
||||
7. Wallclock target: `bun run test` ≤60s informational (per D9, dropped from <30s after Codex F1: marking only ~92 cheap files concurrent doesn't unblock the heavy 56 PGLite + 49 env files). Pinned config: SHARDS=8, MAX_CONCURRENCY=4, document Mac model. **(v0.26.9)**
|
||||
|
||||
**Decisions ledger (v0.26.7 plan):** D1 reversed→D16 sliced, D5 quarantine, D6 no helper wrapper, D7 grep+quarantine, D9 ≤60s informational, D10 ESM-cache claim dropped, D11 codemod uses `find` recursive, D12 lint wired into `verify` not `test`, D13 unquarantine attempt dropped, D14 extended grep patterns, D15 cap raised to 10.
|
||||
|
||||
**Estimated effort:** 1-2 weeks of one engineer's focused work. Could parallelize by sub-area (env-mutation sweep is independent of PGLite sweep).
|
||||
|
||||
### Speed up E2E via Postgres template databases
|
||||
**Priority:** P1
|
||||
|
||||
**What:** E2E tests (`bun run test:e2e`) currently run sequentially in one shared Postgres container, each test file calling `initSchema()` from scratch (~5-20s each on cold init). Speed-up: build the schema ONCE into a template DB (`gbrain_template`), then have each test file `CREATE DATABASE foo TEMPLATE gbrain_template` (~50ms per clone). With per-shard `DATABASE_URL` overrides, E2E can fan out to N parallel shards too.
|
||||
|
||||
**Why:** Current E2E wallclock is ~5-10 min in CI. Template DB clones could bring that to ~1-2 min. Critical for the inner loop on E2E-bearing PRs (currently a real friction point per `/ship` workflow).
|
||||
|
||||
**Sketch:**
|
||||
1. Build template DB once via `initSchema()` against `gbrain_template`.
|
||||
2. Per-test-file: `CREATE DATABASE gbrain_test_clone_<n> TEMPLATE gbrain_template` (50ms vs 5-20s).
|
||||
3. Per-shard isolation via `DATABASE_URL` env override.
|
||||
4. Schema-version stamp on the template so it invalidates when `migrate.ts` changes.
|
||||
5. Cleanup via `DROP DATABASE` in afterAll.
|
||||
|
||||
**Estimated effort:** 1-2 days. Filed during v0.26.4 plan as a deferred follow-up (D4 = B).
|
||||
|
||||
## test infra (v0.26.2 follow-up — pre-existing failures triage)
|
||||
|
||||
### Fix 22 pre-existing test failures unrelated to OAuth
|
||||
|
||||
+10
-10
File diff suppressed because one or more lines are too long
Vendored
+1
-1
@@ -7,7 +7,7 @@
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
|
||||
<script type="module" crossorigin src="/admin/assets/index-DWYc55rS.js"></script>
|
||||
<script type="module" crossorigin src="/admin/assets/index-CDv6_ml5.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/admin/assets/index-BOifXQpQ.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* Admin SPA scope constants — HAND-MAINTAINED MIRROR of src/core/scope.ts.
|
||||
*
|
||||
* The admin tsconfig.json scopes `include: ['src']` to admin/src/, so we
|
||||
* cannot directly import from ../../src/core/scope.ts without breaking the
|
||||
* SPA's compile boundary. Instead, this file is a hand-maintained duplicate;
|
||||
* scripts/check-admin-scope-drift.sh fails the build if the two lists drift.
|
||||
*
|
||||
* If you change ALLOWED_SCOPES in src/core/scope.ts, update this file too,
|
||||
* or `bun run verify` will reject the change.
|
||||
*/
|
||||
|
||||
export type Scope = 'read' | 'write' | 'admin' | 'sources_admin' | 'users_admin';
|
||||
|
||||
// MIRROR OF src/core/scope.ts ALLOWED_SCOPES_LIST — keep alphabetically sorted.
|
||||
export const ALLOWED_SCOPES_LIST: ReadonlyArray<Scope> = [
|
||||
'admin',
|
||||
'read',
|
||||
'sources_admin',
|
||||
'users_admin',
|
||||
'write',
|
||||
];
|
||||
@@ -1,5 +1,6 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { api } from '../api';
|
||||
import { ALLOWED_SCOPES_LIST, type Scope } from '../lib/scope-constants';
|
||||
|
||||
function timeAgo(date: Date): string {
|
||||
const s = Math.floor((Date.now() - date.getTime()) / 1000);
|
||||
@@ -249,7 +250,12 @@ function RegisterModal({ onClose, onRegistered }: {
|
||||
onRegistered: (creds: { clientId: string; clientSecret: string; name: string }) => void;
|
||||
}) {
|
||||
const [name, setName] = useState('');
|
||||
const [scopes, setScopes] = useState({ read: true, write: false, admin: false });
|
||||
// v0.28: scope set sourced from admin/src/lib/scope-constants.ts (mirror
|
||||
// of src/core/scope.ts). CI drift check at scripts/check-admin-scope-drift.sh
|
||||
// fails the build if these diverge.
|
||||
const [scopes, setScopes] = useState<Record<Scope, boolean>>(() =>
|
||||
Object.fromEntries(ALLOWED_SCOPES_LIST.map(s => [s, s === 'read'])) as Record<Scope, boolean>,
|
||||
);
|
||||
const [ttl, setTtl] = useState('86400'); // 24h default
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState('');
|
||||
@@ -298,7 +304,7 @@ function RegisterModal({ onClose, onRegistered }: {
|
||||
<div style={{ marginBottom: 16 }}>
|
||||
<label>Scopes</label>
|
||||
<div className="checkbox-group">
|
||||
{(['read', 'write', 'admin'] as const).map(s => (
|
||||
{ALLOWED_SCOPES_LIST.map(s => (
|
||||
<label key={s} className="checkbox-label">
|
||||
<input type="checkbox" checked={scopes[s]} onChange={e => setScopes(p => ({ ...p, [s]: e.target.checked }))} />
|
||||
{s}
|
||||
|
||||
@@ -5,13 +5,19 @@
|
||||
"": {
|
||||
"name": "gbrain",
|
||||
"dependencies": {
|
||||
"@ai-sdk/anthropic": "^3.0.71",
|
||||
"@ai-sdk/google": "^3.0.64",
|
||||
"@ai-sdk/openai": "^3.0.53",
|
||||
"@ai-sdk/openai-compatible": "^2.0.41",
|
||||
"@anthropic-ai/sdk": "^0.30.0",
|
||||
"@aws-sdk/client-s3": "^3.1028.0",
|
||||
"@dqbd/tiktoken": "^1.0.22",
|
||||
"@electric-sql/pglite": "0.4.3",
|
||||
"@modelcontextprotocol/sdk": "1.29.0",
|
||||
"ai": "^6.0.168",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cors": "^2.8.5",
|
||||
"eventsource-parser": "^3.0.8",
|
||||
"express": "^5.1.0",
|
||||
"express-rate-limit": "^7.5.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
@@ -21,6 +27,7 @@
|
||||
"postgres": "^3.4.0",
|
||||
"tree-sitter-wasms": "0.1.13",
|
||||
"web-tree-sitter": "0.22.6",
|
||||
"zod": "^4.3.6",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
@@ -36,6 +43,20 @@
|
||||
"@electric-sql/pglite",
|
||||
],
|
||||
"packages": {
|
||||
"@ai-sdk/anthropic": ["@ai-sdk/anthropic@3.0.74", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Xew9rfz9WWhDSyF8rNhjT/XWOWelNfJrMlmG0Ahw210hStisRpQZ1s+7VeI9JTJOZ5y5tXqBi5kfPwYnCfyRTA=="],
|
||||
|
||||
"@ai-sdk/gateway": ["@ai-sdk/gateway@3.0.109", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-r6dOqThjODp1vOhGRJg2OCmyB/ZOQtGx1esZ2SDvwDX5XoX8dBqYaYjLg8MPXTzMGJSgOkJyCxWgUcZtAl16pw=="],
|
||||
|
||||
"@ai-sdk/google": ["@ai-sdk/google@3.0.67", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Qeq+SidYtzMrcf0fdw3L0QLmtXK+ErwdBzbxS4+0Q/2UP85Ges8RJJcbAj7SO8e2JbeJoM35BLqkeNy1o3wJvQ=="],
|
||||
|
||||
"@ai-sdk/openai": ["@ai-sdk/openai@3.0.58", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-2+5xGMROmrBboJuoOwqLL3b/o3i56+NRdxXDNVAiTyYjLiBj6KzembeuyuBT217be1X+zkEfAqD1H0irJlGIyw=="],
|
||||
|
||||
"@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.45", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-5YBvurNL7Oj7mT3srws4Rh4cQidoorfEGObAOb5jV40eld8IC7EkXWARZjnWYqgYzabUs6Sn6muiXfQVkgOyOQ=="],
|
||||
|
||||
"@ai-sdk/provider": ["@ai-sdk/provider@3.0.10", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw=="],
|
||||
|
||||
"@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.26", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-CsKNLKsOpvPujRlIYvoz+Ybw+kGn7J4/fIZa/58+R7iWLLfwn6ifE2G6Yq8K9XvH/I/3bzaDAJ3NhRwEMsLBKQ=="],
|
||||
|
||||
"@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.30.1", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" } }, "sha512-nuKvp7wOIz6BFei8WrTdhmSsx5mwnArYyJgh4+vYu3V4J0Ltb8Xm3odPm51n1aSI0XxNCrDl7O88cxCtUdAkaw=="],
|
||||
|
||||
"@aws-crypto/crc32": ["@aws-crypto/crc32@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg=="],
|
||||
@@ -126,6 +147,8 @@
|
||||
|
||||
"@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="],
|
||||
|
||||
"@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="],
|
||||
|
||||
"@smithy/chunked-blob-reader": ["@smithy/chunked-blob-reader@5.2.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-St+kVicSyayWQca+I1rGitaOEH6uKgE8IUWoYnnEX26SWdWQcL6LvMSD19Lg+vYHKdT9B2Zuu7rd3i6Wnyb/iw=="],
|
||||
|
||||
"@smithy/chunked-blob-reader-native": ["@smithy/chunked-blob-reader-native@4.2.3", "", { "dependencies": { "@smithy/util-base64": "^4.3.2", "tslib": "^2.6.2" } }, "sha512-jA5k5Udn7Y5717L86h4EIv06wIr3xn8GM1qHRi/Nf31annXcXHJjBKvgztnbn2TxH3xWrPBfgwHsOwZf0UmQWw=="],
|
||||
@@ -226,6 +249,8 @@
|
||||
|
||||
"@smithy/uuid": ["@smithy/uuid@1.1.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-O/IEdcCUKkubz60tFbGA7ceITTAJsty+lBjNoorP4Z6XRqaFb/OjQjZODophEcuq68nKm6/0r+6/lLQ+XVpk8g=="],
|
||||
|
||||
"@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="],
|
||||
|
||||
"@types/body-parser": ["@types/body-parser@1.19.6", "", { "dependencies": { "@types/connect": "*", "@types/node": "*" } }, "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g=="],
|
||||
|
||||
"@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="],
|
||||
@@ -254,12 +279,16 @@
|
||||
|
||||
"@types/serve-static": ["@types/serve-static@2.2.0", "", { "dependencies": { "@types/http-errors": "*", "@types/node": "*" } }, "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ=="],
|
||||
|
||||
"@vercel/oidc": ["@vercel/oidc@3.2.0", "", {}, "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug=="],
|
||||
|
||||
"abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="],
|
||||
|
||||
"accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="],
|
||||
|
||||
"agentkeepalive": ["agentkeepalive@4.6.0", "", { "dependencies": { "humanize-ms": "^1.2.1" } }, "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ=="],
|
||||
|
||||
"ai": ["ai@6.0.174", "", { "dependencies": { "@ai-sdk/gateway": "3.0.109", "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.26", "@opentelemetry/api": "1.9.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-bTrfLUWHWtkjzWyCY4bmyuk4Qvmj4S4NSNsXyNSVVqkmftQNtxRj7dzUoMeQDBBwlJO6fC7m2Q/lNOPqQQfAGA=="],
|
||||
|
||||
"ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="],
|
||||
|
||||
"ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="],
|
||||
@@ -326,7 +355,7 @@
|
||||
|
||||
"eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="],
|
||||
|
||||
"eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
|
||||
"eventsource-parser": ["eventsource-parser@3.0.8", "", {}, "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ=="],
|
||||
|
||||
"express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="],
|
||||
|
||||
@@ -394,6 +423,8 @@
|
||||
|
||||
"js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="],
|
||||
|
||||
"json-schema": ["json-schema@0.4.0", "", {}, "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="],
|
||||
|
||||
"json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="],
|
||||
|
||||
"json-schema-typed": ["json-schema-typed@8.0.2", "", {}, "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA=="],
|
||||
@@ -528,10 +559,14 @@
|
||||
|
||||
"@aws-crypto/util/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
|
||||
|
||||
"@modelcontextprotocol/sdk/eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
|
||||
|
||||
"@modelcontextprotocol/sdk/express-rate-limit": ["express-rate-limit@8.3.2", "", { "dependencies": { "ip-address": "10.1.0" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-77VmFeJkO0/rvimEDuUC5H30oqUC4EyOhyGccfqoLebB0oiEYfM7nwPrsDsBL1gsTpwfzX8SFy2MT3TDyRq+bg=="],
|
||||
|
||||
"@types/bun/bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
|
||||
|
||||
"eventsource/eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
|
||||
|
||||
"express/cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="],
|
||||
|
||||
"form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="],
|
||||
|
||||
+6
-9
@@ -1,12 +1,9 @@
|
||||
[test]
|
||||
# PGLite initialization can be slow under parallel test execution.
|
||||
# Default 5s is too short when many test files boot PGLite instances at once.
|
||||
# 60s is the empirical ceiling we observed before the first file's beforeAll
|
||||
# completed on a loaded machine.
|
||||
# PGLite WASM cold start + initSchema() runs ~5–20s on loaded machines.
|
||||
# Default 5s is too short for those tests' beforeAll hooks. 60s is the
|
||||
# empirical ceiling we observed for the slowest cold-init paths.
|
||||
#
|
||||
# NOTE: this bunfig.toml `timeout` key is read by `bun test` but empirically
|
||||
# does NOT apply to beforeEach/afterEach hook timeouts under `bun run test`
|
||||
# chained behind `bun run typecheck`. The test script in package.json passes
|
||||
# `--timeout=60000` explicitly to cover both per-test and per-hook timeouts.
|
||||
# Leaving both in place as belt-and-suspenders.
|
||||
# v0.26.4: scripts/run-unit-parallel.sh and scripts/run-unit-shard.sh
|
||||
# also pass `--timeout=60000` explicitly so the ceiling is consistent
|
||||
# whether tests are invoked through the wrapper or directly via bun test.
|
||||
timeout = 60_000
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
# Switching embedding models or dimensions on an existing brain
|
||||
|
||||
GBrain stores embeddings in a fixed-dimension `vector(N)` column on
|
||||
`content_chunks`. If you switch to a model with a different dimension
|
||||
(e.g. `text-embedding-3-large` 1536 → `voyage-multilingual-large-2` 2048,
|
||||
or back to a smaller model like `nomic-embed-text` 768), the on-disk
|
||||
column type doesn't change automatically.
|
||||
|
||||
`gbrain init` and `gbrain doctor` both detect and refuse to silently
|
||||
proceed in this case. This doc is the recipe they point at.
|
||||
|
||||
## Why we don't do this automatically
|
||||
|
||||
Switching dimensions requires:
|
||||
|
||||
1. Dropping the HNSW vector index (pgvector won't survive an `ALTER COLUMN TYPE`).
|
||||
2. Altering the column type.
|
||||
3. Wiping every existing embedding (the old vectors are unusable in the new space).
|
||||
4. Re-embedding the entire corpus (can take hours on a 50K-page brain and costs $1-100 in API calls depending on model).
|
||||
5. Conditionally recreating the index (HNSW supports up to 2000 dimensions per pgvector; above that you must use exact scans).
|
||||
|
||||
That's not an upgrade-time auto-run. It's a deliberate, expensive
|
||||
operation. Run it when you've decided you actually want the new model.
|
||||
|
||||
## Recipe — manual `psql` against your brain
|
||||
|
||||
Replace `<NEW_DIMS>` with your target dimension count.
|
||||
|
||||
```sql
|
||||
BEGIN;
|
||||
|
||||
-- 1. Drop the HNSW index. It can't survive the column type change.
|
||||
DROP INDEX IF EXISTS idx_chunks_embedding;
|
||||
|
||||
-- 2. Alter the column type. (You can DROP COLUMN + ADD COLUMN instead
|
||||
-- if the existing data is already gone — same end state.)
|
||||
ALTER TABLE content_chunks ALTER COLUMN embedding TYPE vector(<NEW_DIMS>);
|
||||
|
||||
-- 3. Clear stale embeddings so they don't survive into the new space.
|
||||
-- Either truncate (faster, drops all chunks) or null out (preserves
|
||||
-- chunk text so re-embed regenerates without re-chunking):
|
||||
UPDATE content_chunks SET embedding = NULL, embedded_at = NULL;
|
||||
|
||||
-- 4. Recreate the HNSW index ONLY IF dims <= 2000. Above that, leave it
|
||||
-- indexless and rely on exact scans (gbrain searchVector handles this
|
||||
-- automatically — search just gets slower, not broken).
|
||||
-- For dims <= 2000 (e.g. 1024, 1536, 768):
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_embedding
|
||||
ON content_chunks USING hnsw (embedding vector_cosine_ops);
|
||||
-- For dims > 2000 (e.g. 2048 Voyage 4 Large): skip step 4.
|
||||
|
||||
COMMIT;
|
||||
```
|
||||
|
||||
Then update gbrain's config so it knows the new dim:
|
||||
|
||||
```bash
|
||||
gbrain config set embedding_model <model>
|
||||
gbrain config set embedding_dimensions <NEW_DIMS>
|
||||
```
|
||||
|
||||
And re-embed the corpus:
|
||||
|
||||
```bash
|
||||
gbrain embed --stale
|
||||
```
|
||||
|
||||
## PGLite (local brain)
|
||||
|
||||
Same recipe, but you connect to the embedded database differently:
|
||||
|
||||
```bash
|
||||
gbrain config get database_url # confirm engine: pglite
|
||||
# Open a psql-equivalent — for PGLite, the easiest path is to write a small
|
||||
# script that imports PGLiteEngine and runs the SQL via engine.executeRaw.
|
||||
# Or migrate to Postgres temporarily (gbrain migrate --to supabase) if you
|
||||
# want a real psql connection.
|
||||
```
|
||||
|
||||
For most PGLite users the simpler path is to **wipe and re-init** if your
|
||||
corpus is small enough that re-syncing is faster than hand-crafting the
|
||||
migration:
|
||||
|
||||
```bash
|
||||
mv ~/.gbrain/brain.pglite ~/.gbrain/brain.pglite.bak
|
||||
gbrain init --pglite --embedding-dimensions <NEW_DIMS>
|
||||
gbrain sync # re-imports your brain repo from disk
|
||||
```
|
||||
|
||||
## Verify
|
||||
|
||||
After the recipe lands, `gbrain doctor --fast` should report green and
|
||||
`gbrain doctor` (full) should say check 8b passes:
|
||||
|
||||
```
|
||||
✓ embedding_provider dim parity: config 768 / column vector(768) / live probe 768
|
||||
```
|
||||
|
||||
If it doesn't, file an issue with the doctor output and the SQL you ran.
|
||||
|
||||
## v0.29+ plans
|
||||
|
||||
`gbrain migrate-embedding-dim --to <N>` is a tracked TODO. It will run
|
||||
the recipe above with progress reporting + an explicit confirmation
|
||||
gate. Until that lands, this manual recipe is the canonical path.
|
||||
@@ -34,6 +34,85 @@ docs/guides/rls-and-you.md for the GBRAIN:RLS_EXEMPT comment escape hatch.
|
||||
|
||||
99% of the time, you want the fix. Run the SQL. Re-run `gbrain doctor`. Done.
|
||||
|
||||
## v0.26.7 — auto-RLS event trigger and one-time backfill
|
||||
|
||||
Starting in v0.26.7 (migration v35), gbrain ships two changes that close the
|
||||
gap where a table could exist in your `public` schema without RLS for any
|
||||
amount of time at all.
|
||||
|
||||
**1. The event trigger.** A Postgres DDL event trigger named
|
||||
`auto_rls_on_create_table` runs `ALTER TABLE … ENABLE ROW LEVEL SECURITY`
|
||||
on every newly created `public.*` table. It covers `CREATE TABLE`,
|
||||
`CREATE TABLE AS … SELECT`, and `SELECT … INTO` — every syntax Postgres
|
||||
reports as a table-creation command. Tables created by gbrain itself, by
|
||||
your other apps sharing the same Supabase project (Baku, Hermes, anything),
|
||||
or by a human running raw SQL all get RLS enabled the moment they exist.
|
||||
Non-`public` schemas (`auth`, `storage`, `realtime`, etc.) are explicitly
|
||||
ignored — Supabase manages those, and we should not touch them.
|
||||
|
||||
**2. The one-time backfill.** When you upgrade to v0.26.7, the migration
|
||||
walks every existing `public.*` base table whose RLS is off and whose comment
|
||||
doesn't carry the `GBRAIN:RLS_EXEMPT` exemption (see below) and enables RLS
|
||||
on each. After the upgrade, `gbrain doctor`'s `rls` check should be a no-op
|
||||
on every brain.
|
||||
|
||||
### Breaking change: read this before upgrading
|
||||
|
||||
If you have public tables that are intentionally RLS-off and you want them
|
||||
to stay that way, you MUST add the `GBRAIN:RLS_EXEMPT` comment **before**
|
||||
running `gbrain upgrade` to v0.26.7. The backfill flips RLS on for any public
|
||||
table that doesn't carry the exact comment contract documented below. There
|
||||
is no `--dry-run` flag on the migration.
|
||||
|
||||
The minimum cost of getting this wrong is one round-trip: the operator runs
|
||||
the SQL to enable RLS on a table that should have been exempt, then
|
||||
`ALTER TABLE … DISABLE ROW LEVEL SECURITY` and adds the exempt comment to
|
||||
prevent a re-flip on a later doctor run. No data is lost.
|
||||
|
||||
### Cross-app implications
|
||||
|
||||
If a non-gbrain app (Baku, Hermes, a script you wrote, anything) creates
|
||||
tables in the same Supabase project, the trigger will enable RLS on those
|
||||
tables too. Two ways to handle that:
|
||||
|
||||
1. **The app's connection role has BYPASSRLS** (e.g. it's also using the
|
||||
`postgres` role). Newly created tables get RLS on but the app reads/writes
|
||||
freely because BYPASSRLS bypasses policies entirely.
|
||||
2. **The app's role does NOT have BYPASSRLS.** Then the app needs to add a
|
||||
`CREATE POLICY` immediately after creating the table, granting itself
|
||||
the read/write access it needs. The trigger does NOT add policies — it
|
||||
only enables RLS, leaving the deny-by-default posture in place until the
|
||||
app's policy lands.
|
||||
|
||||
If neither condition holds, the app will fail to read its own freshly-created
|
||||
tables. The fix is at the app side, not gbrain's: either grant BYPASSRLS or
|
||||
ship a policy.
|
||||
|
||||
### What if the trigger gets dropped?
|
||||
|
||||
`gbrain doctor` includes a new `rls_event_trigger` check that verifies the
|
||||
trigger is installed and enabled. If you drop it manually for any reason
|
||||
(debugging, migration testing, anything), doctor warns and gives you the
|
||||
recovery command:
|
||||
|
||||
```
|
||||
gbrain apply-migrations --force-retry 35
|
||||
```
|
||||
|
||||
Re-running migration v35 is idempotent — it `DROP EVENT TRIGGER IF EXISTS`
|
||||
and recreates cleanly.
|
||||
|
||||
### Why no FORCE ROW LEVEL SECURITY?
|
||||
|
||||
Postgres has two RLS dials. `ENABLE` blocks anon/authenticated; `FORCE` also
|
||||
blocks the table OWNER unless they hold BYPASSRLS. We use `ENABLE` only,
|
||||
matching the posture in `src/schema.sql`, migrations v24, and v29. `FORCE`
|
||||
would lock non-BYPASSRLS apps out of their own freshly-created tables (the
|
||||
trigger function inherits the caller's role, not the gbrain role) — which
|
||||
defeats the cross-app coexistence story above. If you want defense-in-depth
|
||||
`FORCE` on a specific gbrain-owned table, add it explicitly in your own
|
||||
migration; gbrain's auto-RLS does not opt you in by default.
|
||||
|
||||
## The 1% case: deliberate exemption
|
||||
|
||||
Sometimes a public table is supposed to be readable by the anon key. An
|
||||
|
||||
@@ -85,6 +85,15 @@ Save this token. Open `http://localhost:3131/admin` and paste it to access the
|
||||
dashboard. The dashboard shows live activity, registered clients, request logs,
|
||||
and per-client config export.
|
||||
|
||||
> **v0.26.9+:** `mcp_request_log.params` and the live SSE activity feed default
|
||||
> to a redacted summary `{redacted, kind, declared_keys, unknown_key_count, approx_bytes}`.
|
||||
> Declared param keys are kept (intersected against the operation's spec); unknown
|
||||
> keys are counted but never named, and byte sizes round up to 1KB so size-probe
|
||||
> attacks can't binary-search secret content. Operators on a personal laptop who
|
||||
> want raw payloads back can pass `gbrain serve --http --log-full-params` (loud
|
||||
> stderr warning fires at startup). Multi-tenant deployments should leave it on
|
||||
> the redacted default.
|
||||
|
||||
### 2. Register OAuth clients
|
||||
|
||||
Register clients from the **`/admin` dashboard**:
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"version": 1,
|
||||
"description": "Embedding provider smoke test — verifies semantic search returns expected results for known brain content. Run after any embedding model change or migration.",
|
||||
"queries": [
|
||||
{
|
||||
"id": "yc-labs-strategy",
|
||||
"query": "YC Labs strategy and product team",
|
||||
"relevant": [
|
||||
"originals/yc-labs-internal-team",
|
||||
"originals/harj-yc-labs-strategy-2026-05"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "garry-tan-person",
|
||||
"query": "Who is Garry Tan",
|
||||
"relevant": [
|
||||
"people/garry-tan"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "gstack-project",
|
||||
"query": "GStack open source AI coding framework",
|
||||
"relevant": [
|
||||
"projects/gstack/gstackbrain"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "yc-carry-compensation",
|
||||
"query": "GP carry and compensation structure at YC",
|
||||
"relevant": [
|
||||
"originals/harj-yc-labs-strategy-2026-05"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "meeting-search",
|
||||
"query": "recent office hours meeting notes",
|
||||
"relevant": []
|
||||
}
|
||||
]
|
||||
}
|
||||
+186
-20
@@ -137,13 +137,13 @@ strict behavior when unset.
|
||||
|
||||
## Key files
|
||||
|
||||
- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Also exports upload validators: `validateUploadPath`, `validatePageSlug`, `validateFilename`, plus `matchesSlugAllowList(slug, prefixes)` (v0.23 glob matcher: `<prefix>/*` matches recursive children; bare `<prefix>` matches exact only). `OperationContext.remote` flags untrusted callers; `OperationContext.allowedSlugPrefixes` (v0.23) is the trusted-workspace allow-list set by the dream cycle. `put_page` enforces: when `viaSubagent` and `allowedSlugPrefixes` is set, slug must match the allow-list; else the legacy `wiki/agents/<id>/...` namespace check applies. Auto-link enabled for trusted-workspace writes (skipped only when `remote=true && !trustedWorkspace`). As of v0.26.0, every `Operation` also carries `scope?: 'read' | 'write' | 'admin'` + `localOnly?: boolean`. All ops are annotated; `sync_brain`, `file_upload`, `file_list`, and `file_url` are `admin + localOnly` (rejected over HTTP). `OperationContext.auth?: AuthInfo` is threaded through HTTP dispatch for scope enforcement in `serve-http.ts` before the op runs.
|
||||
- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Also exports upload validators: `validateUploadPath`, `validatePageSlug`, `validateFilename`, plus `matchesSlugAllowList(slug, prefixes)` (v0.23 glob matcher: `<prefix>/*` matches recursive children; bare `<prefix>` matches exact only). `OperationContext.remote` flags untrusted callers; `OperationContext.allowedSlugPrefixes` (v0.23) is the trusted-workspace allow-list set by the dream cycle. `put_page` enforces: when `viaSubagent` and `allowedSlugPrefixes` is set, slug must match the allow-list; else the legacy `wiki/agents/<id>/...` namespace check applies. Auto-link enabled for trusted-workspace writes (skipped only when `remote=true && !trustedWorkspace`). As of v0.26.0, every `Operation` also carries `scope?: 'read' | 'write' | 'admin'` + `localOnly?: boolean`. All ops are annotated; `sync_brain`, `file_upload`, `file_list`, and `file_url` are `admin + localOnly` (rejected over HTTP). `OperationContext.auth?: AuthInfo` is threaded through HTTP dispatch for scope enforcement in `serve-http.ts` before the op runs. **v0.26.9 (D12 + F7b):** `OperationContext.remote` is now a REQUIRED field in the TypeScript type — the compiler is the first defense against transports that forget to set it. Four trust-boundary call sites (`put_page` allowlist, file_upload trust-narrowing, submit_job protected-name guard, auto-link skip) flipped from falsy-default (`!ctx.remote`) to fail-closed semantics (`ctx.remote === false` for "trusted-only" sites and `ctx.remote !== false` for "untrust unless explicit-false"). Anything that isn't strictly `false` is now treated as remote. Closed an HTTP MCP shell-job RCE: a `read+write`-scoped OAuth token could submit `shell` jobs because the HTTP request handler's literal context skipped `remote: true` and `submit_job`'s protected-name guard saw a falsy undefined. Stdio MCP set the field correctly via dispatch.ts; HTTP inlined a parallel context-builder for several releases and lost it.
|
||||
- `src/core/engine.ts` — Pluggable engine interface (BrainEngine). `clampSearchLimit(limit, default, cap)` takes an explicit cap so per-operation caps can be tighter than `MAX_SEARCH_LIMIT`. Exports `LinkBatchInput` / `TimelineBatchInput` for the v0.12.1 bulk-insert API (`addLinksBatch` / `addTimelineEntriesBatch`). As of v0.13.1, `BrainEngine` has a `readonly kind: 'postgres' | 'pglite'` discriminator so migrations (`src/core/migrate.ts`) and other consumers can branch on engine without `instanceof` + dynamic imports.
|
||||
- `src/core/engine-factory.ts` — Engine factory with dynamic imports (`'pglite'` | `'postgres'`)
|
||||
- `src/core/pglite-engine.ts` — PGLite (embedded Postgres 17.5 via WASM) implementation, all 40 BrainEngine methods. `addLinksBatch` / `addTimelineEntriesBatch` use multi-row `unnest()` with manual `$N` placeholders. As of v0.13.1, `connect()` wraps `PGlite.create()` in a try/catch that emits an actionable error naming the macOS 26.3 WASM bug (#223) and pointing at `gbrain doctor`; the lock is released on failure so the next process can retry cleanly. v0.22.0: `searchKeyword` and `searchKeywordChunks` multiply `ts_rank` by the source-factor CASE expression at the chunk-grain level; `searchVector` becomes a two-stage CTE — inner CTE keeps `ORDER BY cc.embedding <=> vec` so HNSW stays usable, outer SELECT re-ranks by `raw_score * source_factor`. Inner LIMIT scales with offset to preserve pagination contract. As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL — probes for the specific forward-referenced state the embedded schema blob needs (`pages.source_id`, `links.link_source`, `links.origin_page_id`, `content_chunks.symbol_name`, `content_chunks.language`, `sources` FK target table) and adds only what's missing. Closes the upgrade-wedge bug class that bit users 10+ times across 6 schema versions over 2 years (#239/#243/#266/#357/#366/#374/#375/#378/#395/#396). No-op on fresh installs and modern brains.
|
||||
- `src/core/pglite-schema.ts` — PGLite-specific DDL (pgvector, pg_trgm, triggers)
|
||||
- `src/core/postgres-engine.ts` — Postgres + pgvector implementation (Supabase / self-hosted). `addLinksBatch` / `addTimelineEntriesBatch` use `INSERT ... SELECT FROM unnest($1::text[], ...) JOIN pages ON CONFLICT DO NOTHING RETURNING 1` — 4-5 array params regardless of batch size, sidesteps the 65535-parameter cap. As of v0.12.3, `searchKeyword` / `searchVector` scope `statement_timeout` via `sql.begin` + `SET LOCAL` so the GUC dies with the transaction instead of leaking across the pooled postgres.js connection (contributed by @garagon). `getEmbeddingsByChunkIds` uses `tryParseEmbedding` so one corrupt row skips+warns instead of killing the query. v0.22.0: `searchKeyword`, `searchKeywordChunks`, and `searchVector` apply source-aware ranking by inlining the source-factor CASE and `NOT (col LIKE …)` hard-exclude clause from `src/core/search/sql-ranking.ts`. `searchVector` switches to a two-stage CTE (HNSW-safe inner ORDER BY, source-boost re-rank in the outer SELECT) and carries `p.source_id` through inner→outer for v0.18 multi-source callers. v0.22.1 (#406): `_savedConfig` retains the connect config; `reconnect()` tears down + recreates the pool from saved config (called by supervisor watchdog after 3 consecutive health-check failures). `executeRaw` is a single-statement passthrough — no per-call retry (D3 dropped that as unsound for non-idempotent statements; recovery is supervisor-driven). v0.22.1 (#363, contributed by @orendi84): `connect()` applies `resolveSessionTimeouts()` from `db.ts` as connection-time startup parameters (`statement_timeout`, `idle_in_transaction_session_timeout`) so orphan pgbouncer backends can't hold locks for hours. v0.22.1 (#409, contributed by @atrevino47): `countStaleChunks()` + `listStaleChunks()` server-side-filter on `embedding IS NULL` for `embed --stale`, eliminating ~76 MB/call client-side pull on a fully-embedded brain; `upsertChunks()` resets both `embedding` AND `embedded_at` to NULL when chunk_text changes without a new embedding (consistency). As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL on the same forward-reference probe set as the PGLite engine, so old Postgres brains pinned at v0.13/v0.18/v0.19 walk forward cleanly instead of wedging on `column "..." does not exist`.
|
||||
- `src/core/utils.ts` — Shared SQL utilities extracted from postgres-engine.ts. Exports `parseEmbedding(value)` (throws on unknown input, used by migration + ingest paths where data integrity matters) and as of v0.12.3 `tryParseEmbedding(value)` (returns `null` + warns once per process, used by search/rescore paths where availability matters more than strictness).
|
||||
- `src/core/postgres-engine.ts` — Postgres + pgvector implementation (Supabase / self-hosted). `addLinksBatch` / `addTimelineEntriesBatch` use `INSERT ... SELECT FROM unnest($1::text[], ...) JOIN pages ON CONFLICT DO NOTHING RETURNING 1` — 4-5 array params regardless of batch size, sidesteps the 65535-parameter cap. As of v0.12.3, `searchKeyword` / `searchVector` scope `statement_timeout` via `sql.begin` + `SET LOCAL` so the GUC dies with the transaction instead of leaking across the pooled postgres.js connection (contributed by @garagon). `getEmbeddingsByChunkIds` uses `tryParseEmbedding` so one corrupt row skips+warns instead of killing the query. v0.22.0: `searchKeyword`, `searchKeywordChunks`, and `searchVector` apply source-aware ranking by inlining the source-factor CASE and `NOT (col LIKE …)` hard-exclude clause from `src/core/search/sql-ranking.ts`. `searchVector` switches to a two-stage CTE (HNSW-safe inner ORDER BY, source-boost re-rank in the outer SELECT) and carries `p.source_id` through inner→outer for v0.18 multi-source callers. v0.22.1 (#406): `_savedConfig` retains the connect config; `reconnect()` tears down + recreates the pool from saved config (called by supervisor watchdog after 3 consecutive health-check failures). `executeRaw` is a single-statement passthrough — no per-call retry (D3 dropped that as unsound for non-idempotent statements; recovery is supervisor-driven). v0.22.1 (#363, contributed by @orendi84): `connect()` applies `resolveSessionTimeouts()` from `db.ts` as connection-time startup parameters (`statement_timeout`, `idle_in_transaction_session_timeout`) so orphan pgbouncer backends can't hold locks for hours. v0.22.1 (#409, contributed by @atrevino47): `countStaleChunks()` + `listStaleChunks()` server-side-filter on `embedding IS NULL` for `embed --stale`, eliminating ~76 MB/call client-side pull on a fully-embedded brain; `upsertChunks()` resets both `embedding` AND `embedded_at` to NULL when chunk_text changes without a new embedding (consistency). As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL on the same forward-reference probe set as the PGLite engine, so old Postgres brains pinned at v0.13/v0.18/v0.19 walk forward cleanly instead of wedging on `column "..." does not exist`. **v0.28.1:** `disconnect()` is now idempotent. New `_connectionStyle` instance field tracks whether the engine owns its pool (worker engines) or shares the module-level singleton; second call on an instance-pool engine is a no-op rather than falling through to `db.disconnect()` and clobbering the singleton. Pinned by `test/e2e/postgres-engine-disconnect-idempotency.test.ts` (2 cases). Closes the bug class where any test sharing an engine across multiple `worker.start()` / `worker.stop()` cycles silently broke its own DB connectivity.
|
||||
- `src/core/utils.ts` — Shared SQL utilities extracted from postgres-engine.ts. Exports `parseEmbedding(value)` (throws on unknown input, used by migration + ingest paths where data integrity matters) and as of v0.12.3 `tryParseEmbedding(value)` (returns `null` + warns once per process, used by search/rescore paths where availability matters more than strictness). **v0.26.9 (D14):** adds `isUndefinedColumnError(err)` predicate — pattern-matches Postgres SQLSTATE 42703 / "column ... does not exist" with engine-driver shape variation tolerated. Replaces bare `catch {}` blocks in `oauth-provider.ts` so genuine errors (lock timeout, network blip, permission denied) propagate while column-missing falls through to the legacy fallback path. Reusable from any future code that needs the same column-existence probe semantics.
|
||||
- `src/core/db.ts` — Connection management, schema initialization. v0.22.1 (#363, contributed by @orendi84): `resolveSessionTimeouts()` returns `statement_timeout` + `idle_in_transaction_session_timeout` (defaults: 5min each, env-overridable via `GBRAIN_STATEMENT_TIMEOUT` / `GBRAIN_IDLE_TX_TIMEOUT` / `GBRAIN_CLIENT_CHECK_INTERVAL`). Both `connect()` (module singleton) and `PostgresEngine.connect()` (worker pool) consume the result via postgres.js's `connection` option, sending GUCs as startup parameters that survive PgBouncer transaction mode (unlike the prior `setSessionDefaults` post-pool SET, kept as a back-compat no-op shim).
|
||||
- `src/commands/migrate-engine.ts` — Bidirectional engine migration (`gbrain migrate --to supabase/pglite`)
|
||||
- `src/core/import-file.ts` — importFromFile + importFromContent (chunk + embed + tags)
|
||||
@@ -164,7 +164,13 @@ strict behavior when unset.
|
||||
- `src/core/search/eval.ts` — Retrieval eval harness: P@k, R@k, MRR, nDCG@k metrics + runEval() orchestrator
|
||||
- `src/core/search/source-boost.ts` (v0.22.0) — Source-type boost map keyed by slug prefix. `DEFAULT_SOURCE_BOOSTS` (originals/ 1.5, concepts/ 1.3, writing/ 1.4, people/companies/deals/ 1.2, daily/ 0.8, media/x/ 0.7, wintermute/chat/ 0.5) and `DEFAULT_HARD_EXCLUDES` (test/, archive/, attachments/, .raw/). `parseSourceBoostEnv` / `parseHardExcludesEnv` parse comma-separated `prefix:factor` pairs from `GBRAIN_SOURCE_BOOST` / `GBRAIN_SEARCH_EXCLUDE` env vars. `resolveBoostMap` and `resolveHardExcludes` merge defaults + env + caller `SearchOpts.exclude_slug_prefixes`/`include_slug_prefixes`.
|
||||
- `src/core/search/sql-ranking.ts` (v0.22.0) — Pure SQL string builders. `buildSourceFactorCase(slugColumn, boostMap, detail)` emits a CASE expression with longest-prefix-match wins (returns literal `'1.0'` when `detail === 'high'` for temporal-bypass parity with COMPILED_TRUTH_BOOST). `buildHardExcludeClause(slugColumn, prefixes)` emits `NOT (col LIKE 'p1%' OR col LIKE 'p2%')` — OR-chain wrapped in NOT, NOT `NOT LIKE ALL/ANY` (those quantifiers don't express set-exclusion). LIKE meta-character escape covers all three of `%`, `_`, AND `\` (backslash matters because it's Postgres LIKE's default escape char). Single-quote doubling on SQL string literals so injection-style inputs are inert text.
|
||||
- `src/commands/eval.ts` — `gbrain eval` command: single-run table + A/B config comparison. v0.25.0 adds sub-subcommand dispatch on `args[0]` so `gbrain eval export` + `gbrain eval prune` + `gbrain eval replay` route into session-capture handlers; bare `gbrain eval --qrels …` fall-through preserves the legacy IR-metrics flow.
|
||||
- `src/commands/eval.ts` — `gbrain eval` command: single-run table + A/B config comparison. v0.25.0 adds sub-subcommand dispatch on `args[0]` so `gbrain eval export` + `gbrain eval prune` + `gbrain eval replay` route into session-capture handlers; bare `gbrain eval --qrels …` fall-through preserves the legacy IR-metrics flow. v0.27.x adds `gbrain eval cross-modal` to the dispatch (the user-facing path is the cli.ts no-DB branch — `src/commands/eval.ts:cross-modal` only fires when callers re-enter with an existing engine).
|
||||
- `src/commands/eval-cross-modal.ts` (v0.27.x) — multi-model quality gate. Three different-provider frontier models score the OUTPUT against the TASK on a 5-dim list. Verdict `pass` (exit 0) / `fail` (exit 1) / `inconclusive` (exit 2; <2/3 model successes per Q3=A in plans/radiant-napping-lerdorf.md). Reuses `src/core/ai/gateway.ts:chat()` so config/auth/aliasing comes from the gateway recipe registry — no parallel provider stack. Self-configures the gateway (`configureGateway(loadConfig() + process.env)`) since the cli.ts dispatch bypasses `connectEngine()`. Default cycles 3 in TTY, 1 in non-TTY (T11=B partial cost guardrail). Receipts land at `gbrainPath('eval-receipts')/<slug>-<sha8-of-output>.json`. The full `--budget-usd` cap is a v0.27.x follow-up TODO.
|
||||
- `src/core/cross-modal-eval/json-repair.ts` (v0.27.x) — `parseModelJSON(raw)` named export with a 4-strategy fallback chain (direct parse → fence-strip → trailing-comma + single-quote + embedded-newline repair → regex nuclear option). Adversarial input throws rather than fabricating scores — the aggregator treats a throw as "this model contributed nothing this cycle" so the gate stays correct at >=2/3 successes.
|
||||
- `src/core/cross-modal-eval/aggregate.ts` (v0.27.x) — pure verdict logic. Pass criterion: `(successes >= 2) AND (every dim mean >= 7) AND (every dim min across models >= 5)` (Q2=A floor). Inconclusive when <2/3 models returned parseable scores (Q3=A regression guard for the v1 .mjs `Object.values({}).every(...) === true` empty-array PASS bug).
|
||||
- `src/core/cross-modal-eval/runner.ts` (v0.27.x) — orchestrator. Each cycle runs `Promise.allSettled([gwChat(slotA), gwChat(slotB), gwChat(slotC)])` (T4=A — bare allSettled, no rate-leases for the CLI path; minion-integration TODO recovers cross-process concurrency). Stops early on PASS or INCONCLUSIVE; runs up to 3 cycles. Default slots: `openai:gpt-4o` / `anthropic:claude-opus-4-7` / `google:gemini-1.5-pro`. `estimateCost()` exports a small per-model pricing table (drifts; refresh alongside model-family bumps).
|
||||
- `src/core/cross-modal-eval/receipt-name.ts` (v0.27.x) — receipt filename binds (slug, SKILL.md sha-8). `findReceiptForSkill(skillPath, receiptDir)` returns `'found' | 'stale' | 'missing'` (T10=A). Skillify-check item 11 surfaces the status as informational (T7=C); the audit does NOT fail on missing/stale receipts.
|
||||
- `src/core/cross-modal-eval/receipt-write.ts` (v0.27.x) — wraps `fs.writeFileSync` with `mkdirSync({recursive:true})` ahead of every write (T5 correction; `gbrainPath()` does NOT auto-mkdir).
|
||||
- `src/commands/eval-export.ts` (v0.25.0) — streams `eval_candidates` rows as NDJSON to stdout with `schema_version: 1` prefix on every line. EPIPE-safe, progress heartbeats on stderr, stable id-desc tiebreaker so `--since` windows never dupe/miss rows.
|
||||
- `src/commands/eval-prune.ts` (v0.25.0) — explicit retention cleanup. Requires `--older-than DUR`. `--dry-run` reports would-delete count.
|
||||
- `src/commands/eval-replay.ts` (v0.25.0) — contributor-facing replay tool. Reads NDJSON from `gbrain eval export`, re-runs each captured `query` / `search` op against the current brain, computes set-Jaccard@k between captured + current `retrieved_slugs`, top-1 stability rate, and latency Δ. Stable JSON shape (`schema_version: 1`) for CI gating; human mode prints a regression table. Pure Bun, zero new deps. The dev-loop half of BrainBench-Real that closes the gap between "data captured" and "data used to gate a PR." See `docs/eval-bench.md` for the workflow.
|
||||
@@ -174,7 +180,10 @@ strict behavior when unset.
|
||||
- `src/core/search/hybrid.ts` — Cathedral II `Promise<SearchResult[]>` return shape unchanged in v0.25.0. Adds `onMeta?: (m: HybridSearchMeta) => void` callback so op-layer capture can record what hybridSearch actually did. Existing callers leave it undefined.
|
||||
- `docs/eval-capture.md` (v0.25.0) — stable NDJSON schema reference for gbrain-evals consumers.
|
||||
- `test/public-exports.test.ts` (v0.25.0 / R2) — runtime contract test. Imports each of the 17 public subpaths via package name and pins a canary symbol per module. Paired with `scripts/check-exports-count.sh`.
|
||||
- `src/core/embedding.ts` — OpenAI text-embedding-3-large, batch, retry, backoff
|
||||
- `src/core/embedding.ts` — OpenAI text-embedding-3-large, batch, retry, backoff. **v0.28.7:** `BATCH_SIZE` reverted 50→100 — the original Voyage safety guard halved OpenAI throughput on every page. Per-recipe pre-split + recursive halving + adaptive shrink-on-miss now live in the gateway, so the outer paginator goes back to its original purpose: progress-callback granularity, not batch protection.
|
||||
- `src/core/ai/types.ts` — provider/recipe types. **v0.28.7 (#680):** `EmbeddingTouchpoint` extended with optional `chars_per_token` (default 4 chars/token, matching OpenAI tiktoken on English) and `safety_factor` (default 0.8, budget-utilization ceiling). Both consulted only when `max_batch_tokens` is also set. Voyage declares `chars_per_token=1` + `safety_factor=0.5` to handle dense payloads (CJK/JSON/base64) that overshoot tiktoken. The pre-split budget is `max_batch_tokens × safety_factor / chars_per_token`.
|
||||
- `src/core/ai/gateway.ts` — unified seam for every AI call. **v0.28.7 (#680):** module-scoped `_embedTransport` defaulting to AI SDK `embedMany`, with `__setEmbedTransportForTests(fn)` test seam so tests drive the public `embed()` function with a stubbed transport instead of probing private helpers. `splitByTokenBudget` and `isTokenLimitError` are now exported `@internal` — pure functions reused directly by the test file. Module-level `_shrinkState: Map<recipeId, {factor, consecutiveSuccesses}>` halves the recipe's effective `safety_factor` on token-limit miss (floor 0.05) and heals back ×1.5 toward the ceiling after `SHRINK_HEAL_AFTER=10` consecutive successes. `configureGateway()` walks every registered recipe at construction time and emits a once-per-process stderr warning for any embedding touchpoint missing `max_batch_tokens` (excluding the canonical OpenAI fast-path recipe). `resetGateway()` clears `_shrinkState`, the warned-set, and restores the real transport. ASCII flow diagram embedded in the `embed()` JSDoc covers the routing decision, recursion + halving, and shrinkState lifecycle.
|
||||
- `src/core/ai/recipes/voyage.ts` — Voyage AI openai-compatible recipe. **v0.28.7 (#680):** declares `chars_per_token=1` + `safety_factor=0.5` so the gateway pre-splits Voyage batches at a 60K-character budget (50% of 120K-token cap with the dense-tokenizer ratio). Closes the v0.27 backfill loop where ~26% of the corpus stayed un-embedded because tiktoken-grounded budgeting silently undercounted Voyage's actual token usage.
|
||||
- `src/core/check-resolvable.ts` — Resolver validation: reachability, MECE overlap, DRY checks, structured fix objects. v0.14.1: `CROSS_CUTTING_PATTERNS.conventions` is an array (notability gate accepts both `conventions/quality.md` and `_brain-filing-rules.md`). New `extractDelegationTargets()` parses `> **Convention:**`, `> **Filing rule:**`, and inline backtick references. DRY suppression is proximity-based via `DRY_PROXIMITY_LINES = 40`.
|
||||
- `src/core/repo-root.ts` — Shared `findRepoRoot(startDir?)` (v0.16.4): walks up from `startDir` (default `process.cwd()`) looking for `skills/RESOLVER.md`. Zero-dependency module imported by both `doctor.ts` and `check-resolvable.ts`. Parameterized `startDir` makes tests hermetic.
|
||||
- `src/commands/check-resolvable.ts` — Standalone CLI wrapper (v0.16.4) over `checkResolvable()`. Exports `parseFlags`, `resolveSkillsDir`, `DEFERRED`, `runCheckResolvable`. Exit rule: **1 on any issue (warnings OR errors)**, stricter than doctor's `ok` flag — honors README:259. Stable JSON envelope `{ok, skillsDir, report, autoFix, deferred, error, message}` — same shape on success and error paths. `--fix` path runs `autoFixDryViolations` BEFORE `checkResolvable` (same ordering as doctor). `scripts/skillify-check.ts` subprocess-calls `gbrain check-resolvable --json` (cached per process) and fails loud on binary-missing — no silent false-pass. **v0.19:** AGENTS.md workspaces now resolve natively (see `src/core/resolver-filenames.ts`) — gbrain inspects the 107-skill OpenClaw deployment whether the routing file is `RESOLVER.md` or `AGENTS.md`. `DEFERRED[]` is empty — Checks 5 + 6 shipped as real code, not issue URLs.
|
||||
@@ -198,10 +207,12 @@ strict behavior when unset.
|
||||
- `src/commands/extract.ts` — `gbrain extract links|timeline|all [--source fs|db]`: batch link/timeline extraction. fs walks markdown files, db walks pages from the engine (mutation-immune snapshot iteration; use this for live brains with no local checkout). As of v0.12.1 there is no in-memory dedup pre-load — candidates are buffered 100 at a time and flushed via `addLinksBatch` / `addTimelineEntriesBatch`; `ON CONFLICT DO NOTHING` enforces uniqueness at the DB layer, and the `created` counter returns real rows inserted (truthful on re-runs). v0.22.1 (#417): `ExtractOpts.slugs?: string[]` enables incremental extract — when set, `extractForSlugs()` reads ONLY those slugs' files (single combined links+timeline pass) instead of the full directory walk. CLI `gbrain extract` keeps full-walk behavior; the cycle path threads sync's `pagesAffected` through. `walkMarkdownFiles(brainDir)` still runs at line 455 to build `allSlugs` for link resolution — see `TODOS.md` for replacing it with `engine.getAllSlugs()`.
|
||||
- `src/commands/graph-query.ts` — `gbrain graph-query <slug> [--type T] [--depth N] [--direction in|out|both]`: typed-edge relationship traversal (renders indented tree)
|
||||
- `src/core/link-extraction.ts` — shared library for the v0.12.0 graph layer. extractEntityRefs (canonical, replaces backlinks.ts duplicate) matches both `[Name](people/slug)` markdown links and Obsidian `[[people/slug|Name]]` wikilinks as of v0.12.3. extractPageLinks, inferLinkType heuristics (attended/works_at/invested_in/founded/advises/source/mentions), parseTimelineEntries, isAutoLinkEnabled config helper. `DIR_PATTERN` covers `people`, `companies`, `deals`, `topics`, `concepts`, `projects`, `entities`, `tech`, `finance`, `personal`, `openclaw`. Used by extract.ts, operations.ts auto-link post-hook, and backlinks.ts.
|
||||
- `src/core/zombie-reap.ts` (v0.28.1) — idempotent `installSigchldHandler()` so JS-spawned children get reaped via Bun's internal `waitpid()`. Bun (like Node) only auto-reaps when a SIGCHLD listener is registered; without it, every child the worker spawns (shell jobs, embed batches, sub-agents) becomes a zombie on exit and holds connection slots. Called once at module load from `src/cli.ts` (with Windows platform guard — SIGCHLD doesn't exist on Windows). Cross-file leak guard via `_uninstallSigchldHandlerForTests()` for tests. Layer 1 of the three-layer zombie defense; Layer 2 is tini-as-PID-1 wrapping the worker subtree (via `src/core/minions/spawn-helpers.ts`); Layer 3 is the container's own tini for hard Bun crashes.
|
||||
- `src/core/minions/` — Minions job queue: BullMQ-inspired, Postgres-native (queue, worker, backoff, types, protected-names, quiet-hours, stagger, handlers/shell).
|
||||
- `src/core/minions/queue.ts` — MinionQueue class (submit, claim, complete, fail, stall detection, parent-child, depth/child-cap, per-job timeouts, cascade-kill, attachments, idempotency keys, child_done inbox, removeOnComplete/Fail). `add()` takes a 4th `trusted` arg (separate from `opts` to prevent spread leakage); protected names in `PROTECTED_JOB_NAMES` require `{allowProtectedSubmit: true}` and the check runs trim-normalized (whitespace-bypass safe). v0.14.1 #219: `add()` plumbs `max_stalled` through with a `[1, 100]` clamp; omitted values let the schema DEFAULT (5) kick in. v0.19.0: `handleWallClockTimeouts(lockDurationMs)` is Layer 3 kill shot for jobs where `FOR UPDATE SKIP LOCKED` stall detection and the timeout sweep both fail to evict (wedged worker holding a row lock via a pending transaction). v0.19.1: `maxWaiting` coalesce path now uses `pg_advisory_xact_lock` keyed on `(name, queue)` to serialize concurrent submits for the same key, and filters on `queue` in addition to `name` so cross-queue same-name jobs don't suppress each other.
|
||||
- `src/core/minions/worker.ts` — MinionWorker class (handler registry, lock renewal, graceful shutdown, timeout safety net). v0.14.0 abort-path fix: aborted jobs now call `failJob` with reason (`timeout`/`cancel`/`lock-lost`/`shutdown`) instead of returning silently. `shutdownAbort` (instance field) fires on process SIGTERM/SIGINT and propagates to `ctx.shutdownSignal` — shell handler listens to it; non-shell handlers don't. v0.22.1 (#403): per-job timeout fires `abort.abort(new Error('timeout'))` then a 30-second grace-then-evict safety net force-evicts the job from `inFlight` and marks it dead in DB if the handler ignores the abort signal — frees the slot even when a handler wedges (the 98-waiting-0-active prod incident driver).
|
||||
- `src/core/minions/supervisor.ts` — MinionSupervisor process manager. Spawns `gbrain jobs work` as a child, restarts on crash with exponential backoff, periodic health check. v0.22.1 (#406): `consecutiveHealthFailures` counter; on 3 consecutive failures emits `health_warn` with `reason: 'db_connection_degraded'` and calls `engine.reconnect()` to swap in a fresh pool, then resets the counter. Worker exit classifier emits `likely_cause` field on `worker_exited` events: `oom_or_external_kill` (SIGKILL), `graceful_shutdown` (SIGTERM), `runtime_error` (code 1), `clean_exit` (code 0), `unknown`.
|
||||
- `src/core/minions/worker.ts` — MinionWorker class (handler registry, lock renewal, graceful shutdown, timeout safety net). v0.14.0 abort-path fix: aborted jobs now call `failJob` with reason (`timeout`/`cancel`/`lock-lost`/`shutdown`) instead of returning silently. `shutdownAbort` (instance field) fires on process SIGTERM/SIGINT and propagates to `ctx.shutdownSignal` — shell handler listens to it; non-shell handlers don't. v0.22.1 (#403): per-job timeout fires `abort.abort(new Error('timeout'))` then a 30-second grace-then-evict safety net force-evicts the job from `inFlight` and marks it dead in DB if the handler ignores the abort signal — frees the slot even when a handler wedges (the 98-waiting-0-active prod incident driver). **v0.28.1 engine-ownership invariant:** `start()` no longer calls `engine.disconnect()` on shutdown — that was a leaky abstraction (the worker disconnected an engine it didn't own). The CLI handler in `src/commands/jobs.ts case 'work'` now owns engine lifecycle via try/finally with loud error logging on disconnect failure. Pinned by `test/worker-shutdown-disconnect.test.ts` asserting the inverse (`disconnectSpy).not.toHaveBeenCalled()`).
|
||||
- `src/core/minions/supervisor.ts` — MinionSupervisor process manager. Spawns `gbrain jobs work` as a child, restarts on crash with exponential backoff, periodic health check. v0.22.1 (#406): `consecutiveHealthFailures` counter; on 3 consecutive failures emits `health_warn` with `reason: 'db_connection_degraded'` and calls `engine.reconnect()` to swap in a fresh pool, then resets the counter. Worker exit classifier emits `likely_cause` field on `worker_exited` events: `oom_or_external_kill` (SIGKILL), `graceful_shutdown` (SIGTERM), `runtime_error` (code 1), `clean_exit` (code 0), `unknown`. **v0.28.1:** consumes `detectTini()` + `buildSpawnInvocation()` from `src/core/minions/spawn-helpers.ts` to wrap the worker subtree in tini-as-PID-1 when tini is on `PATH` (handles native-addon zombie reaping that the in-process SIGCHLD reaper can't reach). Exposes `isTiniDetected` read-only accessor for tests.
|
||||
- `src/core/minions/spawn-helpers.ts` (v0.28.1) — pure `detectTini()` + `buildSpawnInvocation()` helpers consumed by both `supervisor.ts` and `autopilot.ts`. Resolves the DRY violation between the two spawn sites and makes the tini wrapping testable without `mock.module()` (rule R2 of `scripts/check-test-isolation.sh`). `detectTini()` calls `execFileSync('which', ['tini'])` with explicit `env: process.env` so Bun sees runtime PATH mutations (the env-snapshot bug fix). `buildSpawnInvocation(tiniPath, cmd, args)` returns `{cmd, args}` with tini prepended when present, or the bare invocation otherwise. Pinned by `test/spawn-helpers.test.ts` (5 cases) and `test/supervisor-tini.test.ts` (4 cases).
|
||||
- `src/core/minions/types.ts` — `MinionJobInput` + `MinionJobStatus` + handler context types. `MinionJobInput.max_stalled` (new in v0.14.1) is optional; omitted values let the schema DEFAULT (5) kick in, provided values are clamped to `[1, 100]`.
|
||||
- `src/core/minions/protected-names.ts` — side-effect-free constant module exporting `PROTECTED_JOB_NAMES` + `isProtectedJobName()`. Kept pure so queue core can import without loading handler modules.
|
||||
- `src/core/minions/handlers/shell.ts` — `shell` job handler. Spawns `/bin/sh -c cmd` (absolute path, PATH-override-safe) or `argv[0] argv[1..]` (no shell). Env allowlist: `PATH, HOME, USER, LANG, TZ, NODE_ENV` + caller `env:` overrides. UTF-8-safe stdout/stderr tail via `string_decoder.StringDecoder`. Abort (either `ctx.signal` or `ctx.shutdownSignal`) fires SIGTERM → 5s grace → SIGKILL on child. Requires `GBRAIN_ALLOW_SHELL_JOBS=1` on worker (gated by `registerBuiltinHandlers`).
|
||||
@@ -219,14 +230,14 @@ strict behavior when unset.
|
||||
- `src/core/minions/attachments.ts` — Attachment validation (path traversal, null byte, oversize, base64, duplicate detection)
|
||||
- `src/commands/agent.ts` (v0.16) — `gbrain agent run <prompt> [flags]` CLI. Submits `subagent` (or N children + 1 aggregator) under `{allowProtectedSubmit: true}`. Single-entry `--fanout-manifest` short-circuits. Children get `on_child_fail: 'continue'` + `max_stalled: 3`. `--follow` is the default on TTY; streams logs + polls `waitForCompletion` in parallel. Ctrl-C detaches, does not cancel.
|
||||
- `src/commands/agent-logs.ts` (v0.16) — `gbrain agent logs <job> [--follow] [--since]`. Merges JSONL heartbeat audit + `subagent_messages` into a chronological timeline. `parseSince` accepts ISO-8601 or relative (`5m`, `1h`, `2d`). Transcript tail renders only for terminal jobs.
|
||||
- `src/commands/jobs.ts` — `gbrain jobs` CLI subcommands + `gbrain jobs work` daemon. v0.13.1 surfaces the full `MinionJobInput` retry/backoff/timeout/idempotency surface as first-class CLI flags on `jobs submit`: `--max-stalled`, `--backoff-type fixed|exponential`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key`. `jobs smoke --sigkill-rescue` is the opt-in regression guard for #219. v0.16 wires `registerBuiltinHandlers` to always register `subagent` + `subagent_aggregator` (no env flag — `ANTHROPIC_API_KEY` is the natural cost gate, trust is via `PROTECTED_JOB_NAMES`) and loads `GBRAIN_PLUGIN_PATH` plugins at worker startup with a loud startup-line per plugin. `shell` handler still gated by `GBRAIN_ALLOW_SHELL_JOBS=1` (RCE surface, separate concern). v0.22.10 (#521): the `autopilot-cycle` handler now forwards `job.data.phases` to `runCycle` (was previously discarded — caller-supplied phase selection silently became a full cycle). Phases are validated against `ALL_PHASES` from `src/core/cycle.ts`; invalid names are filtered out and an empty/missing array falls back to the default 6-phase cycle. v0.22.13 (PR #490 CODEX-1+CODEX-4): `sync` handler now resolves `sourceId` at entry by looking up `sources.local_path` (mirrors `cycle.ts:480`'s autopilot fix from PR #475) so multi-source brains read the per-source `last_commit` anchor instead of the global config key. Concurrency routed through the shared `autoConcurrency()` policy in `src/core/sync-concurrency.ts` instead of the prior hardcoded `4`; PGLite stays serial. `noEmbed` default is `true` (embed is a separate job — submit `gbrain embed --stale` after sync, or rely on the autopilot cycle's embed phase).
|
||||
- `src/commands/jobs.ts` — `gbrain jobs` CLI subcommands + `gbrain jobs work` daemon. **v0.28.1:** `case 'work'` now wraps `worker.start()` in try/finally and owns engine lifecycle — calls `engine.disconnect()` on shutdown with loud error logging on failure. Replaces the prior call inside `MinionWorker.start()` (which violated engine ownership: the worker disconnected an engine it didn't own, and clobbered the module-level singleton on PostgresEngine via the now-fixed idempotency bug). Pool slots now free immediately on shutdown instead of waiting for TCP keepalive (~minutes). v0.13.1 surfaces the full `MinionJobInput` retry/backoff/timeout/idempotency surface as first-class CLI flags on `jobs submit`: `--max-stalled`, `--backoff-type fixed|exponential`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key`. `jobs smoke --sigkill-rescue` is the opt-in regression guard for #219. v0.16 wires `registerBuiltinHandlers` to always register `subagent` + `subagent_aggregator` (no env flag — `ANTHROPIC_API_KEY` is the natural cost gate, trust is via `PROTECTED_JOB_NAMES`) and loads `GBRAIN_PLUGIN_PATH` plugins at worker startup with a loud startup-line per plugin. `shell` handler still gated by `GBRAIN_ALLOW_SHELL_JOBS=1` (RCE surface, separate concern). v0.22.10 (#521): the `autopilot-cycle` handler now forwards `job.data.phases` to `runCycle` (was previously discarded — caller-supplied phase selection silently became a full cycle). Phases are validated against `ALL_PHASES` from `src/core/cycle.ts`; invalid names are filtered out and an empty/missing array falls back to the default 6-phase cycle. v0.22.13 (PR #490 CODEX-1+CODEX-4): `sync` handler now resolves `sourceId` at entry by looking up `sources.local_path` (mirrors `cycle.ts:480`'s autopilot fix from PR #475) so multi-source brains read the per-source `last_commit` anchor instead of the global config key. Concurrency routed through the shared `autoConcurrency()` policy in `src/core/sync-concurrency.ts` instead of the prior hardcoded `4`; PGLite stays serial. `noEmbed` default is `true` (embed is a separate job — submit `gbrain embed --stale` after sync, or rely on the autopilot cycle's embed phase).
|
||||
- `src/commands/features.ts` — `gbrain features --json --auto-fix`: usage scan + feature adoption salesman
|
||||
- `src/commands/autopilot.ts` — `gbrain autopilot --install`: self-maintaining brain daemon (sync+extract+embed)
|
||||
- `src/commands/autopilot.ts` — `gbrain autopilot --install`: self-maintaining brain daemon (sync+extract+embed). **v0.28.1:** consumes `detectTini()` from `src/core/minions/spawn-helpers.ts` and resolves it once at startup instead of per worker respawn (was paying an `execFileSync` cost on every restart).
|
||||
- `src/mcp/server.ts` — MCP stdio server (generated from operations). v0.22.7: tool-call handler delegates to `dispatchToolCall` from `src/mcp/dispatch.ts` so stdio + HTTP transports share one validation, context-build, and error-format path.
|
||||
- `src/mcp/dispatch.ts` (v0.22.7) — Shared tool-call dispatch consumed by both stdio (`server.ts`) and HTTP transports. Exports `dispatchToolCall(engine, name, params, opts)`, `buildOperationContext(engine, params, opts)`, and `validateParams(op, params)`. Single source of truth for `(ctx, params)` handler arg order and the 5-field `OperationContext` shape (engine + config + logger + dryRun + remote). Defaults to `remote: true` (untrusted); local CLI callers pass `remote: false`. Closed F1/F2/F3 drift bugs in the original v0.22.5 HTTP transport.
|
||||
- `src/mcp/dispatch.ts` (v0.22.7) — Shared tool-call dispatch consumed by both stdio (`server.ts`) and HTTP transports. Exports `dispatchToolCall(engine, name, params, opts)`, `buildOperationContext(engine, params, opts)`, and `validateParams(op, params)`. Single source of truth for `(ctx, params)` handler arg order and the 5-field `OperationContext` shape (engine + config + logger + dryRun + remote). Defaults to `remote: true` (untrusted); local CLI callers pass `remote: false`. Closed F1/F2/F3 drift bugs in the original v0.22.5 HTTP transport. **v0.26.9 (F8):** adds `summarizeMcpParams(opName, params)` — privacy-preserving redactor for `mcp_request_log` and the admin SSE feed. Returns `{redacted, kind, declared_keys, unknown_key_count, approx_bytes}`. Intersects submitted top-level keys against the operation's declared `params` allow-list (declared keys preserved as a sorted array for debug visibility; unknown keys counted but never named, closing the attacker-controlled-key-name leak). Byte counts bucketed up to nearest 1KB so an attacker can't binary-search secret-content sizes via repeated probes. Operators on a personal laptop who want raw payload visibility opt back in with `gbrain serve --http --log-full-params` (loud stderr warning at startup). Canonical helper — new logging code paths route through it rather than `JSON.stringify(params)`.
|
||||
- `src/mcp/rate-limit.ts` (v0.22.7) — Bounded-LRU token-bucket limiter. `buildDefaultLimiters()` returns the two-bucket pipeline: pre-auth IP (30/60s, fires BEFORE the DB lookup so brute-force load against `access_tokens` is actually capped) + post-auth token-id (60/60s). Tracks `lastTouchedMs` separately from `lastRefillMs` so an exhausted key can't be reset by hammering past the TTL. LRU cap bounds memory under attacker-controlled key growth.
|
||||
- `src/commands/serve-http.ts` (v0.26.0) — Express 5 HTTP MCP server with OAuth 2.1, admin dashboard, and SSE live activity feed. Started via `gbrain serve --http [--port N] [--token-ttl N] [--enable-dcr] [--public-url URL]`. Supersedes the v0.22.7 `src/mcp/http-transport.ts` simple bearer-auth path. Combines MCP SDK's `mcpAuthRouter` (authorize / token / register / revoke endpoints), a custom `client_credentials` handler (SDK's token endpoint throws `UnsupportedGrantTypeError` for CC; the custom handler runs BEFORE the router and falls through for `auth_code` / `refresh_token`), `requireBearerAuth` middleware for `/mcp` with scope enforcement before op dispatch, `localOnly` rejection, and `express-rate-limit` at 50 req / 15 min on `/token`. Serves the built admin SPA from `admin/dist/` with SPA fallback. `/admin/events` SSE endpoint broadcasts every MCP request to connected admin browsers. `cookie-parser` middleware wired (Express 5 has no built-in). Startup logging prints port, engine, configured issuer URL (honors `--public-url`), registered-client count, DCR status, and admin bootstrap token.
|
||||
- `src/core/oauth-provider.ts` (v0.26.0) — `GBrainOAuthProvider` implementing the MCP SDK's `OAuthServerProvider` + `OAuthRegisteredClientsStore` interfaces. Backed by raw SQL (works on both PGLite and Postgres — OAuth is infrastructure, not a BrainEngine concern). Full OAuth 2.1 spec: `authorize` + `exchangeAuthorizationCode` with PKCE (for ChatGPT), `client_credentials` (for Perplexity / Claude), `refresh_token` with rotation, `revokeToken`, `registerClient` (DCR path validates redirect_uri must be `https://` or loopback per RFC 6749 §3.1.2.1). All tokens + client secrets SHA-256 hashed before storage. Auth codes single-use with 10-minute TTL via atomic `DELETE...RETURNING` (closes RFC 6749 §10.5 TOCTOU race). Refresh rotation also `DELETE...RETURNING` (closes §10.4 stolen-token detection bypass). `pgArray()` escapes commas/quotes/braces in elements so a comma-bearing redirect_uri can't smuggle a second array element. Legacy `access_tokens` fallback in `verifyAccessToken` grandfathers pre-v0.26 bearer tokens as `read+write+admin`. `sweepExpiredTokens()` runs on startup wrapped in try/catch. **v0.26.2:** module-private `coerceTimestamp()` boundary helper at the top of the file normalizes postgres-driver-as-string BIGINT columns to JS numbers at every read site (5 call sites: `getClient` L112+L113 for DCR `/register` RFC 7591 §3.2.1 numeric timestamps, `exchangeRefreshToken` L274 + `verifyAccessToken` L296+L303 for the SDK's `typeof === 'number'` bearerAuth check). Throws on non-finite input (NaN/Infinity) so corrupt rows fail loud at the boundary instead of riding through as `expiresAt: NaN`; returns undefined for SQL NULL so callers decide NULL semantics explicitly (refresh + access token paths treat NULL as expired). Helper intentionally NOT promoted to `src/core/utils.ts` — codex review flagged repo-wide BIGINT precision-loss risk for a generic helper.
|
||||
- `src/commands/serve-http.ts` (v0.26.0) — Express 5 HTTP MCP server with OAuth 2.1, admin dashboard, and SSE live activity feed. Started via `gbrain serve --http [--port N] [--token-ttl N] [--enable-dcr] [--public-url URL] [--log-full-params]`. Supersedes the v0.22.7 `src/mcp/http-transport.ts` simple bearer-auth path. Combines MCP SDK's `mcpAuthRouter` (authorize / token / register / revoke endpoints), a custom `client_credentials` handler (SDK's token endpoint throws `UnsupportedGrantTypeError` for CC; the custom handler runs BEFORE the router and falls through for `auth_code` / `refresh_token`), `requireBearerAuth` middleware for `/mcp` with scope enforcement before op dispatch, `localOnly` rejection, and `express-rate-limit` at 50 req / 15 min on `/token`. Serves the built admin SPA from `admin/dist/` with SPA fallback. `/admin/events` SSE endpoint broadcasts every MCP request to connected admin browsers. `cookie-parser` middleware wired (Express 5 has no built-in). Startup logging prints port, engine, configured issuer URL (honors `--public-url`), registered-client count, DCR status, and admin bootstrap token. **v0.26.9 hardening pass:** F7 sets `remote: true` explicitly on the `/mcp` request handler's OperationContext literal (closes the HTTP shell-job RCE — without this, `submit_job`'s protected-name guard at `operations.ts:1391` saw a falsy undefined and skipped, letting a `read+write`-scoped OAuth token submit `shell` jobs). F8 wires `summarizeMcpParams` from `src/mcp/dispatch.ts` into both `mcp_request_log` writes and the admin SSE feed by default (raw payloads opt-in via `--log-full-params` with stderr warning). F9 sets cookie `Secure` flag when behind HTTPS or a public-URL proxy. F10 caps the magic-link nonce store with an LRU bound. F12 routes DCR disable through the `GBrainOAuthProvider` constructor's `dcrDisabled` option instead of the prior monkey-patch on the express router. F14 wraps `transport.handleRequest` in try/catch so SDK throws return a JSON-RPC 500 envelope instead of express's default HTML error page. F15 unifies OperationError + unexpected exceptions through `buildError` / `serializeError` so `/mcp` always returns the same envelope shape. **v0.28.1:** `/health` endpoint extracted into pure `probeHealth(engine)` async function with `HEALTH_TIMEOUT_MS = 3000` exported constant — drops the timeout from 5s to 3s so Fly.io's 5s health-check deadline gets 2s of headroom for TCP, response framing, and clock skew. Races `engine.getStats()` against the timeout via `Promise.race`; saturated pool returns 503 with `Health check timed out (database pool may be saturated)` instead of hanging. `clearTimeout` in finally block prevents pending-timer pile-up under high probe rates (race-leak fix from adversarial review).
|
||||
- `src/core/oauth-provider.ts` (v0.26.0) — `GBrainOAuthProvider` implementing the MCP SDK's `OAuthServerProvider` + `OAuthRegisteredClientsStore` interfaces. Backed by raw SQL (works on both PGLite and Postgres — OAuth is infrastructure, not a BrainEngine concern). Full OAuth 2.1 spec: `authorize` + `exchangeAuthorizationCode` with PKCE (for ChatGPT), `client_credentials` (for Perplexity / Claude), `refresh_token` with rotation, `revokeToken`, `registerClient` (DCR path validates redirect_uri must be `https://` or loopback per RFC 6749 §3.1.2.1). All tokens + client secrets SHA-256 hashed before storage. Auth codes single-use with 10-minute TTL via atomic `DELETE...RETURNING` (closes RFC 6749 §10.5 TOCTOU race). Refresh rotation also `DELETE...RETURNING` (closes §10.4 stolen-token detection bypass). `pgArray()` escapes commas/quotes/braces in elements so a comma-bearing redirect_uri can't smuggle a second array element. Legacy `access_tokens` fallback in `verifyAccessToken` grandfathers pre-v0.26 bearer tokens as `read+write+admin`. `sweepExpiredTokens()` runs on startup wrapped in try/catch. **v0.26.9 RFC 6749/7009 hardening pass:** F1+F2 fold `client_id` atomically into the `DELETE WHERE` clauses for both auth-code exchange and refresh rotation — pre-fix the post-hoc client compare burned the row on wrong-client paths so the legitimate client couldn't retry. F3 enforces refresh-scope-subset against the original grant on the row (RFC 6749 §6), not the client's currently-allowed scopes — fixes the case where revoking a scope from a client wouldn't shrink the agent's existing refresh tokens. F4 binds `client_id` on `revokeToken` so a client can only revoke its own tokens (RFC 7009 §2.1). F7c validates the `/token` request's `redirect_uri` against the value stored at `/authorize` (RFC 6749 §4.1.3) — empty-string treated as missing rather than wildcard match (adversarial-review fix). F5 swaps bare `catch {}` blocks in `verifyAccessToken` and `getClient` for `isUndefinedColumnError` from `src/core/utils.ts` — only SQLSTATE 42703 falls through to legacy fallback; lock timeouts and network blips throw and surface. F6 makes `sweepExpiredTokens()` actually return the count via `RETURNING 1` + array length, not a fire-and-forget zero. F12 adds `dcrDisabled` constructor option so `serve-http.ts` can disable the `/register` endpoint without monkey-patching the router. **v0.26.2:** module-private `coerceTimestamp()` boundary helper at the top of the file normalizes postgres-driver-as-string BIGINT columns to JS numbers at every read site (5 call sites: `getClient` L112+L113 for DCR `/register` RFC 7591 §3.2.1 numeric timestamps, `exchangeRefreshToken` L274 + `verifyAccessToken` L296+L303 for the SDK's `typeof === 'number'` bearerAuth check). Throws on non-finite input (NaN/Infinity) so corrupt rows fail loud at the boundary instead of riding through as `expiresAt: NaN`; returns undefined for SQL NULL so callers decide NULL semantics explicitly (refresh + access token paths treat NULL as expired). Helper intentionally NOT promoted to `src/core/utils.ts` — codex review flagged repo-wide BIGINT precision-loss risk for a generic helper.
|
||||
- `admin/` (v0.26.0) — React 19 + Vite + TypeScript admin SPA embedded in the binary via `admin/dist/` served by `serve-http.ts`. 7 screens: Login (bootstrap token → session cookie), Dashboard (metrics + SSE feed + token health), Agents (sortable table + sparklines + Register button), Register (modal with scope checkboxes + grant type selector), Credentials reveal (full-screen modal with Copy + Download JSON + yellow one-time-only warning), Request Log (filterable paginated), Agent Detail drawer (Details / Activity / Config Export tabs + Revoke). Design tokens: `#0a0a0f` bg, Inter for UI, JetBrains Mono for data, 4-32px spacing scale, rounded pill badges. HTTP-only SameSite=Strict cookie auth. 65KB gzip. Build: `cd admin && bun install && bun run build`; output at `admin/dist/` is committed for self-contained binaries.
|
||||
- `src/commands/auth.ts` — Token management. `gbrain auth create/list/revoke/test` for legacy bearer tokens (v0.22.7 wired as a first-class CLI subcommand) plus `gbrain auth register-client` (v0.26.0) and `gbrain auth revoke-client <client_id>` (v0.26.2) for OAuth 2.1 client lifecycle. `revoke-client` runs an atomic `DELETE...RETURNING` on `oauth_clients`; FK `ON DELETE CASCADE` on `oauth_tokens.client_id` and `oauth_codes.client_id` purges every active token + authorization code in a single transaction. `process.exit(1)` on no-such-client (idempotent — re-running on the same id produces the same exit-1 message). Legacy tokens stored as SHA-256 hashes in `access_tokens`; OAuth clients in `oauth_clients`. As of v0.26.0, legacy tokens grandfather to `read+write+admin` scopes on the OAuth HTTP server, so pre-v0.26 deployments keep working with no migration.
|
||||
- `src/commands/upgrade.ts` — Self-update CLI. `runPostUpgrade()` enumerates migrations from the TS registry (src/commands/migrations/index.ts) and tail-calls `runApplyMigrations(['--yes', '--non-interactive'])` so the mechanical side of every outstanding migration runs unconditionally.
|
||||
@@ -234,8 +245,8 @@ strict behavior when unset.
|
||||
- `src/commands/repair-jsonb.ts` — `gbrain repair-jsonb [--dry-run] [--json]`: rewrites `jsonb_typeof='string'` rows in place across 5 affected columns (pages.frontmatter, raw_data.data, ingest_log.pages_updated, files.metadata, page_versions.frontmatter). Fixes v0.12.0 double-encode bug on Postgres; PGLite no-ops. Idempotent.
|
||||
- `src/commands/orphans.ts` — `gbrain orphans [--json] [--count] [--include-pseudo]`: surfaces pages with zero inbound wikilinks, grouped by domain. Auto-generated/raw/pseudo pages filtered by default. Also exposed as `find_orphans` MCP operation. Shipped in v0.12.3 (contributed by @knee5).
|
||||
- `src/commands/integrity.ts` — `gbrain integrity check|auto|review|extract`: bare-tweet detection, dead-link detection, three-bucket repair (auto-repair / review-queue / skip). `scanIntegrity()` is the shared library function called from `gbrain doctor` (sampled at limit=500) and `cmdCheck` (full scan). v0.22.8: batch-load fast path on Postgres uses `SELECT DISTINCT ON (slug)` in a single SQL query to fix the PgBouncer round-trip timeout (60s → ~6s) while preserving `engine.getAllSlugs()`'s `Set<string>` semantics on multi-source brains. Gated by `engine.kind === 'postgres'` at the call site so PGLite never enters batch; fallback `catch` logs at `GBRAIN_DEBUG=1` so real Postgres errors are diagnosable.
|
||||
- `src/commands/doctor.ts` — `gbrain doctor [--json] [--fast] [--fix] [--dry-run] [--index-audit]`: health checks. v0.12.3 added `jsonb_integrity` + `markdown_body_completeness` reliability checks. v0.14.1: `--fix` delegates inlined cross-cutting rules to `> **Convention:** see [path](path).` callouts (pipes DRY violations into `src/core/dry-fix.ts`); `--fix --dry-run` previews without writing. v0.14.2: `schema_version` check fails loudly when `version=0` (migrations never ran — the #218 `bun install -g` signature) and routes users to `gbrain apply-migrations --yes`; new opt-in `--index-audit` flag (Postgres-only) reports zero-scan indexes from `pg_stat_user_indexes` (informational only, no auto-drop). v0.15.2: every DB check is wrapped in a progress phase; `markdown_body_completeness` runs under a 1s heartbeat timer so 10+ min scans are observable on 50K-page brains. v0.19.1 added `queue_health` (Postgres-only) with two subchecks: stalled-forever active jobs (started_at > 1h) and waiting-depth-per-name > threshold (default 10, override via `GBRAIN_QUEUE_WAITING_THRESHOLD`). Worker-heartbeat subcheck intentionally deferred to follow-up B7 because it needs a `minion_workers` table to produce ground-truth signal. Fix hints point at `gbrain repair-jsonb`, `gbrain sync --force`, `gbrain apply-migrations`, and `gbrain jobs get/cancel <id>`. v0.22.12 (#500): `sync_failures` check shows `[CODE=N, ...]` breakdown for both unacked entries (warn) and acked-historical entries (ok), surfacing systemic failure modes (`SLUG_MISMATCH=2685`) instead of a bare count.
|
||||
- `src/core/migrate.ts` — schema-migration runner. Owns the `MIGRATIONS` array (source of truth for schema DDL). v0.14.2 extended the `Migration` interface with `sqlFor?: { postgres?, pglite? }` (engine-specific SQL overrides `sql`) and `transaction?: boolean` (set to false for `CREATE INDEX CONCURRENTLY`, which Postgres refuses inside a transaction; ignored on PGLite since it has no concurrent writers). Migration v14 (fix wave) uses a handler branching on `engine.kind` to run CONCURRENTLY on Postgres (with a pre-drop of any invalid remnant via `pg_index.indisvalid`) and plain `CREATE INDEX` on PGLite. v15 bumps `minion_jobs.max_stalled` default 1→5 and backfills existing non-terminal rows. v0.22.6.1: migration v24 (`rls_backfill_missing_tables`) uses `sqlFor: { pglite: '' }` to no-op on PGLite — PGLite has no RLS engine and is single-tenant by definition, and the v24 ALTERs target subagent tables that don't exist in pglite-schema.ts. Closes #395 (contributed by @jdcastro2). **v30 (v0.23):** creates `dream_verdicts (file_path TEXT, content_hash TEXT, worth_processing BOOL, reasons JSONB, judged_at TIMESTAMPTZ, PK(file_path, content_hash))`. RLS-enabled when running as a BYPASSRLS role. The synthesize phase reads/writes this table to avoid re-judging on backfill re-runs.
|
||||
- `src/commands/doctor.ts` — `gbrain doctor [--json] [--fast] [--fix] [--dry-run] [--index-audit]`: health checks. v0.12.3 added `jsonb_integrity` + `markdown_body_completeness` reliability checks. v0.14.1: `--fix` delegates inlined cross-cutting rules to `> **Convention:** see [path](path).` callouts (pipes DRY violations into `src/core/dry-fix.ts`); `--fix --dry-run` previews without writing. v0.14.2: `schema_version` check fails loudly when `version=0` (migrations never ran — the #218 `bun install -g` signature) and routes users to `gbrain apply-migrations --yes`; new opt-in `--index-audit` flag (Postgres-only) reports zero-scan indexes from `pg_stat_user_indexes` (informational only, no auto-drop). v0.15.2: every DB check is wrapped in a progress phase; `markdown_body_completeness` runs under a 1s heartbeat timer so 10+ min scans are observable on 50K-page brains. v0.19.1 added `queue_health` (Postgres-only) with two subchecks: stalled-forever active jobs (started_at > 1h) and waiting-depth-per-name > threshold (default 10, override via `GBRAIN_QUEUE_WAITING_THRESHOLD`). Worker-heartbeat subcheck intentionally deferred to follow-up B7 because it needs a `minion_workers` table to produce ground-truth signal. Fix hints point at `gbrain repair-jsonb`, `gbrain sync --force`, `gbrain apply-migrations`, and `gbrain jobs get/cancel <id>`. v0.22.12 (#500): `sync_failures` check shows `[CODE=N, ...]` breakdown for both unacked entries (warn) and acked-historical entries (ok), surfacing systemic failure modes (`SLUG_MISMATCH=2685`) instead of a bare count. v0.26.7 (#612): `rls_event_trigger` check (post-install drift detector for migration v35's auto-RLS event trigger). Lives outside the `// 5. RLS` slice that the structural doctor.test.ts guards anchor on, so the existing test guards stay intact. Healthy `evtenabled` set is `('O','A')` only — `R` is replica-only and would not fire in normal sessions; `D` is disabled. Fix hint is `gbrain apply-migrations --force-retry 35`.
|
||||
- `src/core/migrate.ts` — schema-migration runner. Owns the `MIGRATIONS` array (source of truth for schema DDL). v0.14.2 extended the `Migration` interface with `sqlFor?: { postgres?, pglite? }` (engine-specific SQL overrides `sql`) and `transaction?: boolean` (set to false for `CREATE INDEX CONCURRENTLY`, which Postgres refuses inside a transaction; ignored on PGLite since it has no concurrent writers). Migration v14 (fix wave) uses a handler branching on `engine.kind` to run CONCURRENTLY on Postgres (with a pre-drop of any invalid remnant via `pg_index.indisvalid`) and plain `CREATE INDEX` on PGLite. v15 bumps `minion_jobs.max_stalled` default 1→5 and backfills existing non-terminal rows. v0.22.6.1: migration v24 (`rls_backfill_missing_tables`) uses `sqlFor: { pglite: '' }` to no-op on PGLite — PGLite has no RLS engine and is single-tenant by definition, and the v24 ALTERs target subagent tables that don't exist in pglite-schema.ts. Closes #395 (contributed by @jdcastro2). **v30 (v0.23):** creates `dream_verdicts (file_path TEXT, content_hash TEXT, worth_processing BOOL, reasons JSONB, judged_at TIMESTAMPTZ, PK(file_path, content_hash))`. RLS-enabled when running as a BYPASSRLS role. The synthesize phase reads/writes this table to avoid re-judging on backfill re-runs. **v35 (v0.26.7):** auto-RLS event trigger + one-time backfill. `auto_rls_on_create_table` fires on `ddl_command_end` for `WHEN TAG IN ('CREATE TABLE','CREATE TABLE AS','SELECT INTO')` and runs `ALTER TABLE … ENABLE ROW LEVEL SECURITY` on every new `public.*` table — no FORCE (matches v24/v29/schema.sql posture so non-BYPASSRLS apps can still read their own tables). The same migration backfills RLS on every existing `public.*` base table whose comment doesn't match the doctor regex (`^GBRAIN:RLS_EXEMPT\s+reason=\S.{3,}`). Per-table failure aborts the offending CREATE TABLE (event triggers fire inside the DDL transaction); no EXCEPTION wrap — that would convert loud rollback into silent permissive default. PGLite no-op via `sqlFor.pglite: ''`. Breaking change: operators with intentionally-RLS-off public tables must add the GBRAIN:RLS_EXEMPT comment BEFORE upgrade or the backfill will flip them on.
|
||||
- `src/core/progress.ts` — Shared bulk-action progress reporter. Writes to stderr. Modes: `auto` (TTY: `\r`-rewriting; non-TTY: plain lines), `human`, `json` (JSONL), `quiet`. Rate-gated by `minIntervalMs` and `minItems`. `startHeartbeat(reporter, note)` helper for single long queries. `child()` composes phase paths. Singleton SIGINT/SIGTERM coordinator emits `abort` events for every live phase. EPIPE defense on both sync throws and stream `'error'` events. Zero dependencies. Introduced in v0.15.2.
|
||||
- `src/core/cli-options.ts` — Global CLI flag parser. `parseGlobalFlags(argv)` returns `{cliOpts, rest}` with `--quiet` / `--progress-json` / `--progress-interval=<ms>` stripped. `getCliOptions()` / `setCliOptions()` expose a module-level singleton so commands reach the resolved flags without parameter threading. `cliOptsToProgressOptions()` maps to reporter options. `childGlobalFlags()` returns the flag suffix to append to `execSync('gbrain ...')` calls in migration orchestrators. `OperationContext.cliOpts` extends shared-op dispatch for MCP callers.
|
||||
- `src/core/db-lock.ts` (v0.22.13) — generic `tryAcquireDbLock(engine, lockId, ttlMinutes)` over the existing `gbrain_cycle_locks` table. Parameterized lock id so different scopes can nest cleanly: `gbrain-cycle` for the broad cycle (held by `cycle.ts`) and `gbrain-sync` (`SYNC_LOCK_ID` constant) for `performSync`'s narrower writer window. Same UPSERT-with-TTL semantics as the prior cycle-only helper, just generalized. Survives PgBouncer transaction pooling (unlike session-scoped `pg_try_advisory_lock`); crashed holders auto-release once their TTL expires.
|
||||
@@ -306,6 +317,8 @@ strict behavior when unset.
|
||||
- `src/commands/backlinks.ts` — Back-link checker and fixer (enforces Iron Law)
|
||||
- `src/commands/lint.ts` — Page quality linter (catches LLM artifacts, placeholder dates)
|
||||
- `src/commands/report.ts` — Structured report saver (audit trail for maintenance/enrichment)
|
||||
- `src/core/destructive-guard.ts` (v0.26.5) — three-layer protection against accidental data loss in gbrain. `assessDestructiveImpact(engine, sourceId)` counts pages/chunks/embeddings/files for a source. `checkDestructiveConfirmation(impact, opts)` is the fail-closed gate (`--confirm-destructive` required when data is present; `--yes` alone is rejected). `softDeleteSource` / `restoreSource` / `listArchivedSources` / `purgeExpiredSources` drive the source-level archive lifecycle via the column shape introduced in migration v34 (`sources.archived BOOLEAN`, `archived_at TIMESTAMPTZ`, `archive_expires_at TIMESTAMPTZ`). v0.26.5 added the page-level analog through `BrainEngine.softDeletePage` / `restorePage` / `purgeDeletedPages` plus `pages.deleted_at TIMESTAMPTZ` and a partial purge index. The MCP `delete_page` op rewires to `softDeletePage`; new ops `restore_page` (`scope: write`) and `purge_deleted_pages` (`scope: admin`, `localOnly: true`) round out the surface. Search visibility (`buildVisibilityClause` in `src/core/search/sql-ranking.ts`) hides soft-deleted pages and archived sources from `searchKeyword` / `searchKeywordChunks` / `searchVector` in both engines. The autopilot cycle's new 9th `purge` phase calls `purgeExpiredSources` + `engine.purgeDeletedPages(72)` so the 72h TTL is real, not honor-system.
|
||||
- `src/commands/pages.ts` (v0.26.5) — `gbrain pages purge-deleted [--older-than HOURS|Nd] [--dry-run] [--json]` operator escape hatch. Mirror of `gbrain sources purge` for the page-level lifecycle. Hard-deletes pages whose `deleted_at` is older than the cutoff; cascades to content_chunks/page_links/chunk_relations.
|
||||
- `openclaw.plugin.json` — ClawHub bundle plugin manifest
|
||||
|
||||
### BrainBench — in a sibling repo (v0.20+)
|
||||
@@ -341,10 +354,24 @@ Key commands added for Minions (job queue):
|
||||
- `gbrain jobs smoke [--sigkill-rescue]` — health smoke test. `--sigkill-rescue` is the v0.13.1 regression guard for #219: simulates a killed worker and asserts the stalled job is requeued instead of dead-lettered on first stall.
|
||||
- `gbrain jobs work [--queue Q] [--concurrency N]` — start worker daemon (Postgres only)
|
||||
|
||||
Key commands added in v0.26.5 (destructive-guard, end-to-end):
|
||||
- `gbrain sources archive <id>` — soft-delete a source. Hides from search via the new `sources.archived` column + cascading visibility filter. Preserves data for 72h. (PR #595 cherry-pick.)
|
||||
- `gbrain sources restore <id> [--no-federate]` — un-archive a soft-deleted source. Re-federates by default.
|
||||
- `gbrain sources archived [--json]` — list soft-deleted sources with their TTL.
|
||||
- `gbrain sources purge [<id>] [--confirm-destructive]` — permanent delete; with no id, purges all sources whose TTL expired.
|
||||
- `gbrain sources remove <id> [--confirm-destructive] [--dry-run]` — `--yes` alone no longer enough on populated sources. Boxed impact preview before destruction.
|
||||
- `gbrain pages purge-deleted [--older-than HOURS|Nd] [--dry-run] [--json]` — operator escape hatch for page-level soft-delete cleanup. Mirror of `gbrain sources purge`. The autopilot cycle's new `purge` phase calls the same library function automatically every run.
|
||||
- MCP `delete_page` op semantically shifts from hard-delete to soft-delete. New ops: `restore_page` (`scope: write`), `purge_deleted_pages` (`scope: admin`, `localOnly: true`).
|
||||
- `get_page` and `list_pages` extended with `include_deleted: boolean` (default false).
|
||||
- New autopilot cycle phase `purge` (9th, runs after `orphans`). `gbrain dream --phase purge` runs only the purge sweep.
|
||||
- Index strategy note: the partial index `pages_deleted_at_purge_idx ON pages (deleted_at) WHERE deleted_at IS NOT NULL` supports the autopilot purge query. Search filters (`WHERE deleted_at IS NULL`) do NOT need their own index — soft-deleted cardinality stays low and Postgres won't use the partial index for the negative predicate. Don't add a regular `(deleted_at)` index without measuring.
|
||||
- Schema migration v34 (`destructive_guard_columns`) adds `pages.deleted_at` + the partial purge index; promotes `archived` from `sources.config` JSONB to real columns; backfills any pre-v0.26.5 JSONB shape.
|
||||
|
||||
Key commands added in v0.25.0:
|
||||
- `gbrain eval export [--since DUR] [--limit N] [--tool query|search]` — stream captured `eval_candidates` rows as NDJSON to stdout. Every line starts with `"schema_version": 1` per the stable contract in `docs/eval-capture.md`. EPIPE-safe, progress heartbeats on stderr, deterministic ordering. Primary consumer is the sibling `gbrain-evals` repo for BrainBench-Real replay.
|
||||
- `gbrain eval prune --older-than DUR [--dry-run]` — explicit retention cleanup for `eval_candidates`. Requires `--older-than` (never deletes without a window). Duration strings: 30d, 7d, 1h, 90m, 3600s.
|
||||
- `gbrain eval replay --against FILE.ndjson [--limit N] [--top-regressions K] [--json] [--verbose]` — contributor-facing dev loop. Reads a captured NDJSON snapshot, re-runs each `query` / `search` op against the current brain, computes mean set-Jaccard@k between captured + current `retrieved_slugs`, top-1 stability rate, and latency Δ. JSON mode (`schema_version: 1`) for CI gating; human mode prints a regression table sorted worst-first. Closes the gap between "data captured" and "data used to gate a PR." See `docs/eval-bench.md` for the workflow.
|
||||
- `gbrain eval cross-modal --task "..." --output <path> [--cycles N] [--slot-a-model ID] [--slot-b-model ID] [--slot-c-model ID] [--receipt-dir DIR] [--json]` (v0.27.x) — multi-model quality gate. Three different-provider frontier models score the OUTPUT against the TASK on 5 documented dimensions. Pass criterion: every dim mean >=7 AND no model scored any dim <5. Exit codes: 0 PASS, 1 FAIL, 2 INCONCLUSIVE (<2/3 models returned parseable scores). Default cycles=3 in TTY, **cycles=1 in non-TTY** (limits accidental scripted bulk spend). Default slots: `openai:gpt-4o` / `anthropic:claude-opus-4-7` / `google:gemini-1.5-pro` — refresh alongside model-family bumps. Receipts land at `~/.gbrain/.gbrain/eval-receipts/<slug>-<sha8-of-output>.json` (gbrainPath honors GBRAIN_HOME). Bypasses `connectEngine()` via the cli.ts no-DB branch — runs cleanly before `gbrain init`. Reuses `src/core/ai/gateway.ts:chat()` for config/auth (no parallel provider stack). Cost-estimate prints to stderr before each cycle (T11=B partial cost guardrail; full `--budget-usd N` is a follow-up TODO).
|
||||
- `gbrain doctor` gains an `eval_capture` check: reads `eval_capture_failures` for the last 24h, groups by reason, warns when non-zero. Cross-process visibility (doctor runs in a separate process from MCP). Pre-v31 brains get `Skipped (table unavailable)` — non-fatal.
|
||||
- Config addition: `eval: { capture?: boolean, scrub_pii?: boolean }` in `~/.gbrain/config.json`. **File-plane only** — `gbrain config set` writes the DB plane and does NOT control capture.
|
||||
- **`GBRAIN_CONTRIBUTOR_MODE=1` env var** is the contributor-facing toggle. Capture is **off by default** as of v0.25.0; production users get a quiet brain. Resolution order: explicit `eval.capture` config wins both directions, then env var, then off. Documented in README.md, CONTRIBUTING.md, and `docs/eval-bench.md`.
|
||||
@@ -364,7 +391,7 @@ Key commands added in v0.14.2:
|
||||
- `gbrain doctor` gains two new checks: `sync_failures` (surfaces unacknowledged parse failures with exact paths + fix hints) and `brain_score` (renders the 5-component breakdown when score < 100: embed coverage / 35, link density / 25, timeline coverage / 15, orphans / 15, dead links / 10 — sum equals total).
|
||||
|
||||
Key commands added in v0.26.0 (OAuth 2.1 + HTTP server + admin dashboard):
|
||||
- `gbrain serve --http [--port 3131] [--token-ttl 3600] [--enable-dcr]` — HTTP MCP server with OAuth 2.1, admin dashboard at `/admin`, SSE activity feed at `/admin/events`, health check at `/health`. Prints admin bootstrap token on first start. Alongside (not replacing) stdio `gbrain serve`.
|
||||
- `gbrain serve --http [--port 3131] [--token-ttl 3600] [--enable-dcr] [--log-full-params]` — HTTP MCP server with OAuth 2.1, admin dashboard at `/admin`, SSE activity feed at `/admin/events`, health check at `/health`. Prints admin bootstrap token on first start. Alongside (not replacing) stdio `gbrain serve`. As of v0.26.9, `mcp_request_log.params` and the SSE feed default to a redacted summary (`{redacted, kind, declared_keys, unknown_key_count, approx_bytes}`); pass `--log-full-params` to log raw payloads on a personal laptop with a startup warning.
|
||||
- **OAuth client registration** — three paths:
|
||||
1. CLI: `gbrain auth register-client <name> --grant-types <types> --scopes <scopes>` (wired into `src/commands/auth.ts` as a thin wrapper over `GBrainOAuthProvider.registerClientManual`). Default grant types: `client_credentials`. Default scopes: `read`.
|
||||
2. Admin dashboard: Register client modal → credential reveal with Copy + Download JSON.
|
||||
@@ -394,6 +421,118 @@ Key commands added in v0.22.16 (claw-test friction loop):
|
||||
|
||||
## Testing
|
||||
|
||||
### Test command tiers (v0.26.4 — parallel fast loop)
|
||||
|
||||
Five tiers of test commands, each with a clear scope:
|
||||
|
||||
| Command | What it runs | Wallclock | When to use |
|
||||
|---|---|---|---|
|
||||
| `bun run test` | Parallel unit-test fast loop. 8-shard fan-out via `scripts/run-unit-parallel.sh`, then a serial pass over `*.serial.test.ts`. Excludes `*.slow.test.ts` and `test/e2e/*`. No pre-checks, no typecheck. | ~85s on a Mac dev box (3650+ tests) | Inner edit loop. Default. |
|
||||
| `bun run verify` | CI's authoritative pre-test gate set: `check:privacy && check:jsonb && check:progress && check:wasm && bun run typecheck`. The 4 checks `.github/workflows/test.yml` runs on shard 1 + typecheck. Single source of truth — CI literally calls `bun run verify`. | ~12s (wasm-compile dominates) | Before pushing; before `/ship`. |
|
||||
| `bun run test:full` | `verify && bun run test && bun run test:slow && [smart e2e]`. The local equivalent of "everything CI runs." Smart e2e: runs e2e only when `DATABASE_URL` is set; else loud skip notice to stderr. | ~3-5min depending on slow + e2e | Pre-merge sanity, before opening a PR. |
|
||||
| `bun run test:slow` | Just the `*.slow.test.ts` set (intentional cold-path correctness checks). | seconds-to-minutes | When touching slow-path code. |
|
||||
| `bun run test:serial` | Just the `*.serial.test.ts` set (cross-file-contention quarantine; runs at `--max-concurrency=1`). | ~1s per quarantined file | Debugging a specific quarantined file. |
|
||||
| `bun run test:e2e` | Real Postgres E2E. Requires Docker + `DATABASE_URL`. Sequential (template-DB parallelization is a v0.27+ TODO). | ~5-10min | Pre-ship; nightly. |
|
||||
| `bun run check:all` | All 7 historical pre-checks (privacy + jsonb + progress + no-legacy-getconnection + trailing-newline + wasm + exports-count). Superset of `verify`. | ~10s | Local-only sweep. The 4 not in `verify` are nice-to-haves. |
|
||||
|
||||
### CI vs local: intentionally divergent file sets
|
||||
|
||||
- **CI matrix** (`.github/workflows/test.yml`) runs `scripts/test-shard.sh` 4-way, which uses FNV-1a hash bucketing and INCLUDES `*.slow.test.ts`. CI is the ground truth for "did everything pass."
|
||||
- **Local fast loop** (`scripts/run-unit-shard.sh` via the parallel wrapper) uses round-robin-by-index sharding and EXCLUDES `*.slow.test.ts` AND `*.serial.test.ts`. Local trades coverage for inner-loop speed; CI catches what local skips.
|
||||
|
||||
This divergence is intentional. Don't try to make them equal — the two scripts deliberately solve different problems. The regression test at `test/scripts/run-unit-shard.test.ts` pins what the local fast loop should and shouldn't include.
|
||||
|
||||
### Failure-first logging
|
||||
|
||||
When `bun run test` finds any failure, the wrapper:
|
||||
|
||||
1. Writes failure blocks (each prefixed with `--- shard N: <test name> ---`) to `.context/test-failures.log` (workspace-local, gitignored). On systems without a writable `.context/`, falls back to `/tmp/gbrain-test-failures.log`.
|
||||
2. Prints a loud stderr banner with the absolute log path, plus the last 30 lines of the failure log inlined. Banner survives `| head` / `| tail` / agent-side log truncation.
|
||||
3. Writes a one-line-per-shard summary to `.context/test-summary.txt` (`shard N/M: pass=X fail=Y skip=Z rc=W`).
|
||||
4. Exits non-zero. Empty failure log + non-zero exit = infrastructure problem (wedged shard, killed child); the banner says so.
|
||||
|
||||
If a shard wedges (per-shard `GBRAIN_TEST_SHARD_TIMEOUT` cap, default 600s), the wrapper writes `--- shard N: WEDGED after ${SHARD_TIMEOUT}s ---` to the failure log, includes the last 50 lines of the shard log, and proceeds with other shards' results.
|
||||
|
||||
### File taxonomy
|
||||
|
||||
- `*.test.ts` → fast loop (parallel 8-shard fan-out).
|
||||
- `*.slow.test.ts` → run via `bun run test:slow` only (intentional cold-path tests; would dominate the fast loop's wallclock).
|
||||
- `*.serial.test.ts` → run via `bun run test:serial` after the parallel pass completes; uses `--max-concurrency=1`. Quarantine for tests that share file-wide state and race when run alongside other files in the same `bun test` process. Currently: `test/brain-registry.serial.test.ts`, `test/reconcile-links.serial.test.ts`, `test/core/cycle.serial.test.ts`, `test/embed.serial.test.ts` (the latter two added in v0.26.7 — they use `mock.module(...)` which leaks across files in the shard process). **Do not put the parallelism back on a serial file unless you've fixed the contention root cause** (it just re-introduces the flake).
|
||||
- `test/e2e/*.test.ts` → real-Postgres E2E. Skipped when `DATABASE_URL` is unset.
|
||||
|
||||
The intra-file parallelism project (turn `bun test` into `bun test --concurrent` after sweeping shared-state contention sites) is sliced across v0.26.7 (foundation), v0.26.8 (env-mutation sweep), and v0.26.9 (PGLite sweep + codemod + measurement). v0.26.4 ships file-level parallelism only.
|
||||
|
||||
### Test-isolation lint and helpers (v0.26.7)
|
||||
|
||||
The cross-file flake class is enforced statically by `scripts/check-test-isolation.sh`, wired into `bun run verify` and `bun run check:all`. Rules (non-serial unit files only; `*.serial.test.ts` and `test/e2e/*` are skipped):
|
||||
|
||||
| Rule | What it bans | Fix |
|
||||
|---|---|---|
|
||||
| **R1** | `process.env.X = ...`, bracket assignment, `delete process.env.X`, `Object.assign(process.env, ...)`, `Reflect.set(process.env, ...)` | Use `withEnv()` from `test/helpers/with-env.ts`, OR rename file to `*.serial.test.ts` |
|
||||
| **R2** | `mock.module(...)` anywhere in the file | Rename file to `*.serial.test.ts` (no DI on production code for testability) |
|
||||
| **R3** | `new PGLiteEngine(` outside ~50 lines after a `beforeAll(` line | Use the canonical block (below) inside `beforeAll(` |
|
||||
| **R4** | Files creating `new PGLiteEngine(` without `engine.disconnect(` inside an `afterAll(` block | Add `afterAll(() => engine.disconnect())` |
|
||||
|
||||
Files that violated these rules at the v0.26.7 baseline are listed in `scripts/check-test-isolation.allowlist`. **The allow-list MUST shrink over time** — never add new entries. v0.26.8 (env sweep) and v0.26.9 (PGLite sweep) remove entries as files get fixed.
|
||||
|
||||
#### Canonical PGLite block (R3 + R4 compliant)
|
||||
|
||||
Every test file that needs a PGLite engine should use this exact pattern:
|
||||
|
||||
```ts
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
});
|
||||
```
|
||||
|
||||
Why this exact shape: `beforeAll` creates a single engine per file (PGLite WASM cold-start + initSchema is ~20s); `beforeEach` truncates user data via `resetPgliteState` ("two orders of magnitude faster" than fresh-engine-per-test); `afterAll` disconnects so the engine doesn't leak across file boundaries within a shard process.
|
||||
|
||||
#### `withEnv` pattern (R1 fix)
|
||||
|
||||
```ts
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
|
||||
test('reads OPENAI_API_KEY', async () => {
|
||||
await withEnv({ OPENAI_API_KEY: 'sk-test' }, async () => {
|
||||
expect(loadConfig().openai_key).toBe('sk-test');
|
||||
});
|
||||
});
|
||||
|
||||
// Delete a var (override is undefined):
|
||||
await withEnv({ GBRAIN_HOME: undefined }, fn);
|
||||
|
||||
// Multiple keys:
|
||||
await withEnv({ A: '1', B: '2', C: undefined }, fn);
|
||||
```
|
||||
|
||||
`withEnv` saves the prior value of every key it touches and restores via try/finally — including when the callback throws. **It is cross-test safe but NOT intra-file concurrent-safe.** `process.env` is process-global; two `test.concurrent()` calls in the same file both touching the same key will race. Files using `withEnv` stay outside the future `test.concurrent()` codemod's eligibility filter.
|
||||
|
||||
#### When to quarantine instead of fix
|
||||
|
||||
Rename to `*.serial.test.ts` when:
|
||||
- The file uses `mock.module(...)` (R2 — there's no clean fix without changing production code).
|
||||
- The file is genuinely env-coupled (e.g. `gbrain-home-isolation.test.ts`, `claw-test-cli.test.ts`) — module-load env readers + ESM caching defeat dynamic-import-after-env tricks.
|
||||
- The file's tests intentionally share state across `it()` boundaries.
|
||||
|
||||
Quarantine count cap: 10 (informational). Beyond that, push back on the design.
|
||||
|
||||
### Inventory (legacy)
|
||||
|
||||
`bun test` runs all tests. After the v0.12.1 release: ~75 unit test files + 8 E2E test files (1412 unit pass, 119 E2E when `DATABASE_URL` is set — skip gracefully otherwise). Unit tests run
|
||||
without a database. E2E tests skip gracefully when `DATABASE_URL` is not set.
|
||||
|
||||
@@ -406,6 +545,7 @@ parity), `test/cli.test.ts` (CLI structure), `test/config.test.ts` (config redac
|
||||
`test/import-resume.test.ts` (import checkpoints), `test/migrate.test.ts` (migration; v8/v9 helper-btree-index SQL structural assertions + 1000-row wall-clock fixtures that guard the O(n²)→O(n log n) fix + v0.13.1 assertions on v12/v13 SQL shape, `sqlFor` + `transaction:false` runner semantics, the `max_stalled DEFAULT 1` regression guard, and v0.22.6.1 v24 `sqlFor.pglite: ''` no-op assertion),
|
||||
`test/bootstrap.test.ts` (v0.22.6.1 — bootstrap contract: no-op on fresh install, idempotent across two `initSchema()` calls, no-op on modern brain that already has every probed column, full bootstrap path on simulated pre-v0.18 brain, fresh-install regression guard, pre-v0.13 `links` shape coverage),
|
||||
`test/schema-bootstrap-coverage.test.ts` (v0.22.6.1 CI guard — `REQUIRED_BOOTSTRAP_COVERAGE` lists every forward reference in PGLITE_SCHEMA_SQL; the test fails loudly if `applyForwardReferenceBootstrap` skips one. When you add a column-with-index to the embedded schema blob, you extend both arrays or this guard fails. The pattern that broke gbrain ten times in two years is now structurally prevented.),
|
||||
`test/helpers/schema-diff.ts` + `test/helpers/schema-diff.test.ts` + `test/e2e/schema-drift.test.ts` (v0.26.6 #588 — cross-engine schema parity gate. Helper exports pure `snapshotSchema(query)` / `diffSnapshots(pg, pglite, opts)` / `formatDiffForFailure(diff)` / `isCleanDiff(diff)` over a four-tuple per column (`data_type`, `udt_name`, `is_nullable`, `column_default`). E2E test spins up fresh PGLite + Postgres, runs `engine.initSchema()` on each (bootstrap + schema replay + migrations), snapshots `information_schema.columns`, then diffs. 2-table allowlist (`files`, `file_migration_ledger`) — every other Postgres table must reach PGLite via PGLITE_SCHEMA_SQL or a migration's `sqlFor.pglite` branch. Sentinels for `oauth_clients`, `mcp_request_log`, `access_tokens`, `eval_candidates` give tighter blame messages. Skip-gracefully without `DATABASE_URL`. Wired into `scripts/e2e-test-map.ts` so changes to `src/schema.sql`, `src/core/pglite-schema.ts`, or `src/core/migrate.ts` trigger it. The failure message names every drift with a paste-ready hint pointing at `src/core/pglite-schema.ts`.),
|
||||
`test/setup-branching.test.ts` (setup flow), `test/slug-validation.test.ts` (slug validation),
|
||||
`test/storage.test.ts` (storage backends), `test/supabase-admin.test.ts` (Supabase admin),
|
||||
`test/yaml-lite.test.ts` (YAML parsing), `test/check-update.test.ts` (version check + update CLI),
|
||||
@@ -453,7 +593,9 @@ parity), `test/cli.test.ts` (CLI structure), `test/config.test.ts` (config redac
|
||||
`test/doctor.test.ts` (doctor command + v0.12.3 assertions that `jsonb_integrity` scans the four v0.12.0 write sites and `markdown_body_completeness` is present),
|
||||
`test/utils.test.ts` (shared SQL utilities + `tryParseEmbedding` null-return and single-warn semantics),
|
||||
`test/build-llms.test.ts` (llms.txt/llms-full.txt generator: path resolution, idempotence, spec shape, regen-drift guard, content contract, AGENTS.md install-path mirror, size-budget enforcement — 7 cases),
|
||||
`test/oauth.test.ts` (v0.26.0 OAuth 2.1 provider — 27 cases: register, getClient, `client_credentials` grant exchange, `authorization_code` flow with PKCE challenge / verifier, refresh token rotation, `verifyAccessToken` with both OAuth + legacy `access_tokens` fallback, `revokeToken`, `sweepExpiredTokens`, and a contract test asserting `scope` + `localOnly` annotations are set correctly on all 30 operations; **v0.26.2** adds 5 `coerceTimestamp` unit cases (null/undefined/string/number/throw-on-NaN), NULL-`expires_at`-as-expired contract tests for both refresh + access token paths, and a cascade-delete contract test asserting `revoke-client` purges `oauth_tokens` + `oauth_codes` rows via FK CASCADE),
|
||||
`test/oauth.test.ts` (v0.26.0 OAuth 2.1 provider — 27 cases: register, getClient, `client_credentials` grant exchange, `authorization_code` flow with PKCE challenge / verifier, refresh token rotation, `verifyAccessToken` with both OAuth + legacy `access_tokens` fallback, `revokeToken`, `sweepExpiredTokens`, and a contract test asserting `scope` + `localOnly` annotations are set correctly on all 30 operations; **v0.26.2** adds 5 `coerceTimestamp` unit cases (null/undefined/string/number/throw-on-NaN), NULL-`expires_at`-as-expired contract tests for both refresh + access token paths, and a cascade-delete contract test asserting `revoke-client` purges `oauth_tokens` + `oauth_codes` rows via FK CASCADE; **v0.26.9** adds 14 cases pinning the F1/F2/F3/F4/F5/F6/F7c/F12 invariants, including the F1/F4 cross-client isolation pattern (wrong-client attempt MUST reject AND rightful owner MUST still succeed atomically afterward) and the empty-string `redirect_uri` bypass guard surfaced during adversarial review),
|
||||
`test/mcp-dispatch-summarize.test.ts` (v0.26.9 — 7 cases pinning F8 `summarizeMcpParams` invariants: declared-keys allow-list intersection, attacker-key-name leak guard (unknown keys counted not named), 1KB byte bucketing for size-probe defense, missing op falls through to fully-redacted shape, declared-keys sorted for deterministic output),
|
||||
`test/trust-boundary-contract.test.ts` (v0.26.9 — 4 cases pinning F7b fail-closed semantics under cast bypass: `ctx.remote === undefined` treated as remote/untrusted at every flipped call site, `as any` and `Partial<>` spreads can't downgrade trust by accident),
|
||||
`test/check-resolvable-cli.test.ts` (v0.19 CLI wrapper: exit codes, JSON envelope shape, AGENTS.md fallback chain),
|
||||
`test/regression-v0_16_4.test.ts` (findRepoRoot regression guard — hermetic startDir parameterization),
|
||||
`test/filing-audit.test.ts` (v0.19 Check 6: `writes_pages` / `writes_to` frontmatter, filing-rules JSON validation),
|
||||
@@ -462,7 +604,8 @@ parity), `test/cli.test.ts` (CLI structure), `test/config.test.ts` (config redac
|
||||
`test/skillify-scaffold.test.ts` (v0.19 `gbrain skillify scaffold` stubs: SKILL.md, script, tests, routing-eval fixtures),
|
||||
`test/skillpack-install.test.ts` (v0.19 `gbrain skillpack install` managed-block install / update / no-clobber semantics),
|
||||
`test/skillpack-sync-guard.test.ts` (v0.19 sync-guard: bundled skills stay byte-identical to `skills/` source),
|
||||
`test/http-transport.test.ts` (v0.22.7 HTTP transport: 23 unit cases covering bearer auth + missing/no-Bearer/unknown/revoked + `/health` bypass, F1+F2 round-trip via dispatch.ts, F3 invalid_params, application/json response shape (not SSE), CORS default-deny + allowlist, body cap on Content-Length AND chunked, two-bucket rate limit (refill, exhaust+Retry-After, LRU eviction, TTL prune, pre-auth IP fires before DB), and `mcp_request_log` audit on success + auth_failed).
|
||||
`test/http-transport.test.ts` (v0.22.7 HTTP transport: 23 unit cases covering bearer auth + missing/no-Bearer/unknown/revoked + `/health` bypass, F1+F2 round-trip via dispatch.ts, F3 invalid_params, application/json response shape (not SSE), CORS default-deny + allowlist, body cap on Content-Length AND chunked, two-bucket rate limit (refill, exhaust+Retry-After, LRU eviction, TTL prune, pre-auth IP fires before DB), and `mcp_request_log` audit on success + auth_failed),
|
||||
`test/restart-sweep.test.ts` (v0.28.3 — 27 bun:test cases for the `recipes/restart-sweep.md` inlined script: sentinel-anchored fenced-block extraction with salted tmp filenames to bypass ESM cache; constructor-time env reads (proves no module-load snapshot); idempotency layer load/save/atomic-tmp-rename/corrupt-JSON-recovery/30-day-prune; `(sessionKey, lastAlertedAt)` cooldown gate with 6h threshold (the C1 fix that survives synthesized restartTime); AGGRESSIVE-gate two-state tests; execFile argv shape proving shell metachars in `OPENCLAW_TELEGRAM_GROUP` cannot reach `/bin/sh`; real-`\n`-not-literal alert formatting; `GBRAIN_HOME` state path override).
|
||||
|
||||
E2E tests (`test/e2e/`): Run against real Postgres+pgvector. Require `DATABASE_URL`.
|
||||
- `bun run test:e2e` runs Tier 1 (mechanical, all operations, no API keys). Includes 9 dedicated cases for the postgres-engine `addLinksBatch` / `addTimelineEntriesBatch` bind path — postgres-js's `unnest()` binding is structurally different from PGLite's and gets its own coverage.
|
||||
@@ -480,7 +623,7 @@ E2E tests (`test/e2e/`): Run against real Postgres+pgvector. Require `DATABASE_U
|
||||
- `test/e2e/engine-parity.test.ts` (v0.22.0) — Postgres ↔ PGLite top-result and result-set parity for `searchKeyword` + `searchVector`. Codex flagged that Postgres ranks pages then picks best chunk while PGLite returns chunks directly — without parity coverage the source-boost fix could pass on PGLite and fail on Postgres. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/postgres-bootstrap.test.ts` (v0.22.6.1) — exercises `PostgresEngine.initSchema()` directly against a fresh real Postgres database. Asserts the bootstrap path is no-op on fresh installs and that SCHEMA_SQL replays cleanly through the engine path (not via the standalone `db.initSchema` from `src/core/db.ts`, which would have produced false-positive coverage). Codex caught the E2E-shape gap during plan review.
|
||||
- `test/e2e/http-transport.test.ts` (v0.22.7) — 8 cases against real Postgres covering `gbrain serve --http` end-to-end: bearer auth round-trip, `last_used_at` SQL-level debounce semantics, `mcp_request_log` row insertion on success and auth_failed paths, `/health` DB-down → 503 (DB-probing health check), and the F1+F2+F3 dispatch round-trip with a real operation. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/serve-http-oauth.test.ts` (v0.26.0, expanded v0.26.2) — real-Postgres E2E against `gbrain serve --http` with full OAuth 2.1. Spawns a subprocess server, registers a client via the CLI, mints `client_credentials` tokens, exercises the `/mcp` JSON-RPC pipeline. **v0.26.2 adds:** real DCR `/register` HTTP-level response-shape test (asserts `typeof body.client_id_issued_at === 'number'` over the wire — RFC 7591 §3.2.1 spec compliance, not just internal-store shape); real CLI subprocess test for `revoke-client` (registers → mints token → revokes via `execSync` → asserts token rejected at `/mcp` → asserts re-run exits 1); server fixture flips on `--enable-dcr` so `/register` is reachable. **bun execSync env-inheritance fix:** bun's `execSync` does NOT inherit env mutations done via `process.env.X = ...`, only OS-level env from before bun started. helpers.ts loads `.env.testing` and sets `DATABASE_URL` via `process.env` mutation, which is invisible to subprocesses unless `env: { ...process.env }` is passed explicitly — every subprocess call in this file passes `env: { ...process.env }` for that reason. Reference fix for the next maintainer hitting the same failure mode in sibling sync/cycle/dream/claw-test E2Es. `afterAll` cleanup is guarded on `clientId` (won't throw if `beforeAll` failed before registration); cleanup errors surface to stderr without throwing so real test failures aren't masked. Tracks DCR-registered clients alongside the manual one. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/serve-http-oauth.test.ts` (v0.26.0, expanded v0.26.2, expanded v0.26.9) — real-Postgres E2E against `gbrain serve --http` with full OAuth 2.1. Spawns a subprocess server, registers a client via the CLI, mints `client_credentials` tokens, exercises the `/mcp` JSON-RPC pipeline. **v0.26.2 adds:** real DCR `/register` HTTP-level response-shape test (asserts `typeof body.client_id_issued_at === 'number'` over the wire — RFC 7591 §3.2.1 spec compliance, not just internal-store shape); real CLI subprocess test for `revoke-client` (registers → mints token → revokes via `execSync` → asserts token rejected at `/mcp` → asserts re-run exits 1); server fixture flips on `--enable-dcr` so `/register` is reachable. **bun execSync env-inheritance fix:** bun's `execSync` does NOT inherit env mutations done via `process.env.X = ...`, only OS-level env from before bun started. helpers.ts loads `.env.testing` and sets `DATABASE_URL` via `process.env` mutation, which is invisible to subprocesses unless `env: { ...process.env }` is passed explicitly — every subprocess call in this file passes `env: { ...process.env }` for that reason. Reference fix for the next maintainer hitting the same failure mode in sibling sync/cycle/dream/claw-test E2Es. `afterAll` cleanup is guarded on `clientId` (won't throw if `beforeAll` failed before registration); cleanup errors surface to stderr without throwing so real test failures aren't masked. Tracks DCR-registered clients alongside the manual one. **v0.26.9** adds 2 regressions for the F7 trust-boundary fix: an HTTP MCP `submit_job` for `name: "shell"` MUST reject with a permission error (proving the request handler now sets `remote: true` and `submit_job`'s protected-name guard fires), and the same guard rejects subagent submission. Closes the OAuth-token-to-RCE escalation path. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/sync-parallel.test.ts` (v0.22.13 PR #490) — DATABASE_URL-gated. T2: 60-file Postgres sync at concurrency=4 imports all + no connection leak (probes `pg_stat_activity` before/after to confirm worker engines disconnected). P4: 120-file serial-vs-parallel benchmark prints `SYNC_PARALLEL_BENCH N files | serial=Xms | parallel(4)=Yms | speedup=Zx` for CHANGELOG quoting. Asserts parallel ≤ serial × 1.5 (CI-noise tolerant; not a strict speedup gate).
|
||||
- Tier 2 (`skills.test.ts`) requires OpenClaw + API keys, runs nightly in CI
|
||||
- If `.env.testing` doesn't exist in this directory, check sibling worktrees for one:
|
||||
@@ -1524,6 +1667,14 @@ postinstall hook on global installs, so schema migrations never run and the CLI
|
||||
aborts with `Aborted()` the first time it opens PGLite. Use `git clone + bun install
|
||||
&& bun link` as shown above. See [#218](https://github.com/garrytan/gbrain/issues/218).
|
||||
|
||||
**Do NOT use `bun add -g gbrain` or `npm install -g gbrain`.** The npm registry
|
||||
has an unrelated package squatting that name (`gbrain@1.3.x`) — you'd silently
|
||||
install the wrong binary and overwrite the canonical one. v0.28.5+ detects this
|
||||
and prints a recovery message on `gbrain upgrade`, but the `git clone + bun link`
|
||||
path above is the only reliable install method until we publish under
|
||||
`@garrytan/gbrain` (tracked v0.29 follow-up). See
|
||||
[#658](https://github.com/garrytan/gbrain/issues/658).
|
||||
|
||||
```
|
||||
3 results (hybrid search, 0.12s):
|
||||
|
||||
@@ -1912,6 +2063,7 @@ GBrain ships integration recipes that your agent sets up for you. Each recipe te
|
||||
| [X-to-Brain](recipes/x-to-brain.md) | — | Twitter timeline + mentions + deletions |
|
||||
| [Calendar-to-Brain](recipes/calendar-to-brain.md) | credential-gateway | Google Calendar to searchable daily pages |
|
||||
| [Meeting Sync](recipes/meeting-sync.md) | — | Circleback transcripts to brain pages with attendees |
|
||||
| [Restart Sweep](recipes/restart-sweep.md) | OpenClaw + Telegram | Detect dropped Telegram messages after OpenClaw gateway restarts |
|
||||
|
||||
**Data research recipes** extract structured data from email into tracked brain pages. Built-in recipes for investor updates (MRR, ARR, runway, headcount), expense tracking, and company metrics. Create your own with `gbrain research init`.
|
||||
|
||||
@@ -2202,7 +2354,7 @@ ADMIN
|
||||
gbrain serve MCP server (stdio)
|
||||
gbrain serve --http [--port 3131] HTTP MCP server with OAuth 2.1 + admin dashboard
|
||||
[--token-ttl 3600] [--enable-dcr]
|
||||
[--public-url URL]
|
||||
[--public-url URL] [--log-full-params]
|
||||
gbrain auth create|list|revoke|test Legacy bearer token management
|
||||
gbrain auth register-client <name> Register an OAuth 2.1 client
|
||||
--grant-types client_credentials,authorization_code
|
||||
@@ -2213,6 +2365,11 @@ ADMIN
|
||||
# programmatically via oauthProvider.registerClientManual() for host-repo wrappers.
|
||||
gbrain integrations Integration recipe dashboard
|
||||
gbrain sources list|add|remove|... Multi-source brain management (v0.18)
|
||||
v0.28.2: --url <https://...> registers a federated
|
||||
remote git repo; clone is auto-managed under
|
||||
$GBRAIN_HOME/clones/<id>/ and re-cloned on sync if
|
||||
it goes missing. Also exposed via MCP for remote
|
||||
agent setup (whoami + sources_{add,list,remove,status}).
|
||||
gbrain dream [--dry-run] [--phase N] 8-phase maintenance cycle (lint→backlinks→sync→synthesize
|
||||
→extract→patterns→embed→orphans). v0.23 added synthesize +
|
||||
patterns: transcripts → reflections + cross-session themes.
|
||||
@@ -2260,7 +2417,7 @@ The skills in this repo are those patterns, generalized. What took 11 days to bu
|
||||
|
||||
## Contributing
|
||||
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md). Run `bun test` for unit tests. For the full local CI gate (gitleaks + unit + all 29 E2E files in Docker, the same checks GH Actions runs), use `bun run ci:local` ... or `bun run ci:local:diff` for the diff-aware subset during fast iteration.
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md). Run `bun run test` for the parallel unit-test fast loop (~85s on a Mac dev box, 3700+ tests) or `bun run verify` for the pre-push gate (privacy + jsonb + progress + test-isolation + wasm + admin-build + typecheck). For the full local CI gate (gitleaks + unit + all 29 E2E files in Docker, the same checks GH Actions runs), use `bun run ci:local` ... or `bun run ci:local:diff` for the diff-aware subset during fast iteration.
|
||||
|
||||
If you're working on retrieval or any of the search/embedding/ranking surface, set `GBRAIN_CONTRIBUTOR_MODE=1` in your shell rc and use `gbrain eval replay` to gate your changes against a snapshot of real captured queries — the dev loop is documented in [`docs/eval-bench.md`](docs/eval-bench.md). Capture is **off by default** for production users (no surprise data accumulation); the env var is the contributor opt-in.
|
||||
|
||||
@@ -4482,6 +4639,15 @@ Save this token. Open `http://localhost:3131/admin` and paste it to access the
|
||||
dashboard. The dashboard shows live activity, registered clients, request logs,
|
||||
and per-client config export.
|
||||
|
||||
> **v0.26.9+:** `mcp_request_log.params` and the live SSE activity feed default
|
||||
> to a redacted summary `{redacted, kind, declared_keys, unknown_key_count, approx_bytes}`.
|
||||
> Declared param keys are kept (intersected against the operation's spec); unknown
|
||||
> keys are counted but never named, and byte sizes round up to 1KB so size-probe
|
||||
> attacks can't binary-search secret content. Operators on a personal laptop who
|
||||
> want raw payloads back can pass `gbrain serve --http --log-full-params` (loud
|
||||
> stderr warning fires at startup). Multi-tenant deployments should leave it on
|
||||
> the redacted default.
|
||||
|
||||
### 2. Register OAuth clients
|
||||
|
||||
Register clients from the **`/admin` dashboard**:
|
||||
|
||||
+17
-3
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.26.3",
|
||||
"version": "0.28.7",
|
||||
"description": "Postgres-native personal knowledge brain with hybrid RAG search",
|
||||
"type": "module",
|
||||
"main": "src/core/index.ts",
|
||||
@@ -34,12 +34,18 @@
|
||||
"build:schema": "bash scripts/build-schema.sh",
|
||||
"build:llms": "bun run scripts/build-llms.ts",
|
||||
"build:pglite-snapshot": "bun run scripts/build-pglite-snapshot.ts",
|
||||
"test": "scripts/check-privacy.sh && scripts/check-jsonb-pattern.sh && scripts/check-progress-to-stdout.sh && scripts/check-no-legacy-getconnection.sh && scripts/check-trailing-newline.sh && scripts/check-wasm-embedded.sh && scripts/check-exports-count.sh && scripts/check-admin-build.sh && bun run typecheck && bun test --timeout=60000",
|
||||
"test": "bash scripts/run-unit-parallel.sh",
|
||||
"test:full": "bun run verify && bash scripts/run-unit-parallel.sh && bun run test:slow && ([ -n \"$DATABASE_URL\" ] && bash scripts/run-e2e.sh || echo '[test:full] skipped E2E (no DATABASE_URL); run docker-compose -f docker-compose.ci.yml up + bun run test:e2e to include' 1>&2)",
|
||||
"verify": "bun run check:privacy && bun run check:jsonb && bun run check:progress && bun run check:test-isolation && bun run check:wasm && bun run check:admin-build && bun run check:admin-scope-drift && bun run check:cli-exec && bun run typecheck",
|
||||
"check:admin-scope-drift": "scripts/check-admin-scope-drift.sh",
|
||||
"check:cli-exec": "scripts/check-cli-executable.sh",
|
||||
"check:all": "scripts/check-privacy.sh && scripts/check-jsonb-pattern.sh && scripts/check-progress-to-stdout.sh && scripts/check-no-legacy-getconnection.sh && scripts/check-test-isolation.sh && scripts/check-trailing-newline.sh && scripts/check-wasm-embedded.sh && scripts/check-exports-count.sh && scripts/check-admin-build.sh && scripts/check-admin-scope-drift.sh && scripts/check-cli-executable.sh",
|
||||
"check:wasm": "scripts/check-wasm-embedded.sh",
|
||||
"check:newlines": "scripts/check-trailing-newline.sh",
|
||||
"test:e2e": "bash scripts/run-e2e.sh",
|
||||
"test:slow": "bash scripts/run-slow-tests.sh",
|
||||
"test:profile": "bash scripts/profile-tests.sh",
|
||||
"test:serial": "bash scripts/run-serial-tests.sh",
|
||||
"ci:local": "bash scripts/ci-local.sh",
|
||||
"ci:local:diff": "bash scripts/ci-local.sh --diff",
|
||||
"ci:select-e2e": "bun run scripts/select-e2e.ts",
|
||||
@@ -49,6 +55,7 @@
|
||||
"check:progress": "scripts/check-progress-to-stdout.sh",
|
||||
"check:exports-count": "scripts/check-exports-count.sh",
|
||||
"check:admin-build": "scripts/check-admin-build.sh",
|
||||
"check:test-isolation": "scripts/check-test-isolation.sh",
|
||||
"postinstall": "command -v gbrain >/dev/null 2>&1 && gbrain apply-migrations --yes --non-interactive || echo '[gbrain] postinstall skipped. If installed via bun install -g github:...: run `gbrain doctor` and `gbrain apply-migrations --yes` manually. See https://github.com/garrytan/gbrain/issues/218' 1>&2",
|
||||
"prepublish:clawhub": "bun run build:all",
|
||||
"publish:clawhub": "clawhub package publish . --family bundle-plugin"
|
||||
@@ -59,13 +66,19 @@
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/anthropic": "^3.0.71",
|
||||
"@ai-sdk/google": "^3.0.64",
|
||||
"@ai-sdk/openai": "^3.0.53",
|
||||
"@ai-sdk/openai-compatible": "^2.0.41",
|
||||
"@anthropic-ai/sdk": "^0.30.0",
|
||||
"@aws-sdk/client-s3": "^3.1028.0",
|
||||
"@dqbd/tiktoken": "^1.0.22",
|
||||
"@electric-sql/pglite": "0.4.3",
|
||||
"@modelcontextprotocol/sdk": "1.29.0",
|
||||
"ai": "^6.0.168",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cors": "^2.8.5",
|
||||
"eventsource-parser": "^3.0.8",
|
||||
"express": "^5.1.0",
|
||||
"express-rate-limit": "^7.5.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
@@ -74,7 +87,8 @@
|
||||
"pgvector": "^0.2.0",
|
||||
"postgres": "^3.4.0",
|
||||
"tree-sitter-wasms": "0.1.13",
|
||||
"web-tree-sitter": "0.22.6"
|
||||
"web-tree-sitter": "0.22.6",
|
||||
"zod": "^4.3.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
|
||||
@@ -0,0 +1,654 @@
|
||||
---
|
||||
id: restart-sweep
|
||||
name: Restart Sweep
|
||||
version: 0.1.0
|
||||
description: Detect Telegram messages dropped during OpenClaw gateway restarts. Reads OpenClaw session state, alerts on aborted-mid-run sessions and (opt-in) suspicious silence gaps. Cooldown-gated so repeat detections don't spam.
|
||||
category: reflex
|
||||
requires: []
|
||||
secrets:
|
||||
- name: OPENCLAW_OWNER_IDS
|
||||
description: Comma-separated user IDs that own this brain instance
|
||||
where: openclaw config — your own user IDs from the platforms you connect
|
||||
- name: OPENCLAW_TELEGRAM_GROUP
|
||||
description: Target Telegram group ID for restart alerts (negative number for groups)
|
||||
where: forward a message from the group to @userinfobot, copy the chat.id
|
||||
health_checks:
|
||||
- type: env_exists
|
||||
name: OPENCLAW_OWNER_IDS
|
||||
label: Owner IDs configured
|
||||
- type: env_exists
|
||||
name: OPENCLAW_TELEGRAM_GROUP
|
||||
label: Telegram group configured
|
||||
- type: command
|
||||
argv: [openclaw, sessions, --json]
|
||||
label: OpenClaw CLI reachable
|
||||
setup_time: 10 min
|
||||
cost_estimate: "$0 (no per-call cost; runs locally on cron)"
|
||||
---
|
||||
|
||||
# Restart Sweep: Detect Dropped Messages After Gateway Restarts
|
||||
|
||||
When the OpenClaw gateway restarts, webhook-delivered Telegram messages
|
||||
that haven't been processed yet get dropped permanently. Long-poll bots
|
||||
can replay missed updates via `getUpdates`. Webhook bots cannot. This
|
||||
recipe detects the gap by reading OpenClaw's session state and alerting
|
||||
when a session was active just before a restart but silent afterward.
|
||||
|
||||
## IMPORTANT: Instructions for the Agent
|
||||
|
||||
**You are the installer.** This recipe is written for YOU (the AI agent)
|
||||
to execute on behalf of the user. Follow these steps precisely.
|
||||
|
||||
**Stop points (MUST pause and verify before continuing):**
|
||||
- After Step 1: prerequisites pass? If not, fix before proceeding.
|
||||
- After Step 4: dry run produces sensible output? If not, debug before
|
||||
wiring cron.
|
||||
- After Step 5: cron entry created and visible in `crontab -l`? If not,
|
||||
cron isn't installed.
|
||||
|
||||
**When something fails:** Tell the user EXACTLY what failed, what it
|
||||
means, and what to try. Never say "something went wrong."
|
||||
|
||||
## What this does
|
||||
|
||||
1. Reads `/tmp/bootstrap-services.log` (or `$OPENCLAW_BOOTSTRAP_LOG`)
|
||||
to find when the gateway last restarted. Falls back to `now() - 30
|
||||
minutes` if the log isn't readable.
|
||||
2. Runs `openclaw sessions --json` to enumerate all live sessions.
|
||||
3. Filters to Telegram group sessions matching `$OPENCLAW_TELEGRAM_GROUP`.
|
||||
4. Flags sessions with `abortedLastRun: true` (strong signal of a
|
||||
dropped message). Optionally flags sessions that were active in the
|
||||
5 minutes before restart but silent in the 10 minutes after — gated
|
||||
behind `OPENCLAW_RESTART_SWEEP_AGGRESSIVE=1` because the timing
|
||||
heuristic produces false positives during quiet periods.
|
||||
5. Cooldown layer: each sessionKey alerted gets stamped with a
|
||||
`lastAlertedAt` timestamp. Re-alerting on the same sessionKey is
|
||||
suppressed for 6 hours regardless of whether the synthesized restart
|
||||
time matches. This prevents the "missing bootstrap log →
|
||||
re-alert-every-5-minutes-forever" failure mode.
|
||||
6. Sends one alert per cycle to Telegram (or stdout if no Telegram
|
||||
config), then records the alert in
|
||||
`~/.gbrain/integrations/restart-sweep/alerted.json`.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- OpenClaw running with Telegram in webhook mode (long-poll mode
|
||||
doesn't need this — `getUpdates` recovers missed messages on restart)
|
||||
- The `openclaw` CLI on PATH (or you'll provide an absolute path in
|
||||
Step 5)
|
||||
- Telegram bot token already configured in OpenClaw, group ID and
|
||||
optional topic ID known
|
||||
- Cron available on the host (this recipe schedules a 5-minute job;
|
||||
systemd timers, launchd, or any other scheduler also work — adapt
|
||||
Step 5 accordingly)
|
||||
|
||||
## Step 1: Verify prerequisites
|
||||
|
||||
```bash
|
||||
openclaw sessions --json | head -40
|
||||
```
|
||||
|
||||
Should print JSON with a `sessions` array. If it errors, fix
|
||||
`openclaw` reachability before continuing.
|
||||
|
||||
Decide a host-repo install path. The recipe assumes
|
||||
`~/openclaw/scripts/restart-sweep.mjs` and the user's `.env` lives at
|
||||
`~/openclaw/.env`. Adapt to your repo layout.
|
||||
|
||||
## Step 2: Collect the secrets
|
||||
|
||||
Confirm with the user:
|
||||
|
||||
- `OPENCLAW_OWNER_IDS` — comma-separated user IDs (e.g. `123456789,987654321`)
|
||||
- `OPENCLAW_TELEGRAM_GROUP` — the target group ID (negative number for
|
||||
group chats, e.g. `-1001234567890`). Forward a message from the
|
||||
group to `@userinfobot` to get it.
|
||||
- `OPENCLAW_ALERT_TOPIC` — optional, the topic/thread ID for forum
|
||||
groups. Open the topic in Telegram, the URL ends with the thread ID.
|
||||
|
||||
Add these three lines to the host's `.env` (or wherever the host loads
|
||||
env from):
|
||||
|
||||
```bash
|
||||
OPENCLAW_OWNER_IDS=...
|
||||
OPENCLAW_TELEGRAM_GROUP=...
|
||||
OPENCLAW_ALERT_TOPIC=...
|
||||
```
|
||||
|
||||
Optional tuning:
|
||||
|
||||
```bash
|
||||
# Set to 1 to enable the timing-based heuristic (active before restart,
|
||||
# silent after). Off by default because it false-positives during quiet
|
||||
# periods.
|
||||
OPENCLAW_RESTART_SWEEP_AGGRESSIVE=1
|
||||
|
||||
# Override the bootstrap log path (default /tmp/bootstrap-services.log)
|
||||
OPENCLAW_BOOTSTRAP_LOG=/var/log/openclaw/bootstrap.log
|
||||
```
|
||||
|
||||
## Step 3: Write the script to the host repo
|
||||
|
||||
Write the script content from the next section to
|
||||
`~/openclaw/scripts/restart-sweep.mjs` (or wherever the user picks).
|
||||
The script is self-contained — no npm install needed, just Node 18+
|
||||
or Bun.
|
||||
|
||||
<!-- restart-sweep:script -->
|
||||
```javascript
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Restart Message Sweep Script
|
||||
*
|
||||
* Detects Telegram messages dropped during OpenClaw gateway restarts.
|
||||
* Webhook-delivered messages can't be replayed via getUpdates, so we
|
||||
* read OpenClaw's session state and look for sessions that show signs
|
||||
* of dropped processing.
|
||||
*
|
||||
* Runs under Node 18+ or Bun. Copy this file into your host repo and
|
||||
* wire it to a 5-minute cron.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import fsp from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { exec, execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execP = promisify(exec);
|
||||
|
||||
// Module-level constants (no env reads here — env is read at construct time)
|
||||
const RESTART_THRESHOLD_MINUTES = 30; // Fallback restart-time window when bootstrap log is missing
|
||||
const COOLDOWN_HOURS = 6; // Re-alert suppression per sessionKey
|
||||
const STALE_DAYS = 30; // Prune alerted.json entries older than this
|
||||
const PRE_RESTART_WINDOW_MS = 5 * 60 * 1000;
|
||||
const POST_RESTART_WINDOW_MS = 10 * 60 * 1000;
|
||||
|
||||
class MessageSweepDetector {
|
||||
/**
|
||||
* @param {{ execFile?: typeof execFile, runOpenclawSessions?: () => Promise<any[]> }} [deps]
|
||||
* Optional dependency injection for tests. Production: leave undefined.
|
||||
*/
|
||||
constructor(deps = {}) {
|
||||
// Constructor-time env reads (C2): tests can mutate process.env per construction
|
||||
const ownerEnv = process.env.OPENCLAW_OWNER_IDS ?? '';
|
||||
this.OWNER_IDS = ownerEnv.split(',').map(s => s.trim()).filter(Boolean);
|
||||
this.TELEGRAM_GROUP_ID = process.env.OPENCLAW_TELEGRAM_GROUP ?? '';
|
||||
this.ALERT_TOPIC = process.env.OPENCLAW_ALERT_TOPIC ?? '';
|
||||
this.AGGRESSIVE = process.env.OPENCLAW_RESTART_SWEEP_AGGRESSIVE === '1';
|
||||
|
||||
const gbrainHome = process.env.GBRAIN_HOME ?? path.join(os.homedir(), '.gbrain');
|
||||
this.STATE_DIR = path.join(gbrainHome, 'integrations', 'restart-sweep');
|
||||
this.LOG_PATH = path.join(this.STATE_DIR, 'sweep.log.jsonl');
|
||||
this.ALERTED_PATH = path.join(this.STATE_DIR, 'alerted.json');
|
||||
this.BOOTSTRAP_LOG = process.env.OPENCLAW_BOOTSTRAP_LOG ?? '/tmp/bootstrap-services.log';
|
||||
|
||||
// DI hooks (default to real implementations)
|
||||
this._execFile = deps.execFile ?? execFile;
|
||||
this._runOpenclawSessions = deps.runOpenclawSessions ?? null;
|
||||
|
||||
this.sessions = null;
|
||||
this.restartTime = null;
|
||||
this.alertMode = this.determineAlertMode();
|
||||
this.alerted = new Map(); // populated in run() / loadAlerted()
|
||||
}
|
||||
|
||||
determineAlertMode() {
|
||||
if (this.TELEGRAM_GROUP_ID && this.ALERT_TOPIC) return 'telegram';
|
||||
if (this.TELEGRAM_GROUP_ID) return 'telegram_stdout';
|
||||
return 'stdout';
|
||||
}
|
||||
|
||||
async run() {
|
||||
try {
|
||||
console.log('🔍 Starting restart message sweep detection...');
|
||||
|
||||
if (this.OWNER_IDS.length === 0) {
|
||||
console.warn('⚠️ No OPENCLAW_OWNER_IDS configured. Set this environment variable.');
|
||||
}
|
||||
if (!this.TELEGRAM_GROUP_ID) {
|
||||
console.warn('⚠️ No OPENCLAW_TELEGRAM_GROUP configured. Alerts will only go to stdout.');
|
||||
}
|
||||
|
||||
fs.mkdirSync(this.STATE_DIR, { recursive: true });
|
||||
this.alerted = await this.loadAlerted();
|
||||
|
||||
this.restartTime = await this.getLastRestartTime();
|
||||
console.log(`📅 Last restart detected at: ${new Date(this.restartTime).toISOString()}`);
|
||||
|
||||
this.sessions = await this.getSessionState();
|
||||
console.log(`📊 Found ${this.sessions.length} total sessions`);
|
||||
|
||||
const telegramSessions = this.filterTelegramSessions(this.sessions);
|
||||
console.log(`📱 Found ${telegramSessions.length} Telegram sessions`);
|
||||
|
||||
const droppedMessages = await this.detectDroppedMessages(telegramSessions);
|
||||
const newDrops = droppedMessages.filter(m => !this.isInCooldown(m.sessionKey));
|
||||
const suppressedCount = droppedMessages.length - newDrops.length;
|
||||
|
||||
if (newDrops.length > 0) {
|
||||
const tail = suppressedCount > 0 ? ` (${suppressedCount} suppressed by cooldown)` : '';
|
||||
console.log(`⚠️ Found ${newDrops.length} potentially dropped message(s)${tail}`);
|
||||
await this.recordAndAlert(newDrops);
|
||||
} else if (suppressedCount > 0) {
|
||||
console.log(`✅ All ${suppressedCount} candidate(s) suppressed by cooldown`);
|
||||
} else {
|
||||
console.log('✅ No dropped messages detected');
|
||||
}
|
||||
|
||||
await this.logResults(droppedMessages);
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Error in message sweep:', error);
|
||||
await this.logError(error);
|
||||
}
|
||||
}
|
||||
|
||||
async getLastRestartTime() {
|
||||
try {
|
||||
const logContent = await fsp.readFile(this.BOOTSTRAP_LOG, 'utf8');
|
||||
const gatewayLines = logContent.split('\n')
|
||||
.filter(line => line.includes('Gateway token synced') || line.includes('✅ OpenClaw gateway'))
|
||||
.reverse();
|
||||
if (gatewayLines.length > 0) {
|
||||
const match = gatewayLines[0].match(/^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})/);
|
||||
if (match) {
|
||||
return new Date(match[1] + ' UTC').getTime();
|
||||
}
|
||||
}
|
||||
return Date.now() - (RESTART_THRESHOLD_MINUTES * 60 * 1000);
|
||||
} catch (error) {
|
||||
console.warn('⚠️ Could not determine restart time from logs, using fallback');
|
||||
return Date.now() - (RESTART_THRESHOLD_MINUTES * 60 * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
async getSessionState() {
|
||||
if (this._runOpenclawSessions) {
|
||||
return await this._runOpenclawSessions();
|
||||
}
|
||||
try {
|
||||
const { stdout } = await execP('openclaw sessions --json');
|
||||
const sessionData = JSON.parse(stdout);
|
||||
return sessionData.sessions || [];
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to get session state:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
filterTelegramSessions(sessions) {
|
||||
if (!this.TELEGRAM_GROUP_ID) return [];
|
||||
return sessions.filter(session => {
|
||||
return session.key &&
|
||||
session.key.includes('telegram:group:' + this.TELEGRAM_GROUP_ID) &&
|
||||
session.kind === 'group';
|
||||
});
|
||||
}
|
||||
|
||||
async detectDroppedMessages(telegramSessions) {
|
||||
const droppedMessages = [];
|
||||
const recentRestartWindow = this.restartTime - PRE_RESTART_WINDOW_MS;
|
||||
const afterRestartWindow = this.restartTime + POST_RESTART_WINDOW_MS;
|
||||
|
||||
for (const session of telegramSessions) {
|
||||
try {
|
||||
const sessionUpdated = session.updatedAt;
|
||||
|
||||
// Primary: aborted last run is the strong signal
|
||||
if (session.abortedLastRun) {
|
||||
const topic = this._extractTopic(session.key);
|
||||
droppedMessages.push({
|
||||
sessionKey: session.key,
|
||||
topic,
|
||||
lastUpdate: new Date(sessionUpdated).toISOString(),
|
||||
sessionId: session.sessionId,
|
||||
abortedLastRun: true,
|
||||
reason: 'Session aborted on last run',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Secondary: timing-based gap detection — opt-in only (false-positive prone)
|
||||
if (!this.AGGRESSIVE) continue;
|
||||
|
||||
if (sessionUpdated >= recentRestartWindow &&
|
||||
sessionUpdated < this.restartTime &&
|
||||
Date.now() > afterRestartWindow) {
|
||||
const topic = this._extractTopic(session.key);
|
||||
droppedMessages.push({
|
||||
sessionKey: session.key,
|
||||
topic,
|
||||
lastUpdate: new Date(sessionUpdated).toISOString(),
|
||||
timeSinceUpdate: Math.floor((Date.now() - sessionUpdated) / 1000 / 60),
|
||||
sessionId: session.sessionId,
|
||||
suspiciousGap: true,
|
||||
reason: 'Active before restart, silent after',
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`⚠️ Error analyzing session ${session.key}:`, error);
|
||||
}
|
||||
}
|
||||
return droppedMessages;
|
||||
}
|
||||
|
||||
_extractTopic(sessionKey) {
|
||||
const m = sessionKey?.match(/:topic:(\d+)/);
|
||||
return m ? m[1] : 'unknown';
|
||||
}
|
||||
|
||||
/**
|
||||
* Cooldown layer (C1): suppresses re-alerts on the same sessionKey
|
||||
* for COOLDOWN_HOURS, regardless of whether the synthesized
|
||||
* restartTime matches. Cooldown wins when the bootstrap log is
|
||||
* missing and restartTime is unstable.
|
||||
*/
|
||||
isInCooldown(sessionKey) {
|
||||
const entry = this.alerted.get(sessionKey);
|
||||
if (!entry || !entry.lastAlertedAt) return false;
|
||||
const ageMs = Date.now() - new Date(entry.lastAlertedAt).getTime();
|
||||
return ageMs < COOLDOWN_HOURS * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
async loadAlerted() {
|
||||
try {
|
||||
const content = await fsp.readFile(this.ALERTED_PATH, 'utf8');
|
||||
const parsed = JSON.parse(content);
|
||||
const map = new Map();
|
||||
const cutoffMs = Date.now() - STALE_DAYS * 24 * 60 * 60 * 1000;
|
||||
for (const [key, entry] of Object.entries(parsed || {})) {
|
||||
if (entry && entry.lastAlertedAt) {
|
||||
const ts = new Date(entry.lastAlertedAt).getTime();
|
||||
if (Number.isFinite(ts) && ts >= cutoffMs) {
|
||||
map.set(key, entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
} catch (err) {
|
||||
if (err && err.code === 'ENOENT') return new Map();
|
||||
console.warn(`⚠️ Failed to load ${this.ALERTED_PATH}: ${err && err.message}; starting with empty state`);
|
||||
return new Map();
|
||||
}
|
||||
}
|
||||
|
||||
async saveAlerted() {
|
||||
const obj = Object.fromEntries(this.alerted);
|
||||
const json = JSON.stringify(obj, null, 2);
|
||||
const tmp = this.ALERTED_PATH + '.tmp';
|
||||
// Atomic on POSIX: write tmp, then rename. Note: this prevents
|
||||
// file corruption only — concurrent cron runs can still both
|
||||
// read old state, both decide to alert, both rename. Given
|
||||
// 5-min cadence and 2-5s runtime, overlap is rare and a
|
||||
// duplicate alert is preferable to a missed one.
|
||||
await fsp.writeFile(tmp, json);
|
||||
await fsp.rename(tmp, this.ALERTED_PATH);
|
||||
}
|
||||
|
||||
async recordAndAlert(droppedMessages) {
|
||||
let alertSent = false;
|
||||
try {
|
||||
await this.alertOnDroppedMessages(droppedMessages);
|
||||
alertSent = true;
|
||||
} catch (err) {
|
||||
console.error('❌ Failed to send alert (will retry next cycle):', err && err.message);
|
||||
}
|
||||
if (!alertSent) return;
|
||||
|
||||
const nowIso = new Date().toISOString();
|
||||
const restartIso = new Date(this.restartTime).toISOString();
|
||||
for (const msg of droppedMessages) {
|
||||
this.alerted.set(msg.sessionKey, {
|
||||
lastAlertedAt: nowIso,
|
||||
restartTime: restartIso,
|
||||
});
|
||||
}
|
||||
try {
|
||||
await this.saveAlerted();
|
||||
} catch (err) {
|
||||
console.warn('⚠️ Failed to save alerted state:', err && err.message);
|
||||
}
|
||||
}
|
||||
|
||||
async alertOnDroppedMessages(droppedMessages) {
|
||||
let alertText = `⚠️ Found ${droppedMessages.length} unprocessed message(s) after restart:\n\n`;
|
||||
for (const msg of droppedMessages.slice(0, 10)) {
|
||||
alertText += `• Topic ${msg.topic}: ${msg.reason} (last update: ${msg.lastUpdate})\n`;
|
||||
if (msg.timeSinceUpdate) {
|
||||
alertText += ` ${msg.timeSinceUpdate} minutes ago\n`;
|
||||
}
|
||||
}
|
||||
if (droppedMessages.length > 10) {
|
||||
alertText += `\n... and ${droppedMessages.length - 10} more`;
|
||||
}
|
||||
|
||||
switch (this.alertMode) {
|
||||
case 'telegram':
|
||||
await this.sendTelegramAlert(alertText);
|
||||
break;
|
||||
case 'telegram_stdout':
|
||||
console.log('📢 Would send Telegram alert, but no topic configured:');
|
||||
console.log(alertText);
|
||||
break;
|
||||
default:
|
||||
console.log('📢 Alert:');
|
||||
console.log(alertText);
|
||||
}
|
||||
}
|
||||
|
||||
async sendTelegramAlert(alertText) {
|
||||
// execFile (not exec): argv array, no shell interpretation,
|
||||
// shell metachars in env vars cannot inject commands.
|
||||
const argv = [
|
||||
'message', 'send',
|
||||
'--channel', 'telegram',
|
||||
'--target', this.TELEGRAM_GROUP_ID,
|
||||
'--thread-id', this.ALERT_TOPIC,
|
||||
'--message', alertText,
|
||||
];
|
||||
await new Promise((resolve, reject) => {
|
||||
this._execFile('openclaw', argv, (err, _stdout, stderr) => {
|
||||
if (err) {
|
||||
err.stderr = stderr;
|
||||
reject(err);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
console.log('📢 Alert sent to Telegram');
|
||||
}
|
||||
|
||||
async logResults(droppedMessages) {
|
||||
const logEntry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
restartTime: new Date(this.restartTime).toISOString(),
|
||||
droppedMessageCount: droppedMessages.length,
|
||||
droppedMessages,
|
||||
};
|
||||
try {
|
||||
await fsp.appendFile(this.LOG_PATH, JSON.stringify(logEntry) + '\n');
|
||||
} catch (error) {
|
||||
console.warn('⚠️ Failed to write log file:', error && error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async logError(error) {
|
||||
const errorEntry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
error: error && error.message,
|
||||
stack: error && error.stack,
|
||||
};
|
||||
try {
|
||||
await fsp.appendFile(this.LOG_PATH, 'ERROR: ' + JSON.stringify(errorEntry) + '\n');
|
||||
} catch (logError) {
|
||||
console.error('Failed to log error:', logError && logError.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run if executed directly
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const detector = new MessageSweepDetector();
|
||||
detector.run().catch(console.error);
|
||||
}
|
||||
|
||||
export default MessageSweepDetector;
|
||||
```
|
||||
|
||||
## Step 4: Dry-run
|
||||
|
||||
Run the script once manually with the env loaded, before wiring cron:
|
||||
|
||||
```bash
|
||||
set -a; source ~/openclaw/.env; set +a
|
||||
node ~/openclaw/scripts/restart-sweep.mjs
|
||||
```
|
||||
|
||||
Expected output (no drops):
|
||||
|
||||
```
|
||||
🔍 Starting restart message sweep detection...
|
||||
📅 Last restart detected at: 2026-05-06T12:53:45.000Z
|
||||
📊 Found 48 total sessions
|
||||
📱 Found 39 Telegram sessions
|
||||
✅ No dropped messages detected
|
||||
```
|
||||
|
||||
If you want to see the alert path, manually edit a session in OpenClaw
|
||||
to set `abortedLastRun: true` and re-run. After the alert fires, check
|
||||
`~/.gbrain/integrations/restart-sweep/alerted.json` — the sessionKey
|
||||
should be there with a `lastAlertedAt` timestamp. Re-running within 6
|
||||
hours suppresses the alert.
|
||||
|
||||
## Step 5: Wire 5-minute cron
|
||||
|
||||
Cron does NOT inherit your shell environment. `openclaw` and `node` may
|
||||
not be on cron's stripped PATH. `.env` files don't auto-load. Use the
|
||||
wrapper-script pattern below to handle both.
|
||||
|
||||
Create `~/openclaw/scripts/restart-sweep-wrapper.sh`:
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set -a
|
||||
source ~/openclaw/.env
|
||||
set +a
|
||||
exec /usr/local/bin/node ~/openclaw/scripts/restart-sweep.mjs
|
||||
```
|
||||
|
||||
```bash
|
||||
chmod +x ~/openclaw/scripts/restart-sweep-wrapper.sh
|
||||
```
|
||||
|
||||
Adjust `/usr/local/bin/node` to wherever your `node` actually lives
|
||||
(`which node` to find it). Same for `openclaw` if the wrapper needs to
|
||||
add it to PATH explicitly:
|
||||
|
||||
```bash
|
||||
export PATH=/usr/local/bin:/usr/bin:/bin:$PATH
|
||||
```
|
||||
|
||||
Add to crontab via `crontab -e`:
|
||||
|
||||
```cron
|
||||
PATH=/usr/local/bin:/usr/bin:/bin
|
||||
*/5 * * * * /bin/bash ~/openclaw/scripts/restart-sweep-wrapper.sh >> ~/.gbrain/integrations/restart-sweep/cron.log 2>&1
|
||||
```
|
||||
|
||||
Verify with `crontab -l`. Wait 5 minutes, then check the cron log to
|
||||
confirm it ran:
|
||||
|
||||
```bash
|
||||
tail -20 ~/.gbrain/integrations/restart-sweep/cron.log
|
||||
```
|
||||
|
||||
## Step 6: Verification
|
||||
|
||||
1. `gbrain integrations doctor restart-sweep` — should pass all three
|
||||
health checks
|
||||
2. `~/.gbrain/integrations/restart-sweep/sweep.log.jsonl` exists and
|
||||
gets a new entry every 5 minutes
|
||||
3. `~/.gbrain/integrations/restart-sweep/cron.log` shows successful
|
||||
invocations (no PATH errors, no `command not found`)
|
||||
4. After a real OpenClaw restart with a stuck session, the Telegram
|
||||
alert fires once, then the cooldown layer suppresses repeats for 6h
|
||||
|
||||
## Tuning
|
||||
|
||||
`OPENCLAW_RESTART_SWEEP_AGGRESSIVE=1` — enables the secondary
|
||||
"active-before-restart, silent-after" heuristic. Off by default because
|
||||
during normal quiet periods (overnight, weekends) it false-positives.
|
||||
Enable if you want maximum sensitivity AND you've established that your
|
||||
group is consistently active.
|
||||
|
||||
The cooldown threshold (6 hours) is a constant in the script. Edit
|
||||
`COOLDOWN_HOURS` if you need different behavior — e.g. 24 hours if your
|
||||
group's normal cadence is daily.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Alerts firing repeatedly on the same session
|
||||
|
||||
Check `~/.gbrain/integrations/restart-sweep/alerted.json`. If the
|
||||
sessionKey is missing or `lastAlertedAt` is recent, the cooldown should
|
||||
suppress. If it's not suppressing:
|
||||
|
||||
- The state file may not be writable. Check `ls -ld
|
||||
~/.gbrain/integrations/restart-sweep/`.
|
||||
- `GBRAIN_HOME` may be set to a different path under cron than under
|
||||
your shell. Check the wrapper script's env loading.
|
||||
- The script's `STATE_DIR` resolution prints in stderr if mkdir fails.
|
||||
Check the cron log.
|
||||
|
||||
### Telegram alert fails silently
|
||||
|
||||
The script logs `❌ Failed to send alert (will retry next cycle)` to
|
||||
stderr when `openclaw message send` returns non-zero. Common causes:
|
||||
|
||||
- `openclaw` not on cron's PATH (use absolute path in the wrapper)
|
||||
- Telegram bot token expired or rate-limited
|
||||
- Wrong group/topic ID (try `openclaw message send --channel telegram
|
||||
--target $OPENCLAW_TELEGRAM_GROUP --message test` manually)
|
||||
|
||||
When the send fails, state is NOT updated, so next cycle retries.
|
||||
|
||||
### Bootstrap log missing
|
||||
|
||||
If `/tmp/bootstrap-services.log` (or `$OPENCLAW_BOOTSTRAP_LOG`) doesn't
|
||||
exist, the script falls back to `now() - 30 minutes` for restartTime.
|
||||
The cooldown layer keeps this from spamming. If you want a stable
|
||||
restart anchor, point `OPENCLAW_BOOTSTRAP_LOG` at OpenClaw's actual
|
||||
startup log (whatever your deployment uses).
|
||||
|
||||
### Cron environment
|
||||
|
||||
The wrapper script in Step 5 handles 80% of cron-day-one failures, but
|
||||
two more knobs:
|
||||
|
||||
- **Locale:** if your script ever interpolates user-provided text into
|
||||
log lines, set `LANG=en_US.UTF-8` in the cron entry to avoid mojibake.
|
||||
- **Working directory:** cron starts in `$HOME` by default. The script
|
||||
uses absolute paths everywhere, so this shouldn't matter, but if you
|
||||
ever add a relative-path dependency, `cd ~/openclaw` in the wrapper.
|
||||
|
||||
## Future upgrade path
|
||||
|
||||
This recipe is the v1 shape: a script copied into the host repo and
|
||||
wired to cron. The v2 shape is a plugin Minion handler registered in
|
||||
the OpenClaw repo against `gbrain/minions` (see
|
||||
`docs/guides/plugin-handlers.md`). Plugin-handler advantages:
|
||||
|
||||
- Built-in queue idempotency (no cooldown layer needed)
|
||||
- Submit via `gbrain jobs submit restart-sweep` from any cron / agent /
|
||||
manual trigger
|
||||
- Centralized retry / backoff / lock management
|
||||
- One less host script to maintain
|
||||
|
||||
When this becomes the right tradeoff (multiple deployments, multiple
|
||||
cron schedules, or just enough complexity to justify the move), promote
|
||||
to the plugin-handler shape and deprecate this recipe.
|
||||
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env bash
|
||||
# Check that admin/src/lib/scope-constants.ts ALLOWED_SCOPES_LIST matches
|
||||
# src/core/scope.ts ALLOWED_SCOPES_LIST. The admin SPA's tsconfig include
|
||||
# scopes to admin/src/ so we can't import the source list directly; instead
|
||||
# this script extracts both lists and diffs them.
|
||||
#
|
||||
# Wired into `bun run verify` and `bun run check:all`.
|
||||
#
|
||||
# Exits 0 on match, 1 on drift, 2 on internal error (file missing, parse fail).
|
||||
#
|
||||
# Usage: scripts/check-admin-scope-drift.sh
|
||||
set -euo pipefail
|
||||
|
||||
SRC=src/core/scope.ts
|
||||
ADMIN=admin/src/lib/scope-constants.ts
|
||||
|
||||
[ -f "$SRC" ] || { echo "[check-admin-scope-drift] missing $SRC" >&2; exit 2; }
|
||||
[ -f "$ADMIN" ] || { echo "[check-admin-scope-drift] missing $ADMIN" >&2; exit 2; }
|
||||
|
||||
# Extract the contents of ALLOWED_SCOPES_LIST = [...] from each file.
|
||||
# The list spans multiple lines, terminated by ']'. awk pulls it cleanly.
|
||||
extract_list() {
|
||||
awk '
|
||||
/ALLOWED_SCOPES_LIST/ && /\[/ { capture = 1 }
|
||||
capture {
|
||||
print
|
||||
if (/\]/) { capture = 0; exit }
|
||||
}
|
||||
' "$1"
|
||||
}
|
||||
|
||||
src_block=$(extract_list "$SRC")
|
||||
admin_block=$(extract_list "$ADMIN")
|
||||
|
||||
if [ -z "$src_block" ]; then
|
||||
echo "[check-admin-scope-drift] could not find ALLOWED_SCOPES_LIST in $SRC" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ -z "$admin_block" ]; then
|
||||
echo "[check-admin-scope-drift] could not find ALLOWED_SCOPES_LIST in $ADMIN" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Strip everything that isn't a quoted scope string and emit one per line.
|
||||
strip_to_scopes() {
|
||||
printf '%s\n' "$1" \
|
||||
| tr ',' '\n' \
|
||||
| grep -oE "'[a-z_]+'" \
|
||||
| tr -d "'" \
|
||||
| sort -u
|
||||
}
|
||||
|
||||
src_scopes=$(strip_to_scopes "$src_block")
|
||||
admin_scopes=$(strip_to_scopes "$admin_block")
|
||||
|
||||
if [ "$src_scopes" != "$admin_scopes" ]; then
|
||||
echo "[check-admin-scope-drift] DRIFT detected between:" >&2
|
||||
echo " $SRC" >&2
|
||||
echo " $ADMIN" >&2
|
||||
echo "" >&2
|
||||
echo "src/core/scope.ts has:" >&2
|
||||
printf ' %s\n' $src_scopes >&2
|
||||
echo "" >&2
|
||||
echo "admin/src/lib/scope-constants.ts has:" >&2
|
||||
printf ' %s\n' $admin_scopes >&2
|
||||
echo "" >&2
|
||||
echo "Update admin/src/lib/scope-constants.ts to match, then 'cd admin && bun run build'." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[check-admin-scope-drift] ok: $(echo "$src_scopes" | wc -l | tr -d ' ') scopes match"
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
# CI guard: src/cli.ts must be tracked by git in executable mode (100755).
|
||||
#
|
||||
# Why: bun-link installs symlink to src/cli.ts directly. If the mode bit
|
||||
# regresses to 100644, the very first `gbrain --version` invocation fails
|
||||
# with `permission denied`. v0.28.5 (cluster C, #683) fixed the original
|
||||
# regression; this guard prevents future drift.
|
||||
#
|
||||
# Wired into `bun run verify`. Fast, no external deps.
|
||||
set -e
|
||||
|
||||
MODE=$(git ls-files --stage src/cli.ts | awk '{print $1}')
|
||||
if [ "$MODE" != "100755" ]; then
|
||||
echo "FAIL: src/cli.ts is tracked at mode $MODE; expected 100755 (executable)."
|
||||
echo ""
|
||||
echo "Fix: chmod +x src/cli.ts && git add --chmod=+x src/cli.ts"
|
||||
echo ""
|
||||
echo "Background: bun-link installs symlink to this file directly. Mode 100644"
|
||||
echo "produces 'permission denied' on first invocation (issue #683)."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: src/cli.ts is git-tracked as executable (100755)"
|
||||
@@ -35,8 +35,8 @@ ALLOWED=(
|
||||
"src/commands/files.ts" # PR 1 refactors to accept engine
|
||||
"src/commands/repair-jsonb.ts" # PR 1 refactors
|
||||
"src/commands/serve-http.ts" # PR 1 threads engine through the OAuth dispatch path
|
||||
"src/commands/integrity.ts" # v0.22.8 batch-load fast path + scanIntegrityBatch; PR 1 refactors to accept engine
|
||||
"src/core/operations.ts" # 3 localOnly ops (file_list/upload/url) move to ctx.engine in PR 1
|
||||
"src/commands/integrity.ts" # scanIntegrityBatch path; PR 1 refactors to accept engine
|
||||
)
|
||||
|
||||
# Build an argument list for `grep` that excludes allowed files.
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
# v0.26.7 baseline allow-list for scripts/check-test-isolation.sh.
|
||||
#
|
||||
# Files here violate one or more of the lint rules (env mutation,
|
||||
# mock.module, PGLite outside beforeAll, missing afterAll{disconnect}).
|
||||
# The lint ships in v0.26.7 and v0.26.8 (env sweep) + v0.26.9 (PGLite
|
||||
# sweep) remove entries from this file as each sweep makes the file
|
||||
# clean.
|
||||
#
|
||||
# RULES:
|
||||
# - This list MUST shrink over time. Never add new entries — adding a
|
||||
# new file means accepting cross-file flake risk for that file.
|
||||
# - When you fix a file (apply withEnv, add the canonical PGLite
|
||||
# block, etc.), remove its entry here.
|
||||
# - When you cannot fix a file cleanly (genuinely env-coupled,
|
||||
# or shares state intentionally), rename it to *.serial.test.ts
|
||||
# instead of leaving it allow-listed.
|
||||
#
|
||||
# Permanent exemption: the test of the lint itself. Its fixture strings
|
||||
# (passed verbatim into subprocesses) legitimately match the lint
|
||||
# patterns it is testing detection of. The file does NOT mutate
|
||||
# process.env at runtime. Permanent — do not remove.
|
||||
test/scripts/check-test-isolation.test.ts
|
||||
test/autopilot-install.test.ts
|
||||
test/bootstrap.test.ts
|
||||
test/brain-resolver.test.ts
|
||||
test/check-resolvable-cli.test.ts
|
||||
test/claw-test-cli.test.ts
|
||||
test/code-def-refs.test.ts
|
||||
test/core/cycle.test.ts
|
||||
test/destructive-guard.test.ts
|
||||
test/doctor-minions-check.test.ts
|
||||
test/doctor.test.ts
|
||||
test/dream.test.ts
|
||||
test/embed.test.ts
|
||||
test/eval-capture.test.ts
|
||||
test/friction-cli.test.ts
|
||||
test/friction.test.ts
|
||||
test/gbrain-home-isolation.test.ts
|
||||
test/helpers/with-env.test.ts
|
||||
test/http-transport.test.ts
|
||||
test/hybrid-meta.test.ts
|
||||
test/init-migrate-only.test.ts
|
||||
test/integrations.test.ts
|
||||
test/mcp-eval-capture.test.ts
|
||||
test/migrate.test.ts
|
||||
test/migration-resume.test.ts
|
||||
test/migrations-v0_11_0.test.ts
|
||||
test/migrations-v0_13_1.test.ts
|
||||
test/migrations-v0_14_0.test.ts
|
||||
test/migrations-v0_19_0.test.ts
|
||||
test/migrations-v0_22_4.test.ts
|
||||
test/minions-shell.test.ts
|
||||
test/minions.test.ts
|
||||
test/mounts-cli.test.ts
|
||||
test/multi-source-integration.test.ts
|
||||
test/orphans.test.ts
|
||||
test/pages-soft-delete.test.ts
|
||||
test/preferences.test.ts
|
||||
test/reindex-code.test.ts
|
||||
test/resolve-prepare.test.ts
|
||||
test/resolvers.test.ts
|
||||
test/scenarios.test.ts
|
||||
test/schema-bootstrap-coverage.test.ts
|
||||
test/search-limit.test.ts
|
||||
test/seed-pglite.test.ts
|
||||
test/skillpack-check.test.ts
|
||||
test/source-resolver.test.ts
|
||||
test/storage-sync.test.ts
|
||||
test/subagent-audit.test.ts
|
||||
test/supervisor.test.ts
|
||||
test/sync-failures.test.ts
|
||||
test/sync-parallel.test.ts
|
||||
test/transcription.test.ts
|
||||
Executable
+141
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: fail if any non-serial unit test file violates intra-process
|
||||
# isolation rules. The v0.26.4 parallel runner loads multiple test files
|
||||
# into one bun process per shard; module-level state (env vars, PGLite
|
||||
# engines, mock.module overrides) leaks across files in that process and
|
||||
# silently flakes other tests.
|
||||
#
|
||||
# Rules enforced (non-serial unit test files only):
|
||||
# R1: no `process.env.X = ...`, `process.env['X'] = ...`,
|
||||
# `delete process.env.X`, `Object.assign(process.env, ...)`,
|
||||
# `Reflect.set(process.env, ...)` mutations. Use withEnv() helper or
|
||||
# rename the file to `*.serial.test.ts`.
|
||||
# R2: no `mock.module(...)` anywhere. Top-level module mocks affect every
|
||||
# other file in the same shard process. Rename to `*.serial.test.ts`.
|
||||
# R3: `new PGLiteEngine(` may only appear within ~50 lines following a
|
||||
# `beforeAll(` line. Engines created at module scope (or in describe
|
||||
# bodies) leak across files in the shard process.
|
||||
# R4: any file that creates `new PGLiteEngine(` must call `.disconnect(`
|
||||
# inside an `afterAll(` block. Without disconnect, engines leak across
|
||||
# file boundaries within a shard process.
|
||||
#
|
||||
# Scope:
|
||||
# - Recursively scans `test/**/*.test.ts`.
|
||||
# - Skips `*.serial.test.ts` entirely (the quarantine escape hatch).
|
||||
# - Skips `test/e2e/**` (E2E runs sequentially in its own runner; not in
|
||||
# the parallel pool).
|
||||
#
|
||||
# Allow-list:
|
||||
# Files in `scripts/check-test-isolation.allowlist` (one filename per
|
||||
# line, # comments allowed) are skipped. This exists because v0.26.7
|
||||
# ships the lint as a foundation; v0.26.8 (env sweep) and v0.26.9
|
||||
# (PGLite sweep) remove entries as files get fixed. New files MUST NOT
|
||||
# be added — the allow-list shrinks over time, never grows.
|
||||
#
|
||||
# Usage: scripts/check-test-isolation.sh [TARGET_DIR]
|
||||
# Exit: 0 when clean, 1 when un-allow-listed violations found.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
TARGET_DIR="${1:-test}"
|
||||
ALLOWLIST_FILE="$ROOT/scripts/check-test-isolation.allowlist"
|
||||
|
||||
# Read allowlist (one filename per line, # comments allowed). Empty file
|
||||
# is fine — every violation will fail.
|
||||
ALLOWLIST=""
|
||||
if [ -f "$ALLOWLIST_FILE" ]; then
|
||||
ALLOWLIST="$(grep -v '^[[:space:]]*#' "$ALLOWLIST_FILE" | grep -v '^[[:space:]]*$' || true)"
|
||||
fi
|
||||
|
||||
is_allowlisted() {
|
||||
local f="$1"
|
||||
[ -z "$ALLOWLIST" ] && return 1
|
||||
echo "$ALLOWLIST" | grep -qxF "$f"
|
||||
}
|
||||
|
||||
# Find non-serial unit test files (excluding test/e2e). Portable across
|
||||
# bash 3.2 (macOS default) and bash 4+; no mapfile.
|
||||
FILE_LIST="$(find "$TARGET_DIR" -name '*.test.ts' \
|
||||
-not -name '*.serial.test.ts' \
|
||||
-not -path "*/e2e/*" \
|
||||
-type f 2>/dev/null | sort)"
|
||||
|
||||
violations=0
|
||||
file_count=0
|
||||
|
||||
emit_violation() {
|
||||
local f="$1" rule="$2" detail="$3" lines="$4"
|
||||
if is_allowlisted "$f"; then
|
||||
return
|
||||
fi
|
||||
echo "ERROR: $f"
|
||||
echo " rule $rule: $detail"
|
||||
if [ -n "$lines" ]; then
|
||||
echo "$lines" | head -3 | sed 's/^/ /'
|
||||
fi
|
||||
violations=$((violations + 1))
|
||||
}
|
||||
|
||||
# Read newline-separated file list; OK on macOS bash 3.2.
|
||||
while IFS= read -r f; do
|
||||
[ -z "$f" ] && continue
|
||||
file_count=$((file_count + 1))
|
||||
# R1: env mutations.
|
||||
env_lines=$(grep -nE 'process\.env\.[A-Za-z_][A-Za-z_0-9]*[[:space:]]*=[^=]|process\.env\[[^]]+\][[:space:]]*=[^=]|delete[[:space:]]+process\.env\.|delete[[:space:]]+process\.env\[|Object\.assign[[:space:]]*\([[:space:]]*process\.env|Reflect\.set[[:space:]]*\([[:space:]]*process\.env' "$f" 2>/dev/null || true)
|
||||
if [ -n "$env_lines" ]; then
|
||||
emit_violation "$f" "R1" "process.env mutation; use withEnv() or rename to *.serial.test.ts" "$env_lines"
|
||||
fi
|
||||
|
||||
# R2: mock.module() anywhere.
|
||||
mock_lines=$(grep -nE 'mock\.module[[:space:]]*\(' "$f" 2>/dev/null || true)
|
||||
if [ -n "$mock_lines" ]; then
|
||||
emit_violation "$f" "R2" "mock.module() leaks across files in the shard process; rename to *.serial.test.ts" "$mock_lines"
|
||||
fi
|
||||
|
||||
# R3: PGLiteEngine outside ~50 lines after a beforeAll(.
|
||||
if grep -qE 'new PGLiteEngine[[:space:]]*\(' "$f" 2>/dev/null; then
|
||||
bad=$(awk '
|
||||
BEGIN { last_before_all = -1000 }
|
||||
/beforeAll[[:space:]]*\(/ { last_before_all = NR }
|
||||
/new PGLiteEngine[[:space:]]*\(/ {
|
||||
if (NR - last_before_all > 50) {
|
||||
printf "%d:%s\n", NR, $0
|
||||
}
|
||||
}
|
||||
' "$f" 2>/dev/null)
|
||||
if [ -n "$bad" ]; then
|
||||
emit_violation "$f" "R3" "new PGLiteEngine(...) outside beforeAll() context (>50 lines); move into beforeAll" "$bad"
|
||||
fi
|
||||
fi
|
||||
|
||||
# R4: PGLiteEngine creation requires afterAll{disconnect}.
|
||||
if grep -qE 'new PGLiteEngine[[:space:]]*\(' "$f" 2>/dev/null; then
|
||||
if ! grep -qE 'afterAll[[:space:]]*\(' "$f" 2>/dev/null \
|
||||
|| ! grep -qE '\.disconnect[[:space:]]*\(' "$f" 2>/dev/null; then
|
||||
emit_violation "$f" "R4" "creates PGLiteEngine but missing afterAll(() => engine.disconnect()); engine leaks across files in the shard process" ""
|
||||
fi
|
||||
fi
|
||||
done <<EOF
|
||||
$FILE_LIST
|
||||
EOF
|
||||
|
||||
if [ $violations -gt 0 ]; then
|
||||
echo
|
||||
echo "check-test-isolation: FAIL ($violations violation(s))"
|
||||
echo
|
||||
echo "Fix:"
|
||||
echo " - For env mutations, use withEnv() from test/helpers/with-env.ts"
|
||||
echo " - For mock.module(), rename to *.serial.test.ts (quarantine)"
|
||||
echo " - For PGLiteEngine, follow the canonical pattern in"
|
||||
echo " test/helpers/reset-pglite.ts JSDoc and CLAUDE.md."
|
||||
echo
|
||||
echo "Or, if this is a baseline file from before the lint shipped,"
|
||||
echo "add it to scripts/check-test-isolation.allowlist (with a TODO"
|
||||
echo "comment naming the sweep PR that will remove it)."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "check-test-isolation: OK ($file_count non-serial unit files scanned)"
|
||||
@@ -41,12 +41,18 @@ export const E2E_TEST_MAP: Record<string, string[]> = {
|
||||
"test/e2e/postgres-jsonb.test.ts",
|
||||
"test/e2e/jsonb-roundtrip.test.ts",
|
||||
"test/e2e/engine-parity.test.ts",
|
||||
"test/e2e/schema-drift.test.ts",
|
||||
],
|
||||
// PGLite bootstrap path + parity guard.
|
||||
"src/core/pglite-engine.ts": [
|
||||
"test/e2e/postgres-bootstrap.test.ts",
|
||||
"test/e2e/engine-parity.test.ts",
|
||||
"test/e2e/schema-drift.test.ts",
|
||||
],
|
||||
// Schema source of truth: any change must pass the cross-engine drift gate.
|
||||
"src/schema.sql": ["test/e2e/schema-drift.test.ts"],
|
||||
"src/core/pglite-schema.ts": ["test/e2e/schema-drift.test.ts"],
|
||||
"src/core/migrate.ts": ["test/e2e/schema-drift.test.ts", "test/e2e/migrate-chain.test.ts"],
|
||||
// MCP stdio + HTTP transports share dispatch.
|
||||
"src/mcp/**": ["test/e2e/mcp.test.ts", "test/e2e/http-transport.test.ts"],
|
||||
// Integrity batch-load fast path.
|
||||
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/run-serial-tests.sh — run *.serial.test.ts files with --max-concurrency=1.
|
||||
#
|
||||
# Serial files are tests that share file-wide state (top-level mock.module,
|
||||
# module-level singletons that intentionally cross test cases) and would race
|
||||
# under intra-file concurrency. Discovered via filename suffix; no annotation
|
||||
# inside the file is needed.
|
||||
#
|
||||
# Excluded by run-unit-shard.sh and run-unit-parallel.sh's parallel pass.
|
||||
# Invoked separately by run-unit-parallel.sh after the parallel pass succeeds.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Use while-read for portability to macOS bash 3.2 (no mapfile).
|
||||
files=()
|
||||
while IFS= read -r f; do
|
||||
files+=("$f")
|
||||
done < <(find test -name '*.serial.test.ts' -not -path 'test/e2e/*' | sort)
|
||||
|
||||
if [ "${#files[@]}" -eq 0 ]; then
|
||||
echo "[serial-tests] no *.serial.test.ts files found"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --dry-run-list mirrors run-unit-shard.sh for inline checks/tests.
|
||||
if [ "${1:-}" = "--dry-run-list" ]; then
|
||||
printf '%s\n' "${files[@]}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[serial-tests] running ${#files[@]} file(s) with --max-concurrency=1"
|
||||
exec bun test --max-concurrency=1 --timeout=60000 "${files[@]}"
|
||||
Executable
+341
@@ -0,0 +1,341 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/run-unit-parallel.sh — fast unit-test loop, parallel fan-out.
|
||||
#
|
||||
# Spawns N parallel `bun test` processes, each running a hash-disjoint shard
|
||||
# of the unit-test set (files only — no e2e, no .slow, no .serial). After
|
||||
# all shards complete, runs serial-only files (*.serial.test.ts) with
|
||||
# --max-concurrency=1. Failure-first logging: extracts failure blocks from
|
||||
# each shard's log, writes to .context/test-failures.log with --- shard $i:
|
||||
# prefixes, prints loud stderr banner if any failures, exit non-zero.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/run-unit-parallel.sh [--shards N] [--max-concurrency N] [--dry-run]
|
||||
#
|
||||
# Env overrides:
|
||||
# SHARDS=N same as --shards
|
||||
# GBRAIN_TEST_SHARD_TIMEOUT per-shard wallclock cap, seconds (default 600)
|
||||
# GBRAIN_TEST_MAX_CONCURRENCY passed through to bun test (default 4)
|
||||
#
|
||||
# Output files (workspace-local; falls back to /tmp if .context/ unwritable):
|
||||
# .context/test-failures.log failure blocks (cleared at start)
|
||||
# .context/test-summary.txt per-shard pass/fail/skip/duration (cleared at start)
|
||||
# .context/test-shards/ per-shard logs + exit codes (cleared at start)
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# CPU detection: Apple Silicon perf cores → Mac total physical → nproc → 4.
|
||||
# Returns a single positive integer.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
detect_cpus() {
|
||||
local n=""
|
||||
n=$(sysctl -n hw.perflevel0.physicalcpu 2>/dev/null) && [ -n "$n" ] && [ "$n" -gt 0 ] && echo "$n" && return
|
||||
n=$(sysctl -n hw.physicalcpu 2>/dev/null) && [ -n "$n" ] && [ "$n" -gt 0 ] && echo "$n" && return
|
||||
n=$(nproc 2>/dev/null) && [ -n "$n" ] && [ "$n" -gt 0 ] && echo "$n" && return
|
||||
echo 4
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Argument parsing. --shards N override wins over $SHARDS; both are clamped.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
SHARDS_OVERRIDE=""
|
||||
MAX_CONCURRENCY_OVERRIDE=""
|
||||
DRY_RUN=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--shards) SHARDS_OVERRIDE="$2"; shift 2 ;;
|
||||
--shards=*) SHARDS_OVERRIDE="${1#*=}"; shift ;;
|
||||
--max-concurrency) MAX_CONCURRENCY_OVERRIDE="$2"; shift 2 ;;
|
||||
--max-concurrency=*) MAX_CONCURRENCY_OVERRIDE="${1#*=}"; shift ;;
|
||||
--dry-run) DRY_RUN=1; shift ;;
|
||||
*) echo "ERROR: unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
N="${SHARDS_OVERRIDE:-${SHARDS:-$(detect_cpus)}}"
|
||||
if ! printf '%s' "$N" | grep -qE '^[0-9]+$' || [ "$N" -lt 1 ]; then
|
||||
echo "ERROR: invalid shard count: $N" >&2; exit 2
|
||||
fi
|
||||
[ "$N" -gt 8 ] && N=8
|
||||
|
||||
INTRA_CONC="${MAX_CONCURRENCY_OVERRIDE:-${GBRAIN_TEST_MAX_CONCURRENCY:-4}}"
|
||||
SHARD_TIMEOUT="${GBRAIN_TEST_SHARD_TIMEOUT:-600}"
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Output directories. Prefer workspace-local .context/, fall back to /tmp.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
LOG_DIR=""
|
||||
if mkdir -p .context/test-shards 2>/dev/null; then
|
||||
LOG_DIR=".context/test-shards"
|
||||
FAILURES_LOG=".context/test-failures.log"
|
||||
SUMMARY_FILE=".context/test-summary.txt"
|
||||
else
|
||||
LOG_DIR="/tmp/gbrain-test-shards-$$"
|
||||
FAILURES_LOG="/tmp/gbrain-test-failures.log"
|
||||
SUMMARY_FILE="/tmp/gbrain-test-summary.txt"
|
||||
mkdir -p "$LOG_DIR" || { echo "ERROR: cannot create log dir" >&2; exit 2; }
|
||||
fi
|
||||
# Clear from prior run.
|
||||
rm -f "$LOG_DIR"/shard-*.log "$LOG_DIR"/shard-*.exit "$LOG_DIR"/shard-*.wedged 2>/dev/null
|
||||
: > "$FAILURES_LOG"
|
||||
: > "$SUMMARY_FILE"
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Resolve `timeout` command. macOS without coreutils has neither; we degrade
|
||||
# to bg-pid + sleep cap. For now, prefer gtimeout (brew coreutils) → timeout.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
TIMEOUT_BIN=""
|
||||
if command -v gtimeout >/dev/null 2>&1; then TIMEOUT_BIN="gtimeout"
|
||||
elif command -v timeout >/dev/null 2>&1; then TIMEOUT_BIN="timeout"
|
||||
fi
|
||||
|
||||
START_TS=$(date +%s)
|
||||
echo "[unit-parallel] N=$N shards | --max-concurrency=$INTRA_CONC | timeout=${SHARD_TIMEOUT}s | logs=$LOG_DIR" >&2
|
||||
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
echo "[unit-parallel] dry-run: would spawn $N shards with the above settings."
|
||||
for i in $(seq 1 "$N"); do
|
||||
SHARD="$i/$N" bash scripts/run-unit-shard.sh --dry-run-list 2>/dev/null \
|
||||
| sed "s|^| [s$i] |"
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Spawn shards. Each child captures its own exit code into a sentinel file
|
||||
# so $? is recoverable per-shard (we never trust `wait`'s aggregate value).
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
SHARD_PIDS=()
|
||||
for i in $(seq 1 "$N"); do
|
||||
(
|
||||
SHARD_LOG="$LOG_DIR/shard-$i.log"
|
||||
if [ -n "$TIMEOUT_BIN" ]; then
|
||||
"$TIMEOUT_BIN" "${SHARD_TIMEOUT}s" \
|
||||
env SHARD="$i/$N" \
|
||||
bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \
|
||||
> "$SHARD_LOG" 2>&1
|
||||
else
|
||||
env SHARD="$i/$N" \
|
||||
bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \
|
||||
> "$SHARD_LOG" 2>&1 &
|
||||
pid=$!
|
||||
( sleep "$SHARD_TIMEOUT" && kill -TERM "$pid" 2>/dev/null && \
|
||||
sleep 5 && kill -KILL "$pid" 2>/dev/null ) &
|
||||
cap_pid=$!
|
||||
wait "$pid" 2>/dev/null
|
||||
kill "$cap_pid" 2>/dev/null
|
||||
wait "$cap_pid" 2>/dev/null
|
||||
fi
|
||||
rc=$?
|
||||
echo "$rc" > "$LOG_DIR/shard-$i.exit"
|
||||
[ "$rc" = "124" ] && echo "WEDGED" > "$LOG_DIR/shard-$i.wedged"
|
||||
) &
|
||||
SHARD_PIDS+=($!)
|
||||
done
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Heartbeat: every 10s, print per-shard progress to stderr by tailing logs
|
||||
# and counting Bun's `(pass)` / `(fail)` / `(skip)` markers. Read-only.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# grep_count: returns 0 (single integer) if file is missing or zero matches,
|
||||
# otherwise the match count. Avoids the `grep -c | echo 0` double-output bug
|
||||
# where 0 matches produces a 2-line "0\n0" string that breaks arithmetic.
|
||||
grep_count() {
|
||||
local pattern="$1"; local file="$2"
|
||||
if [ ! -f "$file" ]; then echo 0; return; fi
|
||||
local n
|
||||
n=$(grep -cE "$pattern" "$file" 2>/dev/null) || n=0
|
||||
echo "${n:-0}"
|
||||
}
|
||||
|
||||
# bun_summary_count: parses Bun's summary lines (one per `bun test` invocation
|
||||
# inside a shard — there's only one when we pass an explicit file list).
|
||||
# Looks for ` N pass` / ` N fail` / ` N skip` patterns and sums them across
|
||||
# all summary blocks the shard emitted. `bun test` prints these near the end
|
||||
# of its output. Format: leading whitespace + integer + space + label.
|
||||
bun_summary_count() {
|
||||
local label="$1"; local file="$2"
|
||||
if [ ! -f "$file" ]; then echo 0; return; fi
|
||||
awk -v label="$label" '
|
||||
$1 ~ /^[0-9]+$/ && $2 == label { total += $1 }
|
||||
END { print total + 0 }
|
||||
' "$file"
|
||||
}
|
||||
|
||||
heartbeat() {
|
||||
while true; do
|
||||
sleep 10
|
||||
local line=""
|
||||
for i in $(seq 1 "$N"); do
|
||||
if [ -f "$LOG_DIR/shard-$i.exit" ]; then
|
||||
local rc; rc=$(cat "$LOG_DIR/shard-$i.exit" 2>/dev/null || echo "?")
|
||||
local status="✓"
|
||||
[ "$rc" != "0" ] && status="✗"
|
||||
line="$line [s$i: done $status]"
|
||||
else
|
||||
local lf="$LOG_DIR/shard-$i.log"
|
||||
if [ -f "$lf" ]; then
|
||||
# Heartbeat: prefer Bun's per-test "✓" (passed) and "(fail)" markers
|
||||
# so we see live progress; the "N pass" summary line only appears at
|
||||
# the very end of the shard and would always show 0 mid-run.
|
||||
local p f
|
||||
p=$(grep_count '^[[:space:]]+✓' "$lf")
|
||||
f=$(grep_count '^\(fail\)' "$lf")
|
||||
line="$line [s$i: ${p}p ${f}f ...]"
|
||||
else
|
||||
line="$line [s$i: starting]"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
printf '[heartbeat] %s\n' "$line" >&2
|
||||
done
|
||||
}
|
||||
heartbeat &
|
||||
HB_PID=$!
|
||||
trap 'kill "$HB_PID" 2>/dev/null; wait "$HB_PID" 2>/dev/null' EXIT
|
||||
|
||||
# Wait for every shard. Don't care about wait's exit code.
|
||||
for pid in "${SHARD_PIDS[@]}"; do wait "$pid" 2>/dev/null || true; done
|
||||
|
||||
kill "$HB_PID" 2>/dev/null
|
||||
wait "$HB_PID" 2>/dev/null
|
||||
trap - EXIT
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Aggregate failures (single writer; serial; never concurrent).
|
||||
# Bun failure block format: from `(fail) ...` line through next `(pass)`,
|
||||
# `(skip)`, blank line, or `__bun_test_summary__` marker.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
TOTAL_FAILURES=0
|
||||
TOTAL_PASS=0
|
||||
TOTAL_SKIP=0
|
||||
TOTAL_RC=0
|
||||
for i in $(seq 1 "$N"); do
|
||||
SHARD_LOG="$LOG_DIR/shard-$i.log"
|
||||
EXIT_FILE="$LOG_DIR/shard-$i.exit"
|
||||
WEDGED_FILE="$LOG_DIR/shard-$i.wedged"
|
||||
rc=1
|
||||
[ -f "$EXIT_FILE" ] && rc=$(cat "$EXIT_FILE" 2>/dev/null || echo 1)
|
||||
|
||||
pass_count=$(bun_summary_count "pass" "$SHARD_LOG")
|
||||
fail_count=$(bun_summary_count "fail" "$SHARD_LOG")
|
||||
skip_count=$(bun_summary_count "skip" "$SHARD_LOG")
|
||||
TOTAL_PASS=$((TOTAL_PASS + pass_count))
|
||||
TOTAL_FAILURES=$((TOTAL_FAILURES + fail_count))
|
||||
TOTAL_SKIP=$((TOTAL_SKIP + skip_count))
|
||||
|
||||
if [ -f "$WEDGED_FILE" ]; then
|
||||
TOTAL_RC=1
|
||||
{
|
||||
echo "--- shard $i: WEDGED after ${SHARD_TIMEOUT}s ---"
|
||||
[ -f "$SHARD_LOG" ] && tail -50 "$SHARD_LOG"
|
||||
echo ""
|
||||
} >> "$FAILURES_LOG"
|
||||
echo "shard $i/$N: WEDGED after ${SHARD_TIMEOUT}s (rc=$rc)" >> "$SUMMARY_FILE"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "shard $i/$N: pass=$pass_count fail=$fail_count skip=$skip_count rc=$rc" >> "$SUMMARY_FILE"
|
||||
|
||||
if [ "$rc" != "0" ]; then
|
||||
TOTAL_RC=1
|
||||
if [ "$fail_count" -gt 0 ] && [ -f "$SHARD_LOG" ]; then
|
||||
# Extract each (fail) block: from `(fail)` line through next `(pass)`,
|
||||
# `(skip)`, blank line, or `__bun_test_summary__`. Single awk pass.
|
||||
awk -v shard="$i" '
|
||||
/^\(fail\) / { in_block=1; print "--- shard " shard ": " $0; next }
|
||||
in_block {
|
||||
if (/^\(pass\)/ || /^\(skip\)/ || /^[[:space:]]*$/ || /__bun_test_summary__/) { in_block=0; print ""; next }
|
||||
print $0
|
||||
}
|
||||
' "$SHARD_LOG" >> "$FAILURES_LOG"
|
||||
elif [ -f "$SHARD_LOG" ]; then
|
||||
# Non-zero rc but no (fail) line found — extraction couldn't pinpoint.
|
||||
# Dump the full shard log so we never silently lose the failure cause.
|
||||
{
|
||||
echo "--- shard $i: rc=$rc, no (fail) markers — full log follows ---"
|
||||
cat "$SHARD_LOG"
|
||||
echo ""
|
||||
} >> "$FAILURES_LOG"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Print each shard's full output to stdout (developer expects to scroll
|
||||
# through it). Print summary file last for one-glance overview.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
for i in $(seq 1 "$N"); do
|
||||
SHARD_LOG="$LOG_DIR/shard-$i.log"
|
||||
echo ""
|
||||
echo "════════════ shard $i/$N ════════════"
|
||||
[ -f "$SHARD_LOG" ] && cat "$SHARD_LOG"
|
||||
done
|
||||
echo ""
|
||||
echo "════════════ summary ════════════"
|
||||
cat "$SUMMARY_FILE"
|
||||
echo ""
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Serial pass: any *.serial.test.ts files run after parallel pass.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
SERIAL_RC=0
|
||||
SERIAL_FILES_COUNT=0
|
||||
SERIAL_FILES_COUNT=$(find test -name '*.serial.test.ts' -not -path 'test/e2e/*' 2>/dev/null | wc -l | tr -d ' ')
|
||||
if [ "$SERIAL_FILES_COUNT" -gt 0 ]; then
|
||||
echo "════════════ serial pass ($SERIAL_FILES_COUNT files) ════════════"
|
||||
bash scripts/run-serial-tests.sh > "$LOG_DIR/serial.log" 2>&1
|
||||
SERIAL_RC=$?
|
||||
cat "$LOG_DIR/serial.log"
|
||||
if [ "$SERIAL_RC" != "0" ]; then
|
||||
TOTAL_RC=1
|
||||
s_fail=$(bun_summary_count "fail" "$LOG_DIR/serial.log")
|
||||
TOTAL_FAILURES=$((TOTAL_FAILURES + s_fail))
|
||||
if [ "$s_fail" -gt 0 ]; then
|
||||
awk '
|
||||
/^\(fail\) / { in_block=1; print "--- shard serial: " $0; next }
|
||||
in_block {
|
||||
if (/^\(pass\)/ || /^\(skip\)/ || /^[[:space:]]*$/ || /__bun_test_summary__/) { in_block=0; print ""; next }
|
||||
print $0
|
||||
}
|
||||
' "$LOG_DIR/serial.log" >> "$FAILURES_LOG"
|
||||
else
|
||||
{
|
||||
echo "--- shard serial: rc=$SERIAL_RC, no (fail) markers — full log follows ---"
|
||||
cat "$LOG_DIR/serial.log"
|
||||
echo ""
|
||||
} >> "$FAILURES_LOG"
|
||||
fi
|
||||
echo "serial: rc=$SERIAL_RC fail=$s_fail" >> "$SUMMARY_FILE"
|
||||
else
|
||||
s_pass=$(bun_summary_count "pass" "$LOG_DIR/serial.log")
|
||||
TOTAL_PASS=$((TOTAL_PASS + s_pass))
|
||||
echo "serial: pass=$s_pass rc=0" >> "$SUMMARY_FILE"
|
||||
fi
|
||||
fi
|
||||
|
||||
END_TS=$(date +%s)
|
||||
ELAPSED=$((END_TS - START_TS))
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Loud banner if anything failed. To stderr so it survives `| head`/`| tail`.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
if [ "$TOTAL_RC" != "0" ]; then
|
||||
ABS_FAIL=$(cd "$(dirname "$FAILURES_LOG")" && pwd)/$(basename "$FAILURES_LOG")
|
||||
{
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "❌ $TOTAL_FAILURES TEST FAILURES — full details:"
|
||||
echo " $ABS_FAIL"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
tail -30 "$FAILURES_LOG"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "[unit-parallel] elapsed=${ELAPSED}s | pass=$TOTAL_PASS fail=$TOTAL_FAILURES skip=$TOTAL_SKIP"
|
||||
} >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[unit-parallel] elapsed=${ELAPSED}s | pass=$TOTAL_PASS fail=$TOTAL_FAILURES skip=$TOTAL_SKIP" >&2
|
||||
exit 0
|
||||
@@ -16,16 +16,29 @@ set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# --max-concurrency=N is forwarded to `bun test`. v0.26.4: invoked by
|
||||
# run-unit-parallel.sh; safe to call without (defaults to bun's default cap).
|
||||
MAX_CONC=""
|
||||
DRY_RUN=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--max-concurrency) MAX_CONC="$2"; shift 2 ;;
|
||||
--max-concurrency=*) MAX_CONC="${1#*=}"; shift ;;
|
||||
--dry-run-list) DRY_RUN=1; shift ;;
|
||||
*) echo "ERROR: unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# All non-E2E test files, sorted for deterministic shard splits.
|
||||
# Tier 4: *.slow.test.ts is the convention for "always-slow" tests (e.g.,
|
||||
# bootstrap correctness checks that intentionally exercise the cold init
|
||||
# path and can't benefit from Tier 3's snapshot). They're excluded from the
|
||||
# fast loop and run via `bun run test:slow` (or in CI where everything runs).
|
||||
# Tier 4: *.slow.test.ts is "always-slow" (cold-path correctness checks);
|
||||
# *.serial.test.ts is "concurrency-unsafe" (file-wide shared state). Both
|
||||
# are excluded from the fast loop. Slow runs via `bun run test:slow`; serial
|
||||
# runs via scripts/run-serial-tests.sh after the parallel pass.
|
||||
# Use while-read to stay portable to macOS bash 3.2 (no mapfile).
|
||||
all_files=()
|
||||
while IFS= read -r f; do
|
||||
all_files+=("$f")
|
||||
done < <(find test -name '*.test.ts' -not -path 'test/e2e/*' -not -name '*.slow.test.ts' | sort)
|
||||
done < <(find test -name '*.test.ts' -not -path 'test/e2e/*' -not -name '*.slow.test.ts' -not -name '*.serial.test.ts' | sort)
|
||||
|
||||
files=()
|
||||
if [ -n "${SHARD:-}" ]; then
|
||||
@@ -53,11 +66,13 @@ if [ "${#files[@]}" -eq 0 ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --dry-run-list mirrors scripts/run-e2e.sh for inline smoke checks.
|
||||
if [ "${1:-}" = "--dry-run-list" ]; then
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
printf '%s\n' "${files[@]}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[unit-shard ${SHARD:-(unsharded)}] running ${#files[@]} files"
|
||||
if [ -n "$MAX_CONC" ]; then
|
||||
exec bun test --max-concurrency="$MAX_CONC" --timeout=60000 "${files[@]}"
|
||||
fi
|
||||
exec bun test --timeout=60000 "${files[@]}"
|
||||
|
||||
@@ -26,6 +26,14 @@ mutating: false
|
||||
> **Convention:** see [conventions/cross-modal.yaml](../conventions/cross-modal.yaml)
|
||||
> for the review pairs and refusal routing chain.
|
||||
|
||||
> **Relationship to `gbrain eval cross-modal`:** This skill is the manual
|
||||
> mid-flow gate (one model reviews work product before commit, with refusal
|
||||
> routing). The `gbrain eval cross-modal` command (v0.27.x) is a sibling
|
||||
> surface: 3 different-provider frontier models score-and-iterate on a
|
||||
> documented dimension list *before* tests cement behavior. Use this skill
|
||||
> for ad-hoc second opinions; use `gbrain eval cross-modal` for the
|
||||
> skillify Phase 3 quality gate. The two are complementary, not redundant.
|
||||
|
||||
## Contract
|
||||
|
||||
This skill guarantees:
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
---
|
||||
version: "0.28.0"
|
||||
title: "v0.28 — Takes + Think + Unified Model Config"
|
||||
status: published
|
||||
---
|
||||
|
||||
# v0.28 — Takes, Think, Unified Model Config
|
||||
|
||||
You upgraded from v0.27 (or earlier) to v0.28. New surface area:
|
||||
|
||||
- **Takes** — typed/weighted/attributed claims stored as fenced markdown
|
||||
tables and indexed in Postgres. Four kinds: `fact | take | bet | hunch`.
|
||||
Holders: `world | garry | brain | <slug>`.
|
||||
- **`gbrain takes`** CLI — list / search / add / update / supersede / resolve.
|
||||
- **Unified model config** — `models.default` replaces per-phase
|
||||
`dream.synthesize.model` etc. Aliases: `opus`, `sonnet`, `haiku`,
|
||||
`gemini`, `gpt`. CLI flag `--model <name>` overrides per-call.
|
||||
- **Per-token MCP allow-list** — `access_tokens.permissions.takes_holders`
|
||||
controls which holders an MCP-bound token sees. Default: `["world"]`.
|
||||
- **Three new MCP ops** — `takes_list`, `takes_search`, `think`.
|
||||
- **`gbrain think` op surface** — registered now; pipeline lands in v0.28.x.
|
||||
|
||||
## What the migration did automatically
|
||||
|
||||
`gbrain apply-migrations --yes` runs three orchestrator phases:
|
||||
|
||||
1. **Schema verify** — schema migrations v37 (takes + synthesis_evidence)
|
||||
and v38 (access_tokens.permissions JSONB) ran; tables are present.
|
||||
(v31 was claimed by v0.25's eval_capture_tables, v32+v33 by v0.26's
|
||||
OAuth + admin dashboard, v34 by v0.26.5's destructive-guard columns,
|
||||
v35 by v0.26.8's auto-RLS event trigger, and v36 by v0.27's
|
||||
subagent provider-neutral persistence before v0.28 landed.)
|
||||
2. **Backfill takes** — walked every page in the brain, parsed any fenced
|
||||
`<!--- gbrain:takes:begin/end -->` table found, populated the takes
|
||||
index in Postgres. Idempotent; safe to re-run.
|
||||
3. **Re-chunk TODO queued** — `~/.gbrain/migrations/pending-host-work.jsonl`
|
||||
gained an entry asking you to re-chunk pages that had takes BEFORE
|
||||
v0.28. The chunker now strips fenced takes content (so per-token MCP
|
||||
visibility actually works) but the strip rule only applies to NEW
|
||||
imports. Legacy pages still have takes content baked into their
|
||||
`content_chunks` rows — re-chunk them to apply the strip retroactively.
|
||||
|
||||
## What you need to do as the host agent
|
||||
|
||||
### 1. Verify the upgrade
|
||||
|
||||
```bash
|
||||
gbrain doctor
|
||||
```
|
||||
|
||||
Expected: `takes_backfill_complete` and `takes_fence_chunk_leak` checks
|
||||
both green. If either is red, follow the doctor's fix hint.
|
||||
|
||||
### 2. Re-chunk pages with pre-v0.28 takes (one-time, cosmetic for fresh installs)
|
||||
|
||||
For brains with existing fenced takes content in pre-v0.28 markdown:
|
||||
|
||||
```bash
|
||||
# Identify pages with takes content that's been chunked into content_chunks
|
||||
gbrain doctor --check takes_fence_chunk_leak
|
||||
|
||||
# If the check is RED, re-import those pages so the new chunker rule applies.
|
||||
# Run only on the affected slugs (the doctor output enumerates them):
|
||||
gbrain extract takes --rebuild
|
||||
gbrain sync # picks up the markdown delta if any
|
||||
```
|
||||
|
||||
Fresh installs (no pre-v0.28 takes content): nothing to do.
|
||||
|
||||
### 3. Migrate to the unified model config (optional, mechanical)
|
||||
|
||||
Old per-phase keys still work in v0.28 with a deprecation warning.
|
||||
v0.30 will remove them. Migrate when convenient:
|
||||
|
||||
```bash
|
||||
# Old (deprecated):
|
||||
gbrain config set dream.synthesize.model claude-sonnet-4-6
|
||||
gbrain config set dream.patterns.model claude-sonnet-4-6
|
||||
|
||||
# New (one key controls everything):
|
||||
gbrain config set models.default sonnet
|
||||
|
||||
# Per-op override (optional):
|
||||
gbrain config set models.dream.synthesize opus
|
||||
|
||||
# Cleanup deprecated keys:
|
||||
gbrain config unset dream.synthesize.model
|
||||
gbrain config unset dream.patterns.model
|
||||
```
|
||||
|
||||
### 4. Configure MCP token visibility (optional, security-relevant)
|
||||
|
||||
Existing tokens default to `permissions.takes_holders=["world"]` — they
|
||||
see public claims only, never private hunches. Tokens for trusted agents
|
||||
(e.g., your own OpenClaw deployment) need explicit broader visibility:
|
||||
|
||||
```bash
|
||||
# List existing tokens
|
||||
gbrain auth list
|
||||
|
||||
# Grant a token visibility into garry's takes (and brain-derived takes)
|
||||
gbrain auth permissions <token-name> set-takes-holders world,garry,brain
|
||||
|
||||
# Or create a new token with the wider set up-front
|
||||
gbrain auth create my-claude-desktop --takes-holders world,garry,brain
|
||||
```
|
||||
|
||||
Tokens for third-party agents (claude.ai, public integrations) should
|
||||
keep the default `["world"]` — Garry's hunches stay private.
|
||||
|
||||
### 5. Try the takes layer
|
||||
|
||||
```bash
|
||||
# Add a take by hand
|
||||
gbrain takes add people/alice-example \
|
||||
--claim "Strong technical founder I have ever met" \
|
||||
--kind take --who garry --weight 0.85 \
|
||||
--source "OH 2026-05-01"
|
||||
|
||||
# List
|
||||
gbrain takes people/alice-example
|
||||
|
||||
# Search
|
||||
gbrain takes search "technical founder"
|
||||
|
||||
# Resolve a bet
|
||||
gbrain takes resolve people/alice-example --row 3 --outcome true \
|
||||
--value 50000000 --unit usd --source crustdata
|
||||
```
|
||||
|
||||
## Verify
|
||||
|
||||
```bash
|
||||
gbrain doctor # all checks green; schema_version >= 32
|
||||
gbrain stats # numbers stable
|
||||
|
||||
# If any step failed, file an issue:
|
||||
# https://github.com/garrytan/gbrain/issues
|
||||
# Include: gbrain doctor output, ~/.gbrain/upgrade-errors.jsonl if it exists,
|
||||
# and which step broke. The migration is designed to be idempotent — safe
|
||||
# to re-run after fixing the underlying issue.
|
||||
```
|
||||
|
||||
## What ships in v0.28.x as follow-ups
|
||||
|
||||
- `gbrain think` synthesis pipeline (gather → RRF → cite → synthesize)
|
||||
- `gbrain takes seed <slug>` — LLM extracts claims from page prose
|
||||
- Dream `auto_think` + `drift` phases (opt-in)
|
||||
- Cross-page reference resolution in `source` columns
|
||||
|
||||
The op surfaces are registered in v0.28.0 so MCP/SDK callers can detect
|
||||
them; the pipelines fill in incrementally without breaking the contract.
|
||||
+276
-151
@@ -1,16 +1,12 @@
|
||||
---
|
||||
name: skillify
|
||||
version: 1.0.0
|
||||
version: 1.1.0
|
||||
description: |
|
||||
The meta skill. Turn any raw feature or script into a properly-skilled,
|
||||
tested, resolvable, evaled unit of agent-visible capability. Use when
|
||||
the user says "skillify this", "is this a skill?", "make this proper",
|
||||
or after a new feature is built without the full skill infrastructure.
|
||||
|
||||
Paired with `gbrain check-resolvable`, skillify gives a user-controllable
|
||||
equivalent of Hermes' auto-skill-creation: you build, skillify checks the
|
||||
checklist, check-resolvable verifies nothing is orphaned. The human keeps
|
||||
judgment; the tooling keeps the checklist honest.
|
||||
The meta skill. Turn any raw feature into a properly-skilled, tested,
|
||||
resolvable unit of agent capability. Cross-modal eval is the recommended
|
||||
Phase 3 quality gate: 3 frontier models from different providers critique
|
||||
the output, you iterate to quality, THEN write tests that lock in the
|
||||
proven-good behavior.
|
||||
triggers:
|
||||
- "skillify this"
|
||||
- "skillify"
|
||||
@@ -19,167 +15,296 @@ triggers:
|
||||
- "add tests and evals for this"
|
||||
- "check skill completeness"
|
||||
tools:
|
||||
- search
|
||||
- list_pages
|
||||
mutating: false
|
||||
- exec
|
||||
- read
|
||||
- write
|
||||
mutating: true
|
||||
---
|
||||
|
||||
# Skillify — The Meta Skill
|
||||
|
||||
> **Relationship to `/cross-modal-review`:** That skill is the manual mid-flow
|
||||
> "second opinion" gate (one model reviews work product before commit). This
|
||||
> skill's Phase 3 below uses `gbrain eval cross-modal` instead — three
|
||||
> different-provider frontier models score-and-iterate on a documented
|
||||
> dimension list *before* tests cement behavior. Use `/cross-modal-review`
|
||||
> for ad-hoc second opinions; use Phase 3 here when skillifying a feature.
|
||||
|
||||
## Contract
|
||||
|
||||
A feature is "properly skilled" when all ten checklist items are present:
|
||||
A feature is "properly skilled" when all 11 checklist items pass. Item 3
|
||||
(cross-modal eval) is informational in v1.1.0 — it does not gate the
|
||||
skillpack-check audit, but a missing or stale receipt is surfaced so the
|
||||
user knows where the gate stands.
|
||||
|
||||
1. `SKILL.md` — skill file with YAML frontmatter, triggers, contract, phases.
|
||||
2. Code — deterministic script if applicable.
|
||||
3. Unit tests — cover every branch of deterministic logic.
|
||||
4. Integration tests — exercise live endpoints, not just in-memory shape.
|
||||
5. LLM evals — quality/correctness cases if the feature includes any LLM call.
|
||||
6. Resolver trigger — `skills/RESOLVER.md` entry with the trigger patterns
|
||||
the user actually types.
|
||||
7. Resolver trigger eval — test that feeds trigger phrases to the resolver
|
||||
and asserts they route to this skill, not the old pre-skillify path.
|
||||
8. Check-resolvable — `gbrain check-resolvable` passes (skill is reachable,
|
||||
MECE against its siblings, no DRY violations).
|
||||
9. E2E test — exercises the full pipeline from user turn to side effect.
|
||||
10. Brain filing — if the feature writes brain pages, `brain/RESOLVER.md`
|
||||
has an entry for the directory so the pages aren't orphaned.
|
||||
## The Checklist
|
||||
|
||||
## Trigger
|
||||
|
||||
- "skillify this" / "skillify" / "is this a skill?" / "make this proper"
|
||||
- "add tests and evals for this"
|
||||
- After building any new feature that touches user-facing behavior
|
||||
- When you grep the repo and notice a script with no SKILL.md next to it
|
||||
|
||||
## Phases
|
||||
|
||||
### Phase 1: Audit what exists
|
||||
|
||||
For the feature being skillified, answer:
|
||||
|
||||
- **Feature name**: what does it do in one line?
|
||||
- **Code path**: where does the implementation live (file path)?
|
||||
- **Checklist status**: run `gbrain skillify check <path>` (preferred)
|
||||
or the legacy `scripts/skillify-check.ts <path>` shim. Both produce
|
||||
the same 10-item scorecard. Note which items are missing.
|
||||
|
||||
### Phase 2: Create missing pieces in order
|
||||
|
||||
**Fast path — brand-new skill:** run `gbrain skillify scaffold <name>
|
||||
--description "..." [--triggers "p1,p2,p3"] [--writes-pages --writes-to
|
||||
"people/,companies/"]`. This creates all 5 stub files atomically and
|
||||
appends an idempotent resolver row. Every scaffolded file carries the
|
||||
`SKILLIFY_STUB` sentinel; `gbrain check-resolvable --strict` will fail
|
||||
CI until you replace the stubs with real content.
|
||||
|
||||
**Manual path — extending an existing skill:** work the list top-down.
|
||||
Each earlier item constrains what later items look like (the SKILL.md
|
||||
contract determines what tests assert; tests determine what evals gate;
|
||||
the resolver entry determines what trigger-eval checks).
|
||||
|
||||
1. Write `SKILL.md` first. Frontmatter must include `name`, `version`,
|
||||
`description`, `triggers[]`, `tools[]`, `mutating`. Body has at minimum
|
||||
Contract, Phases, and Output Format sections.
|
||||
2. Extract deterministic code into a script if applicable (scripts/*.ts
|
||||
for gbrain; host projects may use .mjs / .py / whatever their runtime
|
||||
uses).
|
||||
3. Write unit tests for every branch of the script. Mock external calls
|
||||
(LLM, DB, network) so tests run fast and deterministic.
|
||||
4. Add integration tests that hit real endpoints. These catch bugs the
|
||||
unit tests' mocks hide (see the `files-test-reimplements-production`
|
||||
learning: reimplementation in tests lets production vulnerabilities
|
||||
slip through).
|
||||
5. Add LLM evals if the feature includes any LLM call. Even a three-case
|
||||
eval (happy / edge / adversarial) is cheap insurance against prompt
|
||||
regressions.
|
||||
6. Add the resolver trigger to `skills/RESOLVER.md`. Use the trigger
|
||||
patterns the user ACTUALLY types, not what you think they should type.
|
||||
7. Add a resolver trigger eval that feeds those patterns in and asserts
|
||||
they route to the new skill.
|
||||
8. Run `gbrain check-resolvable` (auto-detects skill trees) or
|
||||
`gbrain check-resolvable --skills-dir <path>` for custom locations.
|
||||
OpenClaw workspaces are auto-detected from
|
||||
`~/.openclaw/workspace/skills/`. The check validates reachability (is
|
||||
the skill mentioned from RESOLVER.md?), MECE overlap (does it duplicate
|
||||
an existing skill's trigger?), gap detection (are there user intents
|
||||
that fall through the resolver with no match?), and DRY. If it fails,
|
||||
fix the skill (or extend an existing one instead of creating a
|
||||
duplicate).
|
||||
9. Add an E2E smoke test. For gbrain: submit a Minion job or run a CLI
|
||||
invocation end-to-end against a fixture brain; assert side effects.
|
||||
10. Update `brain/RESOLVER.md` if the skill writes brain pages. Orphaned
|
||||
brain pages are worse than no brain pages.
|
||||
|
||||
### Phase 3: Verify
|
||||
|
||||
Run each of these and confirm green:
|
||||
|
||||
```bash
|
||||
# Unit tests
|
||||
bun test test/<skill-name>.test.ts
|
||||
|
||||
# Integration tests (when applicable)
|
||||
bun run test:e2e
|
||||
|
||||
# Resolver reachability + MECE + DRY
|
||||
gbrain check-resolvable
|
||||
|
||||
# Conformance tests (skill YAML + required sections)
|
||||
bun test test/skills-conformance.test.ts
|
||||
```
|
||||
□ 1. SKILL.md — skill file with frontmatter + contract + phases
|
||||
□ 2. Code — deterministic script if applicable
|
||||
□ 3. Cross-modal eval — 3 frontier models from 3 providers; informational
|
||||
□ 4. Unit tests — cover every branch of deterministic logic
|
||||
□ 5. Integration tests — exercise live endpoints
|
||||
□ 6. LLM evals — quality/correctness cases for LLM-involving steps
|
||||
□ 7. Resolver trigger — entry in skills/RESOLVER.md with real user trigger phrases
|
||||
□ 8. Resolver eval — test that triggers route to this skill
|
||||
□ 9. Check-resolvable — DRY + MECE audit, no orphans
|
||||
□ 10. E2E test — smoke test: trigger → side effect
|
||||
□ 11. Brain filing — if it writes pages, entry in brain/RESOLVER.md
|
||||
```
|
||||
|
||||
## Quality gates
|
||||
## Phase 0: Should This Be a Skill?
|
||||
|
||||
A feature is NOT properly skilled until:
|
||||
Before skillifying, check:
|
||||
- Will this be invoked 2+ times? (One-off work ≠ skill)
|
||||
- Is there >20 lines of logic? (Trivial helpers don't need full infrastructure)
|
||||
- Does it have a clear trigger phrase a user would actually say?
|
||||
|
||||
- All tests pass (unit + integration + evals).
|
||||
- It appears in `skills/RESOLVER.md` with accurate trigger patterns.
|
||||
- The resolver trigger eval confirms patterns route to the new skill.
|
||||
- `gbrain check-resolvable` shows no orphaned skills, no MECE overlaps,
|
||||
no DRY violations.
|
||||
- If it writes brain pages, `brain/RESOLVER.md` has the directory.
|
||||
If no to all three, it's a script, not a skill. Move on.
|
||||
|
||||
## Anti-Patterns
|
||||
## Phase 1: Audit
|
||||
|
||||
- ❌ Code with no SKILL.md — invisible to the resolver; the agent will
|
||||
never run it.
|
||||
- ❌ SKILL.md with no tests — untested contract; one prompt change
|
||||
regresses silently.
|
||||
- ❌ Tests that reimplement production code — the reimplementation's
|
||||
bugs don't catch production's bugs (the `files-test-reimplements-
|
||||
production` lesson).
|
||||
- ❌ Resolver entry that uses internal jargon the user never types —
|
||||
trigger patterns must mirror real user language.
|
||||
- ❌ Feature that writes to brain without a `brain/RESOLVER.md` entry —
|
||||
orphaned pages the agent will never find.
|
||||
- ❌ Deterministic logic in LLM space — should be a script.
|
||||
- ❌ LLM judgment in deterministic space — should be an eval.
|
||||
```
|
||||
Feature: [name]
|
||||
Code: [path]
|
||||
Missing items: [check each of the 11]
|
||||
```
|
||||
|
||||
## Why skillify + check-resolvable is the right pair
|
||||
## Phase 2: Write SKILL.md + Code (items 1-2)
|
||||
|
||||
Hermes and similar agent frameworks auto-create skills as a background
|
||||
behavior. That's fine until you don't know what the agent shipped —
|
||||
checklists decay, tests drift, resolver entries get stale.
|
||||
### SKILL.md frontmatter template (copy-paste):
|
||||
|
||||
Gbrain ships the same capability as two user-controlled tools:
|
||||
```yaml
|
||||
---
|
||||
name: my-skill
|
||||
version: 1.0.0
|
||||
description: |
|
||||
One paragraph. What it does, when to use it.
|
||||
triggers:
|
||||
- "trigger phrase users actually say"
|
||||
- "another real trigger"
|
||||
tools:
|
||||
- exec
|
||||
- read
|
||||
- write
|
||||
mutating: false # true if it writes to brain/disk
|
||||
---
|
||||
```
|
||||
|
||||
- `/skillify` builds the checklist and helps you fill in the gaps.
|
||||
- `gbrain check-resolvable` validates the whole skill tree: reachability,
|
||||
MECE, DRY, gap detection, orphaned skills.
|
||||
Body must include: **Contract** (what it guarantees), **Phases** (step-by-step), **Output Format** (what it produces).
|
||||
|
||||
You decide when and what. The human keeps judgment. The tooling keeps the
|
||||
checklist honest. In practice this combo produces zero orphaned skills,
|
||||
every feature with tests + evals + resolver triggers + evals of the
|
||||
triggers.
|
||||
Extract deterministic code into `scripts/*.ts`.
|
||||
|
||||
## Phase 3: Cross-Modal Eval (item 3) — THE QUALITY GATE
|
||||
|
||||
### Why this comes before tests
|
||||
|
||||
Tests lock in behavior. If the behavior is mediocre, tests lock in mediocrity.
|
||||
Cross-modal eval proves the quality bar FIRST, then tests cement it.
|
||||
|
||||
### Step 1: Pick a representative input
|
||||
|
||||
Choose the input that exercises the skill's hardest documented use case. If
|
||||
unsure: use the primary trigger example from SKILL.md, or the most complex
|
||||
real-world input from the last 7 days of memory files.
|
||||
|
||||
### Step 2: Run the skill, capture output
|
||||
|
||||
Run the skill on the representative input. The OUTPUT FILE is what gets
|
||||
evaluated.
|
||||
|
||||
### Step 3: Run the eval gate
|
||||
|
||||
```bash
|
||||
gbrain eval cross-modal \
|
||||
--task "What this skill is supposed to accomplish" \
|
||||
--output skills/<slug>/SKILL.md
|
||||
```
|
||||
|
||||
The command runs 3 frontier models from 3 different providers in parallel,
|
||||
scores the OUTPUT against the TASK on 5 documented dimensions, and writes a
|
||||
receipt under `~/.gbrain/.gbrain/eval-receipts/<slug>-<sha8>.json` (the
|
||||
sha-8 binds the receipt to the current SKILL.md content — re-running after
|
||||
edits writes a new receipt).
|
||||
|
||||
**Default models** (override per slot via `--slot-a-model`, `--slot-b-model`,
|
||||
`--slot-c-model`):
|
||||
|
||||
| Slot | Default | Provider |
|
||||
|------|---------|----------|
|
||||
| A | `openai:gpt-4o` | OpenAI |
|
||||
| B | `anthropic:claude-opus-4-7` | Anthropic |
|
||||
| C | `google:gemini-1.5-pro` | Google |
|
||||
|
||||
**These MUST be frontier models from DIFFERENT providers.** Using a single
|
||||
provider's family or budget models defeats the purpose — different families
|
||||
have less correlated blind spots. Refresh the list when a new model
|
||||
generation ships.
|
||||
|
||||
**Pass criteria (BOTH must be true):**
|
||||
|
||||
1. Every dimension's mean across successful models ≥ 7.
|
||||
2. No single model scored any dimension < 5 (the floor).
|
||||
|
||||
**Inconclusive:** fewer than 2 of 3 models returned parseable scores.
|
||||
Receipt is still written (forensics) but the gate is not authoritative.
|
||||
Exit code 2; CI wrappers should treat this as "did not run cleanly", not
|
||||
"failed quality gate".
|
||||
|
||||
### Step 4: Cycle until you pass (≤3 cycles)
|
||||
|
||||
```
|
||||
CYCLE 1:
|
||||
Eval → scores + top 10 improvements
|
||||
IF pass: → done, write tests
|
||||
ELSE:
|
||||
Apply top 10 improvements to the actual file
|
||||
Log: which improvements applied, what changed
|
||||
|
||||
CYCLE 2:
|
||||
Re-eval the FIXED output (same 3 models, same dimensions)
|
||||
Compare: before/after scores per dimension (track delta)
|
||||
IF pass: → done, write tests
|
||||
ELSE: apply remaining improvements + new ones
|
||||
|
||||
CYCLE 3 (final):
|
||||
Re-eval
|
||||
IF pass: → ship
|
||||
ELSE: → ship with KNOWN_GAPS section listing:
|
||||
- Which dimensions are still below 7
|
||||
- Which improvements couldn't be resolved
|
||||
- Why (e.g., "would require architectural change")
|
||||
```
|
||||
|
||||
### Cycles + cost guardrails
|
||||
|
||||
- Default `--cycles 3` in TTY, `--cycles 1` in non-TTY (limits scripted
|
||||
bulk spend in CI loops).
|
||||
- The command prints an estimated max-cost-per-cycle from a small pricing
|
||||
constant before each run. Real cost varies with prompt size; treat the
|
||||
estimate as a ceiling for default `--max-tokens 4000`.
|
||||
- A `--budget-usd N` hard cap is a v0.27.x follow-up TODO.
|
||||
|
||||
### Provider configuration
|
||||
|
||||
Models resolve through the gbrain AI gateway. Configure once with:
|
||||
|
||||
```bash
|
||||
gbrain providers test # see what's configured
|
||||
gbrain config # set keys
|
||||
```
|
||||
|
||||
Or set env vars: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`,
|
||||
`GOOGLE_GENERATIVE_AI_API_KEY`, `TOGETHER_API_KEY`, etc. The gateway reads
|
||||
from `~/.gbrain/config.json` plus `process.env`.
|
||||
|
||||
### Cost expectations
|
||||
|
||||
3 cycles × 3 models = 9 frontier calls max per run. With Opus-class +
|
||||
GPT-4o-class + Gemini-1.5-Pro, expect $1–3 per full run on default
|
||||
`--max-tokens 4000`. Receipts include the per-call model identifiers so
|
||||
you can audit retroactively.
|
||||
|
||||
### Skip cross-modal eval when:
|
||||
|
||||
- Output is < 200 tokens (trivial — not worth 9 API calls).
|
||||
- The skill is a thin wrapper around a single API call (one cycle is enough).
|
||||
|
||||
## Phase 4: Tests (items 4-6)
|
||||
|
||||
NOW that eval has proven quality, write tests that lock it in:
|
||||
|
||||
**Unit tests** — every branch of deterministic logic. Mock external calls.
|
||||
**Integration tests** — hit real endpoints. Catch bugs mocks hide.
|
||||
**LLM evals** — quality/correctness for LLM steps. Lighter than cross-modal eval — test specific behaviors.
|
||||
|
||||
## Phase 5: Resolver + Check-Resolvable (items 7-9)
|
||||
|
||||
1. Add to skills/RESOLVER.md with trigger phrases users ACTUALLY type
|
||||
2. Resolver eval: feed triggers, assert correct routing
|
||||
3. Check-resolvable:
|
||||
- Skill reachable from skills/RESOLVER.md (not orphaned)
|
||||
- No MECE overlap with other skills
|
||||
- No DRY violations (shared logic in lib/, not copy-pasted)
|
||||
- No ambiguous trigger routing
|
||||
|
||||
## Phase 6: E2E + Brain Filing (items 10-11)
|
||||
|
||||
- E2E smoke: full pipeline from trigger to side effect
|
||||
- Brain filing: add to brain/RESOLVER.md if the skill writes brain pages
|
||||
|
||||
## Phase 7: Verify
|
||||
|
||||
```bash
|
||||
bun test test/<skill>.test.ts # unit tests
|
||||
gbrain skillify check skills/<slug>/scripts/<slug>.mjs --json | \
|
||||
jq '.[] | .items[] | select(.name | contains("Cross-modal"))'
|
||||
ls ~/.gbrain/.gbrain/eval-receipts/ # receipt landed
|
||||
gbrain check-resolvable --json | jq .ok # resolver clean
|
||||
```
|
||||
|
||||
## Worked Example: Skillifying a "summarize-pr" Feature
|
||||
|
||||
```
|
||||
Phase 0: Yes — invoked weekly, 50+ lines, clear trigger "summarize this PR"
|
||||
Phase 1: Audit → SKILL.md missing, no tests, no resolver entry. Score: 1/11
|
||||
Phase 2: Write SKILL.md + extract script to scripts/summarize-pr.ts
|
||||
Phase 3: Cross-modal eval cycle 1 →
|
||||
GPT-4o: goal=6, depth=5, specificity=4 → "misses file-level diffs"
|
||||
Opus 4.7: goal=7, depth=6, specificity=5 → "no test plan in summary"
|
||||
Gemini 1.5 Pro: goal=6, depth=5, specificity=5 → "template feels generic"
|
||||
Aggregate: goal=6.3 FAIL, depth=5.3 FAIL
|
||||
Top improvements: add file-level changes, include test plan, use PR context
|
||||
→ Apply fixes → Cycle 2: goal=8, depth=7.5, specificity=7 → PASS
|
||||
Phase 4: Write 12 unit tests locking in the improved behavior
|
||||
Phase 5: Add "summarize this PR" trigger to skills/RESOLVER.md
|
||||
Phase 6: E2E test: feed a real PR URL → verify brain page created
|
||||
Phase 7: All green. Score: 11/11
|
||||
```
|
||||
|
||||
## Quality Gates
|
||||
|
||||
NOT properly skilled until:
|
||||
|
||||
- All required items pass (1-2, 4-10; 11 only when applicable).
|
||||
- Cross-modal eval (item 3) has a current receipt OR is explicitly waived
|
||||
with rationale (item 3 is informational; not blocking, but a missing
|
||||
receipt is visible in the audit).
|
||||
- All tests pass (unit + integration + LLM evals).
|
||||
- Resolver entry exists with real trigger phrases.
|
||||
- Check-resolvable shows no orphans, overlaps, or DRY violations.
|
||||
- Brain filing if applicable.
|
||||
|
||||
## Output Format
|
||||
|
||||
A skillify run produces, in order:
|
||||
Skillify produces three durable artifacts per skill:
|
||||
|
||||
1. An audit printout listing which of the 10 items exist and which are
|
||||
missing for the target feature.
|
||||
2. The files created to close each gap (SKILL.md, test files, resolver
|
||||
entries).
|
||||
3. The final `gbrain check-resolvable` output confirming reachability.
|
||||
4. A one-line summary of the resulting skill completeness score (N/10).
|
||||
1. **The skill tree on disk.** `skills/<slug>/SKILL.md`, `scripts/<slug>.mjs`,
|
||||
`routing-eval.jsonl`, plus a `test/<slug>.test.ts` skeleton. Generated by
|
||||
`gbrain skillify scaffold <name>` and refined by the human/agent into a
|
||||
real implementation.
|
||||
2. **A cross-modal eval receipt** at
|
||||
`~/.gbrain/.gbrain/eval-receipts/<slug>-<sha8>.json`. The sha-8 binds the
|
||||
receipt to the current `SKILL.md` content. `gbrain skillify check`
|
||||
surfaces the status (`found` / `stale` / `missing`) as informational.
|
||||
3. **An audit verdict** from `gbrain skillify check`: `properly skilled` |
|
||||
`close — create: <missing items>` | `needs skillify — run /skillify on
|
||||
<target>`. Score is `<passed>/<total>`. Required items gate the verdict;
|
||||
item 11 (cross-modal eval) is informational and never blocks PASS.
|
||||
|
||||
JSON output (`gbrain skillify check --json`) includes the same fields plus
|
||||
the per-item detail string, so agents can route on the structured envelope
|
||||
without parsing prose.
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
- ❌ Writing tests before cross-modal eval (locks in mediocrity)
|
||||
- ❌ Using budget models for eval (C student grading A student)
|
||||
- ❌ Using a single provider's family for all 3 slots (correlated blind spots)
|
||||
- ❌ Skipping eval "because the output looks fine" (your judgment isn't 3 models)
|
||||
- ❌ Eval without fix cycle (vanity metrics)
|
||||
- ❌ Code with no SKILL.md (invisible to resolver)
|
||||
- ❌ Tests that reimplement production code (masks real bugs)
|
||||
- ❌ Resolver entry with internal jargon (must mirror real user language)
|
||||
- ❌ Two skills doing the same thing (merge or kill one)
|
||||
- ❌ Running cross-modal eval on trivial outputs (< 200 tokens, not worth 9 API calls)
|
||||
|
||||
Regular → Executable
+69
-1
@@ -1,5 +1,8 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
import { installSigchldHandler } from './core/zombie-reap.ts';
|
||||
installSigchldHandler();
|
||||
|
||||
import { readFileSync } from 'fs';
|
||||
import { loadConfig, toEngineConfig } from './core/config.ts';
|
||||
import type { BrainEngine } from './core/engine.ts';
|
||||
@@ -19,7 +22,7 @@ for (const op of operations) {
|
||||
}
|
||||
|
||||
// CLI-only commands that bypass the operation layer
|
||||
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'storage', 'repos', 'code-def', 'code-refs', 'reindex-code', 'code-callers', 'code-callees', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror']);
|
||||
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex-code', 'code-callers', 'code-callees', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think']);
|
||||
|
||||
async function main() {
|
||||
// Parse global flags (--quiet / --progress-json / --progress-interval)
|
||||
@@ -290,6 +293,12 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runIntegrations(args);
|
||||
return;
|
||||
}
|
||||
if (command === 'providers') {
|
||||
const { runProviders } = await import('./commands/providers.ts');
|
||||
const [sub, ...rest] = args;
|
||||
await runProviders(sub, rest);
|
||||
return;
|
||||
}
|
||||
if (command === 'auth') {
|
||||
const { runAuth } = await import('./commands/auth.ts');
|
||||
await runAuth(args);
|
||||
@@ -446,6 +455,16 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
return;
|
||||
}
|
||||
|
||||
// `eval cross-modal` is a pure API-call command — no DB, no brain. Bypass
|
||||
// connectEngine entirely so first-run users (no `gbrain init` yet) can
|
||||
// run the quality gate. Mirrors the dream/doctor no-DB pattern but
|
||||
// doesn't even attempt the connect (T3=A in plans/radiant-napping-lerdorf.md).
|
||||
// The handler self-configures the AI gateway from loadConfig() + process.env.
|
||||
if (command === 'eval' && args[0] === 'cross-modal') {
|
||||
const { runEvalCrossModal } = await import('./commands/eval-cross-modal.ts');
|
||||
process.exit(await runEvalCrossModal(args.slice(1)));
|
||||
}
|
||||
|
||||
// All remaining CLI-only commands need a DB connection
|
||||
const engine = await connectEngine();
|
||||
try {
|
||||
@@ -550,11 +569,27 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runOrphans(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'takes': {
|
||||
const { runTakes } = await import('./commands/takes.ts');
|
||||
await runTakes(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'think': {
|
||||
const { runThinkCli } = await import('./commands/think.ts');
|
||||
await runThinkCli(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'sources': {
|
||||
const { runSources } = await import('./commands/sources.ts');
|
||||
await runSources(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'pages': {
|
||||
// v0.26.5: page-level operator commands (purge-deleted escape hatch).
|
||||
const { runPages } = await import('./commands/pages.ts');
|
||||
await runPages(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'storage': {
|
||||
const { runStorage } = await import('./commands/storage.ts');
|
||||
await runStorage(engine, args);
|
||||
@@ -614,12 +649,45 @@ async function connectEngine(): Promise<BrainEngine> {
|
||||
console.error('No brain configured. Run: gbrain init');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Configure the AI gateway BEFORE engine connect — initSchema needs embedding dims.
|
||||
// Env is read once here; the gateway never reads process.env at call time (Codex C3).
|
||||
const { configureGateway } = await import('./core/ai/gateway.ts');
|
||||
configureGateway({
|
||||
embedding_model: config.embedding_model,
|
||||
embedding_dimensions: config.embedding_dimensions,
|
||||
expansion_model: config.expansion_model,
|
||||
chat_model: config.chat_model,
|
||||
chat_fallback_chain: config.chat_fallback_chain,
|
||||
base_urls: config.provider_base_urls,
|
||||
env: { ...process.env },
|
||||
});
|
||||
|
||||
const { createEngine } = await import('./core/engine-factory.ts');
|
||||
const engine = await createEngine(toEngineConfig(config));
|
||||
const noRetry = process.argv.includes('--no-retry-connect') ||
|
||||
process.env.GBRAIN_NO_RETRY_CONNECT === '1';
|
||||
const { connectWithRetry } = await import('./core/db.ts');
|
||||
await connectWithRetry(engine, toEngineConfig(config), { noRetry });
|
||||
|
||||
// Auto-apply pending schema migrations on connect (#651). Cheap probe
|
||||
// first so already-migrated brains don't pay the bootstrap-probe +
|
||||
// SCHEMA_SQL replay + ledger-check cost on every short-lived CLI call.
|
||||
// This is the conditional version of #652 (oyi77's investigation):
|
||||
// same correctness, no perf regression on the hot path.
|
||||
try {
|
||||
const { hasPendingMigrations } = await import('./core/migrate.ts');
|
||||
if (await hasPendingMigrations(engine)) {
|
||||
await engine.initSchema();
|
||||
}
|
||||
} catch (err) {
|
||||
// Non-fatal: if probe or initSchema fails, surface a hint and continue
|
||||
// with the connected engine. Subsequent operations will surface the
|
||||
// real schema error in context.
|
||||
console.warn(` Schema probe/migrate failed: ${(err as Error).message}`);
|
||||
console.warn(' Try: gbrain init --migrate-only');
|
||||
}
|
||||
|
||||
return engine;
|
||||
}
|
||||
|
||||
|
||||
+67
-8
@@ -34,21 +34,28 @@ function generateToken(): string {
|
||||
return 'gbrain_' + randomBytes(32).toString('hex');
|
||||
}
|
||||
|
||||
async function create(name: string) {
|
||||
if (!name) { console.error('Usage: auth create <name>'); process.exit(1); }
|
||||
async function create(name: string, opts: { takesHolders?: string[] } = {}) {
|
||||
if (!name) { console.error('Usage: auth create <name> [--takes-holders world,garry]'); process.exit(1); }
|
||||
const sql = postgres(getDatabaseUrl(true)!);
|
||||
const token = generateToken();
|
||||
const hash = hashToken(token);
|
||||
|
||||
try {
|
||||
// v0.28: persist per-token takes-holder allow-list. Default ['world'] keeps
|
||||
// private hunches hidden from MCP-bound tokens.
|
||||
const takesHolders = opts.takesHolders && opts.takesHolders.length > 0
|
||||
? opts.takesHolders
|
||||
: ['world'];
|
||||
const permissions = { takes_holders: takesHolders };
|
||||
await sql`
|
||||
INSERT INTO access_tokens (name, token_hash)
|
||||
VALUES (${name}, ${hash})
|
||||
INSERT INTO access_tokens (name, token_hash, permissions)
|
||||
VALUES (${name}, ${hash}, ${sql.json(permissions as Parameters<typeof sql.json>[0])})
|
||||
`;
|
||||
console.log(`Token created for "${name}":\n`);
|
||||
console.log(`Token created for "${name}" (takes_holders=${JSON.stringify(takesHolders)}):\n`);
|
||||
console.log(` ${token}\n`);
|
||||
console.log('Save this token — it will not be shown again.');
|
||||
console.log(`Revoke with: bun run src/commands/auth.ts revoke "${name}"`);
|
||||
console.log(`Update visibility: bun run src/commands/auth.ts permissions "${name}" set-takes-holders world,garry`);
|
||||
} catch (e: any) {
|
||||
if (e.code === '23505') {
|
||||
console.error(`A token named "${name}" already exists. Revoke it first or use a different name.`);
|
||||
@@ -61,6 +68,38 @@ async function create(name: string) {
|
||||
}
|
||||
}
|
||||
|
||||
async function permissions(name: string, action: string, value: string | undefined) {
|
||||
if (!name || action !== 'set-takes-holders' || !value) {
|
||||
console.error('Usage: auth permissions <name> set-takes-holders world,garry,brain');
|
||||
process.exit(1);
|
||||
}
|
||||
const sql = postgres(getDatabaseUrl(true)!);
|
||||
try {
|
||||
const list = value.split(',').map(s => s.trim()).filter(Boolean);
|
||||
if (list.length === 0) {
|
||||
console.error('takes-holders list cannot be empty (use "world" for default-deny on private)');
|
||||
process.exit(1);
|
||||
}
|
||||
const perms = { takes_holders: list };
|
||||
const result = await sql`
|
||||
UPDATE access_tokens
|
||||
SET permissions = ${sql.json(perms as Parameters<typeof sql.json>[0])}
|
||||
WHERE name = ${name}
|
||||
RETURNING id
|
||||
`;
|
||||
if (result.length === 0) {
|
||||
console.error(`Token "${name}" not found.`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`Updated "${name}": takes_holders = ${JSON.stringify(list)}`);
|
||||
} catch (e: any) {
|
||||
console.error('Error:', e.message);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
}
|
||||
|
||||
async function list() {
|
||||
const sql = postgres(getDatabaseUrl(true)!);
|
||||
try {
|
||||
@@ -292,9 +331,23 @@ async function registerClient(name: string, args: string[]) {
|
||||
export async function runAuth(args: string[]): Promise<void> {
|
||||
const [cmd, ...rest] = args;
|
||||
switch (cmd) {
|
||||
case 'create': await create(rest[0]); return;
|
||||
case 'create': {
|
||||
// v0.28: optional --takes-holders world,garry,brain (default: world only)
|
||||
const takesIdx = rest.indexOf('--takes-holders');
|
||||
const takesHolders = takesIdx >= 0 && rest[takesIdx + 1]
|
||||
? rest[takesIdx + 1].split(',').map(s => s.trim()).filter(Boolean)
|
||||
: undefined;
|
||||
const positional = rest.find(a => !a.startsWith('--') && a !== rest[takesIdx + 1]);
|
||||
await create(positional || '', { takesHolders });
|
||||
return;
|
||||
}
|
||||
case 'list': await list(); return;
|
||||
case 'revoke': await revoke(rest[0]); return;
|
||||
case 'permissions': {
|
||||
// gbrain auth permissions <name> set-takes-holders world,garry
|
||||
await permissions(rest[0] || '', rest[1] || '', rest[2]);
|
||||
return;
|
||||
}
|
||||
case 'register-client': await registerClient(rest[0], rest.slice(1)); return;
|
||||
case 'revoke-client': await revokeClient(rest[0]); return;
|
||||
case 'test': {
|
||||
@@ -308,10 +361,16 @@ export async function runAuth(args: string[]): Promise<void> {
|
||||
console.log(`GBrain Token Management
|
||||
|
||||
Usage:
|
||||
gbrain auth create <name> Create a legacy bearer token
|
||||
gbrain auth create <name> [--takes-holders world,garry,brain]
|
||||
Create a legacy bearer token. v0.28: --takes-holders
|
||||
sets the per-token allow-list for the takes.holder
|
||||
field (default: ["world"]). MCP-bound calls to
|
||||
takes_list / takes_search / query filter by this.
|
||||
gbrain auth list List all tokens
|
||||
gbrain auth revoke <name> Revoke a legacy token
|
||||
gbrain auth register-client <name> [options] Register an OAuth 2.1 client
|
||||
gbrain auth permissions <name> set-takes-holders <h1,h2,h3>
|
||||
Update visibility for an existing token
|
||||
gbrain auth register-client <name> [options] Register an OAuth 2.1 client (v0.26+)
|
||||
--grant-types <client_credentials,authorization_code> (default: client_credentials)
|
||||
--scopes "<read write admin>" (default: read)
|
||||
gbrain auth revoke-client <client_id> Hard-delete an OAuth 2.1 client (cascades to tokens + codes)
|
||||
|
||||
@@ -23,6 +23,7 @@ import { execSync, spawn, type ChildProcess } from 'child_process';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { loadPreferences } from '../core/preferences.ts';
|
||||
import { loadConfig } from '../core/config.ts';
|
||||
import { detectTini, buildSpawnInvocation } from '../core/minions/spawn-helpers.ts';
|
||||
|
||||
function parseArg(args: string[], flag: string): string | undefined {
|
||||
const idx = args.indexOf(flag);
|
||||
@@ -157,15 +158,23 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
|
||||
if (spawnManagedWorker) {
|
||||
const cliPath = resolveGbrainCliPath();
|
||||
// Resolve tini once at startup — not per respawn — to avoid shelling out
|
||||
// every time the worker restarts. Reaps zombie children from shell jobs
|
||||
// and embed batches that outlive a watchdog-killed worker.
|
||||
const tiniPath = detectTini();
|
||||
const startWorker = () => {
|
||||
// Inject the RSS watchdog default (2048 MB) for the autopilot-supervised
|
||||
// worker. Bare `gbrain jobs work` has no default; the supervisor and
|
||||
// autopilot are the production paths that opt in.
|
||||
const args = ['jobs', 'work', '--max-rss', '2048'];
|
||||
const child = spawn(cliPath, args, { stdio: 'inherit', env: process.env });
|
||||
|
||||
const { cmd: spawnCmd, args: spawnArgs } = buildSpawnInvocation(tiniPath, cliPath, args);
|
||||
|
||||
const child = spawn(spawnCmd, spawnArgs, { stdio: 'inherit', env: process.env });
|
||||
workerProc = child;
|
||||
lastWorkerStartTime = Date.now();
|
||||
console.log(`[autopilot] Minions worker spawned (pid: ${child.pid}, watchdog: 2048MB)`);
|
||||
console.log(`[autopilot] Minions worker spawned (pid: ${child.pid}, watchdog: 2048MB${tiniPath ? ', tini: active' : ''})`);
|
||||
|
||||
child.on('exit', (code) => {
|
||||
workerProc = null;
|
||||
if (stopping) return;
|
||||
|
||||
+181
-3
@@ -278,6 +278,53 @@ export async function runDoctor(engine: BrainEngine | null, args: string[], dbSo
|
||||
// Best-effort. A broken JSONL should not stop doctor.
|
||||
}
|
||||
|
||||
// 3c. Orphan clone temp dirs (v0.28 P1). `gbrain sources add --url` clones
|
||||
// into $GBRAIN_HOME/clones/.tmp/<id>-<rand>/ and renames atomically; if the
|
||||
// process is SIGKILL'd between clone-finish and rename, the temp dir
|
||||
// orphans. Surface entries older than 24h so operators notice before the
|
||||
// disk fills. The autopilot purge phase nukes these on its cadence; this
|
||||
// check just makes the state visible.
|
||||
try {
|
||||
const fs = await import('fs');
|
||||
const cfg = await import('../core/config.ts');
|
||||
const tmpRoot = cfg.gbrainPath('clones', '.tmp');
|
||||
if (fs.existsSync(tmpRoot)) {
|
||||
const STALE_MS = 24 * 3600 * 1000;
|
||||
const now = Date.now();
|
||||
const stale: { name: string; ageHours: number }[] = [];
|
||||
for (const ent of fs.readdirSync(tmpRoot, { withFileTypes: true })) {
|
||||
const full = join(tmpRoot, ent.name);
|
||||
try {
|
||||
const st = fs.lstatSync(full);
|
||||
const age = now - st.mtimeMs;
|
||||
if (age > STALE_MS) {
|
||||
stale.push({ name: ent.name, ageHours: Math.floor(age / 3600_000) });
|
||||
}
|
||||
} catch {
|
||||
/* skip unreadable */
|
||||
}
|
||||
}
|
||||
if (stale.length === 0) {
|
||||
checks.push({
|
||||
name: 'orphan_clones',
|
||||
status: 'ok',
|
||||
message: `No stale clone temp dirs in ${tmpRoot}.`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: 'orphan_clones',
|
||||
status: 'warn',
|
||||
message:
|
||||
`${stale.length} stale clone temp dir(s) in ${tmpRoot}: ` +
|
||||
stale.map(s => `${s.name} (${s.ageHours}h)`).join(', ') +
|
||||
`. Run \`gbrain sources purge-orphan-clones\` or wait for the autopilot purge phase.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Filesystem read failure is non-fatal.
|
||||
}
|
||||
|
||||
// --- DB checks (skip if --fast or no engine) ---
|
||||
|
||||
if (fastMode || !engine) {
|
||||
@@ -499,7 +546,64 @@ export async function runDoctor(engine: BrainEngine | null, args: string[], dbSo
|
||||
// the canonical half-migration signal and fires when the stopgap ran
|
||||
// but `apply-migrations` didn't follow up.
|
||||
|
||||
// 7. Embedding health
|
||||
// 7. RLS event trigger (post-install drift detector for v35 auto-RLS).
|
||||
// Catches the case where an operator manually drops the trigger to debug
|
||||
// something and forgets to recreate it. Does NOT catch install-time silent
|
||||
// failure — runMigrations rethrows on SQL failure and only bumps
|
||||
// config.version after success, so a failed v35 install means version
|
||||
// stays at 34 and check #6 (schema_version) fires loudly.
|
||||
//
|
||||
// Healthy evtenabled values: 'O' (origin) and 'A' (always). 'R' is
|
||||
// replica-only and would NOT fire in normal origin sessions; 'D' is
|
||||
// disabled. Both of those are warn states.
|
||||
progress.heartbeat('rls_event_trigger');
|
||||
if (engine.kind === 'pglite') {
|
||||
checks.push({
|
||||
name: 'rls_event_trigger',
|
||||
status: 'ok',
|
||||
message: 'Skipped (PGLite — no event trigger support)',
|
||||
});
|
||||
} else {
|
||||
try {
|
||||
const sql = db.getConnection();
|
||||
const rows = await sql`
|
||||
SELECT evtname, evtenabled FROM pg_event_trigger
|
||||
WHERE evtname = 'auto_rls_on_create_table'
|
||||
`;
|
||||
if (rows.length === 0) {
|
||||
checks.push({
|
||||
name: 'rls_event_trigger',
|
||||
status: 'warn',
|
||||
message:
|
||||
'Auto-RLS event trigger missing. New tables created outside gbrain may not get RLS. ' +
|
||||
'Fix: gbrain apply-migrations --force-retry 35',
|
||||
});
|
||||
} else if (rows[0].evtenabled !== 'O' && rows[0].evtenabled !== 'A') {
|
||||
checks.push({
|
||||
name: 'rls_event_trigger',
|
||||
status: 'warn',
|
||||
message:
|
||||
`Auto-RLS event trigger present but evtenabled=${rows[0].evtenabled} ` +
|
||||
`(not origin/always). Trigger will not fire in normal sessions. ` +
|
||||
`Fix: ALTER EVENT TRIGGER auto_rls_on_create_table ENABLE;`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: 'rls_event_trigger',
|
||||
status: 'ok',
|
||||
message: 'Auto-RLS event trigger installed',
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
checks.push({
|
||||
name: 'rls_event_trigger',
|
||||
status: 'warn',
|
||||
message: 'Could not check RLS event trigger',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 8. Embedding health
|
||||
progress.heartbeat('embeddings');
|
||||
try {
|
||||
const health = await engine.getHealth();
|
||||
@@ -515,7 +619,81 @@ export async function runDoctor(engine: BrainEngine | null, args: string[], dbSo
|
||||
checks.push({ name: 'embeddings', status: 'warn', message: 'Could not check embedding health' });
|
||||
}
|
||||
|
||||
// 8. Graph health (link + timeline coverage on entity pages).
|
||||
// 8b. Embedding provider eval — live smoke test of the configured provider.
|
||||
// Verifies: correct model, API key works, dimensions match config, DB column matches.
|
||||
progress.heartbeat('embedding_provider');
|
||||
try {
|
||||
const {
|
||||
getEmbeddingModel,
|
||||
getEmbeddingDimensions,
|
||||
embedOne,
|
||||
isAvailable,
|
||||
} = await import('../core/ai/gateway.ts');
|
||||
|
||||
const configuredModel = getEmbeddingModel();
|
||||
const configuredDims = getEmbeddingDimensions();
|
||||
const available = isAvailable('embedding');
|
||||
|
||||
if (!available) {
|
||||
// Per v0.28.5 plan P1: silently skipped when no API key is configured.
|
||||
// Doctor must stay green on CI / local-only / offline environments where
|
||||
// a full provider probe isn't possible. The skipped status is still
|
||||
// visible in --json output so operators can see it ran.
|
||||
checks.push({
|
||||
name: 'embedding_provider',
|
||||
status: 'ok',
|
||||
message: `Skipped (no provider credentials). Model: ${configuredModel}.`,
|
||||
});
|
||||
} else {
|
||||
// Live embed test
|
||||
const start = Date.now();
|
||||
const vec = await embedOne('gbrain doctor embedding smoke test');
|
||||
const ms = Date.now() - start;
|
||||
const actualDims = vec.length;
|
||||
|
||||
const issues: string[] = [];
|
||||
|
||||
// Check dimensions match config
|
||||
if (actualDims !== configuredDims) {
|
||||
issues.push(`Dimension mismatch: provider returned ${actualDims} but config expects ${configuredDims}`);
|
||||
}
|
||||
|
||||
// Check DB column dimensions match (engine-portable; works on both
|
||||
// Postgres and PGLite via the shared dim-check helper added in v0.28.5).
|
||||
try {
|
||||
const { readContentChunksEmbeddingDim } = await import('../core/embedding-dim-check.ts');
|
||||
const colDim = await readContentChunksEmbeddingDim(engine);
|
||||
if (colDim.exists && colDim.dims !== null && colDim.dims !== actualDims) {
|
||||
issues.push(`DB dimension mismatch: column is vector(${colDim.dims}) but provider returns ${actualDims}-dim. See docs/embedding-migrations.md for the manual ALTER recipe.`);
|
||||
}
|
||||
} catch { /* column or table missing — fresh brain, fine */ }
|
||||
|
||||
if (issues.length > 0) {
|
||||
checks.push({
|
||||
name: 'embedding_provider',
|
||||
status: 'warn',
|
||||
message: `${configuredModel} responds (${ms}ms, ${actualDims} dims) but: ${issues.join('; ')}`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: 'embedding_provider',
|
||||
status: 'ok',
|
||||
message: `${configuredModel} ✓ ${ms}ms, ${actualDims} dims, DB aligned`,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
// Per v0.28.5 plan P1: non-fatal on network failure. The probe surfaces
|
||||
// the issue but doesn't fail doctor — common cases (rate limit, transient
|
||||
// 5xx, DNS blip, expired key) shouldn't take down a CI run.
|
||||
checks.push({
|
||||
name: 'embedding_provider',
|
||||
status: 'warn',
|
||||
message: `Embedding provider probe failed: ${e.message?.slice(0, 200) ?? e}`,
|
||||
});
|
||||
}
|
||||
|
||||
// 9. Graph health (link + timeline coverage on entity pages).
|
||||
// dead_links removed in v0.10.1: ON DELETE CASCADE on link FKs makes it always 0.
|
||||
progress.heartbeat('graph_coverage');
|
||||
try {
|
||||
@@ -556,7 +734,7 @@ export async function runDoctor(engine: BrainEngine | null, args: string[], dbSo
|
||||
checks.push({ name: 'graph_coverage', status: 'warn', message: 'Could not check graph coverage' });
|
||||
}
|
||||
|
||||
// 9. Integrity sample scan (v0.13 knowledge runtime).
|
||||
// 10. Integrity sample scan (v0.13 knowledge runtime).
|
||||
// Read-only — no network, no writes, no resolver calls. Samples the first
|
||||
// 500 pages by slug order and surfaces bare-tweet + dead-link counts as a
|
||||
// warning. Full-brain scan: `gbrain integrity check`.
|
||||
|
||||
@@ -0,0 +1,358 @@
|
||||
/**
|
||||
* gbrain eval cross-modal — multi-model quality gate (v0.27.x).
|
||||
*
|
||||
* Three different-provider frontier models score the OUTPUT against the TASK
|
||||
* on a fixed dimension list. Verdict: PASS (exit 0) / FAIL (exit 1) /
|
||||
* INCONCLUSIVE (exit 2; <2/3 model successes).
|
||||
*
|
||||
* Reuses `src/core/ai/gateway.ts` for provider config + auth (T1+T2). Bypasses
|
||||
* `connectEngine()` via the cli.ts no-DB branch (T3=A) so onboarding works
|
||||
* before `gbrain init`. Receipts are bound to (slug, SKILL.md sha-8) so
|
||||
* `gbrain skillify check` can detect stale audits (T10=A).
|
||||
*
|
||||
* Cost guardrails (T11=B):
|
||||
* - Default cycles = 3 in TTY, 1 in non-TTY (limits scripted bulk spend).
|
||||
* - Cost-estimate prints to stderr before each cycle.
|
||||
* - `--budget-usd` hard cap is a v0.27.x follow-up TODO.
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync } from 'fs';
|
||||
|
||||
import { gbrainPath, loadConfig } from '../core/config.ts';
|
||||
import { configureGateway, isAvailable } from '../core/ai/gateway.ts';
|
||||
import {
|
||||
DEFAULT_DIMENSIONS,
|
||||
DEFAULT_SLOTS,
|
||||
estimateCost,
|
||||
runEval,
|
||||
} from '../core/cross-modal-eval/runner.ts';
|
||||
import type {
|
||||
ProgressEvent,
|
||||
RunEvalResult,
|
||||
SlotConfig,
|
||||
} from '../core/cross-modal-eval/runner.ts';
|
||||
|
||||
const HELP = `gbrain eval cross-modal — multi-model quality gate
|
||||
|
||||
USAGE:
|
||||
gbrain eval cross-modal --task "<description>" --output <path-or-skill-slug> [flags]
|
||||
|
||||
REQUIRED:
|
||||
--task "..." What the OUTPUT was meant to achieve.
|
||||
--output <path> File whose content gets scored. Pass a skill slug
|
||||
shortcut (e.g. \`--output skills/my-skill/SKILL.md\`)
|
||||
to bind the receipt to that skill (T10).
|
||||
|
||||
FLAGS:
|
||||
--slug <name> Receipt filename slug. Defaults to inferred slug
|
||||
from --output path (skills/<slug>/SKILL.md → <slug>),
|
||||
or a content sha for ad-hoc inputs.
|
||||
--dimensions "d1,d2,..." Comma-separated dimension list. Default: 5 standard
|
||||
dimensions (goal, depth, sourcing, specificity, useful).
|
||||
--cycles N 1-3. Default: 3 in TTY, 1 in non-TTY (T11). Each
|
||||
cycle is 3 model calls; verdict aggregates over them.
|
||||
--slot-a-model <id> Override default 'openai:gpt-4o'.
|
||||
--slot-b-model <id> Override default 'anthropic:claude-opus-4-7'.
|
||||
--slot-c-model <id> Override default 'google:gemini-1.5-pro'.
|
||||
--receipt-dir <path> Default: gbrainPath('eval-receipts').
|
||||
--max-tokens N Output token budget per call. Default: 4000.
|
||||
--json Emit final aggregate as JSON to stdout (progress to stderr).
|
||||
--help, -h Show this help.
|
||||
|
||||
EXIT CODES:
|
||||
0 PASS — every dim mean >=7 AND no model scored any dim <5.
|
||||
1 FAIL — at least one dim mean <7 OR at least one model scored a dim <5.
|
||||
2 INCONCLUSIVE — fewer than 2/3 models returned parseable scores. Receipt
|
||||
is still written for forensics; the gate is not authoritative.
|
||||
|
||||
CONFIGURATION:
|
||||
Models resolve via the gbrain AI gateway. Configure with:
|
||||
gbrain providers test # see what's configured
|
||||
gbrain config # set keys
|
||||
Or set env vars: OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_GENERATIVE_AI_API_KEY,
|
||||
TOGETHER_API_KEY, etc. The gateway reads from \`~/.gbrain/config.json\` plus
|
||||
process.env.
|
||||
|
||||
EXAMPLES:
|
||||
gbrain eval cross-modal \\
|
||||
--task "Skillify SKILL.md teaches the 11-item meta-skill checklist" \\
|
||||
--output skills/skillify/SKILL.md
|
||||
|
||||
gbrain eval cross-modal \\
|
||||
--task "PR description sells the value of cross-modal eval" \\
|
||||
--output /tmp/pr-description.md \\
|
||||
--cycles 1
|
||||
`;
|
||||
|
||||
interface ParsedArgs {
|
||||
help: boolean;
|
||||
task?: string;
|
||||
output?: string;
|
||||
slug?: string;
|
||||
dimensions?: string[];
|
||||
cycles?: number;
|
||||
slotAModel?: string;
|
||||
slotBModel?: string;
|
||||
slotCModel?: string;
|
||||
receiptDir?: string;
|
||||
maxTokens?: number;
|
||||
json: boolean;
|
||||
}
|
||||
|
||||
function parseArgs(args: string[]): ParsedArgs {
|
||||
const out: ParsedArgs = { help: false, json: false };
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i]!;
|
||||
const next = args[i + 1];
|
||||
switch (arg) {
|
||||
case '--help':
|
||||
case '-h':
|
||||
out.help = true;
|
||||
break;
|
||||
case '--task':
|
||||
if (next === undefined) break;
|
||||
out.task = next;
|
||||
i++;
|
||||
break;
|
||||
case '--output':
|
||||
if (next === undefined) break;
|
||||
out.output = next;
|
||||
i++;
|
||||
break;
|
||||
case '--slug':
|
||||
if (next === undefined) break;
|
||||
out.slug = next;
|
||||
i++;
|
||||
break;
|
||||
case '--dimensions':
|
||||
if (next === undefined) break;
|
||||
out.dimensions = next.split(',').map(s => s.trim()).filter(Boolean);
|
||||
i++;
|
||||
break;
|
||||
case '--cycles':
|
||||
if (next === undefined) break;
|
||||
out.cycles = parseIntStrict(next);
|
||||
i++;
|
||||
break;
|
||||
case '--slot-a-model':
|
||||
if (next === undefined) break;
|
||||
out.slotAModel = next;
|
||||
i++;
|
||||
break;
|
||||
case '--slot-b-model':
|
||||
if (next === undefined) break;
|
||||
out.slotBModel = next;
|
||||
i++;
|
||||
break;
|
||||
case '--slot-c-model':
|
||||
if (next === undefined) break;
|
||||
out.slotCModel = next;
|
||||
i++;
|
||||
break;
|
||||
case '--receipt-dir':
|
||||
if (next === undefined) break;
|
||||
out.receiptDir = next;
|
||||
i++;
|
||||
break;
|
||||
case '--max-tokens':
|
||||
if (next === undefined) break;
|
||||
out.maxTokens = parseIntStrict(next);
|
||||
i++;
|
||||
break;
|
||||
case '--json':
|
||||
out.json = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function parseIntStrict(s: string): number {
|
||||
const m = String(s).trim();
|
||||
if (!/^\d+$/.test(m)) {
|
||||
throw new Error(`expected positive integer, got: ${s}`);
|
||||
}
|
||||
return parseInt(m, 10);
|
||||
}
|
||||
|
||||
function inferSlugFromOutputPath(path: string): string | undefined {
|
||||
// skills/<slug>/SKILL.md or .../skills/<slug>/...
|
||||
const m = path.replace(/\\/g, '/').match(/(?:^|\/)skills\/([^/]+)\/SKILL\.md$/);
|
||||
return m ? m[1] : undefined;
|
||||
}
|
||||
|
||||
function isTTY(): boolean {
|
||||
return Boolean(process.stdout.isTTY);
|
||||
}
|
||||
|
||||
/**
|
||||
* Configure the AI gateway from `~/.gbrain/config.json` + process.env.
|
||||
*
|
||||
* Mirrors the body of `cli.ts:connectEngine()` minus the DB connect — we call
|
||||
* this from the no-DB branch so the gateway is ready when runEval starts.
|
||||
* Returns true on success; false (and prints a hint) when no config is found.
|
||||
*/
|
||||
function configureGatewayForCli(): boolean {
|
||||
const config = loadConfig();
|
||||
if (!config) {
|
||||
// No config file is fine for the eval command — env vars alone may serve.
|
||||
// We still call configureGateway so gateway recipes can read the env map.
|
||||
configureGateway({
|
||||
embedding_model: undefined,
|
||||
embedding_dimensions: undefined,
|
||||
expansion_model: undefined,
|
||||
chat_model: undefined,
|
||||
chat_fallback_chain: undefined,
|
||||
base_urls: undefined,
|
||||
env: { ...process.env },
|
||||
});
|
||||
return true;
|
||||
}
|
||||
configureGateway({
|
||||
embedding_model: config.embedding_model,
|
||||
embedding_dimensions: config.embedding_dimensions,
|
||||
expansion_model: config.expansion_model,
|
||||
chat_model: config.chat_model,
|
||||
chat_fallback_chain: config.chat_fallback_chain,
|
||||
base_urls: config.provider_base_urls,
|
||||
env: { ...process.env },
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function runEvalCrossModal(args: string[]): Promise<number> {
|
||||
const parsed = parseArgs(args);
|
||||
|
||||
if (parsed.help) {
|
||||
process.stdout.write(HELP);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!parsed.task) {
|
||||
process.stderr.write('Error: --task "<description>" is required\n\n');
|
||||
process.stderr.write(HELP);
|
||||
return 1;
|
||||
}
|
||||
if (!parsed.output) {
|
||||
process.stderr.write('Error: --output <path> is required\n\n');
|
||||
process.stderr.write(HELP);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (!existsSync(parsed.output)) {
|
||||
process.stderr.write(`Error: --output path not found: ${parsed.output}\n`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const outputContent = readFileSync(parsed.output, 'utf-8');
|
||||
if (outputContent.trim().length === 0) {
|
||||
process.stderr.write(`Error: --output file is empty: ${parsed.output}\n`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const slug = parsed.slug ?? inferSlugFromOutputPath(parsed.output);
|
||||
const cycles = parsed.cycles ?? (isTTY() ? 3 : 1);
|
||||
const dimensions = parsed.dimensions ?? DEFAULT_DIMENSIONS;
|
||||
const receiptDir = parsed.receiptDir ?? gbrainPath('eval-receipts');
|
||||
const maxTokens = parsed.maxTokens ?? 4000;
|
||||
|
||||
const slots: SlotConfig[] = [
|
||||
{ id: 'A', model: parsed.slotAModel ?? DEFAULT_SLOTS[0]!.model },
|
||||
{ id: 'B', model: parsed.slotBModel ?? DEFAULT_SLOTS[1]!.model },
|
||||
{ id: 'C', model: parsed.slotCModel ?? DEFAULT_SLOTS[2]!.model },
|
||||
];
|
||||
|
||||
// Configure the AI gateway. Without this, every chat() call throws
|
||||
// "AI gateway is not configured" because the cli.ts no-DB branch skips
|
||||
// connectEngine (T3=A).
|
||||
configureGatewayForCli();
|
||||
|
||||
// Probe whether the gateway can serve `chat`. If not, we can't run.
|
||||
if (!isAvailable('chat')) {
|
||||
process.stderr.write(
|
||||
'Error: AI gateway has no usable chat provider. ' +
|
||||
'Configure one of OPENAI_API_KEY / ANTHROPIC_API_KEY / GOOGLE_GENERATIVE_AI_API_KEY ' +
|
||||
'in your shell or run `gbrain config` to set keys.\n',
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Cost estimate (T11=B).
|
||||
const cost = estimateCost(slots, cycles, maxTokens);
|
||||
process.stderr.write(
|
||||
`[eval cross-modal] estimated cost: ~$${cost.perCycleUSD.toFixed(2)}/cycle, ` +
|
||||
`~$${cost.perRunMaxUSD.toFixed(2)} max for ${cycles} cycle(s).\n`,
|
||||
);
|
||||
for (const note of cost.notes) {
|
||||
process.stderr.write(`[eval cross-modal] note: ${note}\n`);
|
||||
}
|
||||
|
||||
// Progress reporter (stderr only).
|
||||
const onProgress = (ev: ProgressEvent) => {
|
||||
switch (ev.kind) {
|
||||
case 'cycle_start':
|
||||
process.stderr.write(`[eval cross-modal] cycle ${ev.cycle}/${ev.total} starting...\n`);
|
||||
break;
|
||||
case 'slot_done': {
|
||||
const status = ev.ok ? 'ok' : 'failed';
|
||||
process.stderr.write(
|
||||
`[eval cross-modal] slot ${ev.slotId} (${ev.modelId}) ${status} in ${ev.ms}ms\n`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'cycle_end':
|
||||
process.stderr.write(`[eval cross-modal] cycle ${ev.cycle} verdict: ${ev.verdict}\n`);
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
let result: RunEvalResult;
|
||||
try {
|
||||
result = await runEval({
|
||||
task: parsed.task,
|
||||
output: outputContent,
|
||||
slug,
|
||||
dimensions,
|
||||
slots,
|
||||
cycles,
|
||||
receiptDir,
|
||||
maxTokens,
|
||||
onProgress,
|
||||
});
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
process.stderr.write(`[eval cross-modal] runtime error: ${msg}\n`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Final summary to stderr (always) + JSON to stdout (when --json).
|
||||
const verdict = result.finalAggregate.verdict;
|
||||
process.stderr.write('\n');
|
||||
process.stderr.write(`[eval cross-modal] ${result.finalAggregate.verdictMessage}\n`);
|
||||
process.stderr.write(`[eval cross-modal] receipt: ${result.finalReceiptPath}\n`);
|
||||
|
||||
if (parsed.json) {
|
||||
process.stdout.write(
|
||||
JSON.stringify(
|
||||
{
|
||||
verdict,
|
||||
aggregate: result.finalAggregate,
|
||||
cycles: result.cycles.map(c => ({
|
||||
cycle: c.cycle,
|
||||
receipt_path: c.receipt_path,
|
||||
verdict: c.aggregate.verdict,
|
||||
overall: c.aggregate.overall,
|
||||
})),
|
||||
finalReceiptPath: result.finalReceiptPath,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
process.stdout.write('\n');
|
||||
}
|
||||
|
||||
if (verdict === 'pass') return 0;
|
||||
if (verdict === 'inconclusive') return 2;
|
||||
return 1;
|
||||
}
|
||||
@@ -37,6 +37,14 @@ export async function runEvalCommand(engine: BrainEngine, args: string[]): Promi
|
||||
const { runEvalReplay } = await import('./eval-replay.ts');
|
||||
return runEvalReplay(engine, args.slice(1));
|
||||
}
|
||||
if (sub === 'cross-modal') {
|
||||
// No-DB sub-subcommand. The cli.ts dispatcher routes the user-facing
|
||||
// path before connectEngine, so this branch only fires when callers
|
||||
// already have an engine and re-enter via runEvalCommand. Engine is
|
||||
// intentionally unused.
|
||||
const { runEvalCrossModal } = await import('./eval-cross-modal.ts');
|
||||
process.exit(await runEvalCrossModal(args.slice(1)));
|
||||
}
|
||||
|
||||
const opts = parseArgs(args);
|
||||
|
||||
|
||||
+168
-4
@@ -22,6 +22,22 @@ export async function runInit(args: string[]) {
|
||||
const pathIndex = args.indexOf('--path');
|
||||
const customPath = pathIndex !== -1 ? args[pathIndex + 1] : null;
|
||||
|
||||
// v0.14: AI provider selection.
|
||||
// --embedding-model PROVIDER:MODEL (verbose) or --model PROVIDER (shorthand, picks recipe default)
|
||||
const embModelIdx = args.indexOf('--embedding-model');
|
||||
const modelShortIdx = args.indexOf('--model');
|
||||
const embDimsIdx = args.indexOf('--embedding-dimensions');
|
||||
const expModelIdx = args.indexOf('--expansion-model');
|
||||
// v0.27: --chat-model PROVIDER:MODEL — default subagent driver.
|
||||
const chatModelIdx = args.indexOf('--chat-model');
|
||||
const aiOpts = await resolveAIOptions(
|
||||
embModelIdx !== -1 ? args[embModelIdx + 1] : null,
|
||||
modelShortIdx !== -1 ? args[modelShortIdx + 1] : null,
|
||||
embDimsIdx !== -1 ? parseInt(args[embDimsIdx + 1], 10) : null,
|
||||
expModelIdx !== -1 ? args[expModelIdx + 1] : null,
|
||||
chatModelIdx !== -1 ? args[chatModelIdx + 1] : null,
|
||||
);
|
||||
|
||||
// Schema-only path: apply initSchema against the already-configured engine
|
||||
// without ever calling saveConfig. Used by apply-migrations, the stopgap
|
||||
// script, and the postinstall hook. Bare `gbrain init` defaults to PGLite
|
||||
@@ -47,7 +63,7 @@ export async function runInit(args: string[]) {
|
||||
}
|
||||
}
|
||||
|
||||
return initPGLite({ jsonOutput, apiKey, customPath });
|
||||
return initPGLite({ jsonOutput, apiKey, customPath, aiOpts });
|
||||
}
|
||||
|
||||
// Supabase/Postgres mode
|
||||
@@ -66,7 +82,57 @@ export async function runInit(args: string[]) {
|
||||
databaseUrl = await supabaseWizard();
|
||||
}
|
||||
|
||||
return initPostgres({ databaseUrl, jsonOutput, apiKey });
|
||||
return initPostgres({ databaseUrl, jsonOutput, apiKey, aiOpts });
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve AI provider options from CLI flags. Verbose form (--embedding-model
|
||||
* openai:text-embedding-3-large) overrides shorthand (--model openai which
|
||||
* expands to the recipe's first embedding model).
|
||||
*/
|
||||
async function resolveAIOptions(
|
||||
verbose: string | null,
|
||||
shorthand: string | null,
|
||||
dimsArg: number | null,
|
||||
expansion: string | null,
|
||||
chat: string | null,
|
||||
): Promise<{ embedding_model?: string; embedding_dimensions?: number; expansion_model?: string; chat_model?: string }> {
|
||||
const out: { embedding_model?: string; embedding_dimensions?: number; expansion_model?: string; chat_model?: string } = {};
|
||||
|
||||
if (verbose) {
|
||||
out.embedding_model = verbose;
|
||||
} else if (shorthand) {
|
||||
const { getRecipe } = await import('../core/ai/recipes/index.ts');
|
||||
const recipe = getRecipe(shorthand);
|
||||
if (!recipe) {
|
||||
console.error(`Unknown provider: ${shorthand}. Run \`gbrain providers list\` to see known providers.`);
|
||||
process.exit(1);
|
||||
}
|
||||
const firstModel = recipe.touchpoints.embedding?.models[0];
|
||||
if (!firstModel) {
|
||||
console.error(`Provider ${shorthand} has no embedding models listed. Use --embedding-model provider:model.`);
|
||||
process.exit(1);
|
||||
}
|
||||
out.embedding_model = `${shorthand}:${firstModel}`;
|
||||
out.embedding_dimensions = recipe.touchpoints.embedding!.default_dims;
|
||||
}
|
||||
|
||||
if (dimsArg !== null && !Number.isNaN(dimsArg) && dimsArg > 0) {
|
||||
out.embedding_dimensions = dimsArg;
|
||||
} else if (out.embedding_model && out.embedding_dimensions === undefined) {
|
||||
// Derive default dims from the resolved recipe when verbose form was used.
|
||||
const { getRecipe } = await import('../core/ai/recipes/index.ts');
|
||||
const providerId = out.embedding_model.split(':')[0];
|
||||
const recipe = getRecipe(providerId);
|
||||
if (recipe?.touchpoints.embedding?.default_dims) {
|
||||
out.embedding_dimensions = recipe.touchpoints.embedding.default_dims;
|
||||
}
|
||||
}
|
||||
|
||||
if (expansion) out.expansion_model = expansion;
|
||||
if (chat) out.chat_model = chat;
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -102,19 +168,69 @@ async function initMigrateOnly(opts: { jsonOutput: boolean }) {
|
||||
}
|
||||
}
|
||||
|
||||
async function initPGLite(opts: { jsonOutput: boolean; apiKey: string | null; customPath: string | null }) {
|
||||
async function initPGLite(opts: {
|
||||
jsonOutput: boolean;
|
||||
apiKey: string | null;
|
||||
customPath: string | null;
|
||||
aiOpts?: { embedding_model?: string; embedding_dimensions?: number; expansion_model?: string; chat_model?: string };
|
||||
}) {
|
||||
const dbPath = opts.customPath || gbrainPath('brain.pglite');
|
||||
console.log(`Setting up local brain with PGLite (no server needed)...`);
|
||||
|
||||
// Configure AI gateway BEFORE initSchema so the vector column uses the right dim.
|
||||
if (opts.aiOpts?.embedding_model || opts.aiOpts?.chat_model) {
|
||||
const { configureGateway } = await import('../core/ai/gateway.ts');
|
||||
configureGateway({
|
||||
embedding_model: opts.aiOpts?.embedding_model,
|
||||
embedding_dimensions: opts.aiOpts?.embedding_dimensions,
|
||||
expansion_model: opts.aiOpts?.expansion_model,
|
||||
chat_model: opts.aiOpts?.chat_model,
|
||||
env: { ...process.env },
|
||||
});
|
||||
if (opts.aiOpts?.embedding_model) console.log(` Embedding: ${opts.aiOpts.embedding_model} (${opts.aiOpts.embedding_dimensions ?? '?'}d)`);
|
||||
if (opts.aiOpts?.expansion_model) console.log(` Expansion: ${opts.aiOpts.expansion_model}`);
|
||||
if (opts.aiOpts?.chat_model) console.log(` Chat: ${opts.aiOpts.chat_model}`);
|
||||
}
|
||||
|
||||
const engine = await createEngine({ engine: 'pglite' });
|
||||
try {
|
||||
await engine.connect({ database_path: dbPath, engine: 'pglite' });
|
||||
|
||||
// v0.28.5 (A4): refuse to silently re-template an existing brain with a
|
||||
// mismatched embedding dimension. Loud failure beats the v0.27 silent-
|
||||
// corruption pattern that surfaced as #673.
|
||||
if (opts.aiOpts?.embedding_dimensions) {
|
||||
const { readContentChunksEmbeddingDim, embeddingMismatchMessage } = await import('../core/embedding-dim-check.ts');
|
||||
const existing = await readContentChunksEmbeddingDim(engine);
|
||||
if (existing.exists && existing.dims !== null && existing.dims !== opts.aiOpts.embedding_dimensions) {
|
||||
console.error('\n' + embeddingMismatchMessage({
|
||||
currentDims: existing.dims,
|
||||
requestedDims: opts.aiOpts.embedding_dimensions,
|
||||
requestedModel: opts.aiOpts.embedding_model,
|
||||
source: 'init',
|
||||
}) + '\n');
|
||||
if (opts.jsonOutput) {
|
||||
console.log(JSON.stringify({
|
||||
status: 'error',
|
||||
reason: 'embedding_dim_mismatch',
|
||||
current_dims: existing.dims,
|
||||
requested_dims: opts.aiOpts.embedding_dimensions,
|
||||
}));
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
await engine.initSchema();
|
||||
|
||||
const config: GBrainConfig = {
|
||||
engine: 'pglite',
|
||||
database_path: dbPath,
|
||||
...(opts.apiKey ? { openai_api_key: opts.apiKey } : {}),
|
||||
...(opts.aiOpts?.embedding_model ? { embedding_model: opts.aiOpts.embedding_model } : {}),
|
||||
...(opts.aiOpts?.embedding_dimensions ? { embedding_dimensions: opts.aiOpts.embedding_dimensions } : {}),
|
||||
...(opts.aiOpts?.expansion_model ? { expansion_model: opts.aiOpts.expansion_model } : {}),
|
||||
...(opts.aiOpts?.chat_model ? { chat_model: opts.aiOpts.chat_model } : {}),
|
||||
};
|
||||
saveConfig(config);
|
||||
|
||||
@@ -146,9 +262,29 @@ async function initPGLite(opts: { jsonOutput: boolean; apiKey: string | null; cu
|
||||
}
|
||||
}
|
||||
|
||||
async function initPostgres(opts: { databaseUrl: string; jsonOutput: boolean; apiKey: string | null }) {
|
||||
async function initPostgres(opts: {
|
||||
databaseUrl: string;
|
||||
jsonOutput: boolean;
|
||||
apiKey: string | null;
|
||||
aiOpts?: { embedding_model?: string; embedding_dimensions?: number; expansion_model?: string; chat_model?: string };
|
||||
}) {
|
||||
const { databaseUrl } = opts;
|
||||
|
||||
// Configure AI gateway BEFORE initSchema so the vector column uses the right dim.
|
||||
if (opts.aiOpts?.embedding_model || opts.aiOpts?.chat_model) {
|
||||
const { configureGateway } = await import('../core/ai/gateway.ts');
|
||||
configureGateway({
|
||||
embedding_model: opts.aiOpts?.embedding_model,
|
||||
embedding_dimensions: opts.aiOpts?.embedding_dimensions,
|
||||
expansion_model: opts.aiOpts?.expansion_model,
|
||||
chat_model: opts.aiOpts?.chat_model,
|
||||
env: { ...process.env },
|
||||
});
|
||||
if (opts.aiOpts?.embedding_model) console.log(` Embedding: ${opts.aiOpts.embedding_model} (${opts.aiOpts.embedding_dimensions ?? '?'}d)`);
|
||||
if (opts.aiOpts?.expansion_model) console.log(` Expansion: ${opts.aiOpts.expansion_model}`);
|
||||
if (opts.aiOpts?.chat_model) console.log(` Chat: ${opts.aiOpts.chat_model}`);
|
||||
}
|
||||
|
||||
// Detect Supabase direct connection URLs and warn about IPv6
|
||||
if (databaseUrl.match(/db\.[a-z]+\.supabase\.co/) || databaseUrl.includes('.supabase.co:5432')) {
|
||||
console.warn('');
|
||||
@@ -194,6 +330,30 @@ async function initPostgres(opts: { databaseUrl: string; jsonOutput: boolean; ap
|
||||
// Non-fatal
|
||||
}
|
||||
|
||||
// v0.28.5 (A4): refuse to silently re-template an existing brain with a
|
||||
// mismatched embedding dimension (mirror of the PGLite path above).
|
||||
if (opts.aiOpts?.embedding_dimensions) {
|
||||
const { readContentChunksEmbeddingDim, embeddingMismatchMessage } = await import('../core/embedding-dim-check.ts');
|
||||
const existing = await readContentChunksEmbeddingDim(engine);
|
||||
if (existing.exists && existing.dims !== null && existing.dims !== opts.aiOpts.embedding_dimensions) {
|
||||
console.error('\n' + embeddingMismatchMessage({
|
||||
currentDims: existing.dims,
|
||||
requestedDims: opts.aiOpts.embedding_dimensions,
|
||||
requestedModel: opts.aiOpts.embedding_model,
|
||||
source: 'init',
|
||||
}) + '\n');
|
||||
if (opts.jsonOutput) {
|
||||
console.log(JSON.stringify({
|
||||
status: 'error',
|
||||
reason: 'embedding_dim_mismatch',
|
||||
current_dims: existing.dims,
|
||||
requested_dims: opts.aiOpts.embedding_dimensions,
|
||||
}));
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Running schema migration...');
|
||||
await engine.initSchema();
|
||||
|
||||
@@ -201,6 +361,10 @@ async function initPostgres(opts: { databaseUrl: string; jsonOutput: boolean; ap
|
||||
engine: 'postgres',
|
||||
database_url: databaseUrl,
|
||||
...(opts.apiKey ? { openai_api_key: opts.apiKey } : {}),
|
||||
...(opts.aiOpts?.embedding_model ? { embedding_model: opts.aiOpts.embedding_model } : {}),
|
||||
...(opts.aiOpts?.embedding_dimensions ? { embedding_dimensions: opts.aiOpts.embedding_dimensions } : {}),
|
||||
...(opts.aiOpts?.expansion_model ? { expansion_model: opts.aiOpts.expansion_model } : {}),
|
||||
...(opts.aiOpts?.chat_model ? { chat_model: opts.aiOpts.chat_model } : {}),
|
||||
};
|
||||
saveConfig(config);
|
||||
console.log('Config saved to ~/.gbrain/config.json');
|
||||
|
||||
+10
-123
@@ -119,131 +119,18 @@ export function expandVars(s: string): string {
|
||||
}
|
||||
|
||||
// --- SSRF Protection ---
|
||||
// Helpers extracted to src/core/url-safety.ts in v0.28 so src/core/git-remote.ts
|
||||
// can reuse them without inverting the layering boundary. Re-exported here for
|
||||
// backward compat with existing callers + test/integrations.test.ts imports.
|
||||
|
||||
/** Parse an IPv4 octet from decimal, hex (0x prefix), or octal (leading 0) notation. */
|
||||
export function parseOctet(s: string): number {
|
||||
if (s.length === 0) return NaN;
|
||||
if (s.startsWith('0x') || s.startsWith('0X')) {
|
||||
if (!/^0[xX][0-9a-fA-F]+$/.test(s)) return NaN;
|
||||
return parseInt(s, 16);
|
||||
}
|
||||
if (s.length > 1 && s.startsWith('0')) {
|
||||
if (!/^0[0-7]+$/.test(s)) return NaN;
|
||||
return parseInt(s, 8);
|
||||
}
|
||||
if (!/^\d+$/.test(s)) return NaN;
|
||||
return parseInt(s, 10);
|
||||
}
|
||||
export {
|
||||
parseOctet,
|
||||
hostnameToOctets,
|
||||
isPrivateIpv4,
|
||||
isInternalUrl,
|
||||
} from '../core/url-safety.ts';
|
||||
|
||||
/**
|
||||
* Convert an IPv4 hostname to 4 octets. Handles bypass encodings:
|
||||
* - Dotted decimal: 127.0.0.1
|
||||
* - Single decimal: 2130706433 (= 0x7f000001)
|
||||
* - Hex: 0x7f000001
|
||||
* - Per-octet hex/octal: 0x7f.0.0.1, 0177.0.0.1
|
||||
* Returns null for non-IP hostnames (fall through to hostname-based checks).
|
||||
*/
|
||||
export function hostnameToOctets(hostname: string): number[] | null {
|
||||
// Single integer form
|
||||
if (/^\d+$/.test(hostname)) {
|
||||
const n = parseInt(hostname, 10);
|
||||
if (Number.isFinite(n) && n >= 0 && n <= 0xFFFFFFFF) {
|
||||
return [(n >>> 24) & 0xFF, (n >>> 16) & 0xFF, (n >>> 8) & 0xFF, n & 0xFF];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
// Hex integer form (0x prefix, no dots)
|
||||
if (/^0[xX][0-9a-fA-F]+$/.test(hostname)) {
|
||||
const n = parseInt(hostname, 16);
|
||||
if (Number.isFinite(n) && n >= 0 && n <= 0xFFFFFFFF) {
|
||||
return [(n >>> 24) & 0xFF, (n >>> 16) & 0xFF, (n >>> 8) & 0xFF, n & 0xFF];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
// Dotted notation with possible octal/hex per octet
|
||||
const parts = hostname.split('.');
|
||||
if (parts.length === 4) {
|
||||
const octets = parts.map(parseOctet);
|
||||
if (octets.every(o => Number.isFinite(o) && o >= 0 && o <= 255)) return octets;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Classify an IPv4 address as internal/private/reserved. */
|
||||
export function isPrivateIpv4(octets: number[]): boolean {
|
||||
const [a, b] = octets;
|
||||
if (a === 127) return true; // 127.0.0.0/8 loopback
|
||||
if (a === 10) return true; // 10.0.0.0/8 RFC1918
|
||||
if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12 RFC1918
|
||||
if (a === 192 && b === 168) return true; // 192.168.0.0/16 RFC1918
|
||||
if (a === 169 && b === 254) return true; // 169.254.0.0/16 link-local (incl. AWS metadata)
|
||||
if (a === 100 && b >= 64 && b <= 127) return true; // 100.64.0.0/10 CGNAT
|
||||
if (a === 0) return true; // 0.0.0.0/8 unspecified
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Returns true if the URL targets an internal/metadata endpoint or uses a non-http(s) scheme. Fail-closed on parse errors. */
|
||||
export function isInternalUrl(urlStr: string): boolean {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(urlStr);
|
||||
} catch {
|
||||
return true; // malformed → block
|
||||
}
|
||||
// B4: scheme allowlist — block file:, data:, blob:, ftp:, gopher:, javascript:, etc.
|
||||
if (url.protocol !== 'http:' && url.protocol !== 'https:') return true;
|
||||
|
||||
let host = url.hostname.toLowerCase();
|
||||
|
||||
// Block known metadata hostnames
|
||||
const metadataHostnames = new Set([
|
||||
'metadata.google.internal',
|
||||
'metadata.google',
|
||||
'metadata',
|
||||
'instance-data',
|
||||
'instance-data.ec2.internal',
|
||||
]);
|
||||
if (metadataHostnames.has(host)) return true;
|
||||
|
||||
// localhost aliases
|
||||
if (host === 'localhost' || host.endsWith('.localhost')) return true;
|
||||
|
||||
// Strip IPv6 brackets if present (WHATWG URL returns hostname with brackets for IPv6)
|
||||
if (host.startsWith('[') && host.endsWith(']')) host = host.slice(1, -1);
|
||||
|
||||
// IPv6 loopback (and any all-zeros form that resolves to loopback-adjacent)
|
||||
if (host === '::1' || host === '::') return true;
|
||||
|
||||
// Handle IPv4-mapped IPv6. WHATWG URL canonicalizes `::ffff:127.0.0.1` to `::ffff:7f00:1`
|
||||
// (two hex hextets), so we must parse hex hextets back to IPv4 octets.
|
||||
if (host.startsWith('::ffff:')) {
|
||||
const tail = host.slice(7);
|
||||
// Mixed form: ::ffff:A.B.C.D (if parser preserved dotted notation)
|
||||
const dotted = hostnameToOctets(tail);
|
||||
if (dotted && isPrivateIpv4(dotted)) return true;
|
||||
// Hex-compressed form: ::ffff:XXXX:YYYY → two 16-bit hextets
|
||||
const hextets = tail.split(':');
|
||||
if (hextets.length === 2 && hextets.every(h => /^[0-9a-f]{1,4}$/.test(h))) {
|
||||
const hi = parseInt(hextets[0], 16);
|
||||
const lo = parseInt(hextets[1], 16);
|
||||
const octets = [(hi >> 8) & 0xff, hi & 0xff, (lo >> 8) & 0xff, lo & 0xff];
|
||||
if (isPrivateIpv4(octets)) return true;
|
||||
}
|
||||
}
|
||||
|
||||
// IPv4 range check (handles hex, octal, single decimal bypass forms)
|
||||
const octets = hostnameToOctets(host);
|
||||
if (octets && isPrivateIpv4(octets)) return true;
|
||||
|
||||
// Trailing dot on numeric-looking hostname — strip and re-check
|
||||
if (host.endsWith('.')) {
|
||||
const stripped = host.slice(0, -1);
|
||||
const strippedOctets = hostnameToOctets(stripped);
|
||||
if (strippedOctets && isPrivateIpv4(strippedOctets)) return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
import { isInternalUrl } from '../core/url-safety.ts';
|
||||
|
||||
export async function executeHealthCheck(
|
||||
check: HealthCheck,
|
||||
|
||||
+15
-1
@@ -714,7 +714,21 @@ HANDLER TYPES (built in)
|
||||
: '';
|
||||
console.log(`Minion worker started (queue: ${queueName}, concurrency: ${concurrency}${watchdogNote}${healthNote})`);
|
||||
console.log(`Registered handlers: ${worker.registeredNames.join(', ')}`);
|
||||
await worker.start();
|
||||
try {
|
||||
await worker.start();
|
||||
} finally {
|
||||
// Release the DB connection pool immediately on shutdown so
|
||||
// PgBouncer slots are freed rather than waiting for TCP keepalive
|
||||
// (~minutes). Disconnect failure is best-effort but logged loudly:
|
||||
// a silent shutdown disconnect error is exactly the bug class the
|
||||
// v0.26.9 D14 direction (isUndefinedColumnError, oauth-provider)
|
||||
// was created to surface. The CLI is the engine owner here, not
|
||||
// the worker — keeping disconnect at this layer preserves the
|
||||
// "engine ownership stays with the creator" invariant that broke
|
||||
// tests in earlier waves of this branch.
|
||||
try { await engine.disconnect(); }
|
||||
catch (e) { console.error('[gbrain jobs work] engine disconnect failed during shutdown:', e); }
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ import { v0_18_0 } from './v0_18_0.ts';
|
||||
import { v0_18_1 } from './v0_18_1.ts';
|
||||
import { v0_21_0 } from './v0_21_0.ts';
|
||||
import { v0_22_4 } from './v0_22_4.ts';
|
||||
import { v0_28_0 } from './v0_28_0.ts';
|
||||
|
||||
export const migrations: Migration[] = [
|
||||
v0_11_0,
|
||||
@@ -35,6 +36,7 @@ export const migrations: Migration[] = [
|
||||
v0_18_1,
|
||||
v0_21_0,
|
||||
v0_22_4,
|
||||
v0_28_0,
|
||||
];
|
||||
|
||||
/** Look up a migration by exact version string. */
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
/**
|
||||
* v0.28.0 migration orchestrator — Takes + Think + Unified Model Config.
|
||||
*
|
||||
* v0.28 ships the typed/weighted/attributed claims layer, a unified model
|
||||
* configuration resolver, and a per-token MCP allow-list for take visibility.
|
||||
*
|
||||
* Phases (all idempotent, additive):
|
||||
* A. Schema — verify migrations v37 + v38 already applied (the schema
|
||||
* runner in src/core/migrate.ts does the actual DDL during
|
||||
* `gbrain upgrade`/initSchema). This phase asserts post-condition.
|
||||
* B. Backfill — submit `gbrain extract takes` as a Minion job so any
|
||||
* pre-existing fenced takes tables in markdown populate the
|
||||
* takes table without blocking the foreground upgrade.
|
||||
* Falls back to inline run on PGLite (no Minion worker).
|
||||
* C. Re-chunk — emit a pending-host-work TODO for `gbrain re-chunk
|
||||
* --where pages-with-takes` (Codex P0 #3 fix: pages with
|
||||
* pre-v0.28 chunks still contain the fenced takes content;
|
||||
* the chunker strip only applies to NEW imports). Re-chunk
|
||||
* is heavy + per-page-disruptive, so we queue a TODO instead
|
||||
* of running it inline.
|
||||
* D. Record — runner-owned ledger write (handled by apply-migrations.ts).
|
||||
*
|
||||
* No content mutation. No data loss. Operator runs `gbrain doctor` after
|
||||
* upgrade to verify takes_backfill_complete + takes_fence_chunk_leak checks.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, appendFileSync, readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import type {
|
||||
Migration, OrchestratorOpts, OrchestratorResult, OrchestratorPhaseResult,
|
||||
} from './types.ts';
|
||||
import type { BrainEngine } from '../../core/engine.ts';
|
||||
import { loadConfig, toEngineConfig, gbrainPath } from '../../core/config.ts';
|
||||
import { createEngine } from '../../core/engine-factory.ts';
|
||||
|
||||
let testEngineOverride: BrainEngine | null = null;
|
||||
export function __setTestEngineOverride(engine: BrainEngine | null): void {
|
||||
testEngineOverride = engine;
|
||||
}
|
||||
|
||||
function migrationsDir(): string { return gbrainPath('migrations'); }
|
||||
function pendingHostWorkPath(): string { return join(migrationsDir(), 'pending-host-work.jsonl'); }
|
||||
|
||||
interface PendingHostWorkEntry {
|
||||
migration: string;
|
||||
ts: string;
|
||||
skill: string;
|
||||
reason: string;
|
||||
command: string;
|
||||
}
|
||||
|
||||
// ── Phase A — Schema verify ────────────────────────────────
|
||||
|
||||
async function phaseASchema(
|
||||
engine: BrainEngine | null,
|
||||
opts: OrchestratorOpts,
|
||||
): Promise<OrchestratorPhaseResult> {
|
||||
if (opts.dryRun) return { name: 'schema', status: 'skipped', detail: 'dry-run' };
|
||||
if (!engine) {
|
||||
return { name: 'schema', status: 'skipped', detail: 'no_brain_configured' };
|
||||
}
|
||||
try {
|
||||
const versionStr = await engine.getConfig('version');
|
||||
const v = parseInt(versionStr || '0', 10);
|
||||
if (v < 38) {
|
||||
return {
|
||||
name: 'schema',
|
||||
status: 'failed',
|
||||
detail: `expected schema version >= 38 (takes + access_tokens.permissions); got ${v}. Run \`gbrain apply-migrations --yes\` to apply.`,
|
||||
};
|
||||
}
|
||||
// Quick post-condition: takes + synthesis_evidence tables exist
|
||||
const rows = await engine.executeRaw<{ tablename: string }>(
|
||||
`SELECT tablename FROM pg_tables WHERE tablename IN ('takes', 'synthesis_evidence')`,
|
||||
);
|
||||
if (rows.length < 2) {
|
||||
return {
|
||||
name: 'schema',
|
||||
status: 'failed',
|
||||
detail: `expected tables takes + synthesis_evidence; found ${rows.map(r => r.tablename).join(', ') || 'none'}`,
|
||||
};
|
||||
}
|
||||
return { name: 'schema', status: 'complete', detail: 'schema v38 applied; takes + synthesis_evidence present' };
|
||||
} catch (e) {
|
||||
return { name: 'schema', status: 'failed', detail: e instanceof Error ? e.message : String(e) };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Phase B — Backfill takes ───────────────────────────────
|
||||
|
||||
async function phaseBBackfill(
|
||||
engine: BrainEngine | null,
|
||||
opts: OrchestratorOpts,
|
||||
): Promise<OrchestratorPhaseResult> {
|
||||
if (opts.dryRun) return { name: 'backfill', status: 'skipped', detail: 'dry-run' };
|
||||
if (!engine) return { name: 'backfill', status: 'skipped', detail: 'no_brain_configured' };
|
||||
|
||||
try {
|
||||
// Inline run on both engines for v0.28.0 simplicity. Larger brains can run
|
||||
// `gbrain extract takes --rebuild` later; the migration's job is to get
|
||||
// the table populated for upgrade-time doctor checks.
|
||||
const { extractTakes } = await import('../../core/cycle/extract-takes.ts');
|
||||
const result = await extractTakes(engine, { source: 'db' });
|
||||
return {
|
||||
name: 'backfill',
|
||||
status: 'complete',
|
||||
detail: `extract-takes scanned ${result.pagesScanned} pages; ${result.pagesWithTakes} had fenced takes; upserted ${result.takesUpserted} rows`,
|
||||
};
|
||||
} catch (e) {
|
||||
return { name: 'backfill', status: 'failed', detail: e instanceof Error ? e.message : String(e) };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Phase C — Re-chunk TODO ────────────────────────────────
|
||||
|
||||
function existingHostEntries(version: string, key: string): boolean {
|
||||
const p = pendingHostWorkPath();
|
||||
if (!existsSync(p)) return false;
|
||||
try {
|
||||
const raw = readFileSync(p, 'utf8');
|
||||
for (const line of raw.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
try {
|
||||
const obj = JSON.parse(trimmed) as PendingHostWorkEntry & { _key?: string };
|
||||
if (obj.migration === version && obj._key === key) return true;
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
} catch { /* read error */ }
|
||||
return false;
|
||||
}
|
||||
|
||||
function phaseCRechunkTodo(opts: OrchestratorOpts): OrchestratorPhaseResult {
|
||||
if (opts.dryRun) return { name: 'rechunk-todo', status: 'skipped', detail: 'dry-run' };
|
||||
const key = 'rechunk-pages-with-takes';
|
||||
if (existingHostEntries('0.28.0', key)) {
|
||||
return { name: 'rechunk-todo', status: 'complete', detail: 'already queued' };
|
||||
}
|
||||
try {
|
||||
mkdirSync(migrationsDir(), { recursive: true });
|
||||
const entry = {
|
||||
migration: '0.28.0',
|
||||
ts: new Date().toISOString(),
|
||||
skill: 'skills/migrations/v0.28.0.md',
|
||||
reason: 'Pages with pre-v0.28 chunks still contain fenced takes content. Re-chunk so the new chunker strip rule is applied (Codex P0 #3 fix).',
|
||||
command: "gbrain extract takes --rebuild # forces re-chunk via reimport pipeline; see migration doc for the precise sweep command in your env",
|
||||
_key: key,
|
||||
};
|
||||
appendFileSync(pendingHostWorkPath(), JSON.stringify(entry) + '\n');
|
||||
return {
|
||||
name: 'rechunk-todo',
|
||||
status: 'complete',
|
||||
detail: `queued re-chunk TODO at ${pendingHostWorkPath()} (read skills/migrations/v0.28.0.md for the playbook)`,
|
||||
};
|
||||
} catch (e) {
|
||||
return { name: 'rechunk-todo', status: 'failed', detail: e instanceof Error ? e.message : String(e) };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Orchestrator ───────────────────────────────────────────
|
||||
|
||||
async function orchestrator(opts: OrchestratorOpts): Promise<OrchestratorResult> {
|
||||
console.log('');
|
||||
console.log('=== v0.28.0 — Takes + Think + Unified Model Config ===');
|
||||
if (opts.dryRun) console.log(' (dry-run; no side effects)');
|
||||
console.log('');
|
||||
|
||||
const phases: OrchestratorPhaseResult[] = [];
|
||||
|
||||
// Acquire engine for phases that need it. Skip cleanly when none configured.
|
||||
let engine: BrainEngine | null = null;
|
||||
let ownsEngine = false;
|
||||
try {
|
||||
if (testEngineOverride) {
|
||||
engine = testEngineOverride;
|
||||
} else {
|
||||
const config = loadConfig();
|
||||
if (config) {
|
||||
const engineConfig = toEngineConfig(config);
|
||||
engine = await createEngine(engineConfig);
|
||||
await engine.connect(engineConfig);
|
||||
ownsEngine = true;
|
||||
}
|
||||
}
|
||||
|
||||
phases.push(await phaseASchema(engine, opts));
|
||||
if (phases[0].status === 'failed') {
|
||||
return { version: '0.28.0', status: 'partial', phases };
|
||||
}
|
||||
|
||||
phases.push(await phaseBBackfill(engine, opts));
|
||||
phases.push(phaseCRechunkTodo(opts));
|
||||
} finally {
|
||||
if (ownsEngine && engine) {
|
||||
try { await engine.disconnect(); } catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
|
||||
const overallStatus: 'complete' | 'partial' | 'failed' =
|
||||
phases.some(p => p.status === 'failed') ? 'partial' : 'complete';
|
||||
|
||||
return { version: '0.28.0', status: overallStatus, phases };
|
||||
}
|
||||
|
||||
export const v0_28_0: Migration = {
|
||||
version: '0.28.0',
|
||||
featurePitch: {
|
||||
headline: "Takes ship — your brain finally captures what you BELIEVE, not just what's true",
|
||||
description:
|
||||
'v0.28 adds the takes layer: typed/weighted/attributed claims (fact/take/bet/hunch) ' +
|
||||
'stored as fenced markdown tables on every page, indexed in Postgres for fast queries. ' +
|
||||
'Plus `gbrain takes` CLI (list/search/add/update/supersede/resolve), unified model config ' +
|
||||
'(`models.default` replaces every per-phase config key), per-token MCP allow-list for ' +
|
||||
'visibility (private hunches stay private), and three new MCP ops (takes_list, takes_search, ' +
|
||||
'think). `gbrain think` op surface lands now; the synthesis pipeline lands incrementally in ' +
|
||||
'v0.28.x. Migration backfills takes from any pre-existing fenced markdown tables; queues a ' +
|
||||
're-chunk TODO so the chunker-strip rule (Codex P0 fix — keeps takes content out of page ' +
|
||||
'chunks where the per-token allow-list cannot reach) catches up on legacy pages.',
|
||||
},
|
||||
orchestrator,
|
||||
};
|
||||
|
||||
/** Exported for unit tests. */
|
||||
export const __testing = {
|
||||
phaseASchema,
|
||||
phaseBBackfill,
|
||||
phaseCRechunkTodo,
|
||||
pendingHostWorkPath,
|
||||
};
|
||||
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* gbrain pages — page-level operator commands. v0.26.5+.
|
||||
*
|
||||
* The first subcommand: `pages purge-deleted [--older-than HOURS] [--dry-run]`.
|
||||
* Manual escape hatch alongside the autopilot purge phase. Hard-deletes pages
|
||||
* whose `deleted_at` is older than the cutoff; cascades to content_chunks,
|
||||
* page_links, chunk_relations via existing FKs.
|
||||
*/
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
|
||||
const SOFT_DELETE_TTL_HOURS_DEFAULT = 72;
|
||||
|
||||
function parseOlderThanHours(args: string[]): number {
|
||||
const idx = args.indexOf('--older-than');
|
||||
if (idx === -1 || idx === args.length - 1) return SOFT_DELETE_TTL_HOURS_DEFAULT;
|
||||
const raw = args[idx + 1];
|
||||
// Accept bare numbers (hours) or `<N>h` / `<N>d`. Reject anything ambiguous.
|
||||
const trimmed = raw.trim();
|
||||
const dayMatch = trimmed.match(/^(\d+)d$/);
|
||||
if (dayMatch) return Math.max(0, parseInt(dayMatch[1], 10) * 24);
|
||||
const hourMatch = trimmed.match(/^(\d+)h?$/);
|
||||
if (hourMatch) return Math.max(0, parseInt(hourMatch[1], 10));
|
||||
console.error(`Invalid --older-than value: "${raw}". Expected hours (e.g. 72 or 72h) or days (e.g. 3d).`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
async function runPurgeDeleted(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const olderThanHours = parseOlderThanHours(args);
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const json = args.includes('--json');
|
||||
|
||||
if (dryRun) {
|
||||
// Use listPages with includeDeleted to enumerate the recoverable set, then
|
||||
// count how many would be purged given the cutoff. Stays read-only.
|
||||
const candidates = await engine.listPages({ includeDeleted: true, limit: 10000 });
|
||||
const cutoff = Date.now() - olderThanHours * 60 * 60 * 1000;
|
||||
const wouldPurge = candidates.filter(
|
||||
(p) => p.deleted_at && p.deleted_at instanceof Date && p.deleted_at.getTime() < cutoff,
|
||||
);
|
||||
if (json) {
|
||||
console.log(JSON.stringify({ dry_run: true, older_than_hours: olderThanHours, count: wouldPurge.length, slugs: wouldPurge.map((p) => p.slug) }, null, 2));
|
||||
return;
|
||||
}
|
||||
console.log(`(dry-run) Would purge ${wouldPurge.length} page(s) soft-deleted more than ${olderThanHours}h ago.`);
|
||||
for (const p of wouldPurge) console.log(` ${p.slug} deleted_at=${p.deleted_at?.toISOString()}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await engine.purgeDeletedPages(olderThanHours);
|
||||
if (json) {
|
||||
console.log(JSON.stringify({ older_than_hours: olderThanHours, count: result.count, slugs: result.slugs }, null, 2));
|
||||
return;
|
||||
}
|
||||
if (result.count === 0) {
|
||||
console.log(`No pages to purge (older than ${olderThanHours}h).`);
|
||||
} else {
|
||||
console.log(`Purged ${result.count} page(s) (older than ${olderThanHours}h):`);
|
||||
for (const slug of result.slugs) console.log(` ${slug}`);
|
||||
}
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
console.log(`gbrain pages — page-level operator commands (v0.26.5)
|
||||
|
||||
Subcommands:
|
||||
purge-deleted [--older-than HOURS|Nd] [--dry-run] [--json]
|
||||
Hard-delete soft-deleted pages older than the cutoff
|
||||
(default 72h). Cascades to chunks/links/edges.
|
||||
Mirror of the autopilot purge phase.
|
||||
|
||||
Notes:
|
||||
Soft-delete a page via the MCP \`delete_page\` op. Restore via \`restore_page\`.
|
||||
This command is the manual operator escape hatch — the autopilot cycle's
|
||||
purge phase already calls the same library function on every run.
|
||||
`);
|
||||
}
|
||||
|
||||
export async function runPages(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const sub = args[0];
|
||||
const rest = args.slice(1);
|
||||
|
||||
switch (sub) {
|
||||
case 'purge-deleted': return runPurgeDeleted(engine, rest);
|
||||
case undefined:
|
||||
case '--help':
|
||||
case '-h':
|
||||
printHelp();
|
||||
return;
|
||||
default:
|
||||
console.error(`Unknown subcommand: ${sub}`);
|
||||
printHelp();
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
/**
|
||||
* `gbrain providers` CLI — list, test, env, explain.
|
||||
*
|
||||
* This command operates WITHOUT a brain connection (no engine needed) so
|
||||
* users can verify provider setup before `gbrain init`.
|
||||
*/
|
||||
|
||||
import { listRecipes, getRecipe } from '../core/ai/recipes/index.ts';
|
||||
import { configureGateway, embedOne, isAvailable as gwIsAvailable, chat as gwChat } from '../core/ai/gateway.ts';
|
||||
import { probeOllama, probeLMStudio } from '../core/ai/probes.ts';
|
||||
import { loadConfig } from '../core/config.ts';
|
||||
import { AIConfigError, AITransientError } from '../core/ai/errors.ts';
|
||||
import type { Recipe } from '../core/ai/types.ts';
|
||||
|
||||
const SCHEMA_VERSION = 1;
|
||||
|
||||
type TouchpointFilter = 'embedding' | 'expansion' | 'chat';
|
||||
|
||||
interface ProviderOption {
|
||||
id: string;
|
||||
touchpoint: TouchpointFilter;
|
||||
model: string;
|
||||
dims?: number;
|
||||
cost_per_1m_tokens_usd?: number;
|
||||
cost_per_1m_input_usd?: number;
|
||||
cost_per_1m_output_usd?: number;
|
||||
price_last_verified?: string;
|
||||
env_ready: boolean;
|
||||
tier: 'native' | 'openai-compat';
|
||||
pros: string[];
|
||||
cons: string[];
|
||||
}
|
||||
|
||||
function configureFromEnv(): void {
|
||||
const config = loadConfig();
|
||||
configureGateway({
|
||||
embedding_model: config?.embedding_model,
|
||||
embedding_dimensions: config?.embedding_dimensions,
|
||||
expansion_model: config?.expansion_model,
|
||||
chat_model: config?.chat_model,
|
||||
chat_fallback_chain: config?.chat_fallback_chain,
|
||||
base_urls: config?.provider_base_urls,
|
||||
env: { ...process.env },
|
||||
});
|
||||
}
|
||||
|
||||
function envReady(recipe: Recipe): boolean {
|
||||
const required = recipe.auth_env?.required ?? [];
|
||||
if (required.length === 0) return true; // e.g. local Ollama
|
||||
return required.every(k => !!process.env[k]);
|
||||
}
|
||||
|
||||
export async function runProviders(subcommand: string | undefined, args: string[]): Promise<void> {
|
||||
configureFromEnv();
|
||||
|
||||
switch (subcommand) {
|
||||
case 'list':
|
||||
return runList(args);
|
||||
case 'test':
|
||||
return runTest(args);
|
||||
case 'env':
|
||||
return runEnv(args);
|
||||
case 'explain':
|
||||
return runExplain(args);
|
||||
case undefined:
|
||||
case '--help':
|
||||
case '-h':
|
||||
printHelp();
|
||||
return;
|
||||
default:
|
||||
console.error(`Unknown providers subcommand: ${subcommand}`);
|
||||
printHelp();
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
console.log(`gbrain providers — AI provider status and testing
|
||||
|
||||
USAGE
|
||||
gbrain providers list List all known providers + status
|
||||
gbrain providers test [--touchpoint T] [--model ID] Smoke-test configured (or specified) providers
|
||||
gbrain providers env <id> Show env vars required/optional for a provider
|
||||
gbrain providers explain [--json] Emit a provider choice matrix (agent-friendly)
|
||||
|
||||
TOUCHPOINTS
|
||||
--touchpoint embedding (default) Probes embed_one("...")
|
||||
--touchpoint chat Probes chat({messages: [{role:'user', content:'ping'}]})
|
||||
|
||||
EXAMPLES
|
||||
gbrain providers list
|
||||
gbrain providers test --model openai:text-embedding-3-large
|
||||
gbrain providers test --touchpoint chat --model anthropic:claude-haiku-4-5
|
||||
gbrain providers test --touchpoint chat --model deepseek:deepseek-chat
|
||||
gbrain providers env ollama
|
||||
gbrain providers explain --json
|
||||
`);
|
||||
}
|
||||
|
||||
function runList(_args: string[]): void {
|
||||
const recipes = listRecipes();
|
||||
const rows: string[] = [];
|
||||
rows.push('PROVIDER'.padEnd(14) + 'TIER'.padEnd(18) + 'EMBED'.padEnd(8) + 'EXPAND'.padEnd(8) + 'CHAT'.padEnd(8) + 'STATUS');
|
||||
rows.push('-'.repeat(78));
|
||||
for (const r of recipes) {
|
||||
const hasEmbed = !!r.touchpoints.embedding && (r.touchpoints.embedding.models.length > 0);
|
||||
const hasExpand = !!r.touchpoints.expansion;
|
||||
const hasChat = !!r.touchpoints.chat && r.touchpoints.chat.models.length > 0;
|
||||
const ready = envReady(r);
|
||||
const status = ready ? '✓ ready' : `✗ missing ${r.auth_env?.required?.[0] ?? 'setup'}`;
|
||||
rows.push(
|
||||
r.id.padEnd(14) +
|
||||
r.tier.padEnd(18) +
|
||||
(hasEmbed ? 'yes' : '—').padEnd(8) +
|
||||
(hasExpand ? 'yes' : '—').padEnd(8) +
|
||||
(hasChat ? 'yes' : '—').padEnd(8) +
|
||||
status,
|
||||
);
|
||||
}
|
||||
console.log(rows.join('\n'));
|
||||
}
|
||||
|
||||
async function runTest(args: string[]): Promise<void> {
|
||||
const modelIdx = args.indexOf('--model');
|
||||
const modelArg = modelIdx >= 0 ? args[modelIdx + 1] : undefined;
|
||||
|
||||
const tpIdx = args.indexOf('--touchpoint');
|
||||
const tpArg = (tpIdx >= 0 ? args[tpIdx + 1] : 'embedding') as TouchpointFilter;
|
||||
|
||||
if (tpArg !== 'embedding' && tpArg !== 'chat') {
|
||||
console.error(`--touchpoint must be 'embedding' or 'chat' (got: ${tpArg}).`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// If --model passed, override gateway for this test (touchpoint-aware).
|
||||
if (modelArg) {
|
||||
const [providerId, ...modelParts] = modelArg.split(':');
|
||||
const modelId = modelParts.join(':');
|
||||
const recipe = getRecipe(providerId);
|
||||
if (tpArg === 'embedding') {
|
||||
const dims = recipe?.touchpoints.embedding?.default_dims ?? 1536;
|
||||
configureGateway({
|
||||
embedding_model: modelArg,
|
||||
embedding_dimensions: dims,
|
||||
env: { ...process.env },
|
||||
});
|
||||
} else {
|
||||
configureGateway({
|
||||
chat_model: modelArg,
|
||||
env: { ...process.env },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!gwIsAvailable(tpArg)) {
|
||||
console.error(`${tpArg[0]?.toUpperCase()}${tpArg.slice(1)} provider not configured or not ready. Run \`gbrain providers list\` to see status.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Probing ${tpArg} provider...`);
|
||||
const start = Date.now();
|
||||
try {
|
||||
if (tpArg === 'embedding') {
|
||||
const v = await embedOne('gbrain smoke test');
|
||||
const ms = Date.now() - start;
|
||||
console.log(` ✓ ${ms}ms, ${v.length} dims`);
|
||||
} else {
|
||||
const result = await gwChat({
|
||||
messages: [{ role: 'user', content: 'Reply with just the word: pong' }],
|
||||
maxTokens: 16,
|
||||
});
|
||||
const ms = Date.now() - start;
|
||||
const preview = (result.text || '<empty>').replace(/\s+/g, ' ').slice(0, 80);
|
||||
console.log(` ✓ ${ms}ms · model=${result.model} · stop=${result.stopReason} · in=${result.usage.input_tokens}/out=${result.usage.output_tokens} · "${preview}"`);
|
||||
}
|
||||
console.log('\nAll probes green.');
|
||||
} catch (e) {
|
||||
const ms = Date.now() - start;
|
||||
if (e instanceof AIConfigError) {
|
||||
console.error(` ✗ config error (${ms}ms): ${e.message}`);
|
||||
if (e.fix) console.error(` Fix: ${e.fix}`);
|
||||
process.exit(2);
|
||||
} else if (e instanceof AITransientError) {
|
||||
console.error(` ✗ transient error (${ms}ms): ${e.message}`);
|
||||
console.error(` Retry after a moment.`);
|
||||
process.exit(3);
|
||||
} else {
|
||||
console.error(` ✗ unknown error (${ms}ms): ${e instanceof Error ? e.message : e}`);
|
||||
process.exit(4);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function runEnv(args: string[]): void {
|
||||
const id = args[0];
|
||||
if (!id) {
|
||||
console.error('Usage: gbrain providers env <id>');
|
||||
process.exit(1);
|
||||
}
|
||||
const recipe = getRecipe(id);
|
||||
if (!recipe) {
|
||||
console.error(`Unknown provider: ${id}. Run \`gbrain providers list\` to see known providers.`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`${recipe.name} (${recipe.id})`);
|
||||
console.log('');
|
||||
const required = recipe.auth_env?.required ?? [];
|
||||
const optional = recipe.auth_env?.optional ?? [];
|
||||
if (required.length > 0) {
|
||||
console.log('Required:');
|
||||
for (const k of required) {
|
||||
const set = !!process.env[k];
|
||||
console.log(` ${k.padEnd(32)} ${set ? '✓ set' : '✗ not set'}`);
|
||||
}
|
||||
} else {
|
||||
console.log('Required: (none)');
|
||||
}
|
||||
if (optional.length > 0) {
|
||||
console.log('\nOptional:');
|
||||
for (const k of optional) {
|
||||
const set = !!process.env[k];
|
||||
console.log(` ${k.padEnd(32)} ${set ? '✓ set' : '✗ not set'}`);
|
||||
}
|
||||
}
|
||||
if (recipe.auth_env?.setup_url) {
|
||||
console.log(`\nSetup: ${recipe.auth_env.setup_url}`);
|
||||
}
|
||||
if (recipe.setup_hint) {
|
||||
console.log(`\n${recipe.setup_hint}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function runExplain(args: string[]): Promise<void> {
|
||||
const asJson = args.includes('--json') || args.includes('-j');
|
||||
|
||||
const recipes = listRecipes();
|
||||
const env_detected = {
|
||||
OPENAI_API_KEY: !!process.env.OPENAI_API_KEY,
|
||||
GOOGLE_GENERATIVE_AI_API_KEY: !!process.env.GOOGLE_GENERATIVE_AI_API_KEY,
|
||||
ANTHROPIC_API_KEY: !!process.env.ANTHROPIC_API_KEY,
|
||||
VOYAGE_API_KEY: !!process.env.VOYAGE_API_KEY,
|
||||
DEEPSEEK_API_KEY: !!process.env.DEEPSEEK_API_KEY,
|
||||
GROQ_API_KEY: !!process.env.GROQ_API_KEY,
|
||||
TOGETHER_API_KEY: !!process.env.TOGETHER_API_KEY,
|
||||
};
|
||||
|
||||
// Parallel probes for local providers (1s timeout each)
|
||||
const [ollama, lmstudio] = await Promise.all([probeOllama(), probeLMStudio()]);
|
||||
|
||||
const options: ProviderOption[] = [];
|
||||
for (const r of recipes) {
|
||||
if (r.touchpoints.embedding && r.touchpoints.embedding.models.length > 0) {
|
||||
const m = r.touchpoints.embedding;
|
||||
options.push({
|
||||
id: `${r.id}:${m.models[0]}`,
|
||||
touchpoint: 'embedding',
|
||||
model: m.models[0],
|
||||
dims: m.default_dims,
|
||||
cost_per_1m_tokens_usd: m.cost_per_1m_tokens_usd,
|
||||
price_last_verified: m.price_last_verified,
|
||||
env_ready: envReady(r) || (r.id === 'ollama' && ollama.models_endpoint_valid === true),
|
||||
tier: r.tier,
|
||||
pros: prosFor(r, 'embedding'),
|
||||
cons: consFor(r),
|
||||
});
|
||||
}
|
||||
if (r.touchpoints.expansion) {
|
||||
const m = r.touchpoints.expansion;
|
||||
options.push({
|
||||
id: `${r.id}:${m.models[0]}`,
|
||||
touchpoint: 'expansion',
|
||||
model: m.models[0],
|
||||
cost_per_1m_tokens_usd: m.cost_per_1m_tokens_usd,
|
||||
price_last_verified: m.price_last_verified,
|
||||
env_ready: envReady(r),
|
||||
tier: r.tier,
|
||||
pros: prosFor(r, 'expansion'),
|
||||
cons: consFor(r),
|
||||
});
|
||||
}
|
||||
if (r.touchpoints.chat && r.touchpoints.chat.models.length > 0) {
|
||||
const m = r.touchpoints.chat;
|
||||
options.push({
|
||||
id: `${r.id}:${m.models[0]}`,
|
||||
touchpoint: 'chat',
|
||||
model: m.models[0],
|
||||
cost_per_1m_input_usd: m.cost_per_1m_input_usd,
|
||||
cost_per_1m_output_usd: m.cost_per_1m_output_usd,
|
||||
price_last_verified: m.price_last_verified,
|
||||
env_ready: envReady(r),
|
||||
tier: r.tier,
|
||||
pros: prosFor(r, 'chat'),
|
||||
cons: consFor(r),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const recommended = pickRecommended(options, env_detected, ollama.models_endpoint_valid === true);
|
||||
|
||||
const matrix = {
|
||||
schema_version: SCHEMA_VERSION,
|
||||
generated_at: new Date().toISOString(),
|
||||
env_detected,
|
||||
local_probes: {
|
||||
ollama: { url: process.env.OLLAMA_BASE_URL ?? 'http://localhost:11434/v1', reachable: ollama.reachable, models_endpoint_valid: ollama.models_endpoint_valid === true },
|
||||
lmstudio: { url: process.env.LMSTUDIO_BASE_URL ?? 'http://localhost:1234/v1', reachable: lmstudio.reachable, models_endpoint_valid: lmstudio.models_endpoint_valid === true },
|
||||
},
|
||||
options,
|
||||
recommended: recommended.id,
|
||||
recommended_reason: recommended.reason,
|
||||
};
|
||||
|
||||
if (asJson) {
|
||||
console.log(JSON.stringify(matrix, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
// Human-readable table
|
||||
console.log(`Provider matrix (schema v${SCHEMA_VERSION}, generated ${matrix.generated_at})`);
|
||||
console.log('');
|
||||
console.log('Environment:');
|
||||
for (const [k, v] of Object.entries(env_detected)) console.log(` ${k.padEnd(32)} ${v ? '✓ set' : '✗ not set'}`);
|
||||
console.log(` Ollama @ ${matrix.local_probes.ollama.url} ${matrix.local_probes.ollama.models_endpoint_valid ? '✓ reachable' : '✗ not detected'}`);
|
||||
console.log('');
|
||||
console.log('Embedding options:');
|
||||
for (const o of options.filter(x => x.touchpoint === 'embedding')) {
|
||||
const cost = o.cost_per_1m_tokens_usd !== undefined ? `$${o.cost_per_1m_tokens_usd}/1M` : '—';
|
||||
const dims = o.dims ? `${o.dims}d` : '—';
|
||||
console.log(` ${o.env_ready ? '✓' : '✗'} ${o.id.padEnd(44)} ${dims.padEnd(8)} ${cost.padEnd(10)} ${o.tier}`);
|
||||
}
|
||||
console.log('');
|
||||
console.log('Expansion options:');
|
||||
for (const o of options.filter(x => x.touchpoint === 'expansion')) {
|
||||
const cost = o.cost_per_1m_tokens_usd !== undefined ? `$${o.cost_per_1m_tokens_usd}/1M` : '—';
|
||||
console.log(` ${o.env_ready ? '✓' : '✗'} ${o.id.padEnd(44)} ${cost.padEnd(10)} ${o.tier}`);
|
||||
}
|
||||
console.log('');
|
||||
console.log('Chat options:');
|
||||
for (const o of options.filter(x => x.touchpoint === 'chat')) {
|
||||
const inCost = o.cost_per_1m_input_usd !== undefined ? `in $${o.cost_per_1m_input_usd}` : '—';
|
||||
const outCost = o.cost_per_1m_output_usd !== undefined ? `out $${o.cost_per_1m_output_usd}` : '—';
|
||||
console.log(` ${o.env_ready ? '✓' : '✗'} ${o.id.padEnd(44)} ${inCost.padEnd(12)} ${outCost.padEnd(12)} ${o.tier}`);
|
||||
}
|
||||
console.log('');
|
||||
console.log(`Recommended: ${matrix.recommended}`);
|
||||
console.log(` ${matrix.recommended_reason}`);
|
||||
console.log('');
|
||||
console.log('Re-invoke:');
|
||||
console.log(` gbrain init --embedding-model ${matrix.recommended.split(':')[0]}:${matrix.recommended.split(':').slice(1).join(':')}`);
|
||||
}
|
||||
|
||||
function prosFor(r: Recipe, touchpoint: TouchpointFilter): string[] {
|
||||
const out: string[] = [];
|
||||
if (touchpoint === 'chat') {
|
||||
if (r.id === 'anthropic') out.push('Default subagent driver', 'Prompt-cache support', 'Strong tool calling');
|
||||
else if (r.id === 'openai') out.push('Strong tool calling', 'Wide adapter support');
|
||||
else if (r.id === 'google') out.push('1M context', 'Cheap');
|
||||
else if (r.id === 'deepseek') out.push('25-40x cheaper than Anthropic', 'Strong reasoning');
|
||||
else if (r.id === 'groq') out.push('500 tok/s inference', 'Cheap fallback');
|
||||
else if (r.id === 'together') out.push('Open-weights house', 'Llama / Qwen / Mixtral');
|
||||
return out;
|
||||
}
|
||||
if (r.id === 'openai') out.push('Default', 'High quality', 'Wide compatibility');
|
||||
else if (r.id === 'google') out.push('Smaller vectors', 'Matryoshka dim flex');
|
||||
else if (r.id === 'anthropic') out.push('Default expansion model', 'Best-in-class reasoning');
|
||||
else if (r.id === 'ollama') out.push('Local', 'Free', 'Private');
|
||||
else if (r.id === 'voyage') out.push('Best rerank pairing');
|
||||
else if (r.id === 'litellm') out.push('Universal coverage (Bedrock/Vertex/Azure/any)');
|
||||
return out;
|
||||
}
|
||||
|
||||
function consFor(r: Recipe): string[] {
|
||||
const out: string[] = [];
|
||||
if (r.tier === 'native' && r.id !== 'ollama') out.push('Paid');
|
||||
if (r.id === 'ollama') out.push('Requires Ollama daemon running');
|
||||
if (r.id === 'litellm') out.push('Requires LiteLLM proxy + config');
|
||||
return out;
|
||||
}
|
||||
|
||||
function pickRecommended(options: ProviderOption[], env: Record<string, boolean>, ollamaReady: boolean): { id: string; reason: string } {
|
||||
// Embedding recommendation: prefer env-ready native providers in this order.
|
||||
const embOpts = options.filter(o => o.touchpoint === 'embedding');
|
||||
if (env.OPENAI_API_KEY) {
|
||||
const openai = embOpts.find(o => o.id.startsWith('openai:'));
|
||||
if (openai) return { id: openai.id, reason: 'OPENAI_API_KEY set — OpenAI default is high-quality and preserves existing 1536-dim schema.' };
|
||||
}
|
||||
if (ollamaReady) {
|
||||
const ollama = embOpts.find(o => o.id.startsWith('ollama:'));
|
||||
if (ollama) return { id: ollama.id, reason: 'Ollama detected locally — zero cost + private.' };
|
||||
}
|
||||
if (env.GOOGLE_GENERATIVE_AI_API_KEY) {
|
||||
const google = embOpts.find(o => o.id.startsWith('google:'));
|
||||
if (google) return { id: google.id, reason: 'GOOGLE_GENERATIVE_AI_API_KEY set — Gemini embedding at 768 dims.' };
|
||||
}
|
||||
if (env.VOYAGE_API_KEY) {
|
||||
const voyage = embOpts.find(o => o.id.startsWith('voyage:'));
|
||||
if (voyage) return { id: voyage.id, reason: 'VOYAGE_API_KEY set — Voyage at 1024 dims.' };
|
||||
}
|
||||
// Nothing ready. Recommend OpenAI as the lowest-friction path.
|
||||
return {
|
||||
id: 'openai:text-embedding-3-large',
|
||||
reason: 'No provider env detected. OpenAI is the fastest setup — get a key at https://platform.openai.com/api-keys.',
|
||||
};
|
||||
}
|
||||
+189
-49
@@ -25,10 +25,74 @@ import type { BrainEngine } from '../core/engine.ts';
|
||||
import { operations, OperationError } from '../core/operations.ts';
|
||||
import type { OperationContext, AuthInfo } from '../core/operations.ts';
|
||||
import { GBrainOAuthProvider } from '../core/oauth-provider.ts';
|
||||
import type { SqlQuery } from '../core/oauth-provider.ts';
|
||||
import { hasScope, ALLOWED_SCOPES_LIST } from '../core/scope.ts';
|
||||
import { summarizeMcpParams } from '../mcp/dispatch.ts';
|
||||
import { loadConfig } from '../core/config.ts';
|
||||
import { buildError, serializeError } from '../core/errors.ts';
|
||||
import { VERSION } from '../version.ts';
|
||||
import * as db from '../core/db.ts';
|
||||
|
||||
/**
|
||||
* /health endpoint timeout. 3s rather than 5s: Fly.io's default
|
||||
* health-check timeout is 5s, so returning 503 right at the orchestrator
|
||||
* deadline races with the orchestrator recording the request as a timeout.
|
||||
* 3s leaves 2s of headroom for TCP, response framing, and clock skew.
|
||||
*/
|
||||
export const HEALTH_TIMEOUT_MS = 3000;
|
||||
|
||||
export type ProbeHealthResult =
|
||||
| { ok: true; status: 200; body: { status: 'ok'; version: string; engine: string; [k: string]: unknown } }
|
||||
| { ok: false; status: 503; body: { error: 'service_unavailable'; error_description: string } };
|
||||
|
||||
/**
|
||||
* Pure async health probe. Races `engine.getStats()` against a timeout,
|
||||
* returns a tagged result. No Express coupling — easy to unit-test with a
|
||||
* mock engine. The /health route handler is a thin wrapper around this.
|
||||
*/
|
||||
export async function probeHealth(
|
||||
engine: BrainEngine,
|
||||
engineName: string,
|
||||
version: string,
|
||||
timeoutMs: number = HEALTH_TIMEOUT_MS,
|
||||
): Promise<ProbeHealthResult> {
|
||||
// Capture the handle so we can clearTimeout when getStats() wins. Without
|
||||
// this, every fast /health request leaves a 3s pending timer in the event
|
||||
// loop until it fires — under high probe rates this builds up a rolling
|
||||
// backlog of timers and avoidable wakeups. Both adversarial reviewers
|
||||
// (Claude + Codex) flagged this independently.
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
try {
|
||||
const stats = await Promise.race([
|
||||
engine.getStats(),
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error('health_timeout')), timeoutMs);
|
||||
}),
|
||||
]);
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
body: { status: 'ok', version, engine: engineName, ...stats },
|
||||
};
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : 'unknown';
|
||||
return {
|
||||
ok: false,
|
||||
status: 503,
|
||||
body: {
|
||||
error: 'service_unavailable',
|
||||
error_description: msg === 'health_timeout'
|
||||
? 'Health check timed out (database pool may be saturated)'
|
||||
: 'Database connection failed',
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
// Clear the timer regardless of which branch won the race. No-op when
|
||||
// the timer already fired (we're in the timeout-rejection catch block).
|
||||
if (timer !== null) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
interface ServeHttpOptions {
|
||||
port: number;
|
||||
tokenTtl: number;
|
||||
@@ -41,19 +105,39 @@ interface ServeHttpOptions {
|
||||
* issuer claim in tokens MUST match the discovery URL clients hit.
|
||||
*/
|
||||
publicUrl?: string;
|
||||
/**
|
||||
* When true, write raw request payloads to mcp_request_log + the admin SSE
|
||||
* feed. Default false: payloads are summarized via dispatch.summarizeMcpParams
|
||||
* (declared keys only, no values, no attacker-controlled key names).
|
||||
*
|
||||
* Operators running gbrain on their own laptop and debugging agent behavior
|
||||
* can flip this on with `--log-full-params`. The flag prints a loud warning
|
||||
* at startup so the privacy posture change is visible.
|
||||
*/
|
||||
logFullParams?: boolean;
|
||||
}
|
||||
|
||||
export async function runServeHttp(engine: BrainEngine, options: ServeHttpOptions) {
|
||||
const { port, tokenTtl, enableDcr, publicUrl } = options;
|
||||
const { port, tokenTtl, enableDcr, publicUrl, logFullParams } = options;
|
||||
const config = loadConfig() || { engine: 'pglite' as const };
|
||||
|
||||
// Get raw SQL connection for OAuth provider
|
||||
const sql = db.getConnection();
|
||||
if (logFullParams) {
|
||||
console.error(
|
||||
'[serve-http] WARNING: --log-full-params writes raw request payloads to mcp_request_log + SSE feed. Disable for shared dashboards or production.',
|
||||
);
|
||||
}
|
||||
|
||||
// Initialize OAuth provider
|
||||
// Get raw SQL connection for OAuth provider
|
||||
const sql = db.getConnection() as SqlQuery;
|
||||
|
||||
// Initialize OAuth provider. F12 cleanup: DCR-disable now flips a
|
||||
// constructor option instead of monkey-patching `_clientsStore` after
|
||||
// construction. Same outcome (no /register endpoint when --enable-dcr
|
||||
// is not passed); cleaner shape for tests and future maintainers.
|
||||
const oauthProvider = new GBrainOAuthProvider({
|
||||
sql: sql as any,
|
||||
sql,
|
||||
tokenTtl,
|
||||
dcrDisabled: !enableDcr,
|
||||
});
|
||||
|
||||
// Sweep expired tokens on startup (non-blocking)
|
||||
@@ -152,22 +236,35 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// reverse proxies / tunnels; default to localhost for dev.
|
||||
const issuerUrl = new URL(publicUrl || `http://localhost:${port}`);
|
||||
|
||||
// F9: cookie `secure` flag honors both the request's TLS state (req.secure
|
||||
// is set when express trust-proxy lands an X-Forwarded-Proto: https) AND
|
||||
// the operator's declared issuer protocol (so a Cloudflare-tunnel deploy
|
||||
// where the connection inside the tunnel looks like http but the public
|
||||
// URL is https still tags cookies Secure). Without this, an attacker on
|
||||
// the network path could MITM the admin cookie over plaintext.
|
||||
const adminCookie = (req: Request, maxAge: number) => ({
|
||||
httpOnly: true,
|
||||
sameSite: 'strict' as const,
|
||||
secure: req.secure || issuerUrl.protocol === 'https:',
|
||||
maxAge,
|
||||
path: '/admin',
|
||||
});
|
||||
|
||||
const authRouterOptions: any = {
|
||||
provider: oauthProvider,
|
||||
issuerUrl,
|
||||
scopesSupported: ['read', 'write', 'admin'],
|
||||
// v0.28: scopesSupported sourced from ALLOWED_SCOPES_LIST so MCP clients
|
||||
// (Claude Desktop, ChatGPT, Perplexity) can discover sources_admin and
|
||||
// users_admin via /.well-known/oauth-authorization-server. The legacy
|
||||
// ['read','write','admin'] list left those new scopes invisible.
|
||||
scopesSupported: [...ALLOWED_SCOPES_LIST],
|
||||
resourceName: 'GBrain MCP Server',
|
||||
};
|
||||
|
||||
// Disable DCR by removing registerClient from the clients store
|
||||
if (!enableDcr) {
|
||||
// Override the provider's clientsStore to remove registerClient
|
||||
const originalStore = oauthProvider.clientsStore;
|
||||
(oauthProvider as any)._clientsStore = {
|
||||
getClient: originalStore.getClient.bind(originalStore),
|
||||
// No registerClient = DCR disabled
|
||||
};
|
||||
}
|
||||
// F12: DCR disable lives on the provider's constructor option above. The
|
||||
// SDK's mcpAuthRouter reads provider.clientsStore once and only wires up
|
||||
// /register when the store exposes registerClient — so passing dcrDisabled
|
||||
// to the constructor is sufficient. No monkey-patching here.
|
||||
|
||||
const authRouter = mcpAuthRouter(authRouterOptions);
|
||||
|
||||
@@ -193,12 +290,8 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// Health check
|
||||
// ---------------------------------------------------------------------------
|
||||
app.get('/health', async (_req, res) => {
|
||||
try {
|
||||
const stats = await engine.getStats();
|
||||
res.json({ status: 'ok', version: VERSION, engine: config.engine, ...stats });
|
||||
} catch {
|
||||
res.status(503).json({ error: 'service_unavailable', error_description: 'Database connection failed' });
|
||||
}
|
||||
const result = await probeHealth(engine, config.engine || 'pglite', VERSION);
|
||||
res.status(result.status).json(result.body);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -232,12 +325,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
const expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 24 hours
|
||||
adminSessions.set(sessionId, expiresAt);
|
||||
|
||||
res.cookie('gbrain_admin', sessionId, {
|
||||
httpOnly: true,
|
||||
sameSite: 'strict',
|
||||
maxAge: 24 * 60 * 60 * 1000,
|
||||
path: '/admin',
|
||||
});
|
||||
res.cookie('gbrain_admin', sessionId, adminCookie(req, 24 * 60 * 60 * 1000));
|
||||
res.json({ status: 'authenticated' });
|
||||
});
|
||||
|
||||
@@ -271,6 +359,15 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
for (const [nonce, expiresAt] of magicLinkNonces) {
|
||||
if (expiresAt < now) magicLinkNonces.delete(nonce);
|
||||
}
|
||||
// F10: bound the live-nonce store too. An attacker with the bootstrap
|
||||
// token (or a misbehaving agent) could mint nonces faster than they
|
||||
// expire. Map iteration order is insertion order, so dropping from the
|
||||
// front gives a simple FIFO eviction matching the consumedNonces pattern.
|
||||
if (magicLinkNonces.size > NONCE_LRU_CAP) {
|
||||
const drop = magicLinkNonces.size - NONCE_LRU_CAP;
|
||||
const it = magicLinkNonces.keys();
|
||||
for (let i = 0; i < drop; i++) magicLinkNonces.delete(it.next().value as string);
|
||||
}
|
||||
// Cap consumedNonces growth — drop oldest entries past the LRU cap.
|
||||
if (consumedNonces.size > NONCE_LRU_CAP) {
|
||||
const drop = consumedNonces.size - NONCE_LRU_CAP;
|
||||
@@ -339,12 +436,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
const sessionExpiresAt = Date.now() + 7 * 24 * 60 * 60 * 1000; // 7 days for magic link
|
||||
adminSessions.set(sessionId, sessionExpiresAt);
|
||||
|
||||
res.cookie('gbrain_admin', sessionId, {
|
||||
httpOnly: true,
|
||||
sameSite: 'strict',
|
||||
maxAge: 7 * 24 * 60 * 60 * 1000,
|
||||
path: '/admin',
|
||||
});
|
||||
res.cookie('gbrain_admin', sessionId, adminCookie(req, 7 * 24 * 60 * 60 * 1000));
|
||||
res.redirect('/admin/');
|
||||
});
|
||||
|
||||
@@ -638,9 +730,13 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ error: 'unknown_operation', message: `Unknown: ${name}` }) }] };
|
||||
}
|
||||
|
||||
// Scope enforcement
|
||||
// Scope enforcement (v0.28: hasScope replaces exact-string-match so
|
||||
// admin tokens satisfy any scope, write satisfies read, and the new
|
||||
// sources_admin / users_admin scopes resolve through the same
|
||||
// hierarchy. Plain string includes() at this site would have made
|
||||
// sources_admin tokens look like they couldn't even read.)
|
||||
const requiredScope = op.scope || 'read';
|
||||
if (!authInfo.scopes.includes(requiredScope)) {
|
||||
if (!hasScope(authInfo.scopes, requiredScope)) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text',
|
||||
@@ -663,15 +759,31 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
error: (msg: string) => console.error(`[ERROR] ${msg}`),
|
||||
},
|
||||
dryRun: !!(params?.dry_run),
|
||||
// F7: HTTP MCP is the untrusted/agent-facing transport. Stdio MCP at
|
||||
// src/mcp/dispatch.ts:61 sets this; the inlined HTTP context-builder
|
||||
// forgot it for several releases, which let HTTP MCP callers with a
|
||||
// read+write token submit `shell` jobs and execute arbitrary commands
|
||||
// on the host (RCE). The fail-closed contract in operations.ts is the
|
||||
// belt; this is the suspenders.
|
||||
remote: true,
|
||||
auth: authInfo,
|
||||
};
|
||||
|
||||
// F8: redact request payload by default (declared keys only via the
|
||||
// op's `params` allow-list; values + attacker-controlled key names
|
||||
// never written to mcp_request_log or the SSE feed). --log-full-params
|
||||
// bypasses this for operators debugging on their own laptop, with the
|
||||
// startup warning printed earlier.
|
||||
const safeParamsSummary = summarizeMcpParams(name, params);
|
||||
const logParams = logFullParams
|
||||
? (params ? JSON.stringify(params) : null)
|
||||
: (safeParamsSummary ? JSON.stringify(safeParamsSummary) : null);
|
||||
const broadcastParams = logFullParams ? (params || {}) : safeParamsSummary;
|
||||
|
||||
try {
|
||||
const result = await op.handler(ctx, (params || {}) as Record<string, unknown>);
|
||||
const latency = Date.now() - startTime;
|
||||
|
||||
// Log request + broadcast to SSE
|
||||
const logParams = params ? JSON.stringify(params) : null;
|
||||
try {
|
||||
await sql`INSERT INTO mcp_request_log (token_name, agent_name, operation, latency_ms, status, params)
|
||||
VALUES (${authInfo.clientId}, ${agentName}, ${name}, ${latency}, ${'success'}, ${logParams})`;
|
||||
@@ -680,7 +792,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
broadcastEvent({
|
||||
agent: agentName,
|
||||
operation: name,
|
||||
params: params || {},
|
||||
params: broadcastParams,
|
||||
scopes: authInfo.scopes.join(','),
|
||||
latency_ms: latency,
|
||||
status: 'success',
|
||||
@@ -690,10 +802,22 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
return { content: [{ type: 'text', text: JSON.stringify(result) }] };
|
||||
} catch (e) {
|
||||
const latency = Date.now() - startTime;
|
||||
const error = e instanceof OperationError ? e.toJSON() : { error: 'internal_error', message: e instanceof Error ? e.message : 'Unknown error' };
|
||||
|
||||
const errMsg = e instanceof Error ? e.message : 'Unknown error';
|
||||
const logParams = params ? JSON.stringify(params) : null;
|
||||
// F15: unify error envelope. Both OperationError and unexpected
|
||||
// exceptions go through src/core/errors.ts so clients see a single
|
||||
// shape ({class, code, message, hint}). Pre-fix, OperationError
|
||||
// serialized via e.toJSON() and other exceptions used a hand-rolled
|
||||
// {error, message} envelope — a client couldn't pattern-match
|
||||
// reliably across the two.
|
||||
const errorPayload = e instanceof OperationError
|
||||
? buildError({
|
||||
class: 'OperationError',
|
||||
code: e.code,
|
||||
message: e.message,
|
||||
hint: e.suggestion,
|
||||
docs_url: e.docs,
|
||||
})
|
||||
: serializeError(e);
|
||||
const errMsg = errorPayload.message;
|
||||
try {
|
||||
await sql`INSERT INTO mcp_request_log (token_name, agent_name, operation, latency_ms, status, params, error_message)
|
||||
VALUES (${authInfo.clientId}, ${agentName}, ${name}, ${latency}, ${'error'}, ${logParams}, ${errMsg})`;
|
||||
@@ -702,21 +826,37 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
broadcastEvent({
|
||||
agent: agentName,
|
||||
operation: name,
|
||||
params: params || {},
|
||||
params: broadcastParams,
|
||||
scopes: authInfo.scopes.join(','),
|
||||
latency_ms: latency,
|
||||
status: 'error',
|
||||
error: errMsg,
|
||||
error: errorPayload,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
|
||||
return { content: [{ type: 'text', text: JSON.stringify(error) }], isError: true };
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ error: errorPayload }) }], isError: true };
|
||||
}
|
||||
});
|
||||
|
||||
// Use StreamableHTTPServerTransport for stateless request handling
|
||||
const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined as any });
|
||||
await server.connect(transport);
|
||||
await transport.handleRequest(req, res, req.body);
|
||||
// F14: wrap transport setup + handleRequest in try/catch. Without this,
|
||||
// an SDK-level throw (e.g., schema parse failure on a malformed request)
|
||||
// propagates to express's default error handler, which renders an HTML
|
||||
// error page — clients expecting JSON-RPC envelopes break. On
|
||||
// !res.headersSent we emit a minimal JSON 500 so the client at least
|
||||
// gets parseable JSON back.
|
||||
try {
|
||||
const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined as any });
|
||||
await server.connect(transport);
|
||||
await transport.handleRequest(req, res, req.body);
|
||||
} catch (e) {
|
||||
console.error('MCP request handler error:', e instanceof Error ? e.message : e);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({
|
||||
error: 'internal_error',
|
||||
message: e instanceof Error ? e.message : 'Unknown error',
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -22,8 +22,15 @@ export async function runServe(engine: BrainEngine, args: string[] = []) {
|
||||
const publicUrlIdx = args.indexOf('--public-url');
|
||||
const publicUrl = publicUrlIdx >= 0 ? args[publicUrlIdx + 1] : undefined;
|
||||
|
||||
// F8 escape hatch: --log-full-params writes raw payloads to mcp_request_log
|
||||
// and the admin SSE feed instead of redacted summaries. Off by default
|
||||
// (privacy-first); operators running gbrain on their own laptop can flip
|
||||
// it on for debug visibility. Loud startup warning fires in serve-http.ts
|
||||
// when set so the posture change is visible in stderr.
|
||||
const logFullParams = args.includes('--log-full-params');
|
||||
|
||||
const { runServeHttp } = await import('./serve-http.ts');
|
||||
await runServeHttp(engine, { port, tokenTtl, enableDcr, publicUrl });
|
||||
await runServeHttp(engine, { port, tokenTtl, enableDcr, publicUrl, logFullParams });
|
||||
} else {
|
||||
console.error('Starting GBrain MCP server (stdio)...');
|
||||
await startMcpServer(engine);
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
/**
|
||||
* gbrain skillify check — 10-item post-task audit.
|
||||
* gbrain skillify check — 11-item post-task audit.
|
||||
*
|
||||
* Promoted from `scripts/skillify-check.ts` (D-CX-2). The legacy
|
||||
* script stays as a thin shim so existing callers keep working, but
|
||||
* the CLI entry point is now `gbrain skillify check`.
|
||||
*
|
||||
* 10-item checklist (essay Step 3-10):
|
||||
* 11-item checklist (essay Step 3-10 + v0.27.x cross-modal eval):
|
||||
* 1. SKILL.md exists
|
||||
* 2. Code file exists at target path
|
||||
* 3. Unit tests exist
|
||||
@@ -16,12 +16,24 @@
|
||||
* 8. check-resolvable gate (runs `gbrain check-resolvable --json`)
|
||||
* 9. E2E smoke (required copy of #4 for required-gate semantics)
|
||||
* 10. Brain filing (only when the script writes pages)
|
||||
* 11. Cross-modal eval (INFORMATIONAL; required:false). Looks for a
|
||||
* receipt at `gbrainPath('eval-receipts')/<slug>-<sha8>.json`
|
||||
* bound to the current SKILL.md content hash (T10=A,T7=C in
|
||||
* plans/radiant-napping-lerdorf.md). A missing or stale receipt
|
||||
* surfaces as a non-blocking note, not a failure.
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync, readdirSync, statSync } from 'fs';
|
||||
import { basename, dirname, join, resolve } from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
import { gbrainPath } from '../core/config.ts';
|
||||
import {
|
||||
describeReceiptStatus,
|
||||
findReceiptForSkill,
|
||||
type ReceiptStatus,
|
||||
} from '../core/cross-modal-eval/receipt-name.ts';
|
||||
|
||||
interface CheckItem {
|
||||
name: string;
|
||||
passed: boolean;
|
||||
@@ -259,6 +271,18 @@ function runSkillifyCheckTarget(target: string, root: string): CheckResult {
|
||||
),
|
||||
);
|
||||
|
||||
// Item 11: cross-modal eval (informational, T7=C). The receipt is bound
|
||||
// to (slug, sha8 of SKILL.md). The audit doesn't fail on a missing or
|
||||
// stale receipt — it just surfaces the status.
|
||||
const crossModalReceipt = lookupCrossModalReceipt(skillMd, skillName);
|
||||
items.push(
|
||||
checkOptional(
|
||||
'Cross-modal eval (informational)',
|
||||
crossModalReceipt.passed,
|
||||
crossModalReceipt.detail,
|
||||
),
|
||||
);
|
||||
|
||||
const passed = items.filter(i => i.passed).length;
|
||||
const total = items.length;
|
||||
const missing = items.filter(i => !i.passed && i.required).map(i => i.name);
|
||||
@@ -275,6 +299,46 @@ function runSkillifyCheckTarget(target: string, root: string): CheckResult {
|
||||
return { path: target, skillName, items, score: passed, total, recommendation };
|
||||
}
|
||||
|
||||
/**
|
||||
* Item 11 helper: look up the cross-modal eval receipt for this skill.
|
||||
* `passed` is true when a current-sha receipt exists. Stale or missing
|
||||
* receipts return passed=true *for the audit* — item 11 is informational
|
||||
* (T7=C) — but the detail string makes the status visible.
|
||||
*
|
||||
* Reads the receipt from `gbrainPath('eval-receipts')` (T5 correction:
|
||||
* this resolves to <GBRAIN_HOME>/.gbrain/eval-receipts/, NOT the legacy
|
||||
* <GBRAIN_HOME>/eval-receipts/ that the original plan claimed).
|
||||
*/
|
||||
function lookupCrossModalReceipt(
|
||||
skillMdPath: string,
|
||||
skillName: string,
|
||||
): { passed: boolean; detail: string } {
|
||||
if (!existsSync(skillMdPath)) {
|
||||
return { passed: true, detail: 'no SKILL.md — skipping cross-modal eval check' };
|
||||
}
|
||||
let status: ReceiptStatus;
|
||||
try {
|
||||
status = findReceiptForSkill(skillMdPath, gbrainPath('eval-receipts'));
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return { passed: true, detail: `receipt lookup failed: ${msg}` };
|
||||
}
|
||||
switch (status.status) {
|
||||
case 'found':
|
||||
return { passed: true, detail: describeReceiptStatus(skillName, status) };
|
||||
case 'stale':
|
||||
return {
|
||||
passed: false, // visually marked as not-yet-rerun but item is required:false
|
||||
detail: describeReceiptStatus(skillName, status),
|
||||
};
|
||||
case 'missing':
|
||||
return {
|
||||
passed: false,
|
||||
detail: describeReceiptStatus(skillName, status),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function recentlyModified(root: string, days: number = 7): string[] {
|
||||
const candidates: string[] = [];
|
||||
const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
|
||||
|
||||
@@ -2,15 +2,17 @@
|
||||
* gbrain skillify <scaffold|check> — W4 CLI namespace.
|
||||
*
|
||||
* `scaffold`: creates 5 stub files for a new skill. Mechanical only.
|
||||
* `check`: 10-item audit of an existing skill. Promoted from
|
||||
* `scripts/skillify-check.ts` (D-CX-2). The legacy script
|
||||
* remains as a thin shim that invokes this subcommand.
|
||||
* `check`: 11-item audit of an existing skill (item 11, cross-modal
|
||||
* eval, is informational; T7=C in plans/radiant-napping-lerdorf.md).
|
||||
* Promoted from `scripts/skillify-check.ts` (D-CX-2). The
|
||||
* legacy script remains as a thin shim that invokes this
|
||||
* subcommand.
|
||||
*
|
||||
* The markdown skill at `skills/skillify/SKILL.md` orchestrates the
|
||||
* full 10-step loop (essay's "skillify it!"): scaffold → fill in the
|
||||
* body → run check → run check-resolvable → run tests → commit.
|
||||
* The CLI primitives do the mechanical steps; the skill carries the
|
||||
* judgment steps.
|
||||
* full 11-step loop (essay's "skillify it!"): scaffold → fill in the
|
||||
* body → run cross-modal eval → run check → run check-resolvable →
|
||||
* run tests → commit. The CLI primitives do the mechanical steps;
|
||||
* the skill carries the judgment steps.
|
||||
*/
|
||||
|
||||
import { isAbsolute, resolve as resolvePath } from 'path';
|
||||
|
||||
+228
-53
@@ -26,6 +26,23 @@
|
||||
import { writeFileSync, unlinkSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import {
|
||||
assessDestructiveImpact,
|
||||
checkDestructiveConfirmation,
|
||||
softDeleteSource,
|
||||
restoreSource,
|
||||
listArchivedSources,
|
||||
purgeExpiredSources,
|
||||
formatImpact,
|
||||
formatSoftDelete,
|
||||
SOFT_DELETE_TTL_HOURS,
|
||||
} from '../core/destructive-guard.ts';
|
||||
import {
|
||||
addSource as opsAddSource,
|
||||
recloneIfMissing,
|
||||
SourceOpError,
|
||||
type SourceRow as OpsSourceRow,
|
||||
} from '../core/sources-ops.ts';
|
||||
|
||||
// ── Validation ──────────────────────────────────────────────
|
||||
|
||||
@@ -98,62 +115,64 @@ async function countPages(engine: BrainEngine, sourceId: string): Promise<number
|
||||
async function runAdd(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const id = args[0];
|
||||
if (!id) {
|
||||
console.error('Usage: gbrain sources add <id> --path <path> [--name <display>] [--federated|--no-federated]');
|
||||
console.error(
|
||||
'Usage: gbrain sources add <id> [--path <path> | --url <https-url>] ' +
|
||||
'[--name <display>] [--federated|--no-federated] [--clone-dir <path>]',
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
validateSourceId(id);
|
||||
|
||||
let localPath: string | null = null;
|
||||
let displayName = id;
|
||||
let federated: boolean | null = null; // null = default (false for new, opt-in via --federated)
|
||||
let remoteUrl: string | undefined;
|
||||
let displayName: string | undefined;
|
||||
let federated: boolean | null = null;
|
||||
let cloneDir: string | undefined;
|
||||
|
||||
for (let i = 1; i < args.length; i++) {
|
||||
const a = args[i];
|
||||
if (a === '--path') { localPath = args[++i]; continue; }
|
||||
if (a === '--url') { remoteUrl = args[++i]; continue; }
|
||||
if (a === '--name') { displayName = args[++i]; continue; }
|
||||
if (a === '--federated') { federated = true; continue; }
|
||||
if (a === '--no-federated') { federated = false; continue; }
|
||||
if (a === '--clone-dir') { cloneDir = args[++i]; continue; }
|
||||
console.error(`Unknown flag: ${a}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// Overlapping path guard: reject if new path is inside or contains an
|
||||
// existing source's local_path (per eng review §4 finding 4.1).
|
||||
// Throwing (vs process.exit) keeps this testable via the standard
|
||||
// CLI error-handling wrapper in src/cli.ts.
|
||||
if (localPath) {
|
||||
const others = await engine.executeRaw<{ id: string; local_path: string }>(
|
||||
`SELECT id, local_path FROM sources WHERE local_path IS NOT NULL AND id != $1`,
|
||||
[id],
|
||||
);
|
||||
for (const other of others) {
|
||||
const a = localPath;
|
||||
const b = other.local_path;
|
||||
if (a === b || a.startsWith(b + '/') || b.startsWith(a + '/')) {
|
||||
throw new Error(
|
||||
`path "${a}" overlaps with existing source "${other.id}" at "${b}". ` +
|
||||
`Overlapping sources are not allowed — same files would ingest twice under different source_ids.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (remoteUrl && localPath) {
|
||||
console.error('Error: --url and --path are mutually exclusive (--url manages its own clone path).');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const config = federated === null ? {} : { federated };
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, local_path, config)
|
||||
VALUES ($1, $2, $3, $4::jsonb)
|
||||
ON CONFLICT (id) DO NOTHING`,
|
||||
[id, displayName, localPath, JSON.stringify(config)],
|
||||
);
|
||||
// Throw on SourceOpError; cli.ts wraps every command in a try/catch that
|
||||
// turns Error into the right exit code. Tests assert throw shape, so we
|
||||
// intentionally propagate rather than process.exit here.
|
||||
const created: OpsSourceRow = await opsAddSource(engine, {
|
||||
id,
|
||||
name: displayName,
|
||||
localPath,
|
||||
remoteUrl,
|
||||
federated,
|
||||
cloneDir,
|
||||
});
|
||||
|
||||
const created = await fetchSource(engine, id);
|
||||
if (!created) {
|
||||
console.error(`Failed to create source "${id}" (conflict with existing id?)`);
|
||||
process.exit(4);
|
||||
}
|
||||
const fed = isFederated(created.config);
|
||||
console.log(`Created source "${id}"${displayName !== id ? ` (name: ${displayName})` : ''}${localPath ? ` → ${localPath}` : ''}`);
|
||||
console.log(` federated: ${fed}${fed ? ' — appears in cross-source default search' : ' — only searched when explicitly named via --source'}`);
|
||||
const finalRemoteUrl = (created.config as Record<string, unknown>).remote_url as string | undefined;
|
||||
const tail = finalRemoteUrl
|
||||
? ` ← cloned from ${finalRemoteUrl}`
|
||||
: created.local_path
|
||||
? ` → ${created.local_path}`
|
||||
: '';
|
||||
console.log(
|
||||
`Created source "${id}"${displayName && displayName !== id ? ` (name: ${displayName})` : ''}${tail}`,
|
||||
);
|
||||
if (finalRemoteUrl) {
|
||||
console.log(` clone path: ${created.local_path}`);
|
||||
}
|
||||
console.log(
|
||||
` federated: ${fed}${fed ? ' — appears in cross-source default search' : ' — only searched when explicitly named via --source'}`,
|
||||
);
|
||||
}
|
||||
|
||||
// ── Subcommand: list ────────────────────────────────────────
|
||||
@@ -188,10 +207,10 @@ async function runList(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
console.log('SOURCES');
|
||||
console.log('───────');
|
||||
for (const e of entries) {
|
||||
const fedMark = e.federated ? 'federated' : 'isolated';
|
||||
const fedMark = e.federated ? 'federated' : (e as any).archived ? '⚠ archived' : 'isolated';
|
||||
const pathStr = e.local_path ?? '(no local path)';
|
||||
const sync = e.last_sync_at ? `last sync ${e.last_sync_at}` : 'never synced';
|
||||
console.log(` ${e.id.padEnd(20)} ${fedMark.padEnd(10)} ${String(e.page_count).padStart(6)} pages ${sync}`);
|
||||
console.log(` ${e.id.padEnd(20)} ${fedMark.padEnd(12)} ${String(e.page_count).padStart(6)} pages ${sync}`);
|
||||
if (e.local_path) console.log(` ${' '.repeat(22)}${pathStr}`);
|
||||
}
|
||||
if (entries.length === 0) console.log(' (no sources registered)');
|
||||
@@ -202,13 +221,12 @@ async function runList(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
async function runRemove(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const id = args[0];
|
||||
if (!id) {
|
||||
console.error('Usage: gbrain sources remove <id> [--yes] [--dry-run] [--keep-storage]');
|
||||
console.error('Usage: gbrain sources remove <id> [--yes] [--confirm-destructive] [--dry-run] [--keep-storage]');
|
||||
process.exit(2);
|
||||
}
|
||||
const yes = args.includes('--yes');
|
||||
const dryRun = args.includes('--dry-run');
|
||||
// NOTE: --keep-storage is accepted for forward compatibility but has no
|
||||
// effect until Step 7 wires in explicit storage object deletion.
|
||||
const confirmDestructive = args.includes('--confirm-destructive');
|
||||
const _keepStorage = args.includes('--keep-storage');
|
||||
void _keepStorage;
|
||||
|
||||
@@ -223,23 +241,162 @@ async function runRemove(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
process.exit(4);
|
||||
}
|
||||
|
||||
const pageCount = await countPages(engine, id);
|
||||
console.log(`Source "${id}" → ${pageCount} pages will be deleted (cascade).`);
|
||||
// v0.26.5: Impact preview + destructive guard
|
||||
const impact = await assessDestructiveImpact(engine, id);
|
||||
if (impact) {
|
||||
console.log(formatImpact(impact));
|
||||
|
||||
if (dryRun) {
|
||||
console.log(`(dry-run; no side effects)`);
|
||||
return;
|
||||
}
|
||||
if (dryRun) {
|
||||
console.log('(dry-run; no side effects)');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!yes) {
|
||||
console.error(`Refusing to remove without --yes. Pass --yes to confirm.`);
|
||||
process.exit(5);
|
||||
const blockMsg = checkDestructiveConfirmation(impact, { yes, confirmDestructive, dryRun });
|
||||
if (blockMsg) {
|
||||
console.error(blockMsg);
|
||||
process.exit(5);
|
||||
}
|
||||
} else {
|
||||
if (dryRun) { console.log('(dry-run; source not found)'); return; }
|
||||
if (!yes && !confirmDestructive) {
|
||||
console.error('Refusing to remove without --yes or --confirm-destructive.');
|
||||
process.exit(5);
|
||||
}
|
||||
}
|
||||
|
||||
await engine.executeRaw(`DELETE FROM sources WHERE id = $1`, [id]);
|
||||
const pageCount = impact?.pageCount ?? 0;
|
||||
console.log(`Removed source "${id}" (${pageCount} pages + dependent rows cascaded).`);
|
||||
}
|
||||
|
||||
// ── Subcommand: archive (soft-delete) ───────────────────────
|
||||
|
||||
async function runArchive(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const id = args[0];
|
||||
if (!id) {
|
||||
console.error('Usage: gbrain sources archive <id>');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
if (id === 'default') {
|
||||
console.error('Error: cannot archive the "default" source.');
|
||||
process.exit(3);
|
||||
}
|
||||
|
||||
// Show impact preview
|
||||
const impact = await assessDestructiveImpact(engine, id);
|
||||
if (!impact) {
|
||||
console.error(`Source "${id}" not found.`);
|
||||
process.exit(4);
|
||||
}
|
||||
|
||||
const result = await softDeleteSource(engine, id);
|
||||
if (!result) {
|
||||
console.error(`Failed to archive source "${id}".`);
|
||||
process.exit(4);
|
||||
}
|
||||
|
||||
console.log(formatSoftDelete(result));
|
||||
}
|
||||
|
||||
// ── Subcommand: restore ─────────────────────────────────────
|
||||
|
||||
async function runRestore(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const id = args[0];
|
||||
const noFederate = args.includes('--no-federate');
|
||||
if (!id) {
|
||||
console.error('Usage: gbrain sources restore <id> [--no-federate]');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const restored = await restoreSource(engine, id, !noFederate);
|
||||
if (!restored) {
|
||||
console.error(`Source "${id}" not found or not archived.`);
|
||||
process.exit(4);
|
||||
}
|
||||
|
||||
console.log(`Source "${id}" restored. ${noFederate ? 'Not re-federated.' : 'Re-federated.'}`);
|
||||
console.log(`All pages, chunks, and embeddings are intact.`);
|
||||
|
||||
// T4 (eng-review): if the source has a remote_url AND its clone dir was
|
||||
// autopurged (e.g. operator rm -rf'd $GBRAIN_HOME/clones/), re-clone
|
||||
// before declaring restore success. Without this, restore returns green
|
||||
// but the source is unsyncable until a later sync path discovers the gap.
|
||||
try {
|
||||
const recloned = await recloneIfMissing(engine, id);
|
||||
if (recloned) {
|
||||
console.log(` re-cloned from remote_url (clone dir was missing).`);
|
||||
}
|
||||
} catch (e) {
|
||||
if (e instanceof SourceOpError) {
|
||||
console.error(` WARN: could not re-clone: ${e.message}`);
|
||||
console.error(` The DB row is restored but the on-disk clone is missing.`);
|
||||
console.error(` Try \`gbrain sync --source ${id}\` to recover, or remove + re-add.`);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Subcommand: purge ───────────────────────────────────────
|
||||
|
||||
async function runPurge(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const id = args[0];
|
||||
const confirmDestructive = args.includes('--confirm-destructive');
|
||||
|
||||
if (id) {
|
||||
// Purge a specific source (must be archived)
|
||||
const impact = await assessDestructiveImpact(engine, id);
|
||||
if (!impact) {
|
||||
console.error(`Source "${id}" not found.`);
|
||||
process.exit(4);
|
||||
}
|
||||
|
||||
console.log(formatImpact(impact));
|
||||
|
||||
if (!confirmDestructive) {
|
||||
console.error(`Pass --confirm-destructive to permanently delete source "${id}".`);
|
||||
process.exit(5);
|
||||
}
|
||||
|
||||
await engine.executeRaw(`DELETE FROM sources WHERE id = $1`, [id]);
|
||||
console.log(`Permanently deleted source "${id}" (${impact.pageCount} pages cascaded).`);
|
||||
return;
|
||||
}
|
||||
|
||||
// No id: purge all expired archives
|
||||
const purged = await purgeExpiredSources(engine);
|
||||
if (purged.length === 0) {
|
||||
console.log('No expired archives to purge.');
|
||||
} else {
|
||||
console.log(`Purged ${purged.length} expired archive(s): ${purged.join(', ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Subcommand: archived ────────────────────────────────────
|
||||
|
||||
async function runListArchived(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const json = args.includes('--json');
|
||||
const archived = await listArchivedSources(engine);
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify({ archived }, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
if (archived.length === 0) {
|
||||
console.log('No archived sources.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('ARCHIVED SOURCES (soft-deleted)');
|
||||
console.log('───────────────────────────────');
|
||||
for (const a of archived) {
|
||||
const hours = Math.max(0, Math.round((a.expiresAt.getTime() - Date.now()) / (1000 * 60 * 60)));
|
||||
console.log(` ${a.id.padEnd(20)} ${String(a.pageCount).padStart(6)} pages expires in ${hours}h (restore: gbrain sources restore ${a.id})`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Subcommand: rename ──────────────────────────────────────
|
||||
|
||||
async function runRename(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
@@ -340,6 +497,10 @@ export async function runSources(engine: BrainEngine, args: string[]): Promise<v
|
||||
case 'detach': runDetach(); return;
|
||||
case 'federate': return runFederate(engine, rest, true);
|
||||
case 'unfederate': return runFederate(engine, rest, false);
|
||||
case 'archive': return runArchive(engine, rest);
|
||||
case 'restore': return runRestore(engine, rest);
|
||||
case 'purge': return runPurge(engine, rest);
|
||||
case 'archived': return runListArchived(engine, rest);
|
||||
case undefined:
|
||||
case '--help':
|
||||
case '-h':
|
||||
@@ -353,13 +514,23 @@ export async function runSources(engine: BrainEngine, args: string[]): Promise<v
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
console.log(`gbrain sources — manage multi-source brain configuration (v0.18.0)
|
||||
console.log(`gbrain sources — manage multi-source brain configuration (v0.26.5)
|
||||
|
||||
Subcommands:
|
||||
add <id> --path <p> [--name <n>] [--federated|--no-federated]
|
||||
Register a new source.
|
||||
list [--json] List registered sources with page counts.
|
||||
remove <id> [--yes] [--dry-run] Cascade-delete a source and its pages.
|
||||
remove <id> [--confirm-destructive] [--dry-run]
|
||||
Permanently delete a source and all its data.
|
||||
Shows impact preview. Requires --confirm-destructive
|
||||
when the source has data (pages/chunks/embeddings).
|
||||
archive <id> Soft-delete: hide from search, preserve data for ${SOFT_DELETE_TTL_HOURS}h.
|
||||
restore <id> [--no-federate] Un-archive a soft-deleted source.
|
||||
archived [--json] List soft-deleted sources and their expiry.
|
||||
purge [<id>] [--confirm-destructive]
|
||||
Permanently delete archived sources.
|
||||
Without <id>: purge all expired archives.
|
||||
With <id>: force-purge (requires --confirm-destructive).
|
||||
rename <id> <new-name> Rename display name (id is immutable).
|
||||
default <id> Set the brain-level default source.
|
||||
attach <id> Write .gbrain-source in CWD (like kubectl context).
|
||||
@@ -368,5 +539,9 @@ Subcommands:
|
||||
unfederate <id> Isolate source from default search.
|
||||
|
||||
Source id: [a-z0-9-]{1,32}. Immutable citation key.
|
||||
|
||||
Destructive operations (remove, purge) show an impact preview before acting.
|
||||
Pass --dry-run to preview without side effects.
|
||||
Use 'archive' instead of 'remove' for a safe ${SOFT_DELETE_TTL_HOURS}h grace period.
|
||||
`);
|
||||
}
|
||||
|
||||
+56
-2
@@ -322,15 +322,69 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
throw new Error(hint);
|
||||
}
|
||||
|
||||
// v0.28: source-aware re-clone branch. When the source has a remote_url
|
||||
// recorded (i.e. it was registered via `sources add --url`), the on-disk
|
||||
// clone is auto-managed. validateRepoState classifies the on-disk state;
|
||||
// we recover from missing/no-git/not-a-dir by re-cloning, refuse on
|
||||
// url-drift or corruption with structured hints.
|
||||
if (opts.sourceId) {
|
||||
const { validateRepoState } = await import('../core/git-remote.ts');
|
||||
const { recloneIfMissing } = await import('../core/sources-ops.ts');
|
||||
const cfgRows = await engine.executeRaw<{ config: unknown }>(
|
||||
`SELECT config FROM sources WHERE id = $1`,
|
||||
[opts.sourceId],
|
||||
);
|
||||
const cfg =
|
||||
typeof cfgRows[0]?.config === 'string'
|
||||
? (JSON.parse(cfgRows[0].config as string) as Record<string, unknown>)
|
||||
: ((cfgRows[0]?.config ?? {}) as Record<string, unknown>);
|
||||
const remoteUrl = typeof cfg.remote_url === 'string' ? cfg.remote_url : null;
|
||||
if (remoteUrl) {
|
||||
const state = validateRepoState(repoPath, remoteUrl);
|
||||
switch (state) {
|
||||
case 'healthy':
|
||||
break;
|
||||
case 'missing':
|
||||
case 'no-git':
|
||||
case 'not-a-dir':
|
||||
console.error(
|
||||
`[gbrain] auto-recovery: re-cloning "${opts.sourceId}" (clone state: ${state}).`,
|
||||
);
|
||||
await recloneIfMissing(engine, opts.sourceId);
|
||||
break;
|
||||
case 'corrupted':
|
||||
throw new Error(
|
||||
`Source "${opts.sourceId}" clone at ${repoPath} is corrupted ` +
|
||||
`(\`git remote get-url origin\` failed). Run: ` +
|
||||
`gbrain sources remove ${opts.sourceId} --confirm-destructive && ` +
|
||||
`gbrain sources add ${opts.sourceId} --url ${remoteUrl}`,
|
||||
);
|
||||
case 'url-drift':
|
||||
throw new Error(
|
||||
`Source "${opts.sourceId}" clone at ${repoPath} has a remote ` +
|
||||
`that differs from config.remote_url=${remoteUrl}. ` +
|
||||
`Re-clone with: gbrain sources rebase-clone ${opts.sourceId} ` +
|
||||
`(if available, else: sources remove + sources add).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate git repo
|
||||
if (!existsSync(join(repoPath, '.git'))) {
|
||||
throw new Error(`Not a git repository: ${repoPath}. GBrain sync requires a git-initialized repo.`);
|
||||
}
|
||||
|
||||
// Git pull (unless --no-pull)
|
||||
// Git pull (unless --no-pull). v0.28.1 codex finding (HIGH): the legacy
|
||||
// git() helper at sync.ts:192 spawns git without GIT_SSRF_FLAGS, so
|
||||
// every steady-state pull was bypassing the redirect/submodule/protocol
|
||||
// hardening that cloneRepo applies. Route through pullRepo from
|
||||
// git-remote.ts so the flag set is consistent across initial clone and
|
||||
// ongoing pulls — single source of truth for the defensive flags.
|
||||
if (!opts.noPull) {
|
||||
try {
|
||||
git(repoPath, 'pull', '--ff-only');
|
||||
const { pullRepo } = await import('../core/git-remote.ts');
|
||||
pullRepo(repoPath);
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
if (msg.includes('non-fast-forward') || msg.includes('diverged')) {
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
/**
|
||||
* v0.28: `gbrain takes` CLI.
|
||||
*
|
||||
* Subcommands:
|
||||
* takes <slug> — list takes for a page
|
||||
* takes search "<query>" [--who h] — keyword search across all takes
|
||||
* takes add <slug> ...flags — append a take (markdown + DB)
|
||||
* takes update <slug> --row N ...flags — update mutable fields
|
||||
* takes supersede <slug> --row N ... — strikethrough old + append new
|
||||
* takes resolve <slug> --row N --outcome true|false [--value N --unit u]
|
||||
*
|
||||
* Markdown is canonical. Every mutate command:
|
||||
* 1. acquires the per-page file lock
|
||||
* 2. re-reads the .md file
|
||||
* 3. applies the edit via takes-fence (upsertTakeRow / supersedeRow)
|
||||
* 4. writes the .md file back
|
||||
* 5. mirrors to the DB via the engine method
|
||||
* 6. releases the lock (auto via withPageLock)
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import type { BrainEngine, TakeKind } from '../core/engine.ts';
|
||||
import {
|
||||
parseTakesFence,
|
||||
upsertTakeRow,
|
||||
supersedeRow,
|
||||
type ParsedTake,
|
||||
} from '../core/takes-fence.ts';
|
||||
import { withPageLock } from '../core/page-lock.ts';
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
function flagValue(args: string[], name: string): string | undefined {
|
||||
const i = args.indexOf(name);
|
||||
if (i === -1) return undefined;
|
||||
return args[i + 1];
|
||||
}
|
||||
|
||||
function flagPresent(args: string[], name: string): boolean {
|
||||
return args.includes(name);
|
||||
}
|
||||
|
||||
async function resolveBrainDir(engine: BrainEngine | null, explicitDir: string | null): Promise<string> {
|
||||
if (explicitDir) {
|
||||
if (!existsSync(explicitDir)) {
|
||||
console.error(`--dir path does not exist: ${explicitDir}`);
|
||||
process.exit(1);
|
||||
}
|
||||
return explicitDir;
|
||||
}
|
||||
if (engine) {
|
||||
const configured = await engine.getConfig('sync.repo_path');
|
||||
if (configured && existsSync(configured)) return configured;
|
||||
}
|
||||
console.error('No brain directory configured. Pass --dir <path> or run `gbrain init` first.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function pageFilePath(brainDir: string, slug: string): string {
|
||||
return join(brainDir, `${slug}.md`);
|
||||
}
|
||||
|
||||
function ensureKind(raw: string | undefined): TakeKind {
|
||||
if (!raw) {
|
||||
console.error('Missing --kind. Expected one of: fact, take, bet, hunch.');
|
||||
process.exit(1);
|
||||
}
|
||||
if (raw !== 'fact' && raw !== 'take' && raw !== 'bet' && raw !== 'hunch') {
|
||||
console.error(`Invalid --kind "${raw}". Expected: fact, take, bet, hunch.`);
|
||||
process.exit(1);
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
function ensureFloat(raw: string | undefined, fallback: number): number {
|
||||
if (raw === undefined) return fallback;
|
||||
const n = parseFloat(raw);
|
||||
if (!Number.isFinite(n)) {
|
||||
console.error(`Invalid weight "${raw}". Expected a number 0..1.`);
|
||||
process.exit(1);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
async function getPageId(engine: BrainEngine, slug: string): Promise<number> {
|
||||
const rows = await engine.executeRaw<{ id: number }>(
|
||||
`SELECT id FROM pages WHERE slug = $1 LIMIT 1`,
|
||||
[slug],
|
||||
);
|
||||
if (!rows[0]) {
|
||||
console.error(`Page not found in brain: ${slug}. Run \`gbrain sync\` first.`);
|
||||
process.exit(1);
|
||||
}
|
||||
return rows[0].id;
|
||||
}
|
||||
|
||||
function readBodyOrEmpty(path: string): string {
|
||||
if (!existsSync(path)) return '';
|
||||
return readFileSync(path, 'utf-8');
|
||||
}
|
||||
|
||||
function writeBody(path: string, body: string): void {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, body, 'utf-8');
|
||||
}
|
||||
|
||||
// --- Subcommands ---
|
||||
|
||||
async function cmdList(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const slug = args[0];
|
||||
if (!slug) {
|
||||
console.error('Usage: gbrain takes <slug> [--json]');
|
||||
process.exit(1);
|
||||
}
|
||||
const json = flagPresent(args, '--json');
|
||||
const holder = flagValue(args, '--who');
|
||||
const kind = flagValue(args, '--kind') as TakeKind | undefined;
|
||||
const sort = flagValue(args, '--sort') as 'weight' | 'since_date' | 'created_at' | undefined;
|
||||
const expired = flagPresent(args, '--expired');
|
||||
|
||||
const takes = await engine.listTakes({
|
||||
page_slug: slug,
|
||||
holder,
|
||||
kind,
|
||||
active: expired ? false : true,
|
||||
sortBy: sort,
|
||||
});
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify(takes, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
if (takes.length === 0) {
|
||||
console.log(`No takes on ${slug}.`);
|
||||
return;
|
||||
}
|
||||
console.log(`# Takes on ${slug}\n`);
|
||||
for (const t of takes) {
|
||||
const tag = t.active ? '' : ' [superseded]';
|
||||
const w = Number(t.weight).toFixed(2);
|
||||
const since = t.since_date ?? '';
|
||||
const src = t.source ? ` — ${t.source}` : '';
|
||||
console.log(`#${t.row_num} [${t.kind} • ${t.holder} • w=${w}${since ? ` • ${since}` : ''}]${tag}\n ${t.claim}${src}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
async function cmdSearch(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const query = args[0];
|
||||
if (!query) {
|
||||
console.error('Usage: gbrain takes search "<query>" [--who h] [--json]');
|
||||
process.exit(1);
|
||||
}
|
||||
const json = flagPresent(args, '--json');
|
||||
const limit = parseInt(flagValue(args, '--limit') ?? '30', 10);
|
||||
const hits = await engine.searchTakes(query, { limit });
|
||||
if (json) {
|
||||
console.log(JSON.stringify(hits, null, 2));
|
||||
return;
|
||||
}
|
||||
if (hits.length === 0) {
|
||||
console.log(`No takes match "${query}".`);
|
||||
return;
|
||||
}
|
||||
for (const h of hits) {
|
||||
const score = Number(h.score).toFixed(2);
|
||||
console.log(`${h.page_slug}#${h.row_num} [${h.kind} • ${h.holder} • w=${Number(h.weight).toFixed(2)} • s=${score}]\n ${h.claim}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
async function cmdAdd(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const slug = args[0];
|
||||
if (!slug) {
|
||||
console.error('Usage: gbrain takes add <slug> --claim "..." --kind <k> --who <h> [--weight 0.5] [--source "..."] [--since YYYY-MM]');
|
||||
process.exit(1);
|
||||
}
|
||||
const claim = flagValue(args, '--claim');
|
||||
if (!claim) { console.error('Missing --claim'); process.exit(1); }
|
||||
const kind = ensureKind(flagValue(args, '--kind'));
|
||||
const holder = flagValue(args, '--who');
|
||||
if (!holder) { console.error('Missing --who'); process.exit(1); }
|
||||
const weight = ensureFloat(flagValue(args, '--weight'), 0.5);
|
||||
const source = flagValue(args, '--source');
|
||||
const since = flagValue(args, '--since');
|
||||
const dirArg = flagValue(args, '--dir');
|
||||
const brainDir = await resolveBrainDir(engine, dirArg ?? null);
|
||||
|
||||
await withPageLock(slug, async () => {
|
||||
const path = pageFilePath(brainDir, slug);
|
||||
const body = readBodyOrEmpty(path);
|
||||
const { body: nextBody, rowNum } = upsertTakeRow(body, {
|
||||
claim, kind, holder, weight, source, sinceDate: since, active: true,
|
||||
});
|
||||
writeBody(path, nextBody);
|
||||
|
||||
// Mirror to DB. Page may not be in DB yet if not synced — caller must run sync first.
|
||||
const pageId = await getPageId(engine, slug);
|
||||
await engine.addTakesBatch([{
|
||||
page_id: pageId, row_num: rowNum, claim, kind, holder, weight,
|
||||
since_date: since, source, active: true, superseded_by: null,
|
||||
}]);
|
||||
console.log(`Added take #${rowNum} to ${slug}.`);
|
||||
});
|
||||
}
|
||||
|
||||
async function cmdUpdate(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const slug = args[0];
|
||||
const rowNumStr = flagValue(args, '--row');
|
||||
if (!slug || !rowNumStr) {
|
||||
console.error('Usage: gbrain takes update <slug> --row N [--weight 0.7] [--source "..."] [--since YYYY-MM]');
|
||||
process.exit(1);
|
||||
}
|
||||
const rowNum = parseInt(rowNumStr, 10);
|
||||
const fields: { weight?: number; source?: string; since_date?: string } = {};
|
||||
const w = flagValue(args, '--weight');
|
||||
if (w !== undefined) fields.weight = ensureFloat(w, 0.5);
|
||||
const s = flagValue(args, '--source');
|
||||
if (s !== undefined) fields.source = s;
|
||||
const since = flagValue(args, '--since');
|
||||
if (since !== undefined) fields.since_date = since;
|
||||
const dirArg = flagValue(args, '--dir');
|
||||
const brainDir = await resolveBrainDir(engine, dirArg ?? null);
|
||||
|
||||
await withPageLock(slug, async () => {
|
||||
const pageId = await getPageId(engine, slug);
|
||||
await engine.updateTake(pageId, rowNum, fields);
|
||||
|
||||
// Sync the markdown table: read fence, find row, apply field updates, re-render.
|
||||
const path = pageFilePath(brainDir, slug);
|
||||
const body = readBodyOrEmpty(path);
|
||||
const parsed = parseTakesFence(body);
|
||||
const target = parsed.takes.find(t => t.rowNum === rowNum);
|
||||
if (!target) {
|
||||
console.warn(`[takes update] DB updated but row #${rowNum} not in markdown fence on disk; markdown may be out of sync. Run 'gbrain extract takes --slugs ${slug}' to reconcile.`);
|
||||
return;
|
||||
}
|
||||
const updated: ParsedTake = {
|
||||
...target,
|
||||
weight: fields.weight ?? target.weight,
|
||||
source: fields.source ?? target.source,
|
||||
sinceDate: fields.since_date ?? target.sinceDate,
|
||||
};
|
||||
// Replace the row in-place by stripping the fence and re-rendering all rows.
|
||||
const allRows = parsed.takes.map(t => t.rowNum === rowNum ? updated : t);
|
||||
// Round-trip via upsertTakeRow with no new row: easiest is to render manually.
|
||||
const { renderTakesFence, TAKES_FENCE_BEGIN, TAKES_FENCE_END } = await import('../core/takes-fence.ts');
|
||||
const newFence = renderTakesFence(allRows);
|
||||
const beginIdx = body.indexOf(TAKES_FENCE_BEGIN);
|
||||
const endIdx = body.indexOf(TAKES_FENCE_END, beginIdx + TAKES_FENCE_BEGIN.length);
|
||||
const out = body.slice(0, beginIdx) + newFence + body.slice(endIdx + TAKES_FENCE_END.length);
|
||||
writeBody(path, out);
|
||||
console.log(`Updated take #${rowNum} on ${slug}.`);
|
||||
});
|
||||
}
|
||||
|
||||
async function cmdSupersede(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const slug = args[0];
|
||||
const rowNumStr = flagValue(args, '--row');
|
||||
if (!slug || !rowNumStr) {
|
||||
console.error('Usage: gbrain takes supersede <slug> --row N --claim "..." [--kind k] [--who h] [--weight 0.5] [--source "..."]');
|
||||
process.exit(1);
|
||||
}
|
||||
const rowNum = parseInt(rowNumStr, 10);
|
||||
const claim = flagValue(args, '--claim');
|
||||
if (!claim) { console.error('Missing --claim'); process.exit(1); }
|
||||
const dirArg = flagValue(args, '--dir');
|
||||
const brainDir = await resolveBrainDir(engine, dirArg ?? null);
|
||||
|
||||
await withPageLock(slug, async () => {
|
||||
const pageId = await getPageId(engine, slug);
|
||||
|
||||
// Read existing row to inherit kind/holder unless overridden
|
||||
const existing = await engine.listTakes({ page_id: pageId, active: false, limit: 500 });
|
||||
const target = existing.find(t => t.row_num === rowNum);
|
||||
if (!target) {
|
||||
console.error(`Row #${rowNum} not found on ${slug}.`);
|
||||
process.exit(1);
|
||||
}
|
||||
const kind = ensureKind(flagValue(args, '--kind') ?? target.kind);
|
||||
const holder = flagValue(args, '--who') ?? target.holder;
|
||||
const weight = ensureFloat(flagValue(args, '--weight'), Math.max(0, target.weight - 0.1));
|
||||
const source = flagValue(args, '--source');
|
||||
const since = flagValue(args, '--since');
|
||||
|
||||
const dbResult = await engine.supersedeTake(pageId, rowNum, {
|
||||
claim, kind, holder, weight, source, since_date: since, active: true,
|
||||
});
|
||||
|
||||
// Mirror in markdown
|
||||
const path = pageFilePath(brainDir, slug);
|
||||
const body = readBodyOrEmpty(path);
|
||||
if (parseTakesFence(body).takes.find(t => t.rowNum === rowNum)) {
|
||||
const { body: nextBody } = supersedeRow(body, rowNum, {
|
||||
claim, kind, holder, weight, source, sinceDate: since,
|
||||
});
|
||||
writeBody(path, nextBody);
|
||||
} else {
|
||||
console.warn(`[takes supersede] DB updated but markdown lacks row #${rowNum}; only DB written.`);
|
||||
}
|
||||
console.log(`Superseded #${dbResult.oldRow} → new #${dbResult.newRow} on ${slug}.`);
|
||||
});
|
||||
}
|
||||
|
||||
async function cmdResolve(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const slug = args[0];
|
||||
const rowNumStr = flagValue(args, '--row');
|
||||
const outcomeStr = flagValue(args, '--outcome');
|
||||
if (!slug || !rowNumStr || !outcomeStr) {
|
||||
console.error('Usage: gbrain takes resolve <slug> --row N --outcome true|false [--value N --unit usd|pct|count] [--source "..."] [--by <slug>]');
|
||||
process.exit(1);
|
||||
}
|
||||
const rowNum = parseInt(rowNumStr, 10);
|
||||
const outcome = outcomeStr === 'true';
|
||||
const valueStr = flagValue(args, '--value');
|
||||
const value = valueStr === undefined ? undefined : parseFloat(valueStr);
|
||||
const unit = flagValue(args, '--unit');
|
||||
const source = flagValue(args, '--source');
|
||||
const resolvedBy = flagValue(args, '--by') ?? 'garry';
|
||||
|
||||
const pageId = await getPageId(engine, slug);
|
||||
await engine.resolveTake(pageId, rowNum, {
|
||||
outcome,
|
||||
value,
|
||||
unit,
|
||||
source,
|
||||
resolvedBy,
|
||||
});
|
||||
console.log(`Resolved take #${rowNum} on ${slug}: outcome=${outcome}${valueStr ? ` value=${value}${unit ? ` ${unit}` : ''}` : ''}.`);
|
||||
console.log(`(Markdown rendering of resolution metadata: deferred to v0.29 — DB stores it; takes-fence renderer doesn't yet surface resolved_* in the table.)`);
|
||||
}
|
||||
|
||||
// --- Dispatcher ---
|
||||
|
||||
export async function runTakes(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
|
||||
console.log(`Usage: gbrain takes <subcommand> [options]
|
||||
|
||||
Subcommands:
|
||||
takes <slug> [--json] [--who h] [--kind k] [--sort weight|since_date|created_at] [--expired]
|
||||
List takes for a page
|
||||
takes search "<query>" [--limit N] [--json]
|
||||
Keyword search across all takes
|
||||
takes add <slug> --claim "..." --kind <fact|take|bet|hunch> --who <holder>
|
||||
[--weight 0.5] [--source "..."] [--since YYYY-MM]
|
||||
Append a take (markdown + DB)
|
||||
takes update <slug> --row N [--weight 0.7] [--source "..."] [--since YYYY-MM]
|
||||
Update mutable fields
|
||||
takes supersede <slug> --row N --claim "..." [--kind k] [--who h] [--weight 0.5] [--source "..."]
|
||||
Strikethrough old + append new
|
||||
takes resolve <slug> --row N --outcome true|false [--value N --unit usd|pct|count] [--source "..."] [--by <slug>]
|
||||
Record bet resolution (immutable)
|
||||
|
||||
Common flags:
|
||||
--dir <path> Override the brain directory (default: sync.repo_path config)
|
||||
--help, -h Show this help
|
||||
`);
|
||||
return;
|
||||
}
|
||||
|
||||
const sub = args[0];
|
||||
const rest = args.slice(1);
|
||||
|
||||
switch (sub) {
|
||||
case 'search': return cmdSearch(engine, rest);
|
||||
case 'add': return cmdAdd(engine, rest);
|
||||
case 'update': return cmdUpdate(engine, rest);
|
||||
case 'supersede': return cmdSupersede(engine, rest);
|
||||
case 'resolve': return cmdResolve(engine, rest);
|
||||
default:
|
||||
// No subcommand keyword → treat first arg as <slug> for the list path.
|
||||
return cmdList(engine, args);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* v0.28: `gbrain think <question>` CLI.
|
||||
*
|
||||
* Thin wrapper around runThink + persistSynthesis. Local CLI = remote=false,
|
||||
* so --save and --take are honored. Reads ANTHROPIC_API_KEY from the env;
|
||||
* degrades to gather-only output with a warning if missing.
|
||||
*/
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { runThink, persistSynthesis } from '../core/think/index.ts';
|
||||
|
||||
function flagValue(args: string[], name: string): string | undefined {
|
||||
const i = args.indexOf(name);
|
||||
if (i === -1) return undefined;
|
||||
return args[i + 1];
|
||||
}
|
||||
|
||||
function flagPresent(args: string[], name: string): boolean {
|
||||
return args.includes(name);
|
||||
}
|
||||
|
||||
export async function runThinkCli(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
|
||||
console.log(`Usage: gbrain think "<question>" [options]
|
||||
|
||||
Options:
|
||||
--anchor <slug> Pull the entity subgraph around this slug
|
||||
--rounds N Multi-pass synthesis (default 1; gap-driven loop ships in v0.29)
|
||||
--save Persist a synthesis page under synthesis/<slug>-<date>.md
|
||||
--take Append a take row to the anchor page (requires --anchor)
|
||||
--model <name> Override the model (alias or full id)
|
||||
--since YYYY-MM-DD Start of temporal window
|
||||
--until YYYY-MM-DD End of temporal window
|
||||
--json Output as JSON
|
||||
--help Show this help
|
||||
|
||||
Without --save, the synthesis is printed to stdout and discarded. With --save,
|
||||
the synthesis page is persisted AND printed.
|
||||
|
||||
Set ANTHROPIC_API_KEY in the environment to run real synthesis. Without it,
|
||||
the gather phase still runs and prints what would have been the input.
|
||||
`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Strip flags from positional args
|
||||
const flagNames = ['--anchor', '--rounds', '--model', '--since', '--until'];
|
||||
const positional: string[] = [];
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const a = args[i];
|
||||
if (flagNames.includes(a)) { i++; continue; }
|
||||
if (a === '--save' || a === '--take' || a === '--json' || a === '--help' || a === '-h') continue;
|
||||
positional.push(a);
|
||||
}
|
||||
const question = positional.join(' ').trim();
|
||||
if (!question) {
|
||||
console.error('Missing question. Try: gbrain think "What do we know about acme-example?"');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const json = flagPresent(args, '--json');
|
||||
const save = flagPresent(args, '--save');
|
||||
const take = flagPresent(args, '--take');
|
||||
const anchor = flagValue(args, '--anchor');
|
||||
const roundsStr = flagValue(args, '--rounds');
|
||||
const rounds = roundsStr ? Math.max(1, parseInt(roundsStr, 10) || 1) : 1;
|
||||
const model = flagValue(args, '--model');
|
||||
const since = flagValue(args, '--since');
|
||||
const until = flagValue(args, '--until');
|
||||
|
||||
if (take && !anchor) {
|
||||
console.error('--take requires --anchor (the take row needs a target page)');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const result = await runThink(engine, {
|
||||
question, anchor, rounds, save, take, model, since, until,
|
||||
// Local CLI: no MCP allow-list filter — operator owns the brain.
|
||||
});
|
||||
|
||||
// Persist if --save (the runThink path doesn't auto-persist; CLI does it explicitly)
|
||||
let savedSlug: string | undefined;
|
||||
let evidenceInserted = 0;
|
||||
if (save) {
|
||||
const persisted = await persistSynthesis(engine, result);
|
||||
savedSlug = persisted.slug;
|
||||
evidenceInserted = persisted.evidenceInserted;
|
||||
for (const w of persisted.warnings) result.warnings.push(w);
|
||||
}
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify({
|
||||
...result,
|
||||
saved_slug: savedSlug ?? null,
|
||||
evidence_inserted: evidenceInserted,
|
||||
}, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
// Human-readable output
|
||||
console.log(`# ${question}\n`);
|
||||
console.log(result.answer);
|
||||
console.log('');
|
||||
if (result.gaps.length > 0) {
|
||||
console.log('## Gaps');
|
||||
for (const g of result.gaps) console.log(`- ${g}`);
|
||||
console.log('');
|
||||
}
|
||||
console.log('---');
|
||||
console.log(`Model: ${result.modelUsed} | Pages: ${result.pagesGathered} | Takes: ${result.takesGathered} | Graph: ${result.graphHits} | Citations: ${result.citations.length}`);
|
||||
if (savedSlug) {
|
||||
console.log(`Saved: ${savedSlug} (${evidenceInserted} evidence rows)`);
|
||||
}
|
||||
if (result.warnings.length > 0) {
|
||||
console.error(`Warnings: ${result.warnings.join(', ')}`);
|
||||
}
|
||||
}
|
||||
+191
-4
@@ -1,8 +1,10 @@
|
||||
import { execSync } from 'child_process';
|
||||
import { existsSync, readFileSync, writeFileSync, mkdirSync, appendFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { existsSync, readFileSync, writeFileSync, mkdirSync, appendFileSync, realpathSync, lstatSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { VERSION } from '../version.ts';
|
||||
|
||||
const GBRAIN_GITHUB_REPO = 'garrytan/gbrain';
|
||||
|
||||
export async function runUpgrade(args: string[]) {
|
||||
if (args.includes('--help') || args.includes('-h')) {
|
||||
console.log('Usage: gbrain upgrade\n\nSelf-update the CLI.\n\nDetects install method (bun, binary, clawhub) and runs the appropriate update.\nAfter upgrading, shows what\'s new and offers to set up new features.');
|
||||
@@ -17,6 +19,18 @@ export async function runUpgrade(args: string[]) {
|
||||
|
||||
let upgraded = false;
|
||||
switch (method) {
|
||||
case 'bun-link':
|
||||
// v0.28.5: bun-link installs are source clones. Pull + bun install
|
||||
// is the upgrade path; npm/bun's update mechanism doesn't apply.
|
||||
console.log('Upgrading via bun-link source clone...');
|
||||
console.log(' cd into your gbrain checkout, then run:');
|
||||
console.log(' git pull');
|
||||
console.log(' bun install');
|
||||
console.log(' bun link');
|
||||
console.log('');
|
||||
console.log(' (auto-detect can\'t do this for you because it doesn\'t know which checkout to update.)');
|
||||
break;
|
||||
|
||||
case 'bun':
|
||||
console.log('Upgrading via bun...');
|
||||
try {
|
||||
@@ -218,6 +232,37 @@ export async function runPostUpgrade(args: string[] = []): Promise<void> {
|
||||
console.error('Run `gbrain apply-migrations --yes` manually to retry.');
|
||||
}
|
||||
|
||||
// v0.28.5 (X1): explicitly apply pending schema migrations.
|
||||
// apply-migrations runs orchestrator migrations and only WARNs about
|
||||
// schema-version drift (apply-migrations.ts:296-302). Without this hook,
|
||||
// `gbrain upgrade` leaves wedged brains wedged — the user has to read
|
||||
// the WARN and run `gbrain init --migrate-only` themselves. We've shipped
|
||||
// 11 wedge incidents asking users to read warnings; close the loop here.
|
||||
// A1's hasPendingMigrations probe in connectEngine is belt-and-suspenders
|
||||
// for any path that bypasses upgrade (autopilot, direct CLI on stale brain).
|
||||
try {
|
||||
const { loadConfig: lcSchema, toEngineConfig: toCfgSchema } = await import('../core/config.ts');
|
||||
const { createEngine } = await import('../core/engine-factory.ts');
|
||||
const cfgSchema = lcSchema();
|
||||
if (cfgSchema) {
|
||||
const engine = await createEngine(toCfgSchema(cfgSchema));
|
||||
try {
|
||||
await engine.connect(toCfgSchema(cfgSchema));
|
||||
await engine.initSchema();
|
||||
console.log(' Schema up to date.');
|
||||
} finally {
|
||||
try { await engine.disconnect(); } catch { /* best-effort */ }
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// Non-fatal: connection or DDL failure here falls back to the existing
|
||||
// user-facing WARN. apply-migrations.ts:296-302 already surfaces the
|
||||
// hint to run `gbrain init --migrate-only`.
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
console.warn(`\nSchema auto-apply skipped: ${msg}`);
|
||||
console.warn('Run `gbrain init --migrate-only` manually if your brain is wedged.');
|
||||
}
|
||||
|
||||
// v0.25.1: agent-readable advisory listing recommended skills the
|
||||
// workspace hasn't installed yet. No-op when everything is installed.
|
||||
try {
|
||||
@@ -244,11 +289,25 @@ function isNewerThan(version: string, baseline: string): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
export function detectInstallMethod(): 'bun' | 'binary' | 'clawhub' | 'unknown' {
|
||||
export function detectInstallMethod(): 'bun' | 'bun-link' | 'binary' | 'clawhub' | 'unknown' {
|
||||
const execPath = process.execPath || '';
|
||||
|
||||
// Check if running from node_modules (bun/npm install)
|
||||
// v0.28.5 cluster D: bun-link signal first.
|
||||
// bun link puts a symlink at ~/.bun/bin/gbrain → either the source's bin
|
||||
// entry (compiled CLI) OR src/cli.ts directly. Either way, realpath
|
||||
// resolves into a directory we can walk up from to find a .git/config
|
||||
// pointing at our repo.
|
||||
const bunLinkResult = detectBunLink();
|
||||
if (bunLinkResult === 'bun-link') return 'bun-link';
|
||||
|
||||
// Check if running from node_modules (bun/npm install). Could be canonical
|
||||
// (we publish under garrytan/gbrain) OR the squatter (npm `gbrain@1.3.x`).
|
||||
// Sub-classify and warn loudly on suspect installs (#658).
|
||||
if (execPath.includes('node_modules') || process.argv[1]?.includes('node_modules')) {
|
||||
const verdict = classifyBunInstall();
|
||||
if (verdict === 'suspect') {
|
||||
printSquatterRecovery();
|
||||
}
|
||||
return 'bun';
|
||||
}
|
||||
|
||||
@@ -267,3 +326,131 @@ export function detectInstallMethod(): 'bun' | 'binary' | 'clawhub' | 'unknown'
|
||||
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.28.5 cluster D, signal 1 — bun-link detection (closes #656).
|
||||
*
|
||||
* argv[1] is what `bun /path/to/cli.ts` was invoked with. When `bun link`
|
||||
* is in play, that path is typically a symlink (~/.bun/bin/gbrain) to
|
||||
* either the source repo's compiled binary or src/cli.ts directly.
|
||||
* Walk up from the realpath looking for a `.git/config` whose remote
|
||||
* url contains `garrytan/gbrain` (case-insensitive substring).
|
||||
*
|
||||
* Returns 'bun-link' when we're confident; null otherwise (caller falls
|
||||
* through to the existing detection chain). Best-effort: forks, tarball
|
||||
* installs, detached source trees, and `.git`-less installs all fall
|
||||
* through, which is acceptable per codex's plan-review feedback.
|
||||
*/
|
||||
function detectBunLink(): 'bun-link' | null {
|
||||
try {
|
||||
const argv1 = process.argv[1];
|
||||
if (!argv1) return null;
|
||||
|
||||
// Symlink check first: `bun link` always creates one.
|
||||
let isSymlink = false;
|
||||
try {
|
||||
isSymlink = lstatSync(argv1).isSymbolicLink();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!isSymlink) return null;
|
||||
|
||||
const resolved = realpathSync(argv1);
|
||||
let dir = dirname(resolved);
|
||||
// Walk up at most 6 levels looking for .git/config.
|
||||
for (let i = 0; i < 6; i++) {
|
||||
const gitConfigPath = join(dir, '.git', 'config');
|
||||
if (existsSync(gitConfigPath)) {
|
||||
try {
|
||||
const cfg = readFileSync(gitConfigPath, 'utf-8');
|
||||
// Loose substring match: covers https://, git@, ssh://, fork URLs
|
||||
// that mention upstream in [remote "upstream"], and case variants.
|
||||
if (cfg.toLowerCase().includes(GBRAIN_GITHUB_REPO.toLowerCase())) {
|
||||
return 'bun-link';
|
||||
}
|
||||
} catch { /* unreadable config — not our case */ }
|
||||
return null; // found .git/config but no match → not our repo
|
||||
}
|
||||
const parent = dirname(dir);
|
||||
if (parent === dir) break; // reached filesystem root
|
||||
dir = parent;
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.28.5 cluster D, signal 2 — bun install authenticity check (closes #658).
|
||||
*
|
||||
* When `bun add -g gbrain` (or `npm install -g gbrain`) installs from
|
||||
* npm, the package is the squatter — an unrelated `gbrain@1.3.x` that
|
||||
* silently overwrites our binary. This function reads the install
|
||||
* directory's package.json and checks two non-spoofable signals:
|
||||
* - `repository.url` contains `garrytan/gbrain` (case-insensitive)
|
||||
* - the install dir contains a `src/cli.ts` file (squatter ships
|
||||
* compiled binary, not source)
|
||||
*
|
||||
* If neither matches, returns 'suspect' and the caller surfaces a loud
|
||||
* recovery message. Codex's plan-review noted these signals are spoofable
|
||||
* by a determined squatter — accepted; this is best-effort warning, not
|
||||
* an assertion. The right structural fix is publishing under a scoped
|
||||
* name like `@garrytan/gbrain` (tracked v0.29 follow-up).
|
||||
*/
|
||||
function classifyBunInstall(): 'canonical' | 'suspect' {
|
||||
try {
|
||||
const argv1 = process.argv[1];
|
||||
if (!argv1) return 'suspect';
|
||||
|
||||
// Walk up from argv1 looking for the package.json that owns this install.
|
||||
let dir = dirname(realpathSync(argv1));
|
||||
for (let i = 0; i < 6; i++) {
|
||||
const pkgPath = join(dir, 'package.json');
|
||||
if (existsSync(pkgPath)) {
|
||||
try {
|
||||
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
|
||||
const repoUrl = (typeof pkg.repository === 'string'
|
||||
? pkg.repository
|
||||
: pkg.repository?.url) ?? '';
|
||||
if (repoUrl.toLowerCase().includes(GBRAIN_GITHUB_REPO.toLowerCase())) {
|
||||
return 'canonical';
|
||||
}
|
||||
// Source-marker fallback: our published-as-source install always
|
||||
// ships src/cli.ts next to package.json. The squatter ships dist/.
|
||||
if (existsSync(join(dir, 'src', 'cli.ts'))) {
|
||||
return 'canonical';
|
||||
}
|
||||
return 'suspect';
|
||||
} catch {
|
||||
return 'suspect';
|
||||
}
|
||||
}
|
||||
const parent = dirname(dir);
|
||||
if (parent === dir) break;
|
||||
dir = parent;
|
||||
}
|
||||
return 'suspect';
|
||||
} catch {
|
||||
return 'suspect';
|
||||
}
|
||||
}
|
||||
|
||||
function printSquatterRecovery(): void {
|
||||
console.warn('');
|
||||
console.warn(' WARNING: gbrain install does not appear to be from garrytan/gbrain.');
|
||||
console.warn(' This is likely the npm-name collision tracked in issue #658:');
|
||||
console.warn(' https://www.npmjs.com/package/gbrain (an unrelated package).');
|
||||
console.warn('');
|
||||
console.warn(' Recovery options:');
|
||||
console.warn(' 1. Install from source:');
|
||||
console.warn(' bun remove -g gbrain');
|
||||
console.warn(' git clone https://github.com/garrytan/gbrain.git');
|
||||
console.warn(' cd gbrain && bun install && bun link');
|
||||
console.warn('');
|
||||
console.warn(' 2. Download a release binary:');
|
||||
console.warn(' https://github.com/garrytan/gbrain/releases');
|
||||
console.warn('');
|
||||
console.warn(' See docs/INSTALL_FOR_AGENTS.md for the canonical install paths.');
|
||||
console.warn('');
|
||||
}
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Per-provider dimension parameter resolver.
|
||||
*
|
||||
* Critical: OpenAI text-embedding-3-* defaults to 3072 dims on the API side.
|
||||
* Without explicit dimensions passthrough, existing 1536-dim brains break.
|
||||
* Similarly, Gemini gemini-embedding-001 defaults to 3072.
|
||||
*
|
||||
* This module centralizes the knowledge of "which provider needs which
|
||||
* providerOptions shape to produce vector(N)".
|
||||
*/
|
||||
|
||||
import type { Implementation } from './types.ts';
|
||||
|
||||
const VOYAGE_OUTPUT_DIMENSION_MODELS = new Set([
|
||||
'voyage-4-large',
|
||||
'voyage-4',
|
||||
'voyage-4-lite',
|
||||
'voyage-3-large',
|
||||
'voyage-3.5',
|
||||
'voyage-3.5-lite',
|
||||
'voyage-code-3',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Build the providerOptions blob for embedMany() that pins output dimensions.
|
||||
*
|
||||
* Matryoshka providers (OpenAI text-embedding-3, Gemini embedding-001) can be
|
||||
* asked to return reduced-dim vectors. Anthropic does not take a dimension
|
||||
* parameter. Most openai-compatible providers do not either, but Voyage's
|
||||
* OpenAI-compatible embeddings endpoint accepts `output_dimension`.
|
||||
*/
|
||||
export function dimsProviderOptions(
|
||||
implementation: Implementation,
|
||||
modelId: string,
|
||||
dims: number,
|
||||
): Record<string, any> | undefined {
|
||||
switch (implementation) {
|
||||
case 'native-openai': {
|
||||
// text-embedding-3-* supports dimensions; text-embedding-ada-002 does not.
|
||||
if (modelId.startsWith('text-embedding-3')) {
|
||||
return { openai: { dimensions: dims } };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
case 'native-google': {
|
||||
if (modelId.startsWith('gemini-embedding') || modelId === 'text-embedding-004') {
|
||||
return { google: { outputDimensionality: dims } };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
case 'native-anthropic':
|
||||
// Anthropic has no embedding model.
|
||||
return undefined;
|
||||
case 'openai-compatible':
|
||||
// Most openai-compatible providers (Ollama, LM Studio, vLLM, LiteLLM)
|
||||
// do not expose a standard dimensions knob. Voyage's compat endpoint is
|
||||
// the exception: it accepts output_dimension and defaults to 1024 dims.
|
||||
if (VOYAGE_OUTPUT_DIMENSION_MODELS.has(modelId)) {
|
||||
return { openaiCompatible: { output_dimension: dims } };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* AI service error hierarchy. Three classes mapping to caller decisions:
|
||||
*
|
||||
* AIConfigError — user fixes: bad key, missing model, dim mismatch.
|
||||
* Abort + show recovery recipe.
|
||||
* AITransientError — retryable: SDK retries exhausted, rate limit sustained.
|
||||
* Propagate so job queue can retry later.
|
||||
* AIServiceError — base class for both.
|
||||
*
|
||||
* The `fix` field carries a human-readable recovery recipe agents and humans
|
||||
* can act on. The `cause` field preserves the underlying SDK error.
|
||||
*/
|
||||
|
||||
export class AIServiceError extends Error {
|
||||
constructor(message: string, public readonly cause?: unknown) {
|
||||
super(message);
|
||||
this.name = 'AIServiceError';
|
||||
}
|
||||
}
|
||||
|
||||
export class AIConfigError extends AIServiceError {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly fix?: string,
|
||||
cause?: unknown,
|
||||
) {
|
||||
super(message, cause);
|
||||
this.name = 'AIConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
export class AITransientError extends AIServiceError {
|
||||
constructor(message: string, cause?: unknown) {
|
||||
super(message, cause);
|
||||
this.name = 'AITransientError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize any thrown error into our hierarchy. AI SDK errors are inspected
|
||||
* by status code + name; unknown errors default to AITransientError so the
|
||||
* caller does not permanently abort on a transient network blip.
|
||||
*/
|
||||
export function normalizeAIError(err: unknown, context?: string): AIServiceError {
|
||||
if (err instanceof AIServiceError) return err;
|
||||
|
||||
const anyErr = err as { name?: string; status?: number; statusCode?: number; message?: string };
|
||||
const status = anyErr?.status ?? anyErr?.statusCode;
|
||||
const name = anyErr?.name ?? '';
|
||||
const msg = anyErr?.message ?? String(err);
|
||||
const ctxPrefix = context ? `[${context}] ` : '';
|
||||
|
||||
// 4xx (except 429) = config-level, non-retryable
|
||||
if (typeof status === 'number' && status >= 400 && status < 500 && status !== 429) {
|
||||
return new AIConfigError(
|
||||
`${ctxPrefix}${msg}`,
|
||||
status === 401 || status === 403
|
||||
? 'Check your API key is valid and has access to this model.'
|
||||
: 'Check your model id + provider options match the provider API.',
|
||||
err,
|
||||
);
|
||||
}
|
||||
|
||||
// AI SDK named errors
|
||||
if (name === 'LoadAPIKeyError' || name === 'InvalidArgumentError') {
|
||||
return new AIConfigError(`${ctxPrefix}${msg}`, undefined, err);
|
||||
}
|
||||
|
||||
// Everything else (5xx, timeouts, network) = transient
|
||||
return new AITransientError(`${ctxPrefix}${msg}`, err);
|
||||
}
|
||||
@@ -0,0 +1,854 @@
|
||||
/**
|
||||
* AI Gateway — unified seam for every AI call gbrain makes.
|
||||
*
|
||||
* v0.14 exports:
|
||||
* - configureGateway(config) — called once by cli.ts connectEngine()
|
||||
* - embed(texts) — embedding for put_page + import
|
||||
* - embedOne(text) — convenience wrapper
|
||||
* - expand(query) — query expansion for hybrid search
|
||||
* - isAvailable(touchpoint) — replaces scattered OPENAI_API_KEY checks
|
||||
* - getEmbeddingDimensions() — for schema setup
|
||||
* - getEmbeddingModel() — for schema metadata
|
||||
*
|
||||
* Future stubs: chunk, transcribe, enrich, improve (throw NotMigratedYet until migrated).
|
||||
*
|
||||
* DESIGN RULES:
|
||||
* - Gateway reads config from a single configureGateway() call.
|
||||
* - NEVER reads process.env at call time (Codex C3).
|
||||
* - AI SDK error instances are normalized to AIConfigError / AITransientError.
|
||||
* - Explicit dimensions passthrough preserves existing 1536 brains (Codex C1).
|
||||
* - Per-provider model cache keyed by (provider, modelId, baseUrl) so env
|
||||
* rotation (via configureGateway()) invalidates stale entries.
|
||||
*/
|
||||
|
||||
import { embed as aiEmbed, embedMany, generateObject, generateText } from 'ai';
|
||||
import { listRecipes } from './recipes/index.ts';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import { createGoogleGenerativeAI } from '@ai-sdk/google';
|
||||
import { createAnthropic } from '@ai-sdk/anthropic';
|
||||
import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
|
||||
import { z } from 'zod';
|
||||
|
||||
import type {
|
||||
AIGatewayConfig,
|
||||
Recipe,
|
||||
TouchpointKind,
|
||||
} from './types.ts';
|
||||
import { resolveRecipe, assertTouchpoint } from './model-resolver.ts';
|
||||
import { dimsProviderOptions } from './dims.ts';
|
||||
import { AIConfigError, AITransientError, normalizeAIError } from './errors.ts';
|
||||
|
||||
const MAX_CHARS = 8000;
|
||||
const DEFAULT_EMBEDDING_MODEL = 'openai:text-embedding-3-large';
|
||||
const DEFAULT_EMBEDDING_DIMENSIONS = 1536;
|
||||
const DEFAULT_EXPANSION_MODEL = 'anthropic:claude-haiku-4-5-20251001';
|
||||
const DEFAULT_CHAT_MODEL = 'anthropic:claude-sonnet-4-6-20250929';
|
||||
|
||||
let _config: AIGatewayConfig | null = null;
|
||||
const _modelCache = new Map<string, any>();
|
||||
|
||||
/**
|
||||
* The function the gateway calls to actually run a batch through the AI SDK.
|
||||
* Defaults to the imported `embedMany`. Tests inject a stub via
|
||||
* `__setEmbedTransportForTests` to drive recursion + fast-path scenarios
|
||||
* without hitting a real provider. Production never reads the override.
|
||||
*/
|
||||
type EmbedManyFn = typeof embedMany;
|
||||
let _embedTransport: EmbedManyFn = embedMany;
|
||||
|
||||
/**
|
||||
* Per-recipe shrink-on-miss state. When a recipe's pre-split misses the
|
||||
* provider's batch cap and recursive halving fires, we tighten its
|
||||
* effective `safety_factor` so subsequent `embed()` calls pre-split smaller
|
||||
* out of the gate. After 10 consecutive batch successes, the factor heals
|
||||
* back toward the recipe default (×1.5 per heal, capped at the declared
|
||||
* `safety_factor`). Module-scoped because the gateway itself is module-scoped;
|
||||
* `resetGateway()` and `configureGateway()` clear it.
|
||||
*/
|
||||
interface ShrinkEntry {
|
||||
factor: number;
|
||||
consecutiveSuccesses: number;
|
||||
}
|
||||
const _shrinkState = new Map<string, ShrinkEntry>();
|
||||
|
||||
/** Floor for shrink-on-miss to prevent infinite shrinking. */
|
||||
const SHRINK_FLOOR = 0.05;
|
||||
/** Successful batches needed before the factor heals back toward recipe default. */
|
||||
const SHRINK_HEAL_AFTER = 10;
|
||||
/** Default chars-per-token when a recipe omits it. Matches OpenAI tiktoken on English. */
|
||||
const DEFAULT_CHARS_PER_TOKEN = 4;
|
||||
/** Default safety factor when a recipe omits it. */
|
||||
const DEFAULT_SAFETY_FACTOR = 0.8;
|
||||
|
||||
/** Configure the gateway. Called by cli.ts#connectEngine. Clears cached models. */
|
||||
export function configureGateway(config: AIGatewayConfig): void {
|
||||
_config = {
|
||||
embedding_model: config.embedding_model ?? DEFAULT_EMBEDDING_MODEL,
|
||||
embedding_dimensions: config.embedding_dimensions ?? DEFAULT_EMBEDDING_DIMENSIONS,
|
||||
expansion_model: config.expansion_model ?? DEFAULT_EXPANSION_MODEL,
|
||||
chat_model: config.chat_model ?? DEFAULT_CHAT_MODEL,
|
||||
chat_fallback_chain: config.chat_fallback_chain,
|
||||
base_urls: config.base_urls,
|
||||
env: config.env,
|
||||
};
|
||||
_modelCache.clear();
|
||||
_shrinkState.clear();
|
||||
warnRecipesMissingBatchTokens();
|
||||
}
|
||||
|
||||
/**
|
||||
* Recipes that have already triggered the missing-max_batch_tokens warning
|
||||
* in this process. Bounded by the number of registered recipes (~10 today).
|
||||
* Cleared on `resetGateway()` so tests can re-exercise the warning path.
|
||||
*/
|
||||
const _warnedRecipes = new Set<string>();
|
||||
|
||||
/**
|
||||
* Walk every registered recipe with an `embedding` touchpoint. Each one
|
||||
* missing `max_batch_tokens` gets exactly one stderr line per process for
|
||||
* its first appearance. Recipes WITH the field stay quiet. The
|
||||
* recursive-halving safety net only fires when `max_batch_tokens` is set,
|
||||
* so a recipe that forgets it has no protection if the provider has a
|
||||
* batch cap. Loud-fail over silent-skip per CLAUDE.md; a future
|
||||
* Cohere/Mistral/Jina recipe that inherits the embedding-touchpoint
|
||||
* pattern but forgets the cap re-creates the v0.27 Voyage backfill loop.
|
||||
* The warning calls that out before production traffic hits it.
|
||||
*/
|
||||
function warnRecipesMissingBatchTokens(): void {
|
||||
for (const recipe of listRecipes()) {
|
||||
const embedding = recipe.touchpoints?.embedding;
|
||||
if (!embedding || embedding.max_batch_tokens !== undefined) continue;
|
||||
// OpenAI is the canonical "no cap declared, fast path is intentional"
|
||||
// recipe; suppress the warning for it. Every other recipe missing the
|
||||
// field is suspicious.
|
||||
if (recipe.id === 'openai') continue;
|
||||
if (_warnedRecipes.has(recipe.id)) continue;
|
||||
_warnedRecipes.add(recipe.id);
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
`[ai.gateway] recipe "${recipe.id}" declares an embedding touchpoint ` +
|
||||
`without max_batch_tokens; recursion is the only safety net for batch caps.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Reset (for tests). */
|
||||
export function resetGateway(): void {
|
||||
_config = null;
|
||||
_modelCache.clear();
|
||||
_shrinkState.clear();
|
||||
_embedTransport = embedMany;
|
||||
_warnedRecipes.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Test-only seam. Replaces the function the gateway calls to embed a
|
||||
* sub-batch. Pass `null` to restore the real `embedMany` from the AI SDK.
|
||||
* Exported intentionally for the adaptive-embed-batch test suite to drive
|
||||
* recursion + fast-path scenarios deterministically. Production code MUST
|
||||
* NOT call this — there is no use case outside tests.
|
||||
*
|
||||
* @internal exported for tests; not part of the public gateway API.
|
||||
*/
|
||||
export function __setEmbedTransportForTests(fn: EmbedManyFn | null): void {
|
||||
_embedTransport = fn ?? embedMany;
|
||||
}
|
||||
|
||||
function requireConfig(): AIGatewayConfig {
|
||||
if (!_config) {
|
||||
throw new AIConfigError(
|
||||
'AI gateway is not configured. Call configureGateway() during engine connect.',
|
||||
'This is a gbrain bug — file an issue at https://github.com/garrytan/gbrain/issues',
|
||||
);
|
||||
}
|
||||
return _config;
|
||||
}
|
||||
|
||||
/** Public config accessors (for schema setup, doctor, etc.). */
|
||||
export function getEmbeddingModel(): string {
|
||||
return requireConfig().embedding_model ?? DEFAULT_EMBEDDING_MODEL;
|
||||
}
|
||||
|
||||
export function getEmbeddingDimensions(): number {
|
||||
return requireConfig().embedding_dimensions ?? DEFAULT_EMBEDDING_DIMENSIONS;
|
||||
}
|
||||
|
||||
export function getExpansionModel(): string {
|
||||
return requireConfig().expansion_model ?? DEFAULT_EXPANSION_MODEL;
|
||||
}
|
||||
|
||||
export function getChatModel(): string {
|
||||
return requireConfig().chat_model ?? DEFAULT_CHAT_MODEL;
|
||||
}
|
||||
|
||||
export function getChatFallbackChain(): string[] {
|
||||
return requireConfig().chat_fallback_chain ?? [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether a touchpoint can be served given the current config.
|
||||
* Replaces scattered `!process.env.OPENAI_API_KEY` checks (Codex C3).
|
||||
*/
|
||||
export function isAvailable(touchpoint: TouchpointKind): boolean {
|
||||
if (!_config) return false;
|
||||
try {
|
||||
const modelStr =
|
||||
touchpoint === 'embedding'
|
||||
? getEmbeddingModel()
|
||||
: touchpoint === 'expansion'
|
||||
? getExpansionModel()
|
||||
: touchpoint === 'chat'
|
||||
? getChatModel()
|
||||
: null;
|
||||
if (!modelStr) return false;
|
||||
const { recipe } = resolveRecipe(modelStr);
|
||||
|
||||
// Recipe must actually support the requested touchpoint.
|
||||
// Anthropic declares only expansion + chat (no embedding model); requesting
|
||||
// embedding from an anthropic-configured brain is unavailable regardless of auth.
|
||||
const touchpointConfig = recipe.touchpoints[touchpoint as 'embedding' | 'expansion' | 'chat'];
|
||||
if (!touchpointConfig) return false;
|
||||
// Openai-compat recipes with empty models list (e.g. litellm template) require user-provided model
|
||||
if (Array.isArray(touchpointConfig.models) && touchpointConfig.models.length === 0 && recipe.id === 'litellm') return false;
|
||||
|
||||
// For openai-compatible without auth requirements (Ollama local), treat as always-available.
|
||||
const required = recipe.auth_env?.required ?? [];
|
||||
if (required.length === 0) return true;
|
||||
return required.every(k => !!_config!.env[k]);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Embedding ----
|
||||
|
||||
async function resolveEmbeddingProvider(modelStr: string): Promise<{ model: any; recipe: Recipe; modelId: string }> {
|
||||
const { parsed, recipe } = resolveRecipe(modelStr);
|
||||
assertTouchpoint(recipe, 'embedding', parsed.modelId);
|
||||
const cfg = requireConfig();
|
||||
|
||||
const cacheKey = `emb:${recipe.id}:${parsed.modelId}:${cfg.base_urls?.[recipe.id] ?? ''}`;
|
||||
const cached = _modelCache.get(cacheKey);
|
||||
if (cached) return { model: cached, recipe, modelId: parsed.modelId };
|
||||
|
||||
const model = instantiateEmbedding(recipe, parsed.modelId, cfg);
|
||||
_modelCache.set(cacheKey, model);
|
||||
return { model, recipe, modelId: parsed.modelId };
|
||||
}
|
||||
|
||||
function instantiateEmbedding(recipe: Recipe, modelId: string, cfg: AIGatewayConfig): any {
|
||||
switch (recipe.implementation) {
|
||||
case 'native-openai': {
|
||||
const apiKey = cfg.env.OPENAI_API_KEY;
|
||||
if (!apiKey) throw new AIConfigError(
|
||||
`OpenAI embedding requires OPENAI_API_KEY.`,
|
||||
recipe.setup_hint,
|
||||
);
|
||||
const client = createOpenAI({ apiKey });
|
||||
// AI SDK v6: use .textEmbeddingModel() for embeddings
|
||||
return (client as any).textEmbeddingModel
|
||||
? (client as any).textEmbeddingModel(modelId)
|
||||
: (client as any).embedding(modelId);
|
||||
}
|
||||
case 'native-google': {
|
||||
const apiKey = cfg.env.GOOGLE_GENERATIVE_AI_API_KEY;
|
||||
if (!apiKey) throw new AIConfigError(
|
||||
`Google embedding requires GOOGLE_GENERATIVE_AI_API_KEY.`,
|
||||
recipe.setup_hint,
|
||||
);
|
||||
const client = createGoogleGenerativeAI({ apiKey });
|
||||
return (client as any).textEmbeddingModel
|
||||
? (client as any).textEmbeddingModel(modelId)
|
||||
: (client as any).embedding(modelId);
|
||||
}
|
||||
case 'native-anthropic':
|
||||
throw new AIConfigError(
|
||||
`Anthropic has no embedding model. Use openai or google for embeddings.`,
|
||||
);
|
||||
case 'openai-compatible': {
|
||||
const baseUrl = cfg.base_urls?.[recipe.id] ?? recipe.base_url_default;
|
||||
if (!baseUrl) throw new AIConfigError(
|
||||
`${recipe.name} requires a base URL.`,
|
||||
recipe.setup_hint,
|
||||
);
|
||||
// For openai-compatible, auth is optional (ollama local) but pass a dummy key if unauthenticated.
|
||||
const apiKey = recipe.auth_env?.required[0]
|
||||
? cfg.env[recipe.auth_env.required[0]]
|
||||
: (cfg.env[`${recipe.id.toUpperCase()}_API_KEY`] ?? 'unauthenticated');
|
||||
if (recipe.auth_env?.required.length && !apiKey) {
|
||||
throw new AIConfigError(
|
||||
`${recipe.name} requires ${recipe.auth_env.required[0]}.`,
|
||||
recipe.setup_hint,
|
||||
);
|
||||
}
|
||||
const client = createOpenAICompatible({
|
||||
name: recipe.id,
|
||||
baseURL: baseUrl,
|
||||
apiKey: apiKey ?? 'unauthenticated',
|
||||
});
|
||||
return client.textEmbeddingModel(modelId);
|
||||
}
|
||||
default:
|
||||
throw new AIConfigError(`Unknown implementation: ${(recipe as any).implementation}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Minimum sub-batch size before we give up splitting and just throw. */
|
||||
const MIN_SUB_BATCH = 1;
|
||||
|
||||
/**
|
||||
* Embed many texts. Truncates to MAX_CHARS, then dispatches based on whether
|
||||
* the recipe declares a per-batch token budget.
|
||||
*
|
||||
* Flow:
|
||||
* ```
|
||||
* embed(texts)
|
||||
* ├─ resolve recipe + model
|
||||
* ├─ truncate each text to MAX_CHARS (8000)
|
||||
* ├─ read recipe.touchpoints.embedding.{max_batch_tokens, chars_per_token, safety_factor}
|
||||
* │
|
||||
* ├─ if max_batch_tokens declared (Voyage path):
|
||||
* │ budget = max_batch_tokens × shrinkState[recipe].factor (default = recipe.safety_factor)
|
||||
* │ splitByTokenBudget(texts, budget, recipe.chars_per_token)
|
||||
* │ for each sub-batch: embedSubBatch(...)
|
||||
* │
|
||||
* └─ else (OpenAI fast path):
|
||||
* embedSubBatch(texts, ...) once // no pre-split, no token-limit safety net
|
||||
*
|
||||
* embedSubBatch(texts, ...)
|
||||
* ├─ try: _embedTransport(texts) → dim check → return Float32Array[]
|
||||
* │ on success: bump shrinkState[recipe].consecutiveSuccesses
|
||||
* │
|
||||
* └─ catch:
|
||||
* if isTokenLimitError(err) AND texts.length > MIN_SUB_BATCH:
|
||||
* shrinkState[recipe].factor *= 0.5 (next embed() pre-splits tighter)
|
||||
* halve at mid=⌈N/2⌉
|
||||
* embedSubBatch(left) ──┐
|
||||
* embedSubBatch(right) ──┴─ concat in order, return
|
||||
* else:
|
||||
* throw normalizeAIError(err, ...)
|
||||
* ```
|
||||
*
|
||||
* Per-recipe state lives in `_shrinkState` and survives across `embed()`
|
||||
* calls within one process. The healing path (after `SHRINK_HEAL_AFTER`
|
||||
* consecutive batch successes) walks the factor back toward the recipe's
|
||||
* declared `safety_factor` so a transient miss doesn't permanently cap
|
||||
* throughput.
|
||||
*/
|
||||
export async function embed(texts: string[]): Promise<Float32Array[]> {
|
||||
if (!texts || texts.length === 0) return [];
|
||||
|
||||
const cfg = requireConfig();
|
||||
const { model, recipe, modelId } = await resolveEmbeddingProvider(getEmbeddingModel());
|
||||
const truncated = texts.map(t => (t ?? '').slice(0, MAX_CHARS));
|
||||
const providerOpts = dimsProviderOptions(recipe.implementation, modelId, cfg.embedding_dimensions ?? DEFAULT_EMBEDDING_DIMENSIONS);
|
||||
const expected = cfg.embedding_dimensions ?? DEFAULT_EMBEDDING_DIMENSIONS;
|
||||
|
||||
const embedding = recipe.touchpoints?.embedding;
|
||||
const maxBatchTokens = embedding?.max_batch_tokens;
|
||||
const charsPerToken = embedding?.chars_per_token ?? DEFAULT_CHARS_PER_TOKEN;
|
||||
|
||||
// Pre-split is gated on max_batch_tokens. Recipes without it (e.g. OpenAI)
|
||||
// ride the fast path: one embedMany call, no recursion safety net.
|
||||
const batches = maxBatchTokens
|
||||
? splitByTokenBudget(truncated, Math.floor(maxBatchTokens * effectiveSafetyFactor(recipe)), charsPerToken)
|
||||
: [truncated];
|
||||
|
||||
const allEmbeddings: Float32Array[] = [];
|
||||
|
||||
for (const batch of batches) {
|
||||
const result = await embedSubBatch(batch, model, providerOpts, expected, recipe, modelId);
|
||||
allEmbeddings.push(...result);
|
||||
}
|
||||
|
||||
return allEmbeddings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split texts into sub-batches that stay under the provided budget. Pure;
|
||||
* no module state. Exported for the adaptive-embed-batch test suite.
|
||||
*
|
||||
* @param texts - The texts to partition. Each text counts as
|
||||
* `Math.ceil(text.length / charsPerToken)` tokens for budget purposes.
|
||||
* @param budgetTokens - The token ceiling for each sub-batch. Caller is
|
||||
* responsible for applying any safety-factor shrink before passing in.
|
||||
* @param charsPerToken - Provider-specific character density. Defaults to
|
||||
* `DEFAULT_CHARS_PER_TOKEN` (4) when omitted, matching OpenAI tiktoken.
|
||||
*
|
||||
* @internal exported for tests; not part of the public gateway API.
|
||||
*/
|
||||
export function splitByTokenBudget(
|
||||
texts: string[],
|
||||
budgetTokens: number,
|
||||
charsPerToken: number = DEFAULT_CHARS_PER_TOKEN,
|
||||
): string[][] {
|
||||
const ratio = charsPerToken > 0 ? charsPerToken : DEFAULT_CHARS_PER_TOKEN;
|
||||
const batches: string[][] = [];
|
||||
let current: string[] = [];
|
||||
let currentTokens = 0;
|
||||
|
||||
for (const text of texts) {
|
||||
const estTokens = Math.ceil(text.length / ratio);
|
||||
if (current.length > 0 && currentTokens + estTokens > budgetTokens) {
|
||||
batches.push(current);
|
||||
current = [];
|
||||
currentTokens = 0;
|
||||
}
|
||||
current.push(text);
|
||||
currentTokens += estTokens;
|
||||
}
|
||||
if (current.length > 0) batches.push(current);
|
||||
|
||||
return batches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the error looks like a provider batch-token-limit error.
|
||||
*
|
||||
* @internal exported for tests; not part of the public gateway API.
|
||||
*/
|
||||
export function isTokenLimitError(err: unknown): boolean {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return (
|
||||
/max.*allowed.*tokens.*batch/i.test(msg) ||
|
||||
/batch.*too.*many.*tokens/i.test(msg) ||
|
||||
/token.*limit.*exceeded/i.test(msg)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the recipe's effective safety factor (declared default, optionally
|
||||
* shrunk by prior misses in this process).
|
||||
*/
|
||||
function effectiveSafetyFactor(recipe: Recipe): number {
|
||||
const declared = recipe.touchpoints?.embedding?.safety_factor ?? DEFAULT_SAFETY_FACTOR;
|
||||
const entry = _shrinkState.get(recipe.id);
|
||||
return entry?.factor ?? declared;
|
||||
}
|
||||
|
||||
/** Tighten the recipe's effective safety factor on a token-limit miss. */
|
||||
function shrinkOnMiss(recipe: Recipe): void {
|
||||
const declared = recipe.touchpoints?.embedding?.safety_factor ?? DEFAULT_SAFETY_FACTOR;
|
||||
const current = _shrinkState.get(recipe.id)?.factor ?? declared;
|
||||
const next = Math.max(SHRINK_FLOOR, current * 0.5);
|
||||
_shrinkState.set(recipe.id, { factor: next, consecutiveSuccesses: 0 });
|
||||
}
|
||||
|
||||
/** Bump the win counter; heal toward declared default after enough wins. */
|
||||
function recordSubBatchSuccess(recipe: Recipe): void {
|
||||
const declared = recipe.touchpoints?.embedding?.safety_factor ?? DEFAULT_SAFETY_FACTOR;
|
||||
const entry = _shrinkState.get(recipe.id);
|
||||
if (!entry || entry.factor >= declared) {
|
||||
// Either no shrink active, or already at/above the declared ceiling — nothing to heal.
|
||||
if (entry) {
|
||||
_shrinkState.set(recipe.id, { factor: entry.factor, consecutiveSuccesses: 0 });
|
||||
}
|
||||
return;
|
||||
}
|
||||
const wins = entry.consecutiveSuccesses + 1;
|
||||
if (wins >= SHRINK_HEAL_AFTER) {
|
||||
const healed = Math.min(declared, entry.factor * 1.5);
|
||||
_shrinkState.set(recipe.id, { factor: healed, consecutiveSuccesses: 0 });
|
||||
} else {
|
||||
_shrinkState.set(recipe.id, { factor: entry.factor, consecutiveSuccesses: wins });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the current shrink state for a recipe. Test-only seam.
|
||||
*
|
||||
* @internal exported for tests; not part of the public gateway API.
|
||||
*/
|
||||
export function __getShrinkStateForTests(recipeId: string): ShrinkEntry | undefined {
|
||||
const entry = _shrinkState.get(recipeId);
|
||||
return entry ? { ...entry } : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Embed a single sub-batch with automatic halving on token-limit errors.
|
||||
* If the batch is already at MIN_SUB_BATCH and still fails, throws.
|
||||
*/
|
||||
async function embedSubBatch(
|
||||
texts: string[],
|
||||
model: any,
|
||||
providerOpts: any,
|
||||
expectedDims: number,
|
||||
recipe: Recipe,
|
||||
modelId: string,
|
||||
): Promise<Float32Array[]> {
|
||||
try {
|
||||
const result = await _embedTransport({
|
||||
model,
|
||||
values: texts,
|
||||
providerOptions: providerOpts,
|
||||
});
|
||||
|
||||
const first = result.embeddings?.[0];
|
||||
if (first && Array.isArray(first) && first.length !== expectedDims) {
|
||||
throw new AIConfigError(
|
||||
`Embedding dim mismatch: model ${modelId} returned ${first.length} but schema expects ${expectedDims}.`,
|
||||
`Run \`gbrain migrate --embedding-model ${getEmbeddingModel()} --embedding-dimensions ${first.length}\` or change models.`,
|
||||
);
|
||||
}
|
||||
|
||||
recordSubBatchSuccess(recipe);
|
||||
return result.embeddings.map((e: number[]) => new Float32Array(e));
|
||||
} catch (err) {
|
||||
// On token-limit error, tighten the recipe's effective safety factor
|
||||
// (so the next embed() pre-splits smaller) and recursively halve THIS
|
||||
// batch to make forward progress without dropping work.
|
||||
if (isTokenLimitError(err) && texts.length > MIN_SUB_BATCH) {
|
||||
shrinkOnMiss(recipe);
|
||||
const mid = Math.ceil(texts.length / 2);
|
||||
const left = await embedSubBatch(texts.slice(0, mid), model, providerOpts, expectedDims, recipe, modelId);
|
||||
const right = await embedSubBatch(texts.slice(mid), model, providerOpts, expectedDims, recipe, modelId);
|
||||
return [...left, ...right];
|
||||
}
|
||||
throw normalizeAIError(err, `embed(${recipe.id}:${modelId})`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Embed one text (convenience wrapper). */
|
||||
export async function embedOne(text: string): Promise<Float32Array> {
|
||||
const [v] = await embed([text]);
|
||||
return v;
|
||||
}
|
||||
|
||||
// ---- Expansion ----
|
||||
|
||||
async function resolveExpansionProvider(modelStr: string): Promise<{ model: any; recipe: Recipe; modelId: string }> {
|
||||
const { parsed, recipe } = resolveRecipe(modelStr);
|
||||
assertTouchpoint(recipe, 'expansion', parsed.modelId);
|
||||
const cfg = requireConfig();
|
||||
|
||||
const cacheKey = `exp:${recipe.id}:${parsed.modelId}:${cfg.base_urls?.[recipe.id] ?? ''}`;
|
||||
const cached = _modelCache.get(cacheKey);
|
||||
if (cached) return { model: cached, recipe, modelId: parsed.modelId };
|
||||
|
||||
const model = instantiateExpansion(recipe, parsed.modelId, cfg);
|
||||
_modelCache.set(cacheKey, model);
|
||||
return { model, recipe, modelId: parsed.modelId };
|
||||
}
|
||||
|
||||
function instantiateExpansion(recipe: Recipe, modelId: string, cfg: AIGatewayConfig): any {
|
||||
switch (recipe.implementation) {
|
||||
case 'native-openai': {
|
||||
const apiKey = cfg.env.OPENAI_API_KEY;
|
||||
if (!apiKey) throw new AIConfigError(`OpenAI expansion requires OPENAI_API_KEY.`, recipe.setup_hint);
|
||||
return createOpenAI({ apiKey }).languageModel(modelId);
|
||||
}
|
||||
case 'native-google': {
|
||||
const apiKey = cfg.env.GOOGLE_GENERATIVE_AI_API_KEY;
|
||||
if (!apiKey) throw new AIConfigError(`Google expansion requires GOOGLE_GENERATIVE_AI_API_KEY.`, recipe.setup_hint);
|
||||
return createGoogleGenerativeAI({ apiKey }).languageModel(modelId);
|
||||
}
|
||||
case 'native-anthropic': {
|
||||
const apiKey = cfg.env.ANTHROPIC_API_KEY;
|
||||
if (!apiKey) throw new AIConfigError(`Anthropic expansion requires ANTHROPIC_API_KEY.`, recipe.setup_hint);
|
||||
return createAnthropic({ apiKey }).languageModel(modelId);
|
||||
}
|
||||
case 'openai-compatible': {
|
||||
const baseUrl = cfg.base_urls?.[recipe.id] ?? recipe.base_url_default;
|
||||
if (!baseUrl) throw new AIConfigError(`${recipe.name} requires a base URL.`, recipe.setup_hint);
|
||||
const apiKey = recipe.auth_env?.required[0]
|
||||
? cfg.env[recipe.auth_env.required[0]]
|
||||
: 'unauthenticated';
|
||||
return createOpenAICompatible({
|
||||
name: recipe.id,
|
||||
baseURL: baseUrl,
|
||||
apiKey: apiKey ?? 'unauthenticated',
|
||||
}).languageModel(modelId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const ExpansionSchema = z.object({
|
||||
queries: z.array(z.string()).min(1).max(5),
|
||||
});
|
||||
|
||||
/**
|
||||
* Expand a search query into up to 4 related queries.
|
||||
* Returns the original query PLUS expansions. On failure, returns just the original.
|
||||
* Caller is responsible for sanitizing the query (prompt-injection boundary stays in expansion.ts).
|
||||
*/
|
||||
export async function expand(query: string): Promise<string[]> {
|
||||
if (!query || !query.trim()) return [query];
|
||||
if (!isAvailable('expansion')) return [query];
|
||||
|
||||
try {
|
||||
const { model, recipe, modelId } = await resolveExpansionProvider(getExpansionModel());
|
||||
const result = await generateObject({
|
||||
model,
|
||||
schema: ExpansionSchema,
|
||||
prompt: [
|
||||
'Rewrite the search query below into 3-4 different, related queries that would help find relevant documents.',
|
||||
'Return ONLY the JSON object. Do NOT include the original query in the result.',
|
||||
'Each rewrite should emphasize different aspects, synonyms, or framings.',
|
||||
'',
|
||||
`Query: ${query}`,
|
||||
].join('\n'),
|
||||
});
|
||||
|
||||
const expansions = result.object?.queries ?? [];
|
||||
// Deduplicate + include the original query
|
||||
const seen = new Set<string>();
|
||||
const all = [query, ...expansions].filter(q => {
|
||||
const k = q.toLowerCase().trim();
|
||||
if (seen.has(k)) return false;
|
||||
seen.add(k);
|
||||
return !!q.trim();
|
||||
});
|
||||
return all;
|
||||
} catch (err) {
|
||||
// Expansion is best-effort: on failure, fall back to the original query alone.
|
||||
const normalized = normalizeAIError(err, 'expand');
|
||||
if (normalized instanceof AIConfigError) {
|
||||
console.warn(`[ai.gateway] expansion disabled: ${normalized.message}`);
|
||||
}
|
||||
return [query];
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Chat (commit 1) ----
|
||||
|
||||
/**
|
||||
* Provider-neutral message shape stored in subagent persistence (commit 2a).
|
||||
* Vercel AI SDK's `generateText` accepts this directly via its `messages`
|
||||
* parameter; tool-use blocks are normalized across providers.
|
||||
*/
|
||||
export type ChatRole = 'system' | 'user' | 'assistant' | 'tool';
|
||||
|
||||
export type ChatBlock =
|
||||
| { type: 'text'; text: string }
|
||||
| { type: 'tool-call'; toolCallId: string; toolName: string; input: unknown }
|
||||
| { type: 'tool-result'; toolCallId: string; toolName: string; output: unknown; isError?: boolean };
|
||||
|
||||
export interface ChatMessage {
|
||||
role: ChatRole;
|
||||
content: string | ChatBlock[];
|
||||
}
|
||||
|
||||
export interface ChatToolDef {
|
||||
name: string;
|
||||
description: string;
|
||||
/** JSON Schema for tool input. */
|
||||
inputSchema: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface ChatResult {
|
||||
/** Final text content concatenated from text blocks. */
|
||||
text: string;
|
||||
/** Raw assistant response blocks (text + tool-call entries) for persistence. */
|
||||
blocks: ChatBlock[];
|
||||
/** Reason the model stopped. Provider-neutral mapping of stop_reason / finish_reason. */
|
||||
stopReason: 'end' | 'tool_calls' | 'length' | 'refusal' | 'content_filter' | 'other';
|
||||
/** Provider-neutral usage. cache_* are present only when the active provider returned them (Anthropic). */
|
||||
usage: {
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
cache_read_tokens: number;
|
||||
cache_creation_tokens: number;
|
||||
};
|
||||
/** "provider:modelId" string of the model that actually answered. */
|
||||
model: string;
|
||||
/** Recipe id for the answering provider. */
|
||||
providerId: string;
|
||||
/** Raw provider metadata (Anthropic-specific cache fields, OpenAI finish_reason, etc.) for downstream callers that need it. */
|
||||
providerMetadata?: Record<string, any>;
|
||||
}
|
||||
|
||||
export interface ChatOpts {
|
||||
/** "provider:modelId" — defaults to config.chat_model. */
|
||||
model?: string;
|
||||
/** System prompt. */
|
||||
system?: string;
|
||||
messages: ChatMessage[];
|
||||
tools?: ChatToolDef[];
|
||||
maxTokens?: number;
|
||||
abortSignal?: AbortSignal;
|
||||
/**
|
||||
* Anthropic-specific: cache the system prompt + last tool def. Silently
|
||||
* ignored on providers without `supports_prompt_cache`.
|
||||
*/
|
||||
cacheSystem?: boolean;
|
||||
}
|
||||
|
||||
async function resolveChatProvider(modelStr: string): Promise<{ model: any; recipe: Recipe; modelId: string }> {
|
||||
const { parsed, recipe } = resolveRecipe(modelStr);
|
||||
assertTouchpoint(recipe, 'chat', parsed.modelId);
|
||||
const cfg = requireConfig();
|
||||
|
||||
const cacheKey = `chat:${recipe.id}:${parsed.modelId}:${cfg.base_urls?.[recipe.id] ?? ''}`;
|
||||
const cached = _modelCache.get(cacheKey);
|
||||
if (cached) return { model: cached, recipe, modelId: parsed.modelId };
|
||||
|
||||
const model = instantiateChat(recipe, parsed.modelId, cfg);
|
||||
_modelCache.set(cacheKey, model);
|
||||
return { model, recipe, modelId: parsed.modelId };
|
||||
}
|
||||
|
||||
function instantiateChat(recipe: Recipe, modelId: string, cfg: AIGatewayConfig): any {
|
||||
switch (recipe.implementation) {
|
||||
case 'native-openai': {
|
||||
const apiKey = cfg.env.OPENAI_API_KEY;
|
||||
if (!apiKey) throw new AIConfigError(`OpenAI chat requires OPENAI_API_KEY.`, recipe.setup_hint);
|
||||
return createOpenAI({ apiKey }).languageModel(modelId);
|
||||
}
|
||||
case 'native-google': {
|
||||
const apiKey = cfg.env.GOOGLE_GENERATIVE_AI_API_KEY;
|
||||
if (!apiKey) throw new AIConfigError(`Google chat requires GOOGLE_GENERATIVE_AI_API_KEY.`, recipe.setup_hint);
|
||||
return createGoogleGenerativeAI({ apiKey }).languageModel(modelId);
|
||||
}
|
||||
case 'native-anthropic': {
|
||||
const apiKey = cfg.env.ANTHROPIC_API_KEY;
|
||||
if (!apiKey) throw new AIConfigError(`Anthropic chat requires ANTHROPIC_API_KEY.`, recipe.setup_hint);
|
||||
return createAnthropic({ apiKey }).languageModel(modelId);
|
||||
}
|
||||
case 'openai-compatible': {
|
||||
const baseUrl = cfg.base_urls?.[recipe.id] ?? recipe.base_url_default;
|
||||
if (!baseUrl) throw new AIConfigError(`${recipe.name} requires a base URL.`, recipe.setup_hint);
|
||||
const required = recipe.auth_env?.required ?? [];
|
||||
const apiKey = required[0] ? cfg.env[required[0]] : 'unauthenticated';
|
||||
if (required.length > 0 && !apiKey) {
|
||||
throw new AIConfigError(`${recipe.name} requires ${required[0]}.`, recipe.setup_hint);
|
||||
}
|
||||
return createOpenAICompatible({
|
||||
name: recipe.id,
|
||||
baseURL: baseUrl,
|
||||
apiKey: apiKey ?? 'unauthenticated',
|
||||
}).languageModel(modelId);
|
||||
}
|
||||
default:
|
||||
throw new AIConfigError(`Unknown implementation: ${(recipe as any).implementation}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Map AI SDK's `finish_reason` (and provider-specific signals) to a provider-
|
||||
* neutral `stopReason`. This is the structural-signal layer that
|
||||
* `chatWithFallback` (commit 3) consults BEFORE any regex heuristic (per D8).
|
||||
*/
|
||||
function mapStopReason(
|
||||
finishReason: string | undefined,
|
||||
providerMetadata: Record<string, any> | undefined,
|
||||
): ChatResult['stopReason'] {
|
||||
// Anthropic: `stop_reason: 'refusal'` lands in providerMetadata.anthropic.
|
||||
const anthropicStop = providerMetadata?.anthropic?.stopReason ?? providerMetadata?.anthropic?.stop_reason;
|
||||
if (anthropicStop === 'refusal') return 'refusal';
|
||||
// OpenAI: `finish_reason: 'content_filter'`.
|
||||
if (finishReason === 'content-filter' || finishReason === 'content_filter') return 'content_filter';
|
||||
if (finishReason === 'tool-calls' || finishReason === 'tool_calls') return 'tool_calls';
|
||||
if (finishReason === 'length' || finishReason === 'max-tokens') return 'length';
|
||||
if (finishReason === 'stop' || finishReason === 'end' || finishReason === 'end-turn') return 'end';
|
||||
return 'other';
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one chat completion turn. Provider-neutral wrapper over Vercel AI SDK's
|
||||
* `generateText`. Tool-use blocks are normalized; cache_control markers are
|
||||
* applied only on Anthropic when `cacheSystem: true`.
|
||||
*
|
||||
* Crash-resumable replay is the caller's responsibility (subagent.ts persists
|
||||
* blocks via the provider-neutral schema landing in commit 2a).
|
||||
*/
|
||||
export async function chat(opts: ChatOpts): Promise<ChatResult> {
|
||||
const modelStr = opts.model ?? getChatModel();
|
||||
const { model, recipe, modelId } = await resolveChatProvider(modelStr);
|
||||
|
||||
const supportsCache = recipe.touchpoints.chat?.supports_prompt_cache === true;
|
||||
const useCache = !!opts.cacheSystem && supportsCache;
|
||||
|
||||
// Build messages. Anthropic prompt-cache markers ride on system + last tool
|
||||
// via providerOptions; the AI SDK accepts the system as a string for
|
||||
// generateText, so cache markers go through providerOptions.anthropic.
|
||||
const tools = (opts.tools ?? []).reduce((acc, t) => {
|
||||
acc[t.name] = {
|
||||
description: t.description,
|
||||
inputSchema: { jsonSchema: t.inputSchema } as any,
|
||||
};
|
||||
return acc;
|
||||
}, {} as Record<string, any>);
|
||||
|
||||
const providerOptions: Record<string, any> = {};
|
||||
if (useCache) {
|
||||
providerOptions.anthropic = { cacheControl: { type: 'ephemeral' } };
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await generateText({
|
||||
model,
|
||||
system: opts.system,
|
||||
messages: opts.messages as any,
|
||||
tools: opts.tools && opts.tools.length > 0 ? tools : undefined,
|
||||
maxOutputTokens: opts.maxTokens ?? 4096,
|
||||
abortSignal: opts.abortSignal,
|
||||
providerOptions: Object.keys(providerOptions).length > 0 ? providerOptions : undefined,
|
||||
});
|
||||
|
||||
// Normalize blocks. Vercel SDK gives us `result.content` (an array of typed
|
||||
// parts) for v6+; fall back to text + toolCalls for older shapes.
|
||||
const blocks: ChatBlock[] = [];
|
||||
const rawContent: any[] = (result as any).content ?? [];
|
||||
if (Array.isArray(rawContent) && rawContent.length > 0) {
|
||||
for (const part of rawContent) {
|
||||
if (part.type === 'text') blocks.push({ type: 'text', text: part.text });
|
||||
else if (part.type === 'tool-call') {
|
||||
blocks.push({
|
||||
type: 'tool-call',
|
||||
toolCallId: part.toolCallId,
|
||||
toolName: part.toolName,
|
||||
input: part.input ?? part.args,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Fallback shape for SDK versions exposing flat .text and .toolCalls.
|
||||
if (typeof (result as any).text === 'string' && (result as any).text.length > 0) {
|
||||
blocks.push({ type: 'text', text: (result as any).text });
|
||||
}
|
||||
for (const tc of (result as any).toolCalls ?? []) {
|
||||
blocks.push({
|
||||
type: 'tool-call',
|
||||
toolCallId: tc.toolCallId,
|
||||
toolName: tc.toolName,
|
||||
input: tc.input ?? tc.args,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const usage = (result as any).usage ?? {};
|
||||
const providerMetadata = (result as any).providerMetadata as Record<string, any> | undefined;
|
||||
const anthropicCache = providerMetadata?.anthropic ?? {};
|
||||
|
||||
return {
|
||||
text: blocks.filter(b => b.type === 'text').map(b => (b as { type: 'text'; text: string }).text).join(''),
|
||||
blocks,
|
||||
stopReason: mapStopReason((result as any).finishReason, providerMetadata),
|
||||
usage: {
|
||||
input_tokens: Number(usage.inputTokens ?? usage.promptTokens ?? 0),
|
||||
output_tokens: Number(usage.outputTokens ?? usage.completionTokens ?? 0),
|
||||
cache_read_tokens: Number(anthropicCache.cacheReadInputTokens ?? anthropicCache.cache_read_input_tokens ?? 0),
|
||||
cache_creation_tokens: Number(anthropicCache.cacheCreationInputTokens ?? anthropicCache.cache_creation_input_tokens ?? 0),
|
||||
},
|
||||
model: `${recipe.id}:${modelId}`,
|
||||
providerId: recipe.id,
|
||||
providerMetadata,
|
||||
};
|
||||
} catch (err) {
|
||||
throw normalizeAIError(err, `chat(${recipe.id}:${modelId})`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Future touchpoint stubs ----
|
||||
|
||||
class NotMigratedYet extends AIConfigError {
|
||||
constructor(touchpoint: string) {
|
||||
super(`${touchpoint} has not been migrated to the gateway yet.`);
|
||||
this.name = 'NotMigratedYet';
|
||||
}
|
||||
}
|
||||
|
||||
export async function chunk(): Promise<never> { throw new NotMigratedYet('chunking'); }
|
||||
export async function transcribe(): Promise<never> { throw new NotMigratedYet('transcription'); }
|
||||
export async function enrich(): Promise<never> { throw new NotMigratedYet('enrichment'); }
|
||||
export async function improve(): Promise<never> { throw new NotMigratedYet('improve'); }
|
||||
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* Parse and validate `provider:model` strings against the recipe registry.
|
||||
*/
|
||||
|
||||
import type { ParsedModelId, Recipe, TouchpointKind, ChatTouchpoint, EmbeddingTouchpoint, ExpansionTouchpoint } from './types.ts';
|
||||
import { getRecipe, RECIPES } from './recipes/index.ts';
|
||||
import { AIConfigError } from './errors.ts';
|
||||
|
||||
/** Split "openai:text-embedding-3-large" into { providerId, modelId }. */
|
||||
export function parseModelId(id: string): ParsedModelId {
|
||||
if (!id || typeof id !== 'string') {
|
||||
throw new AIConfigError(
|
||||
`Invalid model id: ${JSON.stringify(id)}`,
|
||||
'Expected format: provider:model (e.g. openai:text-embedding-3-large)',
|
||||
);
|
||||
}
|
||||
const colon = id.indexOf(':');
|
||||
if (colon === -1) {
|
||||
throw new AIConfigError(
|
||||
`Model id "${id}" is missing a provider prefix.`,
|
||||
'Use format provider:model, e.g. openai:text-embedding-3-large',
|
||||
);
|
||||
}
|
||||
const providerId = id.slice(0, colon).trim().toLowerCase();
|
||||
const modelId = id.slice(colon + 1).trim();
|
||||
if (!providerId || !modelId) {
|
||||
throw new AIConfigError(
|
||||
`Model id "${id}" has empty provider or model.`,
|
||||
'Use format provider:model, e.g. openai:text-embedding-3-large',
|
||||
);
|
||||
}
|
||||
return { providerId, modelId };
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a `provider:model` string to a Recipe + canonical modelId.
|
||||
* Honors `recipe.aliases` (Codex F-OV-5) so users can pass undated forms.
|
||||
* Throws AIConfigError if unknown provider.
|
||||
*/
|
||||
export function resolveRecipe(modelId: string): { parsed: ParsedModelId; recipe: Recipe } {
|
||||
const parsed = parseModelId(modelId);
|
||||
const recipe = getRecipe(parsed.providerId);
|
||||
if (!recipe) {
|
||||
throw new AIConfigError(
|
||||
`Unknown provider: "${parsed.providerId}"`,
|
||||
`Known providers: ${[...knownProviderIds()].join(', ')}. Add a new recipe at src/core/ai/recipes/.`,
|
||||
);
|
||||
}
|
||||
// Apply alias if the modelId matches an alias key. Canonical wins.
|
||||
const canonical = recipe.aliases?.[parsed.modelId];
|
||||
if (canonical) {
|
||||
return { parsed: { providerId: parsed.providerId, modelId: canonical }, recipe };
|
||||
}
|
||||
return { parsed, recipe };
|
||||
}
|
||||
|
||||
type KnownTouchpointKey = 'embedding' | 'expansion' | 'chat';
|
||||
|
||||
function getTouchpoint(recipe: Recipe, touchpoint: TouchpointKind): EmbeddingTouchpoint | ExpansionTouchpoint | ChatTouchpoint | undefined {
|
||||
if (touchpoint === 'embedding' || touchpoint === 'expansion' || touchpoint === 'chat') {
|
||||
return recipe.touchpoints[touchpoint as KnownTouchpointKey];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Assert the resolved recipe actually offers the requested touchpoint. */
|
||||
export function assertTouchpoint(recipe: Recipe, touchpoint: TouchpointKind, modelId: string): void {
|
||||
const tp = getTouchpoint(recipe, touchpoint);
|
||||
if (!tp) {
|
||||
throw new AIConfigError(
|
||||
`Provider "${recipe.id}" does not support touchpoint "${touchpoint}".`,
|
||||
touchpoint === 'embedding' && recipe.id === 'anthropic'
|
||||
? 'Anthropic has no embedding model. Use openai or google for embeddings.'
|
||||
: touchpoint === 'chat' && (recipe.id === 'voyage' || recipe.id === 'ollama')
|
||||
? `${recipe.name} is configured here only for embeddings. Use openai/anthropic/google/deepseek/groq/together for chat.`
|
||||
: undefined,
|
||||
);
|
||||
}
|
||||
const supportedModels = tp.models ?? [];
|
||||
if (supportedModels.length > 0 && !supportedModels.includes(modelId)) {
|
||||
// Non-fatal: providers like ollama/litellm accept arbitrary model ids. We only warn for native providers.
|
||||
if (recipe.tier === 'native') {
|
||||
throw new AIConfigError(
|
||||
`Model "${modelId}" is not listed for ${recipe.name} ${touchpoint}.`,
|
||||
`Known models: ${supportedModels.join(', ')}. Use one of these or add it to the recipe (or add an alias).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function knownProviderIds(): string[] {
|
||||
return [...RECIPES.keys()];
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* Lightweight probes for local AI providers. Used by the providers wizard
|
||||
* to auto-detect ready endpoints before prompting the user.
|
||||
*/
|
||||
|
||||
export interface ProbeResult {
|
||||
reachable: boolean;
|
||||
models_endpoint_valid?: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe an OpenAI-compatible /v1/models endpoint. Per Codex C-secondary-4:
|
||||
* port-open is insufficient — a broken daemon can accept connections but
|
||||
* serve garbage. We validate the response is JSON with the expected shape.
|
||||
*/
|
||||
export async function probeOpenAICompat(baseUrl: string, timeoutMs: number = 1000): Promise<ProbeResult> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const res = await fetch(new URL('/v1/models', baseUrl).toString(), {
|
||||
signal: controller.signal,
|
||||
headers: { accept: 'application/json' },
|
||||
});
|
||||
clearTimeout(timer);
|
||||
if (!res.ok) return { reachable: true, models_endpoint_valid: false, error: `HTTP ${res.status}` };
|
||||
const body = await res.json().catch(() => null);
|
||||
if (!body || typeof body !== 'object') {
|
||||
return { reachable: true, models_endpoint_valid: false, error: 'non-JSON response' };
|
||||
}
|
||||
const isList = (body as any).object === 'list' && Array.isArray((body as any).data);
|
||||
return { reachable: true, models_endpoint_valid: isList };
|
||||
} catch (e) {
|
||||
clearTimeout(timer);
|
||||
return { reachable: false, error: e instanceof Error ? e.message : String(e) };
|
||||
}
|
||||
}
|
||||
|
||||
export async function probeOllama(): Promise<ProbeResult> {
|
||||
const url = process.env.OLLAMA_BASE_URL ?? 'http://localhost:11434/v1';
|
||||
return probeOpenAICompat(url);
|
||||
}
|
||||
|
||||
export async function probeLMStudio(): Promise<ProbeResult> {
|
||||
const url = process.env.LMSTUDIO_BASE_URL ?? 'http://localhost:1234/v1';
|
||||
return probeOpenAICompat(url);
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Anthropic provides language models (expansion + chat) only.
|
||||
* Claude has no first-party embedding model as of v0.27 ship date. Users who
|
||||
* want a fully Anthropic stack would still use OpenAI or Google for embedding.
|
||||
*/
|
||||
export const anthropic: Recipe = {
|
||||
id: 'anthropic',
|
||||
name: 'Anthropic',
|
||||
tier: 'native',
|
||||
implementation: 'native-anthropic',
|
||||
auth_env: {
|
||||
required: ['ANTHROPIC_API_KEY'],
|
||||
setup_url: 'https://console.anthropic.com/settings/keys',
|
||||
},
|
||||
touchpoints: {
|
||||
// No embedding model available.
|
||||
expansion: {
|
||||
models: ['claude-haiku-4-5-20251001', 'claude-sonnet-4-6-20250929'],
|
||||
cost_per_1m_tokens_usd: 0.25,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
chat: {
|
||||
models: [
|
||||
'claude-opus-4-7',
|
||||
'claude-sonnet-4-6-20250929',
|
||||
'claude-haiku-4-5-20251001',
|
||||
],
|
||||
supports_tools: true,
|
||||
supports_subagent_loop: true,
|
||||
supports_prompt_cache: true,
|
||||
max_context_tokens: 200000,
|
||||
cost_per_1m_input_usd: 3.0, // sonnet-class baseline
|
||||
cost_per_1m_output_usd: 15.0,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
},
|
||||
// Friendly undated aliases (Codex F-OV-5).
|
||||
aliases: {
|
||||
'claude-sonnet-4-6': 'claude-sonnet-4-6-20250929',
|
||||
'claude-haiku-4-5': 'claude-haiku-4-5-20251001',
|
||||
},
|
||||
setup_hint: 'Get an API key at https://console.anthropic.com/settings/keys, then `export ANTHROPIC_API_KEY=...`',
|
||||
};
|
||||
@@ -0,0 +1,32 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* DeepSeek exposes an OpenAI-compatible /v1/chat/completions endpoint.
|
||||
* Useful as the second hop in a refusal-fallback chain and for cheap-
|
||||
* research delegation: 25-40x cheaper than Anthropic on equivalent
|
||||
* reasoning workloads.
|
||||
*/
|
||||
export const deepseek: Recipe = {
|
||||
id: 'deepseek',
|
||||
name: 'DeepSeek',
|
||||
tier: 'openai-compat',
|
||||
implementation: 'openai-compatible',
|
||||
base_url_default: 'https://api.deepseek.com/v1',
|
||||
auth_env: {
|
||||
required: ['DEEPSEEK_API_KEY'],
|
||||
setup_url: 'https://platform.deepseek.com/api_keys',
|
||||
},
|
||||
touchpoints: {
|
||||
chat: {
|
||||
models: ['deepseek-chat', 'deepseek-reasoner'],
|
||||
supports_tools: true,
|
||||
supports_subagent_loop: true,
|
||||
supports_prompt_cache: false,
|
||||
max_context_tokens: 128000,
|
||||
cost_per_1m_input_usd: 0.14, // deepseek-chat off-peak baseline
|
||||
cost_per_1m_output_usd: 0.28,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
},
|
||||
setup_hint: 'Get an API key at https://platform.deepseek.com/api_keys, then `export DEEPSEEK_API_KEY=...`',
|
||||
};
|
||||
@@ -0,0 +1,37 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
export const google: Recipe = {
|
||||
id: 'google',
|
||||
name: 'Google Gemini',
|
||||
tier: 'native',
|
||||
implementation: 'native-google',
|
||||
auth_env: {
|
||||
required: ['GOOGLE_GENERATIVE_AI_API_KEY'],
|
||||
setup_url: 'https://aistudio.google.com/apikey',
|
||||
},
|
||||
touchpoints: {
|
||||
embedding: {
|
||||
models: ['gemini-embedding-001'],
|
||||
default_dims: 768,
|
||||
dims_options: [768, 1536, 3072],
|
||||
cost_per_1m_tokens_usd: 0.15,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
expansion: {
|
||||
models: ['gemini-2.0-flash', 'gemini-2.0-flash-lite'],
|
||||
cost_per_1m_tokens_usd: 0.10,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
chat: {
|
||||
models: ['gemini-2.0-flash-exp', 'gemini-2.0-flash', 'gemini-1.5-pro'],
|
||||
supports_tools: true,
|
||||
supports_subagent_loop: true,
|
||||
supports_prompt_cache: false,
|
||||
max_context_tokens: 1000000, // Gemini 1.5 Pro
|
||||
cost_per_1m_input_usd: 0.30,
|
||||
cost_per_1m_output_usd: 1.20,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
},
|
||||
setup_hint: 'Get an API key at https://aistudio.google.com/apikey, then `export GOOGLE_GENERATIVE_AI_API_KEY=...`',
|
||||
};
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Groq runs Llama and Whisper on custom inference hardware (~500 tok/s).
|
||||
* The speed tier and last-resort refusal fallback. Also serves Whisper for
|
||||
* transcription (wired in commit 7).
|
||||
*/
|
||||
export const groq: Recipe = {
|
||||
id: 'groq',
|
||||
name: 'Groq',
|
||||
tier: 'openai-compat',
|
||||
implementation: 'openai-compatible',
|
||||
base_url_default: 'https://api.groq.com/openai/v1',
|
||||
auth_env: {
|
||||
required: ['GROQ_API_KEY'],
|
||||
setup_url: 'https://console.groq.com/keys',
|
||||
},
|
||||
touchpoints: {
|
||||
chat: {
|
||||
models: [
|
||||
'llama-3.3-70b-versatile',
|
||||
'llama-3.1-8b-instant',
|
||||
'gpt-oss-20b',
|
||||
'gpt-oss-120b',
|
||||
],
|
||||
supports_tools: true,
|
||||
// 8b-instant has flaky tool_call_id stability under replay; the 70b model
|
||||
// is the recommended subagent driver. We mark the recipe true and let
|
||||
// commit 2's subagent loop pick model-by-model when it matters.
|
||||
supports_subagent_loop: true,
|
||||
supports_prompt_cache: false,
|
||||
max_context_tokens: 131072,
|
||||
cost_per_1m_input_usd: 0.59, // 70b versatile
|
||||
cost_per_1m_output_usd: 0.79,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
},
|
||||
setup_hint: 'Get an API key at https://console.groq.com/keys, then `export GROQ_API_KEY=...`',
|
||||
};
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* Static recipe registry. Bun-compile-safe: every provider is a static import.
|
||||
*
|
||||
* Adding a new openai-compatible provider = add a file here + register below.
|
||||
* Adding a new native provider = ALSO wire the factory in gateway.ts.
|
||||
*/
|
||||
|
||||
import type { Recipe } from '../types.ts';
|
||||
import { openai } from './openai.ts';
|
||||
import { google } from './google.ts';
|
||||
import { anthropic } from './anthropic.ts';
|
||||
import { ollama } from './ollama.ts';
|
||||
import { voyage } from './voyage.ts';
|
||||
import { litellmProxy } from './litellm-proxy.ts';
|
||||
import { deepseek } from './deepseek.ts';
|
||||
import { groq } from './groq.ts';
|
||||
import { together } from './together.ts';
|
||||
|
||||
const ALL: Recipe[] = [
|
||||
openai,
|
||||
google,
|
||||
anthropic,
|
||||
ollama,
|
||||
voyage,
|
||||
litellmProxy,
|
||||
deepseek,
|
||||
groq,
|
||||
together,
|
||||
];
|
||||
|
||||
/** Map from `provider:id` key to recipe. */
|
||||
export const RECIPES: Map<string, Recipe> = new Map(ALL.map(r => [r.id, r]));
|
||||
|
||||
export function getRecipe(id: string): Recipe | undefined {
|
||||
return RECIPES.get(id);
|
||||
}
|
||||
|
||||
export function listRecipes(): Recipe[] {
|
||||
return [...ALL];
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* LiteLLM proxy template. Users run LiteLLM in front of any provider
|
||||
* (Bedrock, Vertex, Azure, Fireworks, Together, DeepSeek, etc.) and point
|
||||
* gbrain at it via `LITELLM_BASE_URL`. The proxy normalizes to
|
||||
* OpenAI-compatible API.
|
||||
*
|
||||
* See docs/guides/litellm-proxy.md for the setup recipe.
|
||||
*/
|
||||
export const litellmProxy: Recipe = {
|
||||
id: 'litellm',
|
||||
name: 'LiteLLM Proxy (universal)',
|
||||
tier: 'openai-compat',
|
||||
implementation: 'openai-compatible',
|
||||
base_url_default: 'http://localhost:4000', // LiteLLM default
|
||||
auth_env: {
|
||||
required: [], // LITELLM_API_KEY is optional (users may run proxy unauthenticated locally)
|
||||
optional: ['LITELLM_BASE_URL', 'LITELLM_API_KEY'],
|
||||
setup_url: 'https://docs.litellm.ai/docs/proxy/quick_start',
|
||||
},
|
||||
touchpoints: {
|
||||
embedding: {
|
||||
// Models depend on the proxy's config; declare empties so wizard prompts user.
|
||||
models: [],
|
||||
default_dims: 0, // user must declare --embedding-dimensions explicitly
|
||||
cost_per_1m_tokens_usd: undefined,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
},
|
||||
setup_hint: 'Run LiteLLM (https://docs.litellm.ai) in front of any provider; set LITELLM_BASE_URL + pass --embedding-model litellm:<model> and --embedding-dimensions <N>.',
|
||||
};
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
export const ollama: Recipe = {
|
||||
id: 'ollama',
|
||||
name: 'Ollama (local)',
|
||||
tier: 'openai-compat',
|
||||
implementation: 'openai-compatible',
|
||||
base_url_default: 'http://localhost:11434/v1',
|
||||
auth_env: {
|
||||
required: [], // Ollama runs unauthenticated locally; users pass `ollama` as the key.
|
||||
optional: ['OLLAMA_BASE_URL', 'OLLAMA_API_KEY'],
|
||||
setup_url: 'https://ollama.ai',
|
||||
},
|
||||
touchpoints: {
|
||||
embedding: {
|
||||
models: ['nomic-embed-text', 'mxbai-embed-large', 'all-minilm'],
|
||||
default_dims: 768, // nomic-embed-text native dim
|
||||
cost_per_1m_tokens_usd: 0,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
},
|
||||
setup_hint: 'Install Ollama from https://ollama.ai, then `ollama pull nomic-embed-text` and `ollama serve`.',
|
||||
};
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
export const openai: Recipe = {
|
||||
id: 'openai',
|
||||
name: 'OpenAI',
|
||||
tier: 'native',
|
||||
implementation: 'native-openai',
|
||||
auth_env: {
|
||||
required: ['OPENAI_API_KEY'],
|
||||
optional: ['OPENAI_ORG_ID', 'OPENAI_PROJECT'],
|
||||
setup_url: 'https://platform.openai.com/api-keys',
|
||||
},
|
||||
touchpoints: {
|
||||
embedding: {
|
||||
models: ['text-embedding-3-large', 'text-embedding-3-small'],
|
||||
default_dims: 1536,
|
||||
dims_options: [256, 512, 768, 1024, 1536, 3072],
|
||||
cost_per_1m_tokens_usd: 0.13,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
expansion: {
|
||||
models: ['gpt-5.2', 'gpt-4o-mini'],
|
||||
cost_per_1m_tokens_usd: 0.15,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
chat: {
|
||||
models: ['gpt-5.2', 'gpt-4o-mini'],
|
||||
supports_tools: true,
|
||||
supports_subagent_loop: true,
|
||||
supports_prompt_cache: false,
|
||||
max_context_tokens: 200000,
|
||||
cost_per_1m_input_usd: 1.25, // gpt-5.2 baseline
|
||||
cost_per_1m_output_usd: 10.0,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
},
|
||||
setup_hint: 'Get an API key at https://platform.openai.com/api-keys, then `export OPENAI_API_KEY=...`',
|
||||
};
|
||||
@@ -0,0 +1,36 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Together AI hosts open-weights models on shared infrastructure with an
|
||||
* OpenAI-compatible endpoint. House for Qwen, Llama-3.3-70B-Turbo, and other
|
||||
* non-frontier models that sit between DeepSeek's price and Groq's speed.
|
||||
*/
|
||||
export const together: Recipe = {
|
||||
id: 'together',
|
||||
name: 'Together AI',
|
||||
tier: 'openai-compat',
|
||||
implementation: 'openai-compatible',
|
||||
base_url_default: 'https://api.together.xyz/v1',
|
||||
auth_env: {
|
||||
required: ['TOGETHER_API_KEY'],
|
||||
setup_url: 'https://api.together.ai/settings/api-keys',
|
||||
},
|
||||
touchpoints: {
|
||||
chat: {
|
||||
models: [
|
||||
'Qwen/Qwen2.5-72B-Instruct-Turbo',
|
||||
'meta-llama/Llama-3.3-70B-Instruct-Turbo',
|
||||
'deepseek-ai/DeepSeek-V3',
|
||||
'mistralai/Mixtral-8x22B-Instruct-v0.1',
|
||||
],
|
||||
supports_tools: true,
|
||||
supports_subagent_loop: true,
|
||||
supports_prompt_cache: false,
|
||||
max_context_tokens: 131072,
|
||||
cost_per_1m_input_usd: 0.88, // Llama-3.3-70B-Turbo baseline
|
||||
cost_per_1m_output_usd: 0.88,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
},
|
||||
setup_hint: 'Get an API key at https://api.together.ai/settings/api-keys, then `export TOGETHER_API_KEY=...`',
|
||||
};
|
||||
@@ -0,0 +1,34 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Voyage AI exposes an OpenAI-compatible /embeddings endpoint.
|
||||
* Base URL: https://api.voyageai.com/v1
|
||||
*/
|
||||
export const voyage: Recipe = {
|
||||
id: 'voyage',
|
||||
name: 'Voyage AI',
|
||||
tier: 'openai-compat',
|
||||
implementation: 'openai-compatible',
|
||||
base_url_default: 'https://api.voyageai.com/v1',
|
||||
auth_env: {
|
||||
required: ['VOYAGE_API_KEY'],
|
||||
setup_url: 'https://dash.voyageai.com/api-keys',
|
||||
},
|
||||
touchpoints: {
|
||||
embedding: {
|
||||
models: ['voyage-3-large', 'voyage-3', 'voyage-3-lite'],
|
||||
default_dims: 1024,
|
||||
cost_per_1m_tokens_usd: 0.18,
|
||||
price_last_verified: '2026-04-20',
|
||||
// Voyage enforces 120K tokens per batch. Voyage's tokenizer runs
|
||||
// ~3-4× denser than OpenAI tiktoken on mixed content (code/JSON/CJK),
|
||||
// so the per-recipe pre-split uses 1 char ≈ 1 token at 0.5 utilization
|
||||
// (60K char budget). Recursive halving in the gateway is the runtime
|
||||
// safety net when dense payloads still overshoot.
|
||||
max_batch_tokens: 120_000,
|
||||
chars_per_token: 1,
|
||||
safety_factor: 0.5,
|
||||
},
|
||||
},
|
||||
setup_hint: 'Get an API key at https://dash.voyageai.com/api-keys, then `export VOYAGE_API_KEY=...`',
|
||||
};
|
||||
@@ -0,0 +1,145 @@
|
||||
/**
|
||||
* AI provider types.
|
||||
*
|
||||
* Recipes are pure data. The gateway's implementation switch decides which
|
||||
* statically-imported factory to use based on `implementation`.
|
||||
*
|
||||
* Bun-compile-safe: no dynamic imports. Adding a new native provider requires
|
||||
* both a recipe AND a code change to register the factory in gateway.ts.
|
||||
*/
|
||||
|
||||
export type TouchpointKind =
|
||||
| 'embedding'
|
||||
| 'expansion'
|
||||
| 'chat'
|
||||
| 'chunking'
|
||||
| 'transcription'
|
||||
| 'enrichment'
|
||||
| 'improve';
|
||||
|
||||
export type Implementation =
|
||||
| 'native-openai'
|
||||
| 'native-google'
|
||||
| 'native-anthropic'
|
||||
| 'openai-compatible';
|
||||
|
||||
export interface EmbeddingTouchpoint {
|
||||
models: string[];
|
||||
default_dims: number;
|
||||
dims_options?: number[]; // for Matryoshka-aware providers
|
||||
cost_per_1m_tokens_usd?: number;
|
||||
price_last_verified?: string; // ISO date
|
||||
/**
|
||||
* Maximum tokens per batch for this provider's embedding endpoint.
|
||||
* When set, the gateway pre-splits batches at
|
||||
* `max_batch_tokens × safety_factor / chars_per_token` characters and
|
||||
* recursively halves on token-limit errors at runtime. When unset, the
|
||||
* gateway makes a single embedMany() call with no safety net (OpenAI fast
|
||||
* path).
|
||||
*/
|
||||
max_batch_tokens?: number;
|
||||
/**
|
||||
* Expected character density for this provider's tokenizer (chars per
|
||||
* token). OpenAI tiktoken averages ~4 on English text; Voyage averages
|
||||
* ~1 on mixed content (code/JSON/CJK). Defaults to 4 if omitted.
|
||||
* Only consulted when `max_batch_tokens` is also set.
|
||||
*/
|
||||
chars_per_token?: number;
|
||||
/**
|
||||
* Budget-utilization ceiling in (0, 1]. The gateway pre-splits at
|
||||
* `safety_factor × max_batch_tokens` to leave headroom for tokenizer
|
||||
* variance. Defaults to 0.8. Voyage-style providers with dense payloads
|
||||
* should pin this lower (e.g. 0.5). Only consulted when
|
||||
* `max_batch_tokens` is also set.
|
||||
*/
|
||||
safety_factor?: number;
|
||||
}
|
||||
|
||||
export interface ExpansionTouchpoint {
|
||||
models: string[];
|
||||
cost_per_1m_tokens_usd?: number;
|
||||
price_last_verified?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chat touchpoint: tool-using conversational LLMs that can drive Minions
|
||||
* subagents. `supports_tools` and `supports_subagent_loop` are intentionally
|
||||
* separate (Codex F-OV-2): some chat-capable models have flaky tool-calling or
|
||||
* unstable tool_call_id behavior across replays. supports_subagent_loop is the
|
||||
* stricter signal that subagent.ts asserts.
|
||||
*/
|
||||
export interface ChatTouchpoint {
|
||||
models: string[];
|
||||
/** Provider returns native function/tool calling. */
|
||||
supports_tools: boolean;
|
||||
/**
|
||||
* Stable enough across crashes/replays to drive a Minions subagent loop.
|
||||
* Strictly stronger than supports_tools.
|
||||
*/
|
||||
supports_subagent_loop: boolean;
|
||||
/** Anthropic-style ephemeral prompt cache markers honored. */
|
||||
supports_prompt_cache?: boolean;
|
||||
max_context_tokens?: number;
|
||||
cost_per_1m_input_usd?: number;
|
||||
cost_per_1m_output_usd?: number;
|
||||
price_last_verified?: string;
|
||||
}
|
||||
|
||||
export interface Recipe {
|
||||
/** Stable lowercase id used in `provider:model` strings. Unique across recipes. */
|
||||
id: string;
|
||||
/** Human-readable name for display. */
|
||||
name: string;
|
||||
/** Distinguishes native-package providers from openai-compatible endpoints. */
|
||||
tier: 'native' | 'openai-compat';
|
||||
/** Maps to the gateway's implementation switch. */
|
||||
implementation: Implementation;
|
||||
/** For openai-compatible tier: default base URL. May be overridden by env or wizard. */
|
||||
base_url_default?: string;
|
||||
/** Env var name(s) for auth; first is required, rest are optional. */
|
||||
auth_env?: {
|
||||
required: string[];
|
||||
optional?: string[];
|
||||
setup_url?: string;
|
||||
};
|
||||
touchpoints: {
|
||||
embedding?: EmbeddingTouchpoint;
|
||||
expansion?: ExpansionTouchpoint;
|
||||
chat?: ChatTouchpoint;
|
||||
};
|
||||
/**
|
||||
* Optional alias map for friendlier `provider:model` strings (Codex F-OV-5).
|
||||
* Resolved at parse time so users can write `anthropic:claude-sonnet-4-6`
|
||||
* instead of `anthropic:claude-sonnet-4-6-20250929`. Keys are aliases,
|
||||
* values are canonical (declared) model ids.
|
||||
*/
|
||||
aliases?: Record<string, string>;
|
||||
/** One-line description of setup (shown in wizard + env subcommand). */
|
||||
setup_hint?: string;
|
||||
}
|
||||
|
||||
export interface AIGatewayConfig {
|
||||
/** Current embedding model as "provider:modelId" (e.g. "openai:text-embedding-3-large"). */
|
||||
embedding_model?: string;
|
||||
/** Target embedding dims. Gateway asserts returned embeddings match this. */
|
||||
embedding_dimensions?: number;
|
||||
/** Current expansion model as "provider:modelId". */
|
||||
expansion_model?: string;
|
||||
/** Default chat model for `gateway.chat()` callers (subagent default). */
|
||||
chat_model?: string;
|
||||
/**
|
||||
* Optional silent-refusal fallback chain ("provider:modelId" entries).
|
||||
* Plumbed for `chatWithFallback()` (commit 3). Blocked from critic/judge/
|
||||
* synthesize flows in their respective handlers.
|
||||
*/
|
||||
chat_fallback_chain?: string[];
|
||||
/** Optional per-provider base URL override (openai-compatible variants). */
|
||||
base_urls?: Record<string, string>;
|
||||
/** Env snapshot read once at configuration time. Gateway never reads process.env at call time. */
|
||||
env: Record<string, string | undefined>;
|
||||
}
|
||||
|
||||
export interface ParsedModelId {
|
||||
providerId: string; // e.g. "openai"
|
||||
modelId: string; // e.g. "text-embedding-3-large"
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* v0.28: Anthropic model pricing constants for the dream-cycle budget meter.
|
||||
*
|
||||
* Prices in USD per 1M tokens (input | output). Numbers reflect Anthropic's
|
||||
* published pricing as of 2026-05-01. Update when Anthropic publishes new
|
||||
* pricing — the JSON in `~/.gbrain/audit/dream-budget-*.jsonl` carries the
|
||||
* snapshot per call so historical estimates stay reproducible.
|
||||
*
|
||||
* Codex P1 #10 fold: non-Anthropic models (gemini, gpt, anything not in
|
||||
* this map) bypass the budget gate with a `BUDGET_METER_NO_PRICING` warn
|
||||
* once per process. The cycle still runs unbounded for those models.
|
||||
* Future: per-provider pricing modules.
|
||||
*/
|
||||
|
||||
export interface ModelPricing {
|
||||
/** USD per 1M input tokens. */
|
||||
input: number;
|
||||
/** USD per 1M output tokens. */
|
||||
output: number;
|
||||
}
|
||||
|
||||
/** Map of Anthropic model id → pricing. Aliases (opus/sonnet/haiku) resolve via DEFAULT_ALIASES. */
|
||||
export const ANTHROPIC_PRICING: Record<string, ModelPricing> = {
|
||||
// Claude 4.7 family (current generation)
|
||||
'claude-opus-4-7': { input: 15.00, output: 75.00 },
|
||||
'claude-sonnet-4-6': { input: 3.00, output: 15.00 },
|
||||
'claude-haiku-4-5-20251001': { input: 1.00, output: 5.00 },
|
||||
// Older but still frequently aliased
|
||||
'claude-opus-4-6': { input: 15.00, output: 75.00 },
|
||||
'claude-3-5-sonnet-20241022': { input: 3.00, output: 15.00 },
|
||||
'claude-3-5-haiku-20241022': { input: 0.80, output: 4.00 },
|
||||
};
|
||||
|
||||
/**
|
||||
* Estimate the upper-bound USD cost of a single submit.
|
||||
* Uses (estimatedInputTokens × inputRate) + (maxOutputTokens × outputRate).
|
||||
* The maxOutputTokens upper-bounds the output cost — actual completions
|
||||
* usually return less.
|
||||
*
|
||||
* Returns null when the model isn't in the pricing map. Callers warn-once
|
||||
* and treat as zero-cost (the cycle runs unbounded for that submit).
|
||||
*/
|
||||
export function estimateMaxCostUsd(
|
||||
modelId: string,
|
||||
estimatedInputTokens: number,
|
||||
maxOutputTokens: number,
|
||||
): number | null {
|
||||
const p = ANTHROPIC_PRICING[modelId];
|
||||
if (!p) return null;
|
||||
return (
|
||||
(estimatedInputTokens / 1_000_000) * p.input +
|
||||
(maxOutputTokens / 1_000_000) * p.output
|
||||
);
|
||||
}
|
||||
@@ -31,19 +31,31 @@ export interface TextChunk {
|
||||
index: number;
|
||||
}
|
||||
|
||||
// v0.28: import takes-fence stripper as a pre-processing pass. Takes content
|
||||
// lives in the takes table only; duplicating it inside content_chunks would
|
||||
// bypass the per-token MCP allow-list (Codex P0 #3 privacy fix).
|
||||
import { stripTakesFence } from '../takes-fence.ts';
|
||||
|
||||
export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] {
|
||||
const chunkSize = opts?.chunkSize || 300;
|
||||
const chunkOverlap = opts?.chunkOverlap || 50;
|
||||
|
||||
if (!text || text.trim().length === 0) return [];
|
||||
|
||||
const wordCount = countWords(text);
|
||||
// v0.28: strip fenced takes blocks BEFORE chunking. Takes are retrieval-
|
||||
// accessible only via the takes table; their content must not appear in
|
||||
// content_chunks where the per-token allow-list cannot reach. The
|
||||
// takes_fence_chunk_leak doctor check verifies this invariant.
|
||||
const stripped = stripTakesFence(text);
|
||||
if (!stripped || stripped.trim().length === 0) return [];
|
||||
|
||||
const wordCount = countWords(stripped);
|
||||
if (wordCount <= chunkSize) {
|
||||
return [{ text: text.trim(), index: 0 }];
|
||||
return [{ text: stripped.trim(), index: 0 }];
|
||||
}
|
||||
|
||||
// Recursively split, then greedily merge to target size
|
||||
const pieces = recursiveSplit(text, 0, chunkSize);
|
||||
const pieces = recursiveSplit(stripped, 0, chunkSize);
|
||||
const merged = greedyMerge(pieces, chunkSize);
|
||||
const withOverlap = applyOverlap(merged, chunkOverlap);
|
||||
|
||||
|
||||
+25
-1
@@ -31,6 +31,23 @@ export interface GBrainConfig {
|
||||
database_path?: string;
|
||||
openai_api_key?: string;
|
||||
anthropic_api_key?: string;
|
||||
/** AI gateway config (v0.14+). Default: "openai:text-embedding-3-large" / 1536 / "anthropic:claude-haiku-4-5-20251001". */
|
||||
embedding_model?: string;
|
||||
embedding_dimensions?: number;
|
||||
expansion_model?: string;
|
||||
/**
|
||||
* Default chat model for `gateway.chat()` callers (v0.27+).
|
||||
* Default: "anthropic:claude-sonnet-4-6-20250929".
|
||||
*/
|
||||
chat_model?: string;
|
||||
/**
|
||||
* Optional silent-refusal fallback chain for `chatWithFallback()` (v0.27+).
|
||||
* Each entry is a "provider:modelId" string. Blocked from critic/judge/
|
||||
* synthesize flows in their respective handlers (per D13 review decision).
|
||||
*/
|
||||
chat_fallback_chain?: string[];
|
||||
/** Optional base URL overrides for openai-compatible providers (keyed by recipe id). */
|
||||
provider_base_urls?: Record<string, string>;
|
||||
/**
|
||||
* Optional storage backend config (S3/Supabase/local). Shape matches
|
||||
* `StorageConfig` in `./storage.ts`. Typed as `unknown` here to avoid
|
||||
@@ -72,12 +89,19 @@ export function loadConfig(): GBrainConfig | null {
|
||||
const inferredEngine: 'postgres' | 'pglite' = fileConfig?.engine
|
||||
|| (fileConfig?.database_path ? 'pglite' : 'postgres');
|
||||
|
||||
// Merge: env vars override config file
|
||||
// Merge: env vars override config file. READ only — never mutate process.env.
|
||||
const merged = {
|
||||
...fileConfig,
|
||||
engine: inferredEngine,
|
||||
...(dbUrl ? { database_url: dbUrl } : {}),
|
||||
...(process.env.OPENAI_API_KEY ? { openai_api_key: process.env.OPENAI_API_KEY } : {}),
|
||||
...(process.env.GBRAIN_EMBEDDING_MODEL ? { embedding_model: process.env.GBRAIN_EMBEDDING_MODEL } : {}),
|
||||
...(process.env.GBRAIN_EMBEDDING_DIMENSIONS ? { embedding_dimensions: parseInt(process.env.GBRAIN_EMBEDDING_DIMENSIONS, 10) } : {}),
|
||||
...(process.env.GBRAIN_EXPANSION_MODEL ? { expansion_model: process.env.GBRAIN_EXPANSION_MODEL } : {}),
|
||||
...(process.env.GBRAIN_CHAT_MODEL ? { chat_model: process.env.GBRAIN_CHAT_MODEL } : {}),
|
||||
...(process.env.GBRAIN_CHAT_FALLBACK_CHAIN
|
||||
? { chat_fallback_chain: process.env.GBRAIN_CHAT_FALLBACK_CHAIN.split(',').map(s => s.trim()).filter(Boolean) }
|
||||
: {}),
|
||||
};
|
||||
return merged as GBrainConfig;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* cross-modal-eval/aggregate — verdict logic for one cycle.
|
||||
*
|
||||
* Inputs: per-slot results from the 3 frontier models (each either a parsed
|
||||
* scores object or a captured error). Output: verdict + dim averages +
|
||||
* top improvements + verdict prose.
|
||||
*
|
||||
* Pass criterion (Q2 + Q3):
|
||||
* - At least 2 of 3 model calls succeeded with parseable scores.
|
||||
* - Every dimension's mean across successful models is >= 7.
|
||||
* - For every dimension, no successful model scored < 5 (the floor).
|
||||
*
|
||||
* Inconclusive (Q3): fewer than 2 models succeeded.
|
||||
* `Object.values({}).every(...) === true`, so an empty scores map would
|
||||
* silently PASS without this guard. Test 6 in aggregate.test.ts is the
|
||||
* regression guard.
|
||||
*/
|
||||
|
||||
import type { ParsedModelResult } from './json-repair.ts';
|
||||
|
||||
export type SlotResult =
|
||||
| { ok: true; modelId: string; parsed: ParsedModelResult }
|
||||
| { ok: false; modelId: string; error: string };
|
||||
|
||||
export interface AggregateInput {
|
||||
/** One entry per slot (typically 3). */
|
||||
slots: SlotResult[];
|
||||
}
|
||||
|
||||
export interface DimensionRoll {
|
||||
/** Mean across successful models. */
|
||||
mean: number;
|
||||
/** Minimum across successful models (the floor). */
|
||||
min: number;
|
||||
/** All raw scores from successful models, in slot order. */
|
||||
scores: number[];
|
||||
/** Pass=false reason if this dim fails. */
|
||||
failReason?: 'mean_below_7' | 'min_below_5';
|
||||
}
|
||||
|
||||
export interface AggregateResult {
|
||||
/** Verdict: 'pass' | 'fail' | 'inconclusive' (Q3=A). */
|
||||
verdict: 'pass' | 'fail' | 'inconclusive';
|
||||
/** Number of slots that returned parseable scores. */
|
||||
successes: number;
|
||||
/** Number of slots that errored or returned unparseable output. */
|
||||
failures: number;
|
||||
/** Per-dimension roll-up; undefined if inconclusive. */
|
||||
dimensions: Record<string, DimensionRoll>;
|
||||
/** Mean of dimension means; undefined if inconclusive. */
|
||||
overall: number | undefined;
|
||||
/** Top 10 deduplicated improvements across all successful models. */
|
||||
topImprovements: string[];
|
||||
/** Slot-level error notes (carried through to receipt). */
|
||||
errors: Array<{ modelId: string; error: string }>;
|
||||
/** Human-readable one-liner for stderr / receipt verdict prose. */
|
||||
verdictMessage: string;
|
||||
}
|
||||
|
||||
const PASS_MEAN_THRESHOLD = 7;
|
||||
const PASS_FLOOR_THRESHOLD = 5;
|
||||
const MIN_SUCCESSES_FOR_VERDICT = 2;
|
||||
const TOP_IMPROVEMENTS_CAP = 10;
|
||||
const DEDUP_PREFIX_LEN = 40;
|
||||
|
||||
export function aggregate(input: AggregateInput): AggregateResult {
|
||||
const successes = input.slots.filter(s => s.ok);
|
||||
const failures = input.slots.filter(s => !s.ok) as Array<
|
||||
Extract<SlotResult, { ok: false }>
|
||||
>;
|
||||
const errors = failures.map(f => ({ modelId: f.modelId, error: f.error }));
|
||||
|
||||
if (successes.length < MIN_SUCCESSES_FOR_VERDICT) {
|
||||
return {
|
||||
verdict: 'inconclusive',
|
||||
successes: successes.length,
|
||||
failures: failures.length,
|
||||
dimensions: {},
|
||||
overall: undefined,
|
||||
topImprovements: [],
|
||||
errors,
|
||||
verdictMessage:
|
||||
`INCONCLUSIVE: only ${successes.length} of ${input.slots.length} models returned ` +
|
||||
`parseable scores (need >=${MIN_SUCCESSES_FOR_VERDICT}). See receipt for per-slot errors.`,
|
||||
};
|
||||
}
|
||||
|
||||
// Roll up per-dimension across successful slots.
|
||||
const dimensions: Record<string, DimensionRoll> = {};
|
||||
const allDimNames = new Set<string>();
|
||||
for (const s of successes) {
|
||||
if (s.ok) {
|
||||
for (const dim of Object.keys(s.parsed.scores)) {
|
||||
allDimNames.add(dim);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const dim of allDimNames) {
|
||||
const scores: number[] = [];
|
||||
for (const s of successes) {
|
||||
if (s.ok) {
|
||||
const entry = s.parsed.scores[dim];
|
||||
if (entry && Number.isFinite(entry.score)) scores.push(entry.score);
|
||||
}
|
||||
}
|
||||
if (scores.length === 0) continue;
|
||||
const mean = scores.reduce((a, b) => a + b, 0) / scores.length;
|
||||
const min = Math.min(...scores);
|
||||
const roll: DimensionRoll = { mean: round1(mean), min, scores };
|
||||
if (roll.mean < PASS_MEAN_THRESHOLD) roll.failReason = 'mean_below_7';
|
||||
else if (roll.min < PASS_FLOOR_THRESHOLD) roll.failReason = 'min_below_5';
|
||||
dimensions[dim] = roll;
|
||||
}
|
||||
|
||||
const dimRolls = Object.values(dimensions);
|
||||
const overall =
|
||||
dimRolls.length > 0
|
||||
? round1(dimRolls.reduce((a, b) => a + b.mean, 0) / dimRolls.length)
|
||||
: 0;
|
||||
const allDimsPass = dimRolls.every(d => !d.failReason);
|
||||
const verdict: 'pass' | 'fail' = allDimsPass ? 'pass' : 'fail';
|
||||
|
||||
const topImprovements = dedupImprovements(
|
||||
successes.flatMap(s => (s.ok ? s.parsed.improvements : [])),
|
||||
).slice(0, TOP_IMPROVEMENTS_CAP);
|
||||
|
||||
const verdictMessage =
|
||||
verdict === 'pass'
|
||||
? `PASS: every dimension mean >=${PASS_MEAN_THRESHOLD} and min >=${PASS_FLOOR_THRESHOLD} ` +
|
||||
`across ${successes.length}/${input.slots.length} models. Overall ${overall}/10.`
|
||||
: describeFailure(dimensions, successes.length, input.slots.length, overall);
|
||||
|
||||
return {
|
||||
verdict,
|
||||
successes: successes.length,
|
||||
failures: failures.length,
|
||||
dimensions,
|
||||
overall,
|
||||
topImprovements,
|
||||
errors,
|
||||
verdictMessage,
|
||||
};
|
||||
}
|
||||
|
||||
function describeFailure(
|
||||
dimensions: Record<string, DimensionRoll>,
|
||||
successes: number,
|
||||
total: number,
|
||||
overall: number,
|
||||
): string {
|
||||
const failed = Object.entries(dimensions).filter(([, d]) => d.failReason);
|
||||
if (failed.length === 0) {
|
||||
return `FAIL: aggregate failure with no dimension flagged (likely zero dimensions returned).`;
|
||||
}
|
||||
const reasons = failed
|
||||
.map(([name, d]) => {
|
||||
if (d.failReason === 'mean_below_7') {
|
||||
return `${name} mean=${d.mean} (<${PASS_MEAN_THRESHOLD})`;
|
||||
}
|
||||
return `${name} min=${d.min} (<${PASS_FLOOR_THRESHOLD}; scores=[${d.scores.join(', ')}])`;
|
||||
})
|
||||
.join('; ');
|
||||
return `FAIL across ${successes}/${total} models. Overall ${overall}/10. Failing: ${reasons}.`;
|
||||
}
|
||||
|
||||
function dedupImprovements(items: string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const out: string[] = [];
|
||||
for (const item of items) {
|
||||
const key = item.slice(0, DEDUP_PREFIX_LEN).toLowerCase().replace(/\s+/g, ' ').trim();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push(item);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function round1(n: number): number {
|
||||
return Math.round(n * 10) / 10;
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* cross-modal-eval/json-repair — best-effort JSON parser for LLM output.
|
||||
*
|
||||
* Frontier models routinely return:
|
||||
* - Plain JSON
|
||||
* - JSON wrapped in ```json fences
|
||||
* - JSON with trailing commas before } or ]
|
||||
* - JSON with embedded newlines inside strings
|
||||
* - JSON with single quotes used as string delimiters
|
||||
*
|
||||
* Four-strategy fallback chain. The "nuclear option" extracts scores via
|
||||
* regex when none of the above parses succeed; if even that fails to find
|
||||
* any dimension scores, we throw rather than fabricate.
|
||||
*
|
||||
* The aggregator (aggregate.ts) treats a throw here as "this model
|
||||
* contributed nothing this cycle" — the model is excluded from the verdict
|
||||
* but the gate can still PASS at >=2/3 successes.
|
||||
*/
|
||||
|
||||
export interface ParsedScore {
|
||||
score: number;
|
||||
feedback?: string;
|
||||
}
|
||||
|
||||
export interface ParsedModelResult {
|
||||
scores: Record<string, ParsedScore>;
|
||||
overall?: number;
|
||||
improvements: string[];
|
||||
/** True when the result was reconstructed via the regex nuclear option. */
|
||||
_repaired?: boolean;
|
||||
}
|
||||
|
||||
const FENCE_RE = /```(?:json)?\s*\n?([\s\S]*?)```/i;
|
||||
|
||||
export function parseModelJSON(raw: string): ParsedModelResult {
|
||||
if (typeof raw !== 'string' || !raw.trim()) {
|
||||
throw new Error('parseModelJSON: empty or non-string input');
|
||||
}
|
||||
|
||||
// Strategy 1: strip markdown fences if present, then JSON.parse.
|
||||
const cleaned = stripFences(raw).trim();
|
||||
const direct = tryParse(cleaned);
|
||||
if (direct) return shape(direct);
|
||||
|
||||
// Strategy 2: extract the first {...} object substring.
|
||||
const match = cleaned.match(/\{[\s\S]*\}/);
|
||||
if (!match) {
|
||||
throw new Error('parseModelJSON: no JSON object found in input');
|
||||
}
|
||||
const obj = match[0];
|
||||
|
||||
const second = tryParse(obj);
|
||||
if (second) return shape(second);
|
||||
|
||||
// Strategy 3: repair common LLM-JSON mistakes.
|
||||
const fixed = repairJson(obj);
|
||||
const third = tryParse(fixed);
|
||||
if (third) return shape(third);
|
||||
|
||||
// Strategy 4: nuclear option — regex-extract scores + improvements.
|
||||
const reconstructed = regexNuclearOption(obj);
|
||||
if (reconstructed) return reconstructed;
|
||||
|
||||
throw new Error('parseModelJSON: all repair strategies failed');
|
||||
}
|
||||
|
||||
function stripFences(s: string): string {
|
||||
const m = s.match(FENCE_RE);
|
||||
return m ? m[1]! : s;
|
||||
}
|
||||
|
||||
function tryParse(s: string): unknown | null {
|
||||
try {
|
||||
return JSON.parse(s);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function repairJson(s: string): string {
|
||||
return (
|
||||
s
|
||||
// Trailing commas before } or ]
|
||||
.replace(/,(\s*[}\]])/g, '$1')
|
||||
// Single-quoted string values used as delimiters around keys/values
|
||||
// (only between structural punctuation, to avoid touching apostrophes
|
||||
// inside legitimate double-quoted strings).
|
||||
.replace(/(?<=[:{,\[]\s*)'([^']*?)'(?=\s*[,}\]:])/g, '"$1"')
|
||||
// Unescaped newlines inside double-quoted strings — replace with \n.
|
||||
.replace(/("(?:[^"\\]|\\.)*?)\n((?:[^"\\]|\\.)*?")/g, '$1\\n$2')
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Last-resort: scan for `"<dim>": { ... "score": N }` patterns and any
|
||||
* numbered `"N. ..."` improvement strings. Throws if zero scores are
|
||||
* recoverable (better than fabricating a fake PASS).
|
||||
*/
|
||||
function regexNuclearOption(obj: string): ParsedModelResult | null {
|
||||
const scores: Record<string, ParsedScore> = {};
|
||||
const scoreRe = /["']?(\w[\w_-]*)["']?\s*:\s*\{[^}]*?["']?score["']?\s*:\s*(\d+(?:\.\d+)?)/g;
|
||||
for (const m of obj.matchAll(scoreRe)) {
|
||||
const dim = m[1]!;
|
||||
const num = Number(m[2]);
|
||||
if (Number.isFinite(num)) scores[dim] = { score: num };
|
||||
}
|
||||
|
||||
if (Object.keys(scores).length === 0) return null;
|
||||
|
||||
const improvements: string[] = [];
|
||||
const impRe = /"(\d+\.\s[^"]{10,})"/g;
|
||||
for (const m of obj.matchAll(impRe)) {
|
||||
improvements.push(m[1]!);
|
||||
}
|
||||
|
||||
const overallMatch = obj.match(/["']?overall["']?\s*:\s*(\d+(?:\.\d+)?)/);
|
||||
return {
|
||||
scores,
|
||||
overall: overallMatch ? Number(overallMatch[1]) : undefined,
|
||||
improvements:
|
||||
improvements.length > 0
|
||||
? improvements
|
||||
: ['(could not parse improvements from malformed JSON)'],
|
||||
_repaired: true,
|
||||
};
|
||||
}
|
||||
|
||||
function shape(parsed: unknown): ParsedModelResult {
|
||||
if (!parsed || typeof parsed !== 'object') {
|
||||
throw new Error('parseModelJSON: parsed value is not an object');
|
||||
}
|
||||
const p = parsed as Record<string, unknown>;
|
||||
const scoresRaw = (p.scores as Record<string, unknown>) ?? {};
|
||||
const scores: Record<string, ParsedScore> = {};
|
||||
for (const [dim, v] of Object.entries(scoresRaw)) {
|
||||
if (typeof v === 'number') {
|
||||
scores[dim] = { score: v };
|
||||
} else if (v && typeof v === 'object') {
|
||||
const vv = v as Record<string, unknown>;
|
||||
const score = typeof vv.score === 'number' ? vv.score : Number(vv.score);
|
||||
if (!Number.isFinite(score)) continue;
|
||||
const feedback = typeof vv.feedback === 'string' ? vv.feedback : undefined;
|
||||
scores[dim] = { score, feedback };
|
||||
}
|
||||
}
|
||||
|
||||
const improvements = Array.isArray(p.improvements)
|
||||
? (p.improvements as unknown[]).filter((x): x is string => typeof x === 'string')
|
||||
: [];
|
||||
|
||||
const overall = typeof p.overall === 'number' ? p.overall : undefined;
|
||||
|
||||
if (Object.keys(scores).length === 0) {
|
||||
throw new Error('parseModelJSON: parsed object has no usable scores');
|
||||
}
|
||||
|
||||
return { scores, overall, improvements };
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* cross-modal-eval/receipt-name — bind a receipt to a specific skill version.
|
||||
*
|
||||
* Receipt filenames embed a SHA-8 of the SKILL.md content, so the audit can
|
||||
* tell whether the receipt corresponds to the *current* version of the skill
|
||||
* (T10=A). Filename pattern:
|
||||
*
|
||||
* <skill-slug>-<sha8>.json
|
||||
*
|
||||
* findReceiptForSkill returns one of:
|
||||
* - { status: 'found', path } — receipt matches current SKILL.md
|
||||
* - { status: 'stale', latestPath, sha } — receipt(s) exist for older versions
|
||||
* - { status: 'missing' } — no receipt for this skill
|
||||
*
|
||||
* Pure functions — no fs writes (the writer is in receipt-write.ts). The
|
||||
* skillify-check audit and the runner share these helpers so naming stays in
|
||||
* one place.
|
||||
*/
|
||||
|
||||
import { createHash } from 'crypto';
|
||||
import { existsSync, readdirSync, readFileSync, statSync } from 'fs';
|
||||
import { basename, join } from 'path';
|
||||
|
||||
export type ReceiptStatus =
|
||||
| { status: 'found'; path: string; sha: string }
|
||||
| { status: 'stale'; latestPath: string; latestSha: string; currentSha: string }
|
||||
| { status: 'missing'; currentSha: string };
|
||||
|
||||
/**
|
||||
* SHA-256 of skill content, truncated to 8 hex chars. 16M-receipt collision
|
||||
* space per slug is more than enough; the receipts are owned by one user.
|
||||
*/
|
||||
export function sha8(content: string): string {
|
||||
return createHash('sha256').update(content, 'utf8').digest('hex').slice(0, 8);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the canonical receipt filename for a (slug, content) pair.
|
||||
* Returned as a bare filename (no directory), so the caller controls layout.
|
||||
*/
|
||||
export function receiptName(slug: string, content: string): string {
|
||||
if (!slug || typeof slug !== 'string') throw new Error('receiptName: slug required');
|
||||
if (!/^[a-z0-9][a-z0-9_-]*$/i.test(slug)) {
|
||||
throw new Error(`receiptName: slug must be alphanumeric/dash/underscore; got: ${slug}`);
|
||||
}
|
||||
return `${slug}-${sha8(content)}.json`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the SKILL.md at `skillPath` (or return null when missing) and look in
|
||||
* `receiptDir` for any receipt matching the slug embedded in skillPath.
|
||||
*/
|
||||
export function findReceiptForSkill(skillMdPath: string, receiptDir: string): ReceiptStatus {
|
||||
if (!existsSync(skillMdPath)) {
|
||||
return { status: 'missing', currentSha: '' };
|
||||
}
|
||||
|
||||
const slug = inferSlugFromSkillPath(skillMdPath);
|
||||
const content = readFileSync(skillMdPath, 'utf-8');
|
||||
const currentSha = sha8(content);
|
||||
const expectedName = `${slug}-${currentSha}.json`;
|
||||
const expectedPath = join(receiptDir, expectedName);
|
||||
|
||||
if (existsSync(expectedPath)) {
|
||||
return { status: 'found', path: expectedPath, sha: currentSha };
|
||||
}
|
||||
|
||||
if (!existsSync(receiptDir)) {
|
||||
return { status: 'missing', currentSha };
|
||||
}
|
||||
|
||||
// Look for stale receipts (same slug, different sha).
|
||||
const prefix = `${slug}-`;
|
||||
const matches: Array<{ path: string; sha: string; mtime: number }> = [];
|
||||
for (const entry of readdirSync(receiptDir)) {
|
||||
if (!entry.startsWith(prefix) || !entry.endsWith('.json')) continue;
|
||||
const sha = entry.slice(prefix.length, -'.json'.length);
|
||||
if (sha === currentSha) continue;
|
||||
if (!/^[0-9a-f]{8}$/i.test(sha)) continue;
|
||||
const path = join(receiptDir, entry);
|
||||
try {
|
||||
const mtime = statSync(path).mtimeMs;
|
||||
matches.push({ path, sha, mtime });
|
||||
} catch {
|
||||
// Skip files we can't stat — they'll be missing-by-effect.
|
||||
}
|
||||
}
|
||||
if (matches.length === 0) return { status: 'missing', currentSha };
|
||||
|
||||
matches.sort((a, b) => b.mtime - a.mtime);
|
||||
const latest = matches[0]!;
|
||||
return {
|
||||
status: 'stale',
|
||||
latestPath: latest.path,
|
||||
latestSha: latest.sha,
|
||||
currentSha,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the slug out of a SKILL.md path. We accept:
|
||||
* - skills/<slug>/SKILL.md
|
||||
* - <skills-root>/<slug>/SKILL.md
|
||||
* - <slug>/SKILL.md (relative)
|
||||
* The slug is the immediate parent directory name.
|
||||
*/
|
||||
export function inferSlugFromSkillPath(skillMdPath: string): string {
|
||||
const parts = skillMdPath.replace(/\\/g, '/').split('/');
|
||||
const last = parts[parts.length - 1];
|
||||
if (last !== 'SKILL.md') {
|
||||
throw new Error(
|
||||
`inferSlugFromSkillPath: expected path ending in SKILL.md; got: ${skillMdPath}`,
|
||||
);
|
||||
}
|
||||
const parent = parts[parts.length - 2];
|
||||
if (!parent) {
|
||||
throw new Error(
|
||||
`inferSlugFromSkillPath: cannot infer slug — no parent directory in: ${skillMdPath}`,
|
||||
);
|
||||
}
|
||||
return parent;
|
||||
}
|
||||
|
||||
export function describeReceiptStatus(slug: string, status: ReceiptStatus): string {
|
||||
switch (status.status) {
|
||||
case 'found':
|
||||
return `cross-modal eval receipt found for ${slug} (sha ${status.sha}; matches current SKILL.md)`;
|
||||
case 'stale':
|
||||
return (
|
||||
`cross-modal eval receipt for ${slug} exists for an older SKILL.md ` +
|
||||
`(receipt sha ${status.latestSha}, current sha ${status.currentSha}). ` +
|
||||
`Re-run \`gbrain eval cross-modal\` against the current skill output.`
|
||||
);
|
||||
case 'missing':
|
||||
return `no cross-modal eval receipt for ${slug} yet — run \`gbrain eval cross-modal\` to add one`;
|
||||
}
|
||||
}
|
||||
|
||||
/** For tests + tools: pull all receipts for a slug, ordered newest first. */
|
||||
export function listReceiptsForSlug(slug: string, receiptDir: string): string[] {
|
||||
if (!existsSync(receiptDir)) return [];
|
||||
const prefix = `${slug}-`;
|
||||
const out: Array<{ path: string; mtime: number }> = [];
|
||||
for (const entry of readdirSync(receiptDir)) {
|
||||
if (!entry.startsWith(prefix) || !entry.endsWith('.json')) continue;
|
||||
const path = join(receiptDir, entry);
|
||||
try {
|
||||
out.push({ path, mtime: statSync(path).mtimeMs });
|
||||
} catch {
|
||||
// Skip unreadable.
|
||||
}
|
||||
}
|
||||
out.sort((a, b) => b.mtime - a.mtime);
|
||||
return out.map(o => o.path);
|
||||
}
|
||||
|
||||
/** Used by skillify-check to fall back to basename matching when needed. */
|
||||
export function isReceiptFile(path: string): boolean {
|
||||
const name = basename(path);
|
||||
return /^[a-z0-9][a-z0-9_-]*-[0-9a-f]{8}\.json$/i.test(name);
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* cross-modal-eval/receipt-write — auto-mkdir receipt writer.
|
||||
*
|
||||
* `gbrainPath()` from `src/core/config.ts` does NOT auto-mkdir (Codex T5
|
||||
* correction). Every receipt write needs an explicit `mkdirSync({recursive})`
|
||||
* ahead of the write so first-run users don't get `ENOENT: no such file or
|
||||
* directory` from a fresh `~/.gbrain/`.
|
||||
*/
|
||||
|
||||
import { mkdirSync, writeFileSync } from 'fs';
|
||||
import { dirname } from 'path';
|
||||
|
||||
export function writeReceipt(path: string, content: string | object): void {
|
||||
const body = typeof content === 'string' ? content : JSON.stringify(content, null, 2);
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, body, 'utf-8');
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
/**
|
||||
* cross-modal-eval/runner — orchestrate one or more eval cycles.
|
||||
*
|
||||
* Each cycle: 3 different-provider models score the OUTPUT against the TASK
|
||||
* on a fixed dimension list. `Promise.allSettled` so a single-provider 5xx
|
||||
* doesn't kill the cycle (T4=A — bare allSettled, no rate-leases for the
|
||||
* CLI path; future minion-integration TODO recovers cross-process
|
||||
* concurrency control).
|
||||
*
|
||||
* Pass / FAIL / INCONCLUSIVE verdict per `aggregate()`. The receipt schema
|
||||
* (schema_version: 1) is stable: timestamps, model strings, raw scores, and
|
||||
* dim rolls. Receipt filename binds skill slug + content sha-8 (T10=A) so
|
||||
* `gbrain skillify check` can tell whether a receipt is current or stale.
|
||||
*/
|
||||
|
||||
import { join } from 'path';
|
||||
|
||||
import { chat as gwChat } from '../ai/gateway.ts';
|
||||
import type { ChatMessage } from '../ai/gateway.ts';
|
||||
import { aggregate } from './aggregate.ts';
|
||||
import type { AggregateResult, SlotResult } from './aggregate.ts';
|
||||
import { parseModelJSON } from './json-repair.ts';
|
||||
import { receiptName, sha8 } from './receipt-name.ts';
|
||||
import { writeReceipt } from './receipt-write.ts';
|
||||
|
||||
export const RECEIPT_SCHEMA_VERSION = 1;
|
||||
|
||||
/** Default dimensions match the v1.1.0 SKILL.md. */
|
||||
export const DEFAULT_DIMENSIONS: string[] = [
|
||||
'GOAL_ACHIEVEMENT — Does the output actually accomplish what the task asked for?',
|
||||
'DEPTH — Is the output substantive, or surface-level / thin?',
|
||||
'SOURCING — Are claims backed by evidence, links, or citations?',
|
||||
'SPECIFICITY — Are there concrete details, data, quotes, examples?',
|
||||
'USEFULNESS — Would the intended audience find this valuable?',
|
||||
];
|
||||
|
||||
/**
|
||||
* Default 3-provider slot configuration. Implementer should refresh the
|
||||
* model strings alongside model-family bumps in CLAUDE.md.
|
||||
*
|
||||
* The model strings here resolve through `src/core/ai/recipes/`. Each slot
|
||||
* uses a distinct family so blind spots don't correlate. Override via
|
||||
* `--slot-a-model`, `--slot-b-model`, `--slot-c-model` on the CLI.
|
||||
*/
|
||||
export const DEFAULT_SLOTS: SlotConfig[] = [
|
||||
{ id: 'A', model: 'openai:gpt-4o' },
|
||||
{ id: 'B', model: 'anthropic:claude-opus-4-7' },
|
||||
{ id: 'C', model: 'google:gemini-1.5-pro' },
|
||||
];
|
||||
|
||||
export interface SlotConfig {
|
||||
id: string;
|
||||
/** "<provider>:<modelId>" string consumed by gateway.ts:resolveChatProvider. */
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface RunEvalOpts {
|
||||
task: string;
|
||||
output: string;
|
||||
/** Optional skill slug for receipt naming (T10). Falls back to a content sha. */
|
||||
slug?: string;
|
||||
/** Override default dimensions list. */
|
||||
dimensions?: string[];
|
||||
/** Override default 3 slots. */
|
||||
slots?: SlotConfig[];
|
||||
/** 1-3. CLI defaults to 3 in TTY, 1 in non-TTY (T11=B). */
|
||||
cycles?: number;
|
||||
/** Where receipts are written. CLI defaults to gbrainPath('eval-receipts'). */
|
||||
receiptDir: string;
|
||||
/** Per-call max output tokens (default 4000). */
|
||||
maxTokens?: number;
|
||||
/** Optional abort signal threaded into gateway calls. */
|
||||
abortSignal?: AbortSignal;
|
||||
/** Stderr progress callback (cycle 1/3, slot A done, etc.). */
|
||||
onProgress?: (event: ProgressEvent) => void;
|
||||
}
|
||||
|
||||
export type ProgressEvent =
|
||||
| { kind: 'cycle_start'; cycle: number; total: number }
|
||||
| { kind: 'slot_done'; cycle: number; slotId: string; modelId: string; ok: boolean; ms: number }
|
||||
| { kind: 'cycle_end'; cycle: number; verdict: 'pass' | 'fail' | 'inconclusive' };
|
||||
|
||||
export interface CycleReceipt {
|
||||
schema_version: 1;
|
||||
cycle: number;
|
||||
task: string;
|
||||
output_sha8: string;
|
||||
/** Slug used in receipt filename. */
|
||||
slug: string;
|
||||
/** Skill SHA-8 used in receipt filename — caller-supplied via skill_sha. */
|
||||
skill_sha8?: string;
|
||||
timestamp: string;
|
||||
dimensions: string[];
|
||||
slots: Array<{
|
||||
id: string;
|
||||
model: string;
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
raw?: string;
|
||||
parsed?: unknown;
|
||||
}>;
|
||||
aggregate: AggregateResult;
|
||||
/** Path the receipt was written to. */
|
||||
receipt_path: string;
|
||||
}
|
||||
|
||||
export interface RunEvalResult {
|
||||
/** Last cycle's aggregate (the verdict that drives exit code). */
|
||||
finalAggregate: AggregateResult;
|
||||
/** Receipt for each cycle that ran. */
|
||||
cycles: CycleReceipt[];
|
||||
/** Path of the LAST cycle's receipt (the one binding the current sha). */
|
||||
finalReceiptPath: string;
|
||||
}
|
||||
|
||||
/** Run up to `cycles` cycles. Stops early on PASS. */
|
||||
export async function runEval(opts: RunEvalOpts): Promise<RunEvalResult> {
|
||||
const dimensions = opts.dimensions ?? DEFAULT_DIMENSIONS;
|
||||
const slots = opts.slots ?? DEFAULT_SLOTS;
|
||||
const cycles = clampCycles(opts.cycles);
|
||||
const slug = opts.slug ?? `eval-${sha8(opts.output).slice(0, 6)}`;
|
||||
|
||||
const cycleReceipts: CycleReceipt[] = [];
|
||||
let finalAggregate: AggregateResult | null = null;
|
||||
let finalReceiptPath = '';
|
||||
|
||||
for (let cycle = 1; cycle <= cycles; cycle++) {
|
||||
opts.onProgress?.({ kind: 'cycle_start', cycle, total: cycles });
|
||||
|
||||
const slotResults = await runOneCycle({
|
||||
task: opts.task,
|
||||
output: opts.output,
|
||||
dimensions,
|
||||
slots,
|
||||
maxTokens: opts.maxTokens ?? 4000,
|
||||
abortSignal: opts.abortSignal,
|
||||
cycle,
|
||||
onProgress: opts.onProgress,
|
||||
});
|
||||
|
||||
const agg = aggregate({ slots: slotResults });
|
||||
finalAggregate = agg;
|
||||
|
||||
// Receipt filename: <slug>-<sha8 of output>.json on cycle 1; subsequent
|
||||
// cycles append `.cycle<N>` so we don't clobber.
|
||||
const baseName = receiptName(slug, opts.output);
|
||||
const receiptFile =
|
||||
cycle === 1 ? baseName : baseName.replace(/\.json$/, `.cycle${cycle}.json`);
|
||||
const receiptPath = join(opts.receiptDir, receiptFile);
|
||||
|
||||
const receipt: CycleReceipt = {
|
||||
schema_version: RECEIPT_SCHEMA_VERSION,
|
||||
cycle,
|
||||
task: opts.task,
|
||||
output_sha8: sha8(opts.output),
|
||||
slug,
|
||||
timestamp: new Date().toISOString(),
|
||||
dimensions,
|
||||
slots: slotResults.map(s => ({
|
||||
id: s.modelId.split(':')[0]!.toUpperCase().slice(0, 1),
|
||||
model: s.modelId,
|
||||
ok: s.ok,
|
||||
error: s.ok ? undefined : s.error,
|
||||
raw: s.ok ? undefined : undefined, // raw is large; skip from receipt by default
|
||||
parsed: s.ok ? s.parsed : undefined,
|
||||
})),
|
||||
aggregate: agg,
|
||||
receipt_path: receiptPath,
|
||||
};
|
||||
|
||||
writeReceipt(receiptPath, receipt);
|
||||
cycleReceipts.push(receipt);
|
||||
finalReceiptPath = receiptPath;
|
||||
|
||||
opts.onProgress?.({ kind: 'cycle_end', cycle, verdict: agg.verdict });
|
||||
|
||||
if (agg.verdict === 'pass' || agg.verdict === 'inconclusive') break;
|
||||
}
|
||||
|
||||
if (!finalAggregate) {
|
||||
throw new Error('runEval: no cycles ran');
|
||||
}
|
||||
|
||||
return { finalAggregate, cycles: cycleReceipts, finalReceiptPath };
|
||||
}
|
||||
|
||||
interface OneCycleOpts {
|
||||
task: string;
|
||||
output: string;
|
||||
dimensions: string[];
|
||||
slots: SlotConfig[];
|
||||
maxTokens: number;
|
||||
abortSignal?: AbortSignal;
|
||||
cycle: number;
|
||||
onProgress?: (event: ProgressEvent) => void;
|
||||
}
|
||||
|
||||
async function runOneCycle(opts: OneCycleOpts): Promise<SlotResult[]> {
|
||||
const prompt = buildPrompt(opts.task, opts.dimensions, opts.output);
|
||||
|
||||
const tasks = opts.slots.map(slot => callSlot(slot, prompt, opts));
|
||||
const settled = await Promise.allSettled(tasks);
|
||||
|
||||
const slotResults: SlotResult[] = settled.map((s, idx) => {
|
||||
const slot = opts.slots[idx]!;
|
||||
if (s.status === 'fulfilled') return s.value;
|
||||
return { ok: false, modelId: slot.model, error: errorMessage(s.reason) };
|
||||
});
|
||||
|
||||
return slotResults;
|
||||
}
|
||||
|
||||
async function callSlot(
|
||||
slot: SlotConfig,
|
||||
prompt: string,
|
||||
opts: OneCycleOpts,
|
||||
): Promise<SlotResult> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
const messages: ChatMessage[] = [
|
||||
{ role: 'user', content: prompt },
|
||||
];
|
||||
const result = await gwChat({
|
||||
model: slot.model,
|
||||
system: SYSTEM_PROMPT,
|
||||
messages,
|
||||
maxTokens: opts.maxTokens,
|
||||
abortSignal: opts.abortSignal,
|
||||
});
|
||||
|
||||
const parsed = parseModelJSON(result.text ?? '');
|
||||
const ms = Date.now() - start;
|
||||
opts.onProgress?.({
|
||||
kind: 'slot_done',
|
||||
cycle: opts.cycle,
|
||||
slotId: slot.id,
|
||||
modelId: slot.model,
|
||||
ok: true,
|
||||
ms,
|
||||
});
|
||||
return { ok: true, modelId: slot.model, parsed };
|
||||
} catch (err) {
|
||||
const ms = Date.now() - start;
|
||||
const msg = errorMessage(err);
|
||||
opts.onProgress?.({
|
||||
kind: 'slot_done',
|
||||
cycle: opts.cycle,
|
||||
slotId: slot.id,
|
||||
modelId: slot.model,
|
||||
ok: false,
|
||||
ms,
|
||||
});
|
||||
return { ok: false, modelId: slot.model, error: msg };
|
||||
}
|
||||
}
|
||||
|
||||
function buildPrompt(task: string, dimensions: string[], output: string): string {
|
||||
const dimList = dimensions.map((d, i) => `${i + 1}. ${d}`).join('\n');
|
||||
return [
|
||||
'You are a strict quality evaluator. Given a TASK and an OUTPUT, evaluate whether the output achieves the task goals.',
|
||||
'',
|
||||
'TASK:',
|
||||
task,
|
||||
'',
|
||||
`Score the OUTPUT 1-10 on each dimension:`,
|
||||
dimList,
|
||||
'',
|
||||
'Scoring calibration:',
|
||||
' 9-10: Exceptional — would impress a domain expert',
|
||||
' 7-8: Solid — accomplishes the goal, no major gaps',
|
||||
' 5-6: Mediocre — obvious weaknesses',
|
||||
' 3-4: Poor — missing important elements',
|
||||
' 1-2: Failed',
|
||||
'',
|
||||
'Then list exactly 10 specific, actionable improvements — concrete changes with examples, prioritized by impact.',
|
||||
'',
|
||||
'Respond in JSON only (no markdown fences):',
|
||||
'{',
|
||||
' "scores": {',
|
||||
' "dim_1_name": { "score": N, "feedback": "..." },',
|
||||
' ...',
|
||||
' },',
|
||||
' "overall": N,',
|
||||
' "improvements": ["1. ...", "2. ...", ... "10. ..."]',
|
||||
'}',
|
||||
'',
|
||||
'OUTPUT:',
|
||||
output,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
const SYSTEM_PROMPT =
|
||||
'You are a strict quality evaluator. Reply with JSON only. Do not wrap in markdown fences. ' +
|
||||
'Each score must be an integer 1-10. Improvements must be concrete and actionable.';
|
||||
|
||||
function clampCycles(n: number | undefined): number {
|
||||
if (typeof n !== 'number' || !Number.isFinite(n)) return 1;
|
||||
if (n < 1) return 1;
|
||||
if (n > 3) return 3;
|
||||
return Math.floor(n);
|
||||
}
|
||||
|
||||
function errorMessage(err: unknown): string {
|
||||
if (err instanceof Error) return err.message;
|
||||
return String(err);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cost estimation table. Used by the CLI to print a per-run upper-bound
|
||||
* before each cycle (T11=B). Source: gateway recipes' price_last_verified
|
||||
* fields. Prices drift; this is intentionally rough.
|
||||
*/
|
||||
export interface CostEstimate {
|
||||
perCycleUSD: number;
|
||||
perRunMaxUSD: number;
|
||||
perCallTokens: number;
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
export function estimateCost(slots: SlotConfig[], cycles: number, maxTokens: number): CostEstimate {
|
||||
// Per-call cost = (input_tokens × input_price + output_tokens × output_price) / 1e6.
|
||||
// Without knowing prompt size, estimate input ~5k tokens (a SKILL.md + scoring rubric).
|
||||
const ESTIMATED_INPUT_TOKENS = 5000;
|
||||
const PRICING: Record<string, { in: number; out: number } | undefined> = {
|
||||
'openai:gpt-4o': { in: 2.5, out: 10.0 },
|
||||
'openai:gpt-4o-mini': { in: 0.15, out: 0.6 },
|
||||
'anthropic:claude-opus-4-7': { in: 15.0, out: 75.0 },
|
||||
'anthropic:claude-sonnet-4-6-20250929': { in: 3.0, out: 15.0 },
|
||||
'anthropic:claude-haiku-4-5-20251001': { in: 0.25, out: 1.25 },
|
||||
'google:gemini-1.5-pro': { in: 1.25, out: 5.0 },
|
||||
'google:gemini-2.0-flash': { in: 0.1, out: 0.4 },
|
||||
'together:meta-llama/Llama-3.3-70B-Instruct-Turbo': { in: 0.88, out: 0.88 },
|
||||
'deepseek:deepseek-chat': { in: 0.14, out: 0.28 },
|
||||
};
|
||||
|
||||
const notes: string[] = [];
|
||||
let perCycle = 0;
|
||||
for (const slot of slots) {
|
||||
const p = PRICING[slot.model];
|
||||
if (!p) {
|
||||
notes.push(`(${slot.model}): no pricing on file; cost estimate may be low`);
|
||||
continue;
|
||||
}
|
||||
const cost = (ESTIMATED_INPUT_TOKENS * p.in + maxTokens * p.out) / 1_000_000;
|
||||
perCycle += cost;
|
||||
}
|
||||
return {
|
||||
perCycleUSD: round2(perCycle),
|
||||
perRunMaxUSD: round2(perCycle * cycles),
|
||||
perCallTokens: ESTIMATED_INPUT_TOKENS + maxTokens,
|
||||
notes,
|
||||
};
|
||||
}
|
||||
|
||||
function round2(n: number): number {
|
||||
return Math.round(n * 100) / 100;
|
||||
}
|
||||
+136
-2
@@ -52,7 +52,7 @@ import { getCliOptions, cliOptsToProgressOptions } from './cli-options.ts';
|
||||
|
||||
// ─── Types ─────────────────────────────────────────────────────────
|
||||
|
||||
export type CyclePhase = 'lint' | 'backlinks' | 'sync' | 'synthesize' | 'extract' | 'patterns' | 'embed' | 'orphans';
|
||||
export type CyclePhase = 'lint' | 'backlinks' | 'sync' | 'synthesize' | 'extract' | 'patterns' | 'embed' | 'orphans' | 'purge';
|
||||
|
||||
export const ALL_PHASES: CyclePhase[] = [
|
||||
'lint',
|
||||
@@ -63,13 +63,18 @@ export const ALL_PHASES: CyclePhase[] = [
|
||||
'patterns',
|
||||
'embed',
|
||||
'orphans',
|
||||
// v0.26.5: hard-deletes soft-deleted pages and expired archived sources past
|
||||
// the 72h recovery window. Runs last so the rest of the cycle sees the
|
||||
// recoverable set; the purge then drops what's expired.
|
||||
'purge',
|
||||
];
|
||||
|
||||
/**
|
||||
* Phases that mutate state (filesystem or DB) and therefore should
|
||||
* coordinate via the cycle lock. Only orphans is truly read-only
|
||||
* and skips the lock. patterns mutates DB (writes pattern pages) so
|
||||
* it acquires the lock; synthesize too.
|
||||
* it acquires the lock; synthesize too. v0.26.5 adds purge (DELETE-cascade
|
||||
* across pages and sources).
|
||||
*/
|
||||
const NEEDS_LOCK_PHASES: ReadonlySet<CyclePhase> = new Set([
|
||||
'lint',
|
||||
@@ -79,6 +84,7 @@ const NEEDS_LOCK_PHASES: ReadonlySet<CyclePhase> = new Set([
|
||||
'extract',
|
||||
'patterns',
|
||||
'embed',
|
||||
'purge',
|
||||
]);
|
||||
|
||||
export type PhaseStatus = 'ok' | 'warn' | 'fail' | 'skipped';
|
||||
@@ -138,6 +144,10 @@ export interface CycleReport {
|
||||
synth_pages_written: number;
|
||||
/** v0.23: number of pattern pages written/updated by patterns phase. */
|
||||
patterns_written: number;
|
||||
/** v0.26.5: number of source rows hard-deleted by the purge phase. */
|
||||
purged_sources_count: number;
|
||||
/** v0.26.5: number of page rows hard-deleted by the purge phase. */
|
||||
purged_pages_count: number;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -657,6 +667,101 @@ async function runPhaseEmbed(engine: BrainEngine, dryRun: boolean): Promise<Phas
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.26.5 — purge phase. Hard-deletes:
|
||||
* - source rows where `archived = true AND archive_expires_at <= now()`
|
||||
* (paired with the cascade FK to `pages`, this also drops the source's pages)
|
||||
* - page rows where `deleted_at` is older than 72h
|
||||
*
|
||||
* Cascade on `pages` covers `content_chunks`, `page_links`, `chunk_relations`.
|
||||
* `dryRun` short-circuits — no DELETEs are issued.
|
||||
*
|
||||
* Mirrors the operator escape hatches: `gbrain sources purge` (no id) and
|
||||
* `gbrain pages purge-deleted` both call the same library functions, so
|
||||
* scripted purges and the autopilot phase converge on a single behavior.
|
||||
*/
|
||||
/**
|
||||
* v0.28 P1: sweep $GBRAIN_HOME/clones/.tmp/ for entries older than the
|
||||
* configured TTL. addSource / recloneIfMissing clone into temp first then
|
||||
* rename atomically; if the process is SIGKILL'd between clone and rename,
|
||||
* the temp dir orphans. Without this sweep, a brain server accumulates
|
||||
* gigabytes over months. Mirrors the page/source soft-delete TTL pattern
|
||||
* so behavior is uniform across the purge phase.
|
||||
*/
|
||||
async function purgeOrphanClones(staleHours: number): Promise<{ count: number; bytes: number; names: string[] }> {
|
||||
const fs = await import('fs');
|
||||
const cfg = await import('./config.ts');
|
||||
const tmpRoot = cfg.gbrainPath('clones', '.tmp');
|
||||
if (!fs.existsSync(tmpRoot)) return { count: 0, bytes: 0, names: [] };
|
||||
const STALE_MS = staleHours * 3600 * 1000;
|
||||
const now = Date.now();
|
||||
const removed: string[] = [];
|
||||
let bytes = 0;
|
||||
for (const ent of fs.readdirSync(tmpRoot, { withFileTypes: true })) {
|
||||
const full = `${tmpRoot}/${ent.name}`;
|
||||
try {
|
||||
const st = fs.lstatSync(full);
|
||||
if (now - st.mtimeMs <= STALE_MS) continue;
|
||||
// Approximate size via stat (rough — recursive walk would be slow on
|
||||
// a stuck-clone with thousands of files; the bytes field is just
|
||||
// operator-visible feedback, not load-bearing).
|
||||
try { bytes += st.size; } catch { /* skip */ }
|
||||
fs.rmSync(full, { recursive: true, force: true });
|
||||
removed.push(ent.name);
|
||||
} catch {
|
||||
/* skip unreadable / racing-with-another-process */
|
||||
}
|
||||
}
|
||||
return { count: removed.length, bytes, names: removed };
|
||||
}
|
||||
|
||||
async function runPhasePurge(engine: BrainEngine, dryRun: boolean): Promise<PhaseResult> {
|
||||
try {
|
||||
if (dryRun) {
|
||||
return {
|
||||
phase: 'purge',
|
||||
status: 'ok',
|
||||
duration_ms: 0,
|
||||
summary: 'dry-run: skipped purge sweep',
|
||||
details: { dry_run: true, purged_sources_count: 0, purged_pages_count: 0, purged_orphan_clones_count: 0 },
|
||||
};
|
||||
}
|
||||
const { purgeExpiredSources } = await import('./destructive-guard.ts');
|
||||
const purgedSources = await purgeExpiredSources(engine);
|
||||
const purgedPages = await engine.purgeDeletedPages(SOFT_DELETE_TTL_HOURS_FOR_PURGE);
|
||||
const purgedClones = await purgeOrphanClones(SOFT_DELETE_TTL_HOURS_FOR_PURGE);
|
||||
return {
|
||||
phase: 'purge',
|
||||
status: 'ok',
|
||||
duration_ms: 0,
|
||||
summary:
|
||||
`purged ${purgedSources.length} source(s), ${purgedPages.count} page(s), and ` +
|
||||
`${purgedClones.count} orphan clone temp dir(s) past the 72h recovery window`,
|
||||
details: {
|
||||
purged_sources_count: purgedSources.length,
|
||||
purged_pages_count: purgedPages.count,
|
||||
purged_orphan_clones_count: purgedClones.count,
|
||||
purged_orphan_clone_names: purgedClones.names,
|
||||
purged_sources: purgedSources,
|
||||
purged_page_slugs: purgedPages.slugs,
|
||||
},
|
||||
};
|
||||
} catch (e) {
|
||||
return {
|
||||
phase: 'purge',
|
||||
status: 'fail',
|
||||
duration_ms: 0,
|
||||
summary: 'purge phase failed',
|
||||
details: {},
|
||||
error: makeErrorFromException(e),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** v0.26.5: matches SOFT_DELETE_TTL_HOURS in destructive-guard.ts. Inlined here
|
||||
* to avoid a static import (purge phase is only loaded in the autopilot path). */
|
||||
const SOFT_DELETE_TTL_HOURS_FOR_PURGE = 72;
|
||||
|
||||
async function runPhaseOrphans(engine: BrainEngine): Promise<PhaseResult> {
|
||||
try {
|
||||
const { findOrphans } = await import('../commands/orphans.ts');
|
||||
@@ -931,6 +1036,30 @@ export async function runCycle(
|
||||
}
|
||||
await safeYield(opts.yieldBetweenPhases);
|
||||
}
|
||||
|
||||
// ── Phase 9: purge (v0.26.5) ────────────────────────────────
|
||||
// Hard-delete soft-deleted pages and expired archived sources past the
|
||||
// 72h recovery window. Runs last so the rest of the cycle sees the
|
||||
// recoverable set; the purge then drops what's truly expired.
|
||||
if (phases.includes('purge')) {
|
||||
checkAborted(opts.signal);
|
||||
if (!engine) {
|
||||
phaseResults.push({
|
||||
phase: 'purge',
|
||||
status: 'skipped',
|
||||
duration_ms: 0,
|
||||
summary: 'no database connected',
|
||||
details: { reason: 'no_database' },
|
||||
});
|
||||
} else {
|
||||
progress.start('cycle.purge');
|
||||
const { result, duration_ms } = await timePhase(() => runPhasePurge(engine, dryRun));
|
||||
result.duration_ms = duration_ms;
|
||||
phaseResults.push(result);
|
||||
progress.finish();
|
||||
}
|
||||
await safeYield(opts.yieldBetweenPhases);
|
||||
}
|
||||
} finally {
|
||||
if (lock) {
|
||||
try { await lock.release(); } catch { /* best-effort */ }
|
||||
@@ -965,6 +1094,8 @@ function emptyTotals(): CycleReport['totals'] {
|
||||
transcripts_processed: 0,
|
||||
synth_pages_written: 0,
|
||||
patterns_written: 0,
|
||||
purged_sources_count: 0,
|
||||
purged_pages_count: 0,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -992,6 +1123,9 @@ function extractTotals(phases: PhaseResult[]): CycleReport['totals'] {
|
||||
t.synth_pages_written = Number(p.details.pages_written ?? 0);
|
||||
} else if (p.phase === 'patterns' && p.details) {
|
||||
t.patterns_written = Number(p.details.patterns_written ?? 0);
|
||||
} else if (p.phase === 'purge' && p.details) {
|
||||
t.purged_sources_count = Number(p.details.purged_sources_count ?? 0);
|
||||
t.purged_pages_count = Number(p.details.purged_pages_count ?? 0);
|
||||
}
|
||||
}
|
||||
return t;
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* v0.28: auto-think dream phase.
|
||||
*
|
||||
* Reads `dream.auto_think.questions[]` from config, runs `gbrain think` on
|
||||
* each one, persists the result as a synthesis page if `auto_commit=true`
|
||||
* (default false — write to a draft staging area instead). Capped by
|
||||
* `max_per_cycle` and the BudgetMeter's USD cap.
|
||||
*
|
||||
* Cooldown: `dream.auto_think.last_completion_ts` written ONLY on success
|
||||
* so retries after partial failures pick back up.
|
||||
*
|
||||
* Default-disabled. Operator opts in:
|
||||
* gbrain config set dream.auto_think.enabled true
|
||||
* gbrain config set dream.auto_think.questions '["What patterns ...","Who ..."]'
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
import { runThink, persistSynthesis, type ThinkLLMClient } from '../think/index.ts';
|
||||
import { resolveModel } from '../model-config.ts';
|
||||
import { BudgetMeter } from './budget-meter.ts';
|
||||
|
||||
/**
|
||||
* Local phase-result type for auto-think/drift. These phases are not yet
|
||||
* wired into cycle.ts's main dispatcher (deferred to v0.28.x); they ship
|
||||
* standalone for now and are invoked via `gbrain dream --phase auto_think`
|
||||
* once the dispatcher integration lands. Adopting cycle.ts's PhaseResult
|
||||
* shape forces premature CyclePhase enum extension.
|
||||
*/
|
||||
export interface DreamPhaseResult {
|
||||
name: 'auto_think' | 'drift';
|
||||
status: 'complete' | 'partial' | 'failed' | 'skipped';
|
||||
detail: string;
|
||||
totals?: Record<string, number>;
|
||||
duration_ms: number;
|
||||
}
|
||||
|
||||
export interface AutoThinkPhaseOpts {
|
||||
brainDir?: string;
|
||||
dryRun: boolean;
|
||||
/** Inject LLM client (tests). Defaults to the real Anthropic SDK. */
|
||||
client?: ThinkLLMClient;
|
||||
/** Override the audit-ledger path (tests). */
|
||||
auditPath?: string;
|
||||
}
|
||||
|
||||
export interface AutoThinkConfig {
|
||||
enabled: boolean;
|
||||
questions: string[];
|
||||
maxPerCycle: number;
|
||||
budgetUsd: number;
|
||||
cooldownDays: number;
|
||||
autoCommit: boolean;
|
||||
}
|
||||
|
||||
async function loadConfig(engine: BrainEngine): Promise<AutoThinkConfig> {
|
||||
const enabledStr = await engine.getConfig('dream.auto_think.enabled');
|
||||
const questionsStr = await engine.getConfig('dream.auto_think.questions');
|
||||
const maxPerStr = await engine.getConfig('dream.auto_think.max_per_cycle');
|
||||
const budgetStr = await engine.getConfig('dream.auto_think.budget');
|
||||
const cooldownStr = await engine.getConfig('dream.auto_think.cooldown_days');
|
||||
const autoCommitStr = await engine.getConfig('dream.auto_think.auto_commit');
|
||||
|
||||
let questions: string[] = [];
|
||||
if (questionsStr) {
|
||||
try {
|
||||
const parsed = JSON.parse(questionsStr);
|
||||
if (Array.isArray(parsed)) questions = parsed.filter(q => typeof q === 'string');
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
return {
|
||||
enabled: enabledStr === 'true',
|
||||
questions,
|
||||
maxPerCycle: maxPerStr ? Math.max(1, parseInt(maxPerStr, 10) || 5) : 5,
|
||||
budgetUsd: budgetStr ? Math.max(0, parseFloat(budgetStr) || 2.0) : 2.0,
|
||||
cooldownDays: cooldownStr ? Math.max(0, parseInt(cooldownStr, 10) || 30) : 30,
|
||||
autoCommit: autoCommitStr === 'true',
|
||||
};
|
||||
}
|
||||
|
||||
async function isCoolingDown(engine: BrainEngine, days: number): Promise<boolean> {
|
||||
if (days <= 0) return false;
|
||||
const last = await engine.getConfig('dream.auto_think.last_completion_ts');
|
||||
if (!last) return false;
|
||||
const lastMs = Date.parse(last);
|
||||
if (!Number.isFinite(lastMs)) return false;
|
||||
return (Date.now() - lastMs) < days * 86_400_000;
|
||||
}
|
||||
|
||||
function skipped(_reason: string, detail: string): DreamPhaseResult {
|
||||
return { name: 'auto_think', status: 'skipped', detail, duration_ms: 0 };
|
||||
}
|
||||
|
||||
export async function runPhaseAutoThink(
|
||||
engine: BrainEngine,
|
||||
opts: AutoThinkPhaseOpts,
|
||||
): Promise<DreamPhaseResult> {
|
||||
const start = Date.now();
|
||||
const config = await loadConfig(engine);
|
||||
|
||||
if (!config.enabled) {
|
||||
return skipped('not_configured', 'dream.auto_think.enabled is false');
|
||||
}
|
||||
if (config.questions.length === 0) {
|
||||
return skipped('no_questions', 'dream.auto_think.questions is empty');
|
||||
}
|
||||
if (await isCoolingDown(engine, config.cooldownDays)) {
|
||||
return skipped('cooldown_active', `auto_think cooled down (${config.cooldownDays}d cooldown)`);
|
||||
}
|
||||
|
||||
const meter = new BudgetMeter({
|
||||
budgetUsd: config.budgetUsd,
|
||||
phase: 'auto_think',
|
||||
auditPath: opts.auditPath,
|
||||
});
|
||||
|
||||
const modelId = await resolveModel(engine, {
|
||||
configKey: 'models.auto_think',
|
||||
deprecatedConfigKey: 'dream.auto_think.model',
|
||||
fallback: 'opus',
|
||||
});
|
||||
|
||||
const limit = Math.min(config.questions.length, config.maxPerCycle);
|
||||
const results: Array<{ question: string; status: string; slug?: string; warnings?: string[] }> = [];
|
||||
|
||||
for (let i = 0; i < limit; i++) {
|
||||
const q = config.questions[i];
|
||||
|
||||
// Pre-check budget for the planned synthesize call. Estimate ~5K input tokens
|
||||
// (system + ~30 takes + 20 page chunks) and 4K output cap.
|
||||
const check = meter.check({
|
||||
modelId,
|
||||
estimatedInputTokens: 5_000,
|
||||
maxOutputTokens: 4_000,
|
||||
label: `auto_think:${q.slice(0, 40)}`,
|
||||
});
|
||||
if (!check.allowed) {
|
||||
results.push({ question: q, status: 'budget_exhausted' });
|
||||
break;
|
||||
}
|
||||
|
||||
if (opts.dryRun) {
|
||||
results.push({ question: q, status: 'dry_run' });
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await runThink(engine, {
|
||||
question: q,
|
||||
save: config.autoCommit,
|
||||
client: opts.client,
|
||||
model: modelId,
|
||||
});
|
||||
let slug: string | undefined;
|
||||
if (config.autoCommit) {
|
||||
const persisted = await persistSynthesis(engine, result);
|
||||
slug = persisted.slug;
|
||||
}
|
||||
results.push({
|
||||
question: q,
|
||||
status: 'complete',
|
||||
slug,
|
||||
warnings: result.warnings.length ? result.warnings : undefined,
|
||||
});
|
||||
} catch (e) {
|
||||
results.push({
|
||||
question: q,
|
||||
status: 'failed',
|
||||
warnings: [(e as Error).message],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Update cooldown timestamp ONLY when at least one synthesis completed.
|
||||
const anyComplete = results.some(r => r.status === 'complete');
|
||||
if (anyComplete && !opts.dryRun) {
|
||||
await engine.setConfig('dream.auto_think.last_completion_ts', new Date().toISOString());
|
||||
}
|
||||
|
||||
const detail = `${results.filter(r => r.status === 'complete').length} synthesized, ` +
|
||||
`${results.filter(r => r.status === 'budget_exhausted').length} skipped (budget), ` +
|
||||
`${results.filter(r => r.status === 'failed').length} failed. ` +
|
||||
`Cumulative cost: $${meter.totalSpent.toFixed(4)} / $${config.budgetUsd.toFixed(2)}`;
|
||||
|
||||
return {
|
||||
name: 'auto_think',
|
||||
status: anyComplete ? 'complete' : (results.length === 0 ? 'skipped' : 'partial'),
|
||||
detail,
|
||||
totals: { questions_run: results.length, synthesized: results.filter(r => r.status === 'complete').length },
|
||||
duration_ms: Date.now() - start,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
/**
|
||||
* v0.28: cumulative cost meter for dream-cycle phases (auto-think + drift).
|
||||
*
|
||||
* Per Codex P1 #10: each subagent submit estimates max-cost from
|
||||
* `model + max_output_tokens`, accumulates per-cycle, refuses next submit
|
||||
* if cumulative > budget. Non-Anthropic models bypass the gate with a
|
||||
* `BUDGET_METER_NO_PRICING` warn (once per process).
|
||||
*
|
||||
* Ledger lives at `~/.gbrain/audit/dream-budget-YYYY-Www.jsonl` (ISO-week
|
||||
* rotation, same pattern as shell-audit). Each line is one submit's cost
|
||||
* estimate + actual usage when reported back.
|
||||
*/
|
||||
|
||||
import { mkdirSync, appendFileSync } from 'node:fs';
|
||||
import { dirname } from 'node:path';
|
||||
import { gbrainPath } from '../config.ts';
|
||||
import { estimateMaxCostUsd, ANTHROPIC_PRICING } from '../anthropic-pricing.ts';
|
||||
|
||||
export interface BudgetMeterOpts {
|
||||
/** USD cap for the whole cycle. 0 or negative disables the gate. */
|
||||
budgetUsd: number;
|
||||
/** Phase label for telemetry: 'auto_think' | 'drift'. */
|
||||
phase: string;
|
||||
/** Optional override for the audit file path (tests). */
|
||||
auditPath?: string;
|
||||
}
|
||||
|
||||
export interface SubmitEstimate {
|
||||
/** Resolved Anthropic model id (e.g. 'claude-opus-4-7'). */
|
||||
modelId: string;
|
||||
/** Best-guess input token count. Caller computes from prompt size. */
|
||||
estimatedInputTokens: number;
|
||||
/** Max output tokens passed to the LLM call. Upper-bounds the output cost. */
|
||||
maxOutputTokens: number;
|
||||
/** Logical label for the submit (synthesize / verdict / drift / ...). */
|
||||
label?: string;
|
||||
}
|
||||
|
||||
export interface BudgetCheckResult {
|
||||
allowed: boolean;
|
||||
estimatedCostUsd: number;
|
||||
cumulativeCostUsd: number;
|
||||
budgetUsd: number;
|
||||
reason?: string;
|
||||
/** True when the model wasn't in the pricing map (cycle runs unbounded for that submit). */
|
||||
unpriced?: boolean;
|
||||
}
|
||||
|
||||
/** One-process memo: warn-once on missing pricing per model. */
|
||||
const _unpricedWarnings = new Set<string>();
|
||||
|
||||
function auditFilePath(override?: string): string {
|
||||
if (override) return override;
|
||||
// ISO week format: YYYY-Www (2026-W18)
|
||||
const now = new Date();
|
||||
const year = now.getUTCFullYear();
|
||||
// ISO week: Thursday's week. Approximated for filename only.
|
||||
const oneJan = new Date(Date.UTC(year, 0, 1));
|
||||
const diffDays = Math.floor((now.getTime() - oneJan.getTime()) / 86_400_000);
|
||||
const week = Math.ceil((diffDays + oneJan.getUTCDay() + 1) / 7);
|
||||
const weekStr = String(week).padStart(2, '0');
|
||||
return gbrainPath(`audit/dream-budget-${year}-W${weekStr}.jsonl`);
|
||||
}
|
||||
|
||||
function writeLedgerLine(path: string, entry: object): void {
|
||||
try {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
appendFileSync(path, JSON.stringify(entry) + '\n');
|
||||
} catch {
|
||||
// Best-effort. Audit failure must not gate the cycle.
|
||||
}
|
||||
}
|
||||
|
||||
export class BudgetMeter {
|
||||
private cumulativeUsd = 0;
|
||||
private readonly auditPath: string;
|
||||
private unpricedSubmitsThisCycle = 0;
|
||||
|
||||
constructor(private readonly opts: BudgetMeterOpts) {
|
||||
this.auditPath = auditFilePath(opts.auditPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether a planned submit fits within the remaining budget.
|
||||
* Records the attempt to the ledger regardless of allow/deny.
|
||||
* Caller is responsible for skipping the actual LLM call when allowed=false.
|
||||
*/
|
||||
check(estimate: SubmitEstimate): BudgetCheckResult {
|
||||
const cost = estimateMaxCostUsd(estimate.modelId, estimate.estimatedInputTokens, estimate.maxOutputTokens);
|
||||
|
||||
// Codex P1 #10: non-Anthropic / unpriced models bypass the gate.
|
||||
if (cost === null) {
|
||||
this.unpricedSubmitsThisCycle++;
|
||||
if (!_unpricedWarnings.has(estimate.modelId)) {
|
||||
_unpricedWarnings.add(estimate.modelId);
|
||||
process.stderr.write(
|
||||
`[budget] BUDGET_METER_NO_PRICING: model "${estimate.modelId}" not in ANTHROPIC_PRICING. ` +
|
||||
`Budget gate disabled for this submit. (Per-provider pricing modules: TODO v0.29.)\n`,
|
||||
);
|
||||
}
|
||||
writeLedgerLine(this.auditPath, {
|
||||
phase: this.opts.phase,
|
||||
ts: new Date().toISOString(),
|
||||
event: 'submit_unpriced',
|
||||
model: estimate.modelId,
|
||||
label: estimate.label,
|
||||
estimated_input_tokens: estimate.estimatedInputTokens,
|
||||
max_output_tokens: estimate.maxOutputTokens,
|
||||
});
|
||||
return {
|
||||
allowed: true,
|
||||
estimatedCostUsd: 0,
|
||||
cumulativeCostUsd: this.cumulativeUsd,
|
||||
budgetUsd: this.opts.budgetUsd,
|
||||
unpriced: true,
|
||||
};
|
||||
}
|
||||
|
||||
// Budget disabled (<= 0)
|
||||
if (this.opts.budgetUsd <= 0) {
|
||||
this.cumulativeUsd += cost;
|
||||
writeLedgerLine(this.auditPath, {
|
||||
phase: this.opts.phase,
|
||||
ts: new Date().toISOString(),
|
||||
event: 'submit',
|
||||
model: estimate.modelId,
|
||||
label: estimate.label,
|
||||
estimated_cost_usd: cost,
|
||||
cumulative_cost_usd: this.cumulativeUsd,
|
||||
budget_usd: this.opts.budgetUsd,
|
||||
});
|
||||
return { allowed: true, estimatedCostUsd: cost, cumulativeCostUsd: this.cumulativeUsd, budgetUsd: this.opts.budgetUsd };
|
||||
}
|
||||
|
||||
const projected = this.cumulativeUsd + cost;
|
||||
if (projected > this.opts.budgetUsd) {
|
||||
writeLedgerLine(this.auditPath, {
|
||||
phase: this.opts.phase,
|
||||
ts: new Date().toISOString(),
|
||||
event: 'submit_denied',
|
||||
model: estimate.modelId,
|
||||
label: estimate.label,
|
||||
estimated_cost_usd: cost,
|
||||
cumulative_cost_usd: this.cumulativeUsd,
|
||||
budget_usd: this.opts.budgetUsd,
|
||||
});
|
||||
return {
|
||||
allowed: false,
|
||||
estimatedCostUsd: cost,
|
||||
cumulativeCostUsd: this.cumulativeUsd,
|
||||
budgetUsd: this.opts.budgetUsd,
|
||||
reason: `BUDGET_EXHAUSTED: projected $${projected.toFixed(4)} > cap $${this.opts.budgetUsd.toFixed(2)}`,
|
||||
};
|
||||
}
|
||||
|
||||
this.cumulativeUsd += cost;
|
||||
writeLedgerLine(this.auditPath, {
|
||||
phase: this.opts.phase,
|
||||
ts: new Date().toISOString(),
|
||||
event: 'submit',
|
||||
model: estimate.modelId,
|
||||
label: estimate.label,
|
||||
estimated_cost_usd: cost,
|
||||
cumulative_cost_usd: this.cumulativeUsd,
|
||||
budget_usd: this.opts.budgetUsd,
|
||||
});
|
||||
return { allowed: true, estimatedCostUsd: cost, cumulativeCostUsd: this.cumulativeUsd, budgetUsd: this.opts.budgetUsd };
|
||||
}
|
||||
|
||||
/** Cumulative cost spent so far this cycle. */
|
||||
get totalSpent(): number { return this.cumulativeUsd; }
|
||||
|
||||
/** Count of submits that bypassed the gate due to missing pricing. */
|
||||
get unpricedSubmits(): number { return this.unpricedSubmitsThisCycle; }
|
||||
}
|
||||
|
||||
/** Test helper: reset the once-per-process warning memo. */
|
||||
export function _resetBudgetMeterWarningsForTest(): void {
|
||||
_unpricedWarnings.clear();
|
||||
}
|
||||
|
||||
/** Re-export the pricing map for callers that need to introspect it. */
|
||||
export { ANTHROPIC_PRICING };
|
||||
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* v0.28: drift dream phase.
|
||||
*
|
||||
* Detects takes where the underlying evidence has shifted since the take
|
||||
* was made. v0.28 ships the SCAFFOLD: the phase iterates active takes,
|
||||
* runs a lightweight check against recent timeline entries on the same
|
||||
* page, and writes a drift-report-<date>.md if any takes look stale.
|
||||
*
|
||||
* The full LLM-driven drift detection (compare each take's claim to recent
|
||||
* page evidence and propose a weight adjustment) is the v0.29 follow-up.
|
||||
* v0.28 lays the phase orchestration so the contract is stable.
|
||||
*
|
||||
* Default-disabled. Operator opts in:
|
||||
* gbrain config set dream.drift.enabled true
|
||||
* gbrain config set dream.drift.lookback_days 30
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
import { BudgetMeter } from './budget-meter.ts';
|
||||
import { resolveModel } from '../model-config.ts';
|
||||
import type { DreamPhaseResult } from './auto-think.ts';
|
||||
|
||||
export interface DriftPhaseOpts {
|
||||
brainDir?: string;
|
||||
dryRun: boolean;
|
||||
/** Override the audit ledger path (tests). */
|
||||
auditPath?: string;
|
||||
}
|
||||
|
||||
export interface DriftConfig {
|
||||
enabled: boolean;
|
||||
lookbackDays: number;
|
||||
budgetUsd: number;
|
||||
autoUpdate: boolean;
|
||||
}
|
||||
|
||||
async function loadDriftConfig(engine: BrainEngine): Promise<DriftConfig> {
|
||||
const enabledStr = await engine.getConfig('dream.drift.enabled');
|
||||
const lookbackStr = await engine.getConfig('dream.drift.lookback_days');
|
||||
const budgetStr = await engine.getConfig('dream.drift.budget');
|
||||
const autoStr = await engine.getConfig('dream.drift.auto_update');
|
||||
return {
|
||||
enabled: enabledStr === 'true',
|
||||
lookbackDays: lookbackStr ? Math.max(1, parseInt(lookbackStr, 10) || 30) : 30,
|
||||
budgetUsd: budgetStr ? Math.max(0, parseFloat(budgetStr) || 1.0) : 1.0,
|
||||
autoUpdate: autoStr === 'true',
|
||||
};
|
||||
}
|
||||
|
||||
interface DriftCandidate {
|
||||
takeId: number;
|
||||
pageSlug: string;
|
||||
rowNum: number;
|
||||
claim: string;
|
||||
weight: number;
|
||||
/** Number of timeline entries within the lookback window for the same page. */
|
||||
recentEvidenceCount: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cheap pre-LLM heuristic: takes that have substantial recent timeline
|
||||
* evidence on the same page MAY have drifted. Surface them; the v0.29
|
||||
* LLM judge will decide if the weight should move.
|
||||
*/
|
||||
async function findDriftCandidates(
|
||||
engine: BrainEngine,
|
||||
lookbackDays: number,
|
||||
): Promise<DriftCandidate[]> {
|
||||
const cutoffMs = Date.now() - lookbackDays * 86_400_000;
|
||||
const cutoffIso = new Date(cutoffMs).toISOString().slice(0, 10);
|
||||
// Only consider takes with weight in the "soft" middle band (0.3..0.85)
|
||||
// — facts (1.0) don't drift, very-low hunches (<0.3) aren't actionable yet.
|
||||
const rows = await engine.executeRaw<{
|
||||
take_id: number; page_slug: string; row_num: number;
|
||||
claim: string; weight: number; recent_evidence: number;
|
||||
}>(`
|
||||
SELECT t.id AS take_id, p.slug AS page_slug, t.row_num,
|
||||
t.claim, t.weight,
|
||||
(SELECT count(*)::int FROM timeline_entries te
|
||||
WHERE te.page_id = p.id
|
||||
AND te.date >= $1::date)
|
||||
AS recent_evidence
|
||||
FROM takes t
|
||||
JOIN pages p ON p.id = t.page_id
|
||||
WHERE t.active
|
||||
AND t.weight >= 0.3 AND t.weight <= 0.85
|
||||
AND t.resolved_at IS NULL
|
||||
ORDER BY recent_evidence DESC, t.weight DESC
|
||||
LIMIT 200
|
||||
`, [cutoffIso]);
|
||||
return rows
|
||||
.filter(r => Number(r.recent_evidence) >= 1)
|
||||
.map(r => ({
|
||||
takeId: Number(r.take_id),
|
||||
pageSlug: String(r.page_slug),
|
||||
rowNum: Number(r.row_num),
|
||||
claim: String(r.claim),
|
||||
weight: Number(r.weight),
|
||||
recentEvidenceCount: Number(r.recent_evidence),
|
||||
}));
|
||||
}
|
||||
|
||||
function skipped(_reason: string, detail: string): DreamPhaseResult {
|
||||
return { name: 'drift', status: 'skipped', detail, duration_ms: 0 };
|
||||
}
|
||||
|
||||
export async function runPhaseDrift(
|
||||
engine: BrainEngine,
|
||||
opts: DriftPhaseOpts,
|
||||
): Promise<DreamPhaseResult> {
|
||||
const start = Date.now();
|
||||
const config = await loadDriftConfig(engine);
|
||||
if (!config.enabled) {
|
||||
return skipped('not_configured', 'dream.drift.enabled is false');
|
||||
}
|
||||
|
||||
const candidates = await findDriftCandidates(engine, config.lookbackDays);
|
||||
if (candidates.length === 0) {
|
||||
return {
|
||||
name: 'drift',
|
||||
status: 'complete',
|
||||
detail: 'no candidates: no soft-band takes with recent timeline evidence',
|
||||
totals: { candidates: 0 },
|
||||
duration_ms: Date.now() - start,
|
||||
};
|
||||
}
|
||||
|
||||
// Resolve model for the (future v0.29) LLM judge. For v0.28 we just
|
||||
// surface the candidates — the meter call is a no-op when we don't actually
|
||||
// submit, but resolveModel sets the right pricing key when v0.29 ships.
|
||||
const modelId = await resolveModel(engine, {
|
||||
configKey: 'models.drift',
|
||||
deprecatedConfigKey: 'dream.drift.model',
|
||||
fallback: 'sonnet',
|
||||
});
|
||||
const meter = new BudgetMeter({
|
||||
budgetUsd: config.budgetUsd,
|
||||
phase: 'drift',
|
||||
auditPath: opts.auditPath,
|
||||
});
|
||||
|
||||
// v0.28 scaffold: write a candidate report. v0.29 wires LLM-driven weight
|
||||
// adjustment through autoUpdate. modelId + meter are wired now so the
|
||||
// ledger captures the gate state even when we don't submit.
|
||||
void modelId; void meter;
|
||||
|
||||
if (opts.dryRun) {
|
||||
return {
|
||||
name: 'drift',
|
||||
status: 'skipped',
|
||||
detail: `dry-run: ${candidates.length} candidates would be evaluated`,
|
||||
totals: { candidates: candidates.length },
|
||||
duration_ms: Date.now() - start,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'drift',
|
||||
status: 'complete',
|
||||
detail: `surfaced ${candidates.length} drift candidates (LLM judge: v0.29 follow-up). autoUpdate=${config.autoUpdate}`,
|
||||
totals: { candidates: candidates.length },
|
||||
duration_ms: Date.now() - start,
|
||||
};
|
||||
}
|
||||
|
||||
/** Test helper: expose findDriftCandidates without running the full phase. */
|
||||
export const __testing = { findDriftCandidates };
|
||||
@@ -0,0 +1,219 @@
|
||||
/**
|
||||
* v0.28: extract-takes phase. Parses fenced takes blocks out of markdown
|
||||
* pages and upserts them into the `takes` table.
|
||||
*
|
||||
* Two paths (mirror src/commands/extract.ts dual-path pattern):
|
||||
* - fs: walk *.md files under repoPath; parse each fence; batch upsert
|
||||
* - db: iterate engine.getAllSlugs(); fetch each page's compiled_truth +
|
||||
* timeline; parse fence; batch upsert
|
||||
*
|
||||
* Source-of-truth contract: markdown is canonical. The takes table is a
|
||||
* derived index. `gbrain extract takes --rebuild` deletes all takes for
|
||||
* the affected pages first, then re-inserts. Without --rebuild, ON CONFLICT
|
||||
* (page_id, row_num) DO UPDATE keeps the table in sync incrementally.
|
||||
*
|
||||
* Sync-failure surfacing: malformed table rows produce
|
||||
* `TAKES_TABLE_MALFORMED` and `TAKES_ROW_NUM_COLLISION` warnings. v0.28
|
||||
* threads them through as ExtractTakesResult.warnings; the v0_28_0
|
||||
* orchestrator persists to ~/.gbrain/sync-failures.jsonl via the existing
|
||||
* v0.22.12 classifier path (extension follow-up — not blocking v0.28).
|
||||
*/
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join, relative, sep } from 'node:path';
|
||||
import type { BrainEngine, TakeBatchInput } from '../engine.ts';
|
||||
import { parseTakesFence, type ParsedTake } from '../takes-fence.ts';
|
||||
import { walkMarkdownFiles } from '../../commands/extract.ts';
|
||||
|
||||
export interface ExtractTakesOpts {
|
||||
/** Brain repo root. Required for source='fs'. */
|
||||
repoPath?: string;
|
||||
/** Source: 'fs' walks markdown files; 'db' iterates engine pages. Default 'fs'. */
|
||||
source?: 'fs' | 'db';
|
||||
/**
|
||||
* Optional incremental list of slugs to re-extract (used by sync→extract
|
||||
* pipe). Empty/undefined = full walk.
|
||||
*/
|
||||
slugs?: string[];
|
||||
/** Dry-run: parse + count, don't write. */
|
||||
dryRun?: boolean;
|
||||
/** When true, deletes existing takes for affected pages first. */
|
||||
rebuild?: boolean;
|
||||
}
|
||||
|
||||
export interface ExtractTakesResult {
|
||||
pagesScanned: number;
|
||||
pagesWithTakes: number;
|
||||
takesUpserted: number;
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a slug to its DB page_id. Returns null when no row exists for
|
||||
* that slug (e.g. file on disk that hasn't been imported yet).
|
||||
*/
|
||||
async function getPageIdForSlug(engine: BrainEngine, slug: string): Promise<number | null> {
|
||||
const rows = await engine.executeRaw<{ id: number }>(
|
||||
`SELECT id FROM pages WHERE slug = $1 LIMIT 1`,
|
||||
[slug],
|
||||
);
|
||||
return rows[0]?.id ?? null;
|
||||
}
|
||||
|
||||
function parsedTakeToBatchInput(pageId: number, t: ParsedTake): TakeBatchInput {
|
||||
return {
|
||||
page_id: pageId,
|
||||
row_num: t.rowNum,
|
||||
claim: t.claim,
|
||||
kind: t.kind,
|
||||
holder: t.holder,
|
||||
weight: t.weight,
|
||||
since_date: t.sinceDate,
|
||||
until_date: t.untilDate,
|
||||
source: t.source,
|
||||
active: t.active,
|
||||
superseded_by: null,
|
||||
};
|
||||
}
|
||||
|
||||
const BATCH_SIZE = 100;
|
||||
|
||||
async function flushBatch(
|
||||
engine: BrainEngine,
|
||||
buffer: TakeBatchInput[],
|
||||
result: ExtractTakesResult,
|
||||
dryRun: boolean,
|
||||
): Promise<void> {
|
||||
if (buffer.length === 0) return;
|
||||
if (dryRun) {
|
||||
result.takesUpserted += buffer.length;
|
||||
} else {
|
||||
const inserted = await engine.addTakesBatch(buffer);
|
||||
result.takesUpserted += inserted;
|
||||
}
|
||||
buffer.length = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk the repo's markdown files and extract takes from any fenced blocks.
|
||||
* Pages without a fence are no-ops.
|
||||
*/
|
||||
export async function extractTakesFromFs(
|
||||
engine: BrainEngine,
|
||||
opts: { repoPath: string; slugs?: string[]; dryRun?: boolean; rebuild?: boolean },
|
||||
): Promise<ExtractTakesResult> {
|
||||
const result: ExtractTakesResult = {
|
||||
pagesScanned: 0, pagesWithTakes: 0, takesUpserted: 0, warnings: [],
|
||||
};
|
||||
const dryRun = opts.dryRun ?? false;
|
||||
const slugFilter = opts.slugs && opts.slugs.length > 0 ? new Set(opts.slugs) : null;
|
||||
|
||||
const files = walkMarkdownFiles(opts.repoPath);
|
||||
const buffer: TakeBatchInput[] = [];
|
||||
|
||||
for (const { path, relPath } of files) {
|
||||
const slug = relPath.replace(/\.md$/, '').split(sep).join('/');
|
||||
if (slugFilter && !slugFilter.has(slug)) continue;
|
||||
result.pagesScanned++;
|
||||
|
||||
let body: string;
|
||||
try {
|
||||
body = readFileSync(path, 'utf-8');
|
||||
} catch (e) {
|
||||
result.warnings.push(`TAKES_FILE_READ_FAILED: ${relPath}: ${(e as Error).message}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const { takes, warnings } = parseTakesFence(body);
|
||||
if (warnings.length) {
|
||||
for (const w of warnings) result.warnings.push(`${slug}: ${w}`);
|
||||
}
|
||||
if (takes.length === 0) continue;
|
||||
|
||||
const pageId = await getPageIdForSlug(engine, slug);
|
||||
if (pageId === null) {
|
||||
result.warnings.push(`TAKES_PAGE_NOT_IN_DB: slug=${slug} has takes fence but no page row; run 'gbrain sync' first`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (opts.rebuild && !dryRun) {
|
||||
await engine.executeRaw(`DELETE FROM takes WHERE page_id = $1`, [pageId]);
|
||||
}
|
||||
|
||||
result.pagesWithTakes++;
|
||||
for (const t of takes) {
|
||||
buffer.push(parsedTakeToBatchInput(pageId, t));
|
||||
if (buffer.length >= BATCH_SIZE) await flushBatch(engine, buffer, result, dryRun);
|
||||
}
|
||||
}
|
||||
await flushBatch(engine, buffer, result, dryRun);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate engine pages and re-extract takes from each `compiled_truth` body.
|
||||
* Snapshot-stable (uses getAllSlugs). Doesn't read disk — works on
|
||||
* Postgres-only deployments without a local checkout.
|
||||
*/
|
||||
export async function extractTakesFromDb(
|
||||
engine: BrainEngine,
|
||||
opts: { slugs?: string[]; dryRun?: boolean; rebuild?: boolean } = {},
|
||||
): Promise<ExtractTakesResult> {
|
||||
const result: ExtractTakesResult = {
|
||||
pagesScanned: 0, pagesWithTakes: 0, takesUpserted: 0, warnings: [],
|
||||
};
|
||||
const dryRun = opts.dryRun ?? false;
|
||||
const slugs = opts.slugs && opts.slugs.length > 0
|
||||
? opts.slugs
|
||||
: Array.from(await engine.getAllSlugs());
|
||||
const buffer: TakeBatchInput[] = [];
|
||||
|
||||
for (const slug of slugs) {
|
||||
result.pagesScanned++;
|
||||
const page = await engine.getPage(slug);
|
||||
if (!page) continue;
|
||||
const body = `${page.compiled_truth ?? ''}\n${page.timeline ?? ''}`;
|
||||
const { takes, warnings } = parseTakesFence(body);
|
||||
if (warnings.length) {
|
||||
for (const w of warnings) result.warnings.push(`${slug}: ${w}`);
|
||||
}
|
||||
if (takes.length === 0) continue;
|
||||
|
||||
if (opts.rebuild && !dryRun) {
|
||||
await engine.executeRaw(`DELETE FROM takes WHERE page_id = $1`, [page.id]);
|
||||
}
|
||||
|
||||
result.pagesWithTakes++;
|
||||
for (const t of takes) {
|
||||
buffer.push(parsedTakeToBatchInput(page.id, t));
|
||||
if (buffer.length >= BATCH_SIZE) await flushBatch(engine, buffer, result, dryRun);
|
||||
}
|
||||
}
|
||||
await flushBatch(engine, buffer, result, dryRun);
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Single-entry dispatch for `gbrain extract takes` and the v0_28_0 orchestrator. */
|
||||
export async function extractTakes(
|
||||
engine: BrainEngine,
|
||||
opts: ExtractTakesOpts,
|
||||
): Promise<ExtractTakesResult> {
|
||||
const source = opts.source ?? (opts.repoPath ? 'fs' : 'db');
|
||||
if (source === 'fs') {
|
||||
if (!opts.repoPath) throw new Error('extractTakes: source=fs requires repoPath');
|
||||
return extractTakesFromFs(engine, {
|
||||
repoPath: opts.repoPath,
|
||||
slugs: opts.slugs,
|
||||
dryRun: opts.dryRun,
|
||||
rebuild: opts.rebuild,
|
||||
});
|
||||
}
|
||||
return extractTakesFromDb(engine, {
|
||||
slugs: opts.slugs,
|
||||
dryRun: opts.dryRun,
|
||||
rebuild: opts.rebuild,
|
||||
});
|
||||
}
|
||||
|
||||
/** Re-export so callers don't have to import from the relative path. */
|
||||
export { join, relative };
|
||||
@@ -140,7 +140,13 @@ async function loadPatternsConfig(engine: BrainEngine): Promise<PatternsConfig>
|
||||
const enabled = enabledStr === null ? true : enabledStr === 'true';
|
||||
const lookbackStr = await engine.getConfig('dream.patterns.lookback_days');
|
||||
const minEvidenceStr = await engine.getConfig('dream.patterns.min_evidence');
|
||||
const model = (await engine.getConfig('dream.patterns.model')) || 'claude-sonnet-4-6';
|
||||
// v0.28: unified model resolution
|
||||
const { resolveModel } = await import('../model-config.ts');
|
||||
const model = await resolveModel(engine, {
|
||||
configKey: 'models.dream.patterns',
|
||||
deprecatedConfigKey: 'dream.patterns.model',
|
||||
fallback: 'sonnet',
|
||||
});
|
||||
return {
|
||||
enabled,
|
||||
lookbackDays: lookbackStr ? Math.max(1, parseInt(lookbackStr, 10) || 30) : 30,
|
||||
|
||||
@@ -273,8 +273,18 @@ async function loadSynthConfig(engine: BrainEngine): Promise<SynthConfig> {
|
||||
const meetingTranscriptsDir = await engine.getConfig('dream.synthesize.meeting_transcripts_dir');
|
||||
const minCharsStr = await engine.getConfig('dream.synthesize.min_chars');
|
||||
const excludeStr = await engine.getConfig('dream.synthesize.exclude_patterns');
|
||||
const model = (await engine.getConfig('dream.synthesize.model')) || 'claude-sonnet-4-6';
|
||||
const verdictModel = (await engine.getConfig('dream.synthesize.verdict_model')) || 'claude-haiku-4-5-20251001';
|
||||
// v0.28: resolveModel() unifies CLI flag > new key > deprecated key > models.default > env > fallback
|
||||
const { resolveModel } = await import('../model-config.ts');
|
||||
const model = await resolveModel(engine, {
|
||||
configKey: 'models.dream.synthesize',
|
||||
deprecatedConfigKey: 'dream.synthesize.model',
|
||||
fallback: 'sonnet',
|
||||
});
|
||||
const verdictModel = await resolveModel(engine, {
|
||||
configKey: 'models.dream.synthesize_verdict',
|
||||
deprecatedConfigKey: 'dream.synthesize.verdict_model',
|
||||
fallback: 'haiku',
|
||||
});
|
||||
const cooldownHoursStr = await engine.getConfig('dream.synthesize.cooldown_hours');
|
||||
|
||||
let excludePatterns: string[] = ['medical', 'therapy'];
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
/**
|
||||
* Destructive operation guard — v0.26.5
|
||||
*
|
||||
* Protects against accidental data loss in gbrain by requiring explicit
|
||||
* confirmation for operations that cascade-delete pages, chunks, or embeddings.
|
||||
*
|
||||
* Three layers:
|
||||
* 1. Impact preview — always shown before destructive actions
|
||||
* 2. Confirmation gate — requires --confirm-destructive or interactive "type source name"
|
||||
* 3. Soft-delete with TTL — sources are tombstoned for 72h before permanent deletion
|
||||
*
|
||||
* Design principle: the blast radius should be visible BEFORE you pull the trigger,
|
||||
* and recoverable AFTER you pull it (within a grace period).
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────
|
||||
|
||||
export interface DestructiveImpact {
|
||||
sourceId: string;
|
||||
sourceName: string;
|
||||
pageCount: number;
|
||||
chunkCount: number;
|
||||
embeddingCount: number;
|
||||
fileCount: number;
|
||||
/** Human-readable summary line */
|
||||
summary: string;
|
||||
}
|
||||
|
||||
export interface SoftDeletedSource {
|
||||
id: string;
|
||||
name: string;
|
||||
deletedAt: Date;
|
||||
expiresAt: Date;
|
||||
pageCount: number;
|
||||
}
|
||||
|
||||
// ── Constants ───────────────────────────────────────────────
|
||||
|
||||
/** Hours before a soft-deleted source is permanently purged. */
|
||||
export const SOFT_DELETE_TTL_HOURS = 72;
|
||||
|
||||
/** Threshold: operations affecting this many pages or more require confirmation. */
|
||||
export const CONFIRM_THRESHOLD_PAGES = 1;
|
||||
|
||||
// ── Impact Assessment ───────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Compute the blast radius of deleting a source.
|
||||
*/
|
||||
export async function assessDestructiveImpact(
|
||||
engine: BrainEngine,
|
||||
sourceId: string,
|
||||
): Promise<DestructiveImpact | null> {
|
||||
// Fetch source metadata
|
||||
const sources = await engine.executeRaw<{ id: string; name: string }>(
|
||||
`SELECT id, name FROM sources WHERE id = $1`,
|
||||
[sourceId],
|
||||
);
|
||||
if (sources.length === 0) return null;
|
||||
|
||||
const src = sources[0];
|
||||
|
||||
// Count pages
|
||||
const pageRows = await engine.executeRaw<{ n: number }>(
|
||||
`SELECT COUNT(*)::int AS n FROM pages WHERE source_id = $1`,
|
||||
[sourceId],
|
||||
);
|
||||
const pageCount = pageRows[0]?.n ?? 0;
|
||||
|
||||
// Count chunks
|
||||
const chunkRows = await engine.executeRaw<{ n: number }>(
|
||||
`SELECT COUNT(*)::int AS n FROM content_chunks cc
|
||||
JOIN pages p ON cc.page_id = p.id
|
||||
WHERE p.source_id = $1`,
|
||||
[sourceId],
|
||||
);
|
||||
const chunkCount = chunkRows[0]?.n ?? 0;
|
||||
|
||||
// Count embeddings (chunks with non-null embedding vectors)
|
||||
const embedRows = await engine.executeRaw<{ n: number }>(
|
||||
`SELECT COUNT(*)::int AS n FROM content_chunks cc
|
||||
JOIN pages p ON cc.page_id = p.id
|
||||
WHERE p.source_id = $1 AND cc.embedding IS NOT NULL`,
|
||||
[sourceId],
|
||||
);
|
||||
const embeddingCount = embedRows[0]?.n ?? 0;
|
||||
|
||||
// Count files in storage (if any). PGLite has no `files` table — that
|
||||
// surface is Postgres-only (CLAUDE.md: "No files table" for PGLite). Probe
|
||||
// the table existence via information_schema so this works on both engines.
|
||||
let fileCount = 0;
|
||||
const filesTableRows = await engine.executeRaw<{ exists: boolean }>(
|
||||
`SELECT EXISTS (
|
||||
SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name = 'files'
|
||||
) AS exists`,
|
||||
);
|
||||
if (filesTableRows[0]?.exists) {
|
||||
const fileRows = await engine.executeRaw<{ n: number }>(
|
||||
`SELECT COUNT(*)::int AS n FROM files WHERE source_id = $1`,
|
||||
[sourceId],
|
||||
);
|
||||
fileCount = fileRows[0]?.n ?? 0;
|
||||
}
|
||||
|
||||
const parts: string[] = [];
|
||||
if (pageCount > 0) parts.push(`${pageCount.toLocaleString()} pages`);
|
||||
if (chunkCount > 0) parts.push(`${chunkCount.toLocaleString()} chunks`);
|
||||
if (embeddingCount > 0) parts.push(`${embeddingCount.toLocaleString()} embeddings`);
|
||||
if (fileCount > 0) parts.push(`${fileCount.toLocaleString()} files`);
|
||||
|
||||
const summary = parts.length > 0
|
||||
? `⚠️ This will permanently delete: ${parts.join(', ')}`
|
||||
: `Source "${sourceId}" has no data (safe to remove).`;
|
||||
|
||||
return {
|
||||
sourceId,
|
||||
sourceName: src.name,
|
||||
pageCount,
|
||||
chunkCount,
|
||||
embeddingCount,
|
||||
fileCount,
|
||||
summary,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Confirmation Gate ───────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Check whether the caller has provided sufficient confirmation for a
|
||||
* destructive operation. Returns an error message if blocked, or null if OK.
|
||||
*/
|
||||
export function checkDestructiveConfirmation(
|
||||
impact: DestructiveImpact,
|
||||
opts: {
|
||||
yes?: boolean;
|
||||
confirmDestructive?: boolean;
|
||||
dryRun?: boolean;
|
||||
},
|
||||
): string | null {
|
||||
// Dry run always passes (no side effects)
|
||||
if (opts.dryRun) return null;
|
||||
|
||||
// No data = no risk
|
||||
if (impact.pageCount === 0 && impact.chunkCount === 0 && impact.fileCount === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// --confirm-destructive is the explicit "I know what I'm doing" flag
|
||||
if (opts.confirmDestructive) return null;
|
||||
|
||||
// --yes alone is NOT sufficient for destructive operations with data.
|
||||
// This is the key behavior change: --yes used to be enough, now you
|
||||
// need --confirm-destructive when there's actual data at stake.
|
||||
if (opts.yes && impact.pageCount === 0) return null;
|
||||
|
||||
return (
|
||||
`\n${impact.summary}\n\n` +
|
||||
`To proceed, pass --confirm-destructive (or use soft-delete: gbrain sources archive ${impact.sourceId}).\n` +
|
||||
`To preview without side effects: --dry-run`
|
||||
);
|
||||
}
|
||||
|
||||
// ── Soft Delete ─────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Soft-delete a source: mark `archived = true` with a 72h TTL. Pages remain
|
||||
* in DB; the source is hidden from search via `buildVisibilityClause` and
|
||||
* federation is disabled via the existing `config.federated` JSONB key. After
|
||||
* TTL expires, the autopilot purge phase or manual `gbrain sources purge`
|
||||
* permanently removes the row (cascade delete to pages + chunks).
|
||||
*
|
||||
* v0.26.5: archive state moved from `config` JSONB keys to real columns
|
||||
* (`archived`, `archived_at`, `archive_expires_at`). Migration v34 backfills
|
||||
* pre-v0.26.5 rows. Faster filter, no reserved-key footgun. The `federated`
|
||||
* key stays in JSONB because federation has its own toggle path.
|
||||
*/
|
||||
export async function softDeleteSource(
|
||||
engine: BrainEngine,
|
||||
sourceId: string,
|
||||
): Promise<SoftDeletedSource | null> {
|
||||
// Atomic: only flip rows that are currently active. Returns the metadata
|
||||
// we need without a follow-up SELECT. RETURNING projects the columns the
|
||||
// caller cares about; pageCount is a separate count.
|
||||
const expiresClause = `now() + (${SOFT_DELETE_TTL_HOURS} || ' hours')::interval`;
|
||||
const rows = await engine.executeRaw<{ id: string; name: string; archived_at: string; archive_expires_at: string }>(
|
||||
`UPDATE sources
|
||||
SET archived = true,
|
||||
archived_at = now(),
|
||||
archive_expires_at = ${expiresClause},
|
||||
config = COALESCE(config, '{}'::jsonb) || '{"federated": false}'::jsonb
|
||||
WHERE id = $1 AND archived = false
|
||||
RETURNING id, name, archived_at, archive_expires_at`,
|
||||
[sourceId],
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
const row = rows[0];
|
||||
|
||||
const pageRows = await engine.executeRaw<{ n: number }>(
|
||||
`SELECT COUNT(*)::int AS n FROM pages WHERE source_id = $1`,
|
||||
[sourceId],
|
||||
);
|
||||
const pageCount = pageRows[0]?.n ?? 0;
|
||||
|
||||
return {
|
||||
id: sourceId,
|
||||
name: row.name,
|
||||
deletedAt: new Date(row.archived_at),
|
||||
expiresAt: new Date(row.archive_expires_at),
|
||||
pageCount,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore a soft-deleted source (un-archive). Returns true iff a row was
|
||||
* restored. Idempotent-as-false on "already active" or "not found".
|
||||
*
|
||||
* v0.26.5: clears the column-based archive state and (by default) flips
|
||||
* `config.federated = true` so the source re-enters federated search. The
|
||||
* `--no-federate` operator opt-out keeps federation disabled.
|
||||
*/
|
||||
export async function restoreSource(
|
||||
engine: BrainEngine,
|
||||
sourceId: string,
|
||||
refederate: boolean = true,
|
||||
): Promise<boolean> {
|
||||
const federatedPatch = refederate ? '{"federated": true}' : '{"federated": false}';
|
||||
const rows = await engine.executeRaw<{ id: string }>(
|
||||
`UPDATE sources
|
||||
SET archived = false,
|
||||
archived_at = NULL,
|
||||
archive_expires_at = NULL,
|
||||
config = COALESCE(config, '{}'::jsonb) || $1::jsonb
|
||||
WHERE id = $2 AND archived = true
|
||||
RETURNING id`,
|
||||
[federatedPatch, sourceId],
|
||||
);
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* List all soft-deleted (archived) sources.
|
||||
*
|
||||
* v0.26.5: filters via the real `archived` column instead of JSONB
|
||||
* containment. Faster, indexable on demand, no JSONB reserved-key collision
|
||||
* with future config schemas.
|
||||
*/
|
||||
export async function listArchivedSources(
|
||||
engine: BrainEngine,
|
||||
): Promise<SoftDeletedSource[]> {
|
||||
const rows = await engine.executeRaw<{
|
||||
id: string;
|
||||
name: string;
|
||||
archived_at: string;
|
||||
archive_expires_at: string;
|
||||
page_count: number;
|
||||
}>(
|
||||
`SELECT
|
||||
s.id, s.name, s.archived_at, s.archive_expires_at,
|
||||
COALESCE((SELECT COUNT(*)::int FROM pages p WHERE p.source_id = s.id), 0) AS page_count
|
||||
FROM sources s
|
||||
WHERE s.archived = true
|
||||
ORDER BY s.archived_at DESC`,
|
||||
);
|
||||
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
deletedAt: new Date(row.archived_at),
|
||||
expiresAt: new Date(row.archive_expires_at),
|
||||
pageCount: row.page_count,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Permanently purge sources whose 72h TTL has expired. Cascades to pages
|
||||
* (and content_chunks via existing FKs). Returns the ids of purged sources.
|
||||
*
|
||||
* v0.26.5: moved from JSONB-driven iteration to a single set-based DELETE
|
||||
* with `archived = true AND archive_expires_at <= now()`. Server-side
|
||||
* filter; one round-trip; cascade-friendly.
|
||||
*/
|
||||
export async function purgeExpiredSources(
|
||||
engine: BrainEngine,
|
||||
): Promise<string[]> {
|
||||
const rows = await engine.executeRaw<{ id: string }>(
|
||||
`DELETE FROM sources
|
||||
WHERE archived = true
|
||||
AND archive_expires_at IS NOT NULL
|
||||
AND archive_expires_at <= now()
|
||||
RETURNING id`,
|
||||
);
|
||||
return rows.map((r) => r.id);
|
||||
}
|
||||
|
||||
// ── Display Helpers ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Format an impact assessment for terminal display.
|
||||
*/
|
||||
export function formatImpact(impact: DestructiveImpact): string {
|
||||
const lines: string[] = [
|
||||
``,
|
||||
`╔══════════════════════════════════════════════════════════╗`,
|
||||
`║ DESTRUCTIVE OPERATION — Impact Preview ║`,
|
||||
`╠══════════════════════════════════════════════════════════╣`,
|
||||
`║ Source: ${impact.sourceName.padEnd(42)}║`,
|
||||
`║ Source ID: ${impact.sourceId.padEnd(42)}║`,
|
||||
`║ ║`,
|
||||
`║ Pages: ${String(impact.pageCount.toLocaleString()).padEnd(42)}║`,
|
||||
`║ Chunks: ${String(impact.chunkCount.toLocaleString()).padEnd(42)}║`,
|
||||
`║ Embeddings: ${String(impact.embeddingCount.toLocaleString()).padEnd(42)}║`,
|
||||
`║ Files: ${String(impact.fileCount.toLocaleString()).padEnd(42)}║`,
|
||||
`╠══════════════════════════════════════════════════════════╣`,
|
||||
`║ ${impact.summary.padEnd(56)}║`,
|
||||
`╚══════════════════════════════════════════════════════════╝`,
|
||||
``,
|
||||
];
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export function formatSoftDelete(sd: SoftDeletedSource): string {
|
||||
const hours = Math.round((sd.expiresAt.getTime() - Date.now()) / (1000 * 60 * 60));
|
||||
return [
|
||||
``,
|
||||
`Source "${sd.id}" archived (soft-deleted).`,
|
||||
` ${sd.pageCount.toLocaleString()} pages preserved for ${SOFT_DELETE_TTL_HOURS}h.`,
|
||||
` Expires: ${sd.expiresAt.toISOString()} (~${hours}h from now)`,
|
||||
` Removed from search. Data intact.`,
|
||||
``,
|
||||
` Restore: gbrain sources restore ${sd.id}`,
|
||||
` Purge now: gbrain sources purge ${sd.id} --confirm-destructive`,
|
||||
``,
|
||||
].join('\n');
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/**
|
||||
* Detect existing-brain embedding-dimension mismatch (v0.28.5 — A4).
|
||||
*
|
||||
* `gbrain init --embedding-dimensions N` on an existing brain whose
|
||||
* `content_chunks.embedding` column is a different `vector(M)` would
|
||||
* silently create a config/column drift: the config gets templated to N
|
||||
* but the column stays at M. The first sync write blows up with
|
||||
* "expected M, got N" — the silent-corruption pattern v0.28.5 is shipped
|
||||
* to kill.
|
||||
*
|
||||
* Loud-failure path: `gbrain init` AND `gbrain doctor` both consult this
|
||||
* helper. On mismatch they emit the same inline ALTER recipe (see
|
||||
* `embeddingMismatchMessage`) plus a pointer to `docs/embedding-migrations.md`.
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import { PGVECTOR_HNSW_VECTOR_MAX_DIMS } from './vector-index.ts';
|
||||
|
||||
export interface ColumnDimResult {
|
||||
/** Whether the `content_chunks.embedding` column exists. False on a fresh brain. */
|
||||
exists: boolean;
|
||||
/** Parsed `vector(N)` dimension if known. null when the column doesn't exist or the type isn't vector. */
|
||||
dims: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the actual dimension of `content_chunks.embedding` from the engine.
|
||||
*
|
||||
* Uses information_schema + a vector-specific catalog query. Returns
|
||||
* { exists: false, dims: null } on a fresh brain that doesn't have the
|
||||
* column yet. Returns { exists: true, dims: null } on a brain whose
|
||||
* column type isn't `vector` (shouldn't happen but defensive).
|
||||
*/
|
||||
export async function readContentChunksEmbeddingDim(engine: BrainEngine): Promise<ColumnDimResult> {
|
||||
// Probe column existence first to avoid noisy errors on fresh brains.
|
||||
const existsRows = await engine.executeRaw<{ exists: boolean }>(
|
||||
`SELECT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'content_chunks'
|
||||
AND column_name = 'embedding'
|
||||
) AS exists`,
|
||||
);
|
||||
const exists = !!existsRows?.[0]?.exists;
|
||||
if (!exists) return { exists: false, dims: null };
|
||||
|
||||
// pgvector stores dim in pg_type.typmod when atttypmod is set; format_type
|
||||
// returns the human-readable `vector(N)`. We parse N out of that.
|
||||
const formatRows = await engine.executeRaw<{ formatted: string | null }>(
|
||||
`SELECT format_type(a.atttypid, a.atttypmod) AS formatted
|
||||
FROM pg_attribute a
|
||||
JOIN pg_class c ON c.oid = a.attrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public'
|
||||
AND c.relname = 'content_chunks'
|
||||
AND a.attname = 'embedding'
|
||||
AND NOT a.attisdropped`,
|
||||
);
|
||||
const formatted = formatRows?.[0]?.formatted ?? null;
|
||||
if (!formatted) return { exists: true, dims: null };
|
||||
|
||||
const m = formatted.match(/vector\((\d+)\)/i);
|
||||
return { exists: true, dims: m ? parseInt(m[1], 10) : null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the human-readable ALTER recipe printed inline to stderr (or
|
||||
* delivered via `gbrain doctor` output) when an existing brain's column
|
||||
* dim doesn't match the requested dim.
|
||||
*
|
||||
* Steps cover the four-step contract from `docs/embedding-migrations.md`:
|
||||
* 1. DROP INDEX (HNSW can't survive ALTER COLUMN TYPE)
|
||||
* 2. ALTER COLUMN TYPE
|
||||
* 3. Wipe stale embeddings
|
||||
* 4. Conditional reindex (HNSW only when dims <= 2000)
|
||||
*/
|
||||
export function embeddingMismatchMessage(opts: {
|
||||
currentDims: number;
|
||||
requestedDims: number;
|
||||
requestedModel?: string;
|
||||
source?: 'init' | 'doctor';
|
||||
}): string {
|
||||
const { currentDims, requestedDims, requestedModel, source } = opts;
|
||||
const supportsHnsw = requestedDims <= PGVECTOR_HNSW_VECTOR_MAX_DIMS;
|
||||
const reindexLine = supportsHnsw
|
||||
? `CREATE INDEX IF NOT EXISTS idx_chunks_embedding\n ON content_chunks USING hnsw (embedding vector_cosine_ops);`
|
||||
: `-- Skip reindex. dims=${requestedDims} exceeds pgvector's HNSW cap of ${PGVECTOR_HNSW_VECTOR_MAX_DIMS};\n-- searchVector falls back to exact scan.`;
|
||||
|
||||
const header = source === 'doctor'
|
||||
? `Embedding dimension mismatch detected.`
|
||||
: `Refusing to silently re-template existing brain.`;
|
||||
|
||||
const lines = [
|
||||
header,
|
||||
``,
|
||||
` Existing column: vector(${currentDims})`,
|
||||
` Requested: vector(${requestedDims})${requestedModel ? ` (${requestedModel})` : ''}`,
|
||||
``,
|
||||
`Switching dims is destructive: it drops every embedding in your brain and`,
|
||||
`requires a full re-embed (potentially hours and $1-100 in API calls).`,
|
||||
``,
|
||||
`If you actually want to switch, run this manually against your brain's DB:`,
|
||||
``,
|
||||
` BEGIN;`,
|
||||
` DROP INDEX IF EXISTS idx_chunks_embedding;`,
|
||||
` ALTER TABLE content_chunks ALTER COLUMN embedding TYPE vector(${requestedDims});`,
|
||||
` UPDATE content_chunks SET embedding = NULL, embedded_at = NULL;`,
|
||||
` ${reindexLine.split('\n').join('\n ')}`,
|
||||
` COMMIT;`,
|
||||
``,
|
||||
`Then re-embed:`,
|
||||
` gbrain config set embedding_dimensions ${requestedDims}`,
|
||||
requestedModel ? ` gbrain config set embedding_model ${requestedModel}` : '',
|
||||
` gbrain embed --stale`,
|
||||
``,
|
||||
`Full guide: docs/embedding-migrations.md`,
|
||||
].filter(Boolean);
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
+42
-88
@@ -1,120 +1,74 @@
|
||||
/**
|
||||
* Embedding Service
|
||||
* Ported from production Ruby implementation (embedding_service.rb, 190 LOC)
|
||||
* Embedding Service — v0.14+ thin delegation to src/core/ai/gateway.ts.
|
||||
*
|
||||
* OpenAI text-embedding-3-large at 1536 dimensions.
|
||||
* Retry with exponential backoff (4s base, 120s cap, 5 retries).
|
||||
* 8000 character input truncation.
|
||||
* The gateway handles provider resolution, retry, error normalization, and
|
||||
* dimension-parameter passthrough (preserving existing 1536-dim brains).
|
||||
*/
|
||||
|
||||
import OpenAI from 'openai';
|
||||
|
||||
const MODEL = 'text-embedding-3-large';
|
||||
const DIMENSIONS = 1536;
|
||||
const MAX_CHARS = 8000;
|
||||
const MAX_RETRIES = 5;
|
||||
const BASE_DELAY_MS = 4000;
|
||||
const MAX_DELAY_MS = 120000;
|
||||
const BATCH_SIZE = 100;
|
||||
|
||||
let client: OpenAI | null = null;
|
||||
|
||||
function getClient(): OpenAI {
|
||||
if (!client) {
|
||||
client = new OpenAI();
|
||||
}
|
||||
return client;
|
||||
}
|
||||
import {
|
||||
embed as gatewayEmbed,
|
||||
embedOne as gatewayEmbedOne,
|
||||
getEmbeddingModel as gatewayGetModel,
|
||||
getEmbeddingDimensions as gatewayGetDims,
|
||||
} from './ai/gateway.ts';
|
||||
|
||||
/** Embed one text. */
|
||||
export async function embed(text: string): Promise<Float32Array> {
|
||||
const truncated = text.slice(0, MAX_CHARS);
|
||||
const result = await embedBatch([truncated]);
|
||||
return result[0];
|
||||
return gatewayEmbedOne(text);
|
||||
}
|
||||
|
||||
export interface EmbedBatchOptions {
|
||||
/**
|
||||
* Optional callback fired after each 100-item sub-batch completes.
|
||||
* CLI wrappers tick a reporter; Minion handlers can call
|
||||
* job.updateProgress here instead of hooking the per-page callback.
|
||||
* Optional callback fired after each sub-batch completes. CLI wrappers
|
||||
* tick a reporter; Minion handlers can call job.updateProgress here.
|
||||
*/
|
||||
onBatchComplete?: (done: number, total: number) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Embed a batch of texts via the gateway. Sub-batches of 100 so upstream
|
||||
* progress callbacks fire incrementally on large imports. The gateway owns
|
||||
* adaptive batch splitting and per-recipe token-budget logic; this paginator
|
||||
* is purely about progress-callback granularity.
|
||||
*/
|
||||
const BATCH_SIZE = 100;
|
||||
export async function embedBatch(
|
||||
texts: string[],
|
||||
options: EmbedBatchOptions = {},
|
||||
): Promise<Float32Array[]> {
|
||||
const truncated = texts.map(t => t.slice(0, MAX_CHARS));
|
||||
if (!texts || texts.length === 0) return [];
|
||||
// Fast path: small batch, no progress callback — single gateway call.
|
||||
if (texts.length <= BATCH_SIZE && !options.onBatchComplete) {
|
||||
return gatewayEmbed(texts);
|
||||
}
|
||||
const results: Float32Array[] = [];
|
||||
|
||||
// Process in batches of BATCH_SIZE
|
||||
for (let i = 0; i < truncated.length; i += BATCH_SIZE) {
|
||||
const batch = truncated.slice(i, i + BATCH_SIZE);
|
||||
const batchResults = await embedBatchWithRetry(batch);
|
||||
results.push(...batchResults);
|
||||
options.onBatchComplete?.(results.length, truncated.length);
|
||||
for (let i = 0; i < texts.length; i += BATCH_SIZE) {
|
||||
const slice = texts.slice(i, i + BATCH_SIZE);
|
||||
const out = await gatewayEmbed(slice);
|
||||
results.push(...out);
|
||||
options.onBatchComplete?.(results.length, texts.length);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async function embedBatchWithRetry(texts: string[]): Promise<Float32Array[]> {
|
||||
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
const response = await getClient().embeddings.create({
|
||||
model: MODEL,
|
||||
input: texts,
|
||||
dimensions: DIMENSIONS,
|
||||
});
|
||||
|
||||
// Sort by index to maintain order
|
||||
const sorted = response.data.sort((a, b) => a.index - b.index);
|
||||
return sorted.map(d => new Float32Array(d.embedding));
|
||||
} catch (e: unknown) {
|
||||
if (attempt === MAX_RETRIES - 1) throw e;
|
||||
|
||||
// Check for rate limit with Retry-After header
|
||||
let delay = exponentialDelay(attempt);
|
||||
|
||||
if (e instanceof OpenAI.APIError && e.status === 429) {
|
||||
const retryAfter = e.headers?.['retry-after'];
|
||||
if (retryAfter) {
|
||||
const parsed = parseInt(retryAfter, 10);
|
||||
if (!isNaN(parsed)) {
|
||||
delay = parsed * 1000;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(delay);
|
||||
}
|
||||
}
|
||||
|
||||
// Should not reach here
|
||||
throw new Error('Embedding failed after all retries');
|
||||
/** Currently-configured embedding model (short form without provider prefix). */
|
||||
export function getEmbeddingModelName(): string {
|
||||
return gatewayGetModel().split(':').slice(1).join(':') || 'text-embedding-3-large';
|
||||
}
|
||||
|
||||
function exponentialDelay(attempt: number): number {
|
||||
const delay = BASE_DELAY_MS * Math.pow(2, attempt);
|
||||
return Math.min(delay, MAX_DELAY_MS);
|
||||
/** Currently-configured embedding dimensions. */
|
||||
export function getEmbeddingDimensions(): number {
|
||||
return gatewayGetDims();
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export { MODEL as EMBEDDING_MODEL, DIMENSIONS as EMBEDDING_DIMENSIONS };
|
||||
// Back-compat exports for tests that imported these from v0.13.
|
||||
export const EMBEDDING_MODEL = 'text-embedding-3-large';
|
||||
export const EMBEDDING_DIMENSIONS = 1536;
|
||||
|
||||
/**
|
||||
* v0.20.0 Cathedral II Layer 8 (D1): USD cost per 1k tokens for
|
||||
* text-embedding-3-large. Used by `gbrain sync --all` cost preview and
|
||||
* the reindex-code backfill command to surface expected spend before
|
||||
* the agent/user accepts an expensive operation.
|
||||
*
|
||||
* Value: $0.00013 / 1k tokens as of 2026. Update when OpenAI changes
|
||||
* pricing. Single source of truth — every cost-preview surface reads
|
||||
* this constant, so a pricing change is a one-line edit.
|
||||
* USD cost per 1k tokens for text-embedding-3-large. Used by
|
||||
* `gbrain sync --all` cost preview and `reindex-code` to surface
|
||||
* expected spend before accepting expensive operations.
|
||||
*/
|
||||
export const EMBEDDING_COST_PER_1K_TOKENS = 0.00013;
|
||||
|
||||
|
||||
+220
-2
@@ -1,5 +1,5 @@
|
||||
import type {
|
||||
Page, PageInput, PageFilters,
|
||||
Page, PageInput, PageFilters, GetPageOpts,
|
||||
Chunk, ChunkInput, StaleChunkRow,
|
||||
SearchResult, SearchOpts,
|
||||
Link, GraphNode, GraphPath,
|
||||
@@ -88,6 +88,107 @@ export interface ReservedConnection {
|
||||
executeRaw<T = Record<string, unknown>>(sql: string, params?: unknown[]): Promise<T[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.28: Takes — typed/weighted/attributed claims, indexed in Postgres.
|
||||
* Markdown is source of truth (fenced table on the page); this row is the
|
||||
* derived index. Page-scoped via page_id (NOT slug — slug is unique only
|
||||
* within a source). `(page_id, row_num)` is the natural unique key.
|
||||
*/
|
||||
export interface TakeKindLiteral { kind: 'fact' | 'take' | 'bet' | 'hunch' }
|
||||
export type TakeKind = TakeKindLiteral['kind'];
|
||||
|
||||
/** Input row for addTakesBatch. */
|
||||
export interface TakeBatchInput {
|
||||
page_id: number;
|
||||
row_num: number;
|
||||
claim: string;
|
||||
kind: TakeKind;
|
||||
holder: string;
|
||||
weight?: number; // 0..1, default 0.5; clamped server-side
|
||||
since_date?: string; // ISO date 'YYYY-MM-DD'
|
||||
until_date?: string;
|
||||
source?: string;
|
||||
superseded_by?: number | null;
|
||||
active?: boolean; // default true
|
||||
}
|
||||
|
||||
/** Take row as returned by listTakes / searchTakes. */
|
||||
export interface Take {
|
||||
id: number;
|
||||
page_id: number;
|
||||
page_slug: string; // joined from pages
|
||||
row_num: number;
|
||||
claim: string;
|
||||
kind: TakeKind;
|
||||
holder: string;
|
||||
weight: number;
|
||||
since_date: string | null;
|
||||
until_date: string | null;
|
||||
source: string | null;
|
||||
superseded_by: number | null;
|
||||
active: boolean;
|
||||
resolved_at: string | null;
|
||||
resolved_outcome: boolean | null;
|
||||
resolved_value: number | null;
|
||||
resolved_unit: string | null;
|
||||
resolved_source: string | null;
|
||||
resolved_by: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface TakesListOpts {
|
||||
page_id?: number;
|
||||
page_slug?: string; // resolved via JOIN
|
||||
holder?: string;
|
||||
kind?: TakeKind;
|
||||
active?: boolean; // default true (only active rows)
|
||||
resolved?: boolean; // true = only resolved; false = only unresolved; undefined = both
|
||||
/** Per-token MCP allow-list. Server applies AND holder = ANY($takesHoldersAllowList) when set. */
|
||||
takesHoldersAllowList?: string[];
|
||||
sortBy?: 'weight' | 'since_date' | 'created_at';
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}
|
||||
|
||||
/** Search result row from searchTakes / searchTakesVector. */
|
||||
export interface TakeHit {
|
||||
take_id: number;
|
||||
page_id: number;
|
||||
page_slug: string;
|
||||
row_num: number;
|
||||
claim: string;
|
||||
kind: TakeKind;
|
||||
holder: string;
|
||||
weight: number;
|
||||
score: number; // search rank score (ts_rank for keyword, 1-cos_dist for vector)
|
||||
}
|
||||
|
||||
/** v0.28 stale-takes row (mirrors StaleChunkRow shape). Embedding column intentionally omitted. */
|
||||
export interface StaleTakeRow {
|
||||
take_id: number;
|
||||
page_slug: string;
|
||||
row_num: number;
|
||||
claim: string;
|
||||
}
|
||||
|
||||
/** Resolution metadata for resolveTake. */
|
||||
export interface TakeResolution {
|
||||
outcome: boolean;
|
||||
value?: number;
|
||||
unit?: string; // 'usd' | 'pct' | 'count' | other
|
||||
source?: string;
|
||||
resolvedBy: string; // slug or 'garry'
|
||||
}
|
||||
|
||||
/** Synthesis evidence row input (provenance from think synthesis pages). */
|
||||
export interface SynthesisEvidenceInput {
|
||||
synthesis_page_id: number;
|
||||
take_page_id: number;
|
||||
take_row_num: number;
|
||||
citation_index: number;
|
||||
}
|
||||
|
||||
/** Dream-cycle Haiku verdict on whether a transcript is worth processing. */
|
||||
export interface DreamVerdict {
|
||||
worth_processing: boolean;
|
||||
@@ -128,9 +229,48 @@ export interface BrainEngine {
|
||||
withReservedConnection<T>(fn: (conn: ReservedConnection) => Promise<T>): Promise<T>;
|
||||
|
||||
// Pages CRUD
|
||||
getPage(slug: string): Promise<Page | null>;
|
||||
/**
|
||||
* Fetch a page by slug.
|
||||
* v0.26.5: by default soft-deleted rows return null (matches the search
|
||||
* filter contract). Pass `opts.includeDeleted: true` to surface them with
|
||||
* `deleted_at` populated — used by `gbrain pages purge-deleted` listing,
|
||||
* by `restore_page` flow, and by operator diagnostics.
|
||||
*/
|
||||
getPage(slug: string, opts?: GetPageOpts): Promise<Page | null>;
|
||||
putPage(slug: string, page: PageInput): Promise<Page>;
|
||||
/**
|
||||
* Hard-delete a page row. Cascades to content_chunks, page_links,
|
||||
* chunk_relations via existing FK ON DELETE CASCADE.
|
||||
*
|
||||
* v0.26.5: this is no longer the public-facing `delete_page` op handler —
|
||||
* the op now soft-deletes via `softDeletePage` instead. `deletePage` stays
|
||||
* as the underlying primitive used by `purgeDeletedPages` and by callers
|
||||
* that explicitly want hard-delete semantics (e.g. test setup teardown).
|
||||
*/
|
||||
deletePage(slug: string): Promise<void>;
|
||||
/**
|
||||
* v0.26.5 — set `deleted_at = now()` on a page. Returns the slug if a row
|
||||
* was soft-deleted, null if no row matched (already soft-deleted OR not found).
|
||||
* Idempotent-as-null. The page stays in the DB and cascade rows (chunks,
|
||||
* links) stay intact; the autopilot purge phase hard-deletes after 72h.
|
||||
*/
|
||||
softDeletePage(slug: string, opts?: { sourceId?: string }): Promise<{ slug: string } | null>;
|
||||
/**
|
||||
* v0.26.5 — clear `deleted_at` on a soft-deleted page. Returns true iff a
|
||||
* row was restored. False if the slug is unknown OR the page is not
|
||||
* currently soft-deleted (idempotent-as-false).
|
||||
*/
|
||||
restorePage(slug: string, opts?: { sourceId?: string }): Promise<boolean>;
|
||||
/**
|
||||
* v0.26.5 — hard-delete pages whose `deleted_at` is older than the cutoff.
|
||||
* Called by the autopilot purge phase and by the `gbrain pages purge-deleted`
|
||||
* CLI escape hatch. Cascades through existing FKs.
|
||||
*/
|
||||
purgeDeletedPages(olderThanHours: number): Promise<{ slugs: string[]; count: number }>;
|
||||
/**
|
||||
* v0.26.5: by default `listPages` excludes soft-deleted rows. Set
|
||||
* `filters.includeDeleted: true` to surface them.
|
||||
*/
|
||||
listPages(filters?: PageFilters): Promise<Page[]>;
|
||||
resolveSlugs(partial: string): Promise<string[]>;
|
||||
/**
|
||||
@@ -273,6 +413,84 @@ export interface BrainEngine {
|
||||
putRawData(slug: string, source: string, data: object): Promise<void>;
|
||||
getRawData(slug: string, source?: string): Promise<RawData[]>;
|
||||
|
||||
// ============================================================
|
||||
// v0.28: Takes (typed/weighted/attributed claims) + synthesis evidence
|
||||
// ============================================================
|
||||
/**
|
||||
* Bulk insert/upsert takes. Uses `unnest()` (Postgres) or manual `$N`
|
||||
* placeholders (PGLite). Idempotency: ON CONFLICT (page_id, row_num) DO UPDATE
|
||||
* — re-extract on a changed claim/weight updates the row in place.
|
||||
* Returns the number of rows inserted OR updated.
|
||||
*
|
||||
* Weight outside [0, 1] is clamped server-side and surfaces a stderr
|
||||
* warning per call (`TAKES_WEIGHT_CLAMPED`). Invalid `kind` values
|
||||
* fail the whole batch via the CHECK constraint — caller is responsible
|
||||
* for parser validation upstream.
|
||||
*/
|
||||
addTakesBatch(rows: TakeBatchInput[]): Promise<number>;
|
||||
|
||||
/** List takes filtered by holder/kind/active/etc. Resolves page_slug via JOIN. */
|
||||
listTakes(opts?: TakesListOpts): Promise<Take[]>;
|
||||
|
||||
/**
|
||||
* Keyword search across active takes. Uses pg_trgm similarity over claim text.
|
||||
* Honors `takesHoldersAllowList` via WHERE filter so MCP-bound calls cannot
|
||||
* retrieve holders outside the token's allow-list.
|
||||
*/
|
||||
searchTakes(query: string, opts?: SearchOpts & { takesHoldersAllowList?: string[] }): Promise<TakeHit[]>;
|
||||
|
||||
/**
|
||||
* Vector search across active takes. Cosine distance against `embedding`.
|
||||
* Skipped (returns []) when no embedding column has been populated yet.
|
||||
*/
|
||||
searchTakesVector(
|
||||
embedding: Float32Array,
|
||||
opts?: SearchOpts & { takesHoldersAllowList?: string[] },
|
||||
): Promise<TakeHit[]>;
|
||||
|
||||
/** Look up embeddings by take id (mirrors getEmbeddingsByChunkIds). */
|
||||
getTakeEmbeddings(ids: number[]): Promise<Map<number, Float32Array>>;
|
||||
|
||||
/** Pre-flight count for `gbrain embed --stale`. WHERE active AND embedding IS NULL. */
|
||||
countStaleTakes(): Promise<number>;
|
||||
|
||||
/** List stale takes (no embedding column in payload — same pattern as listStaleChunks). */
|
||||
listStaleTakes(): Promise<StaleTakeRow[]>;
|
||||
|
||||
/**
|
||||
* Update a take's mutable fields. May NOT change claim/kind/holder per the
|
||||
* supersession invariants — those route through supersedeTake. Throws
|
||||
* `TAKE_ROW_NOT_FOUND` when (page_id, row_num) doesn't exist.
|
||||
*/
|
||||
updateTake(
|
||||
pageId: number,
|
||||
rowNum: number,
|
||||
fields: { weight?: number; since_date?: string; source?: string },
|
||||
): Promise<void>;
|
||||
|
||||
/**
|
||||
* Supersede the take at (page_id, oldRow). Marks old row active=false +
|
||||
* sets superseded_by; appends new row at the next row_num for the page;
|
||||
* returns both row_nums. Atomic (transactional). Cycle prevention: if newRow
|
||||
* sets superseded_by pointing to a chain that comes back to oldRow, throws
|
||||
* `TAKES_SUPERSEDE_CYCLE`. Resolved bets (`resolved_at IS NOT NULL`) cannot
|
||||
* be superseded — throws `TAKE_RESOLVED_IMMUTABLE`.
|
||||
*/
|
||||
supersedeTake(
|
||||
pageId: number,
|
||||
oldRow: number,
|
||||
newRow: Omit<TakeBatchInput, 'page_id' | 'row_num' | 'superseded_by'>,
|
||||
): Promise<{ oldRow: number; newRow: number }>;
|
||||
|
||||
/**
|
||||
* Resolve a bet (or take). Sets resolved_* columns. Immutable: re-resolve
|
||||
* attempts throw `TAKE_ALREADY_RESOLVED`. Use supersede to express a new bet.
|
||||
*/
|
||||
resolveTake(pageId: number, rowNum: number, resolution: TakeResolution): Promise<void>;
|
||||
|
||||
/** Persist think provenance. ON CONFLICT DO NOTHING; returns rows inserted. */
|
||||
addSynthesisEvidence(rows: SynthesisEvidenceInput[]): Promise<number>;
|
||||
|
||||
// Dream-cycle significance verdict cache (v0.23).
|
||||
// Keyed by (file_path, content_hash). Distinct from raw_data, which is
|
||||
// page-scoped — transcripts being judged aren't pages yet.
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
/**
|
||||
* gbrain remote-source git helpers (v0.28).
|
||||
*
|
||||
* Single source of SSRF-defensive git invocations. parseRemoteUrl delegates
|
||||
* to isInternalUrl from src/core/url-safety.ts (covers scheme allowlist,
|
||||
* IPv6 loopback, IPv4-mapped IPv6, metadata hostnames, hex/octal bypass,
|
||||
* and CGNAT 100.64/10).
|
||||
*
|
||||
* cloneRepo and pullRepo both spread GIT_SSRF_FLAGS so a future flag added
|
||||
* to one path lands on both — single source of truth.
|
||||
*
|
||||
* Tailscale 100.64/10 trips the integrations.ts allowlist (CGNAT line in
|
||||
* url-safety.ts isPrivateIpv4). For self-hosted internal git servers
|
||||
* reachable only via Tailscale, set GBRAIN_ALLOW_PRIVATE_REMOTES=1; loud
|
||||
* stderr warning at use site is the operator's signal.
|
||||
*/
|
||||
import { execFileSync } from 'child_process';
|
||||
import { lstatSync, existsSync, readdirSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { isInternalUrl } from './url-safety.ts';
|
||||
|
||||
/**
|
||||
* SSRF-defensive flag set. Used by both cloneRepo and pullRepo.
|
||||
* - http.followRedirects=false: closes DNS rebinding via redirect chains
|
||||
* - protocol.file.allow=never: no local-file URLs (defense in depth)
|
||||
* - protocol.ext.allow=never: no external helpers (`git-remote-foo`)
|
||||
* - --no-recurse-submodules: .gitmodules cannot become a second fetch surface
|
||||
*/
|
||||
export const GIT_SSRF_FLAGS = [
|
||||
'-c', 'http.followRedirects=false',
|
||||
'-c', 'protocol.file.allow=never',
|
||||
'-c', 'protocol.ext.allow=never',
|
||||
'--no-recurse-submodules',
|
||||
] as const;
|
||||
|
||||
export type RemoteUrlErrorCode =
|
||||
| 'invalid_url'
|
||||
| 'unsupported_scheme'
|
||||
| 'embedded_credentials'
|
||||
| 'path_traversal'
|
||||
| 'internal_target';
|
||||
|
||||
export class RemoteUrlError extends Error {
|
||||
constructor(public code: RemoteUrlErrorCode, message: string) {
|
||||
super(message);
|
||||
this.name = 'RemoteUrlError';
|
||||
}
|
||||
}
|
||||
|
||||
export interface ParsedRemoteUrl {
|
||||
url: string;
|
||||
hostname: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a remote git URL for clone safety. https:// only.
|
||||
* Rejects: non-https schemes, embedded credentials, path traversal, and
|
||||
* internal/private targets via isInternalUrl.
|
||||
*
|
||||
* GBRAIN_ALLOW_PRIVATE_REMOTES=1 lets the URL through with a stderr warning.
|
||||
* Needed for self-hosted git over Tailscale (CGNAT 100.64/10) and similar.
|
||||
*/
|
||||
export function parseRemoteUrl(s: string): ParsedRemoteUrl {
|
||||
if (!s || typeof s !== 'string') {
|
||||
throw new RemoteUrlError('invalid_url', 'URL is empty or not a string');
|
||||
}
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(s);
|
||||
} catch {
|
||||
throw new RemoteUrlError('invalid_url', `URL malformed: ${s}`);
|
||||
}
|
||||
if (url.protocol !== 'https:') {
|
||||
throw new RemoteUrlError(
|
||||
'unsupported_scheme',
|
||||
`URL scheme not supported (https:// only): ${url.protocol}`,
|
||||
);
|
||||
}
|
||||
if (url.username || url.password) {
|
||||
throw new RemoteUrlError(
|
||||
'embedded_credentials',
|
||||
'URL must not contain embedded credentials (https://user:pass@host)',
|
||||
);
|
||||
}
|
||||
if (s.includes('..')) {
|
||||
throw new RemoteUrlError('path_traversal', 'URL must not contain path-traversal (..)');
|
||||
}
|
||||
if (isInternalUrl(s)) {
|
||||
if (process.env.GBRAIN_ALLOW_PRIVATE_REMOTES === '1') {
|
||||
console.error(
|
||||
`[gbrain] WARN: GBRAIN_ALLOW_PRIVATE_REMOTES=1, accepting internal/private URL: ${url.hostname}`,
|
||||
);
|
||||
} else {
|
||||
throw new RemoteUrlError(
|
||||
'internal_target',
|
||||
`URL targets internal/private network: ${url.hostname} ` +
|
||||
`(set GBRAIN_ALLOW_PRIVATE_REMOTES=1 for self-hosted git over Tailscale or similar)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return { url: s, hostname: url.hostname };
|
||||
}
|
||||
|
||||
export interface CloneOpts {
|
||||
depth?: number; // default 1; 0 means full clone
|
||||
branch?: string;
|
||||
timeoutMs?: number; // default 600_000 (10 min)
|
||||
}
|
||||
|
||||
export class GitOperationError extends Error {
|
||||
constructor(
|
||||
public op: 'clone' | 'pull' | 'remote_get_url',
|
||||
message: string,
|
||||
public cause?: unknown,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'GitOperationError';
|
||||
}
|
||||
}
|
||||
|
||||
const GIT_ENV = {
|
||||
// Confine to the gbrain SSRF model — no credential helpers, no SSH askpass,
|
||||
// no GUI prompts. Inherit PATH so git itself is findable.
|
||||
GIT_TERMINAL_PROMPT: '0',
|
||||
GCM_INTERACTIVE: 'never',
|
||||
GIT_ASKPASS: '/bin/false',
|
||||
SSH_ASKPASS: '/bin/false',
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Clone a remote git repo with SSRF-defensive flags.
|
||||
* - destDir must NOT exist or must be empty.
|
||||
* - Default --depth=1 (no history); pass {depth: 0} for full clone.
|
||||
* - Throws GitOperationError on failure; caller is responsible for cleanup.
|
||||
*/
|
||||
export function cloneRepo(url: string, destDir: string, opts: CloneOpts = {}): void {
|
||||
if (existsSync(destDir)) {
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = readdirSync(destDir);
|
||||
} catch (e) {
|
||||
throw new GitOperationError(
|
||||
'clone',
|
||||
`Cannot inspect destination ${destDir}: ${(e as Error).message}`,
|
||||
e,
|
||||
);
|
||||
}
|
||||
if (entries.length > 0) {
|
||||
throw new GitOperationError(
|
||||
'clone',
|
||||
`Destination ${destDir} exists and is not empty; refusing to clone`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const args: string[] = [...GIT_SSRF_FLAGS, 'clone'];
|
||||
if (opts.depth !== 0) {
|
||||
args.push(`--depth=${opts.depth ?? 1}`);
|
||||
}
|
||||
if (opts.branch) {
|
||||
args.push('--branch', opts.branch);
|
||||
}
|
||||
args.push(url, destDir);
|
||||
|
||||
try {
|
||||
execFileSync('git', args, {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
timeout: opts.timeoutMs ?? 600_000,
|
||||
env: { ...process.env, ...GIT_ENV },
|
||||
});
|
||||
} catch (e) {
|
||||
throw new GitOperationError(
|
||||
'clone',
|
||||
`git clone failed for ${url}: ${(e as Error).message}`,
|
||||
e,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Pull a repo with --ff-only and the same SSRF-defensive flags as cloneRepo. */
|
||||
export function pullRepo(repoPath: string, opts: { timeoutMs?: number } = {}): void {
|
||||
const args: string[] = ['-C', repoPath, ...GIT_SSRF_FLAGS, 'pull', '--ff-only'];
|
||||
try {
|
||||
execFileSync('git', args, {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
timeout: opts.timeoutMs ?? 300_000,
|
||||
env: { ...process.env, ...GIT_ENV },
|
||||
});
|
||||
} catch (e) {
|
||||
throw new GitOperationError(
|
||||
'pull',
|
||||
`git pull failed in ${repoPath}: ${(e as Error).message}`,
|
||||
e,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export type RepoState =
|
||||
| 'healthy'
|
||||
| 'missing'
|
||||
| 'not-a-dir'
|
||||
| 'no-git'
|
||||
| 'url-drift'
|
||||
| 'corrupted';
|
||||
|
||||
/**
|
||||
* Classify the on-disk state of a clone. Used by performSync to decide
|
||||
* whether to run pull (healthy), re-clone (missing/no-git/not-a-dir),
|
||||
* refuse with corruption error (corrupted), or refuse with rebase-clone
|
||||
* hint (url-drift).
|
||||
*/
|
||||
export function validateRepoState(
|
||||
repoPath: string,
|
||||
expectedRemoteUrl?: string,
|
||||
): RepoState {
|
||||
let stat;
|
||||
try {
|
||||
stat = lstatSync(repoPath);
|
||||
} catch (e: any) {
|
||||
if (e?.code === 'ENOENT') return 'missing';
|
||||
return 'not-a-dir';
|
||||
}
|
||||
if (!stat.isDirectory()) return 'not-a-dir';
|
||||
if (!existsSync(join(repoPath, '.git'))) return 'no-git';
|
||||
|
||||
let remoteUrl: string;
|
||||
try {
|
||||
const out = execFileSync('git', ['-C', repoPath, 'remote', 'get-url', 'origin'], {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
timeout: 10_000,
|
||||
env: { ...process.env, ...GIT_ENV },
|
||||
});
|
||||
remoteUrl = out.toString().trim();
|
||||
} catch {
|
||||
return 'corrupted';
|
||||
}
|
||||
|
||||
if (expectedRemoteUrl !== undefined && remoteUrl !== expectedRemoteUrl) {
|
||||
return 'url-drift';
|
||||
}
|
||||
return 'healthy';
|
||||
}
|
||||
+8
-14
@@ -242,26 +242,20 @@ export async function importFromContent(
|
||||
}
|
||||
|
||||
// v0.20.0 Cathedral II Layer 8 D2 — extract fenced code blocks from
|
||||
// compiled_truth as first-class code chunks. A markdown page like
|
||||
// `docs/hybrid-search.md` with embedded TypeScript examples now ranks
|
||||
// the TS fence directly in code-aware queries instead of burying it
|
||||
// inside prose. Fences that carry an unrecognized lang tag (or no tag)
|
||||
// fall through — the prose chunker above already chunked them as text.
|
||||
// compiled_truth as first-class code chunks.
|
||||
if (parsed.compiled_truth.trim()) {
|
||||
const fenceChunks = await extractFencedChunks(parsed.compiled_truth, chunks.length);
|
||||
chunks.push(...fenceChunks);
|
||||
}
|
||||
|
||||
// Embed BEFORE the transaction (external API call)
|
||||
// Embed BEFORE the transaction (external API call).
|
||||
// v0.14+ (Codex C2): embedding failure PROPAGATES. Silent drop accumulates
|
||||
// unembedded pages invisibly. Caller can pass opts.noEmbed=true to skip.
|
||||
if (!opts.noEmbed && chunks.length > 0) {
|
||||
try {
|
||||
const embeddings = await embedBatch(chunks.map(c => c.chunk_text));
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
chunks[i].embedding = embeddings[i];
|
||||
chunks[i].token_count = Math.ceil(chunks[i].chunk_text.length / 4);
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
console.warn(`[gbrain] embedding failed for ${slug} (${chunks.length} chunks): ${e instanceof Error ? e.message : String(e)}`);
|
||||
const embeddings = await embedBatch(chunks.map(c => c.chunk_text));
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
chunks[i].embedding = embeddings[i];
|
||||
chunks[i].token_count = Math.ceil(chunks[i].chunk_text.length / 4);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1073,6 +1073,160 @@ export const MIGRATIONS: Migration[] = [
|
||||
},
|
||||
sql: '',
|
||||
},
|
||||
// NOTE: v37 + v38 are the v0.28 takes migrations. Renumbered four times during
|
||||
// the long-lived v0.28 branch as master shipped:
|
||||
// v0.28 originally targeted v31/v32
|
||||
// master v0.25 claimed v31 (eval_capture_tables) → renumbered to v32/v33
|
||||
// master v0.26 claimed v32 (oauth_infrastructure) and v33
|
||||
// (admin_dashboard_columns_v0_26_3) → renumbered to v34/v35
|
||||
// master v0.26.5 claimed v34 (destructive_guard_columns) → renumbered to v35/v36
|
||||
// master v0.26.8 + v0.27 claimed v35 (auto_rls_event_trigger) and v36
|
||||
// (subagent_provider_neutral_persistence_v0_27) → renumbered to v37/v38
|
||||
// Runtime sort by version ascending means source-order doesn't matter.
|
||||
{
|
||||
version: 37,
|
||||
name: 'takes_and_synthesis_evidence',
|
||||
// v0.28: typed/weighted/attributed claims ("takes") + synthesis provenance.
|
||||
// Spec: docs/designs (CEO plan) + plan file. Schema decisions:
|
||||
// - page_id FK (not page_slug) — pages.slug is unique only within source
|
||||
// - (page_id, row_num) is the natural unique key (composite, append-only)
|
||||
// - synthesis_evidence FK ON DELETE CASCADE — when a source take is hard-deleted,
|
||||
// provenance rows go with it; synthesis renderer marks citations as removed
|
||||
// - HNSW index on embedding (pgvector 0.7+ supports both Postgres + PGLite)
|
||||
// - resolved_* columns ship now per CEO-review D4 + Codex P1 #13 (immutable)
|
||||
sql: `
|
||||
CREATE TABLE IF NOT EXISTS takes (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
row_num INTEGER NOT NULL,
|
||||
claim TEXT NOT NULL,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('fact','take','bet','hunch')),
|
||||
holder TEXT NOT NULL,
|
||||
weight REAL NOT NULL DEFAULT 0.5 CHECK (weight >= 0 AND weight <= 1),
|
||||
since_date TEXT,
|
||||
until_date TEXT,
|
||||
source TEXT,
|
||||
superseded_by INTEGER,
|
||||
active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
resolved_at TIMESTAMPTZ,
|
||||
resolved_outcome BOOLEAN,
|
||||
resolved_value REAL,
|
||||
resolved_unit TEXT,
|
||||
resolved_source TEXT,
|
||||
resolved_by TEXT,
|
||||
embedding VECTOR(1536),
|
||||
embedded_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT takes_page_row_key UNIQUE (page_id, row_num)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_takes_page ON takes(page_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_takes_kind_active ON takes(kind) WHERE active;
|
||||
CREATE INDEX IF NOT EXISTS idx_takes_holder_active ON takes(holder) WHERE active;
|
||||
CREATE INDEX IF NOT EXISTS idx_takes_weight_active ON takes(weight DESC) WHERE active;
|
||||
CREATE INDEX IF NOT EXISTS idx_takes_resolved_at ON takes(resolved_at) WHERE resolved_at IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_takes_embedding_hnsw ON takes
|
||||
USING hnsw (embedding vector_cosine_ops)
|
||||
WHERE active AND embedding IS NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS synthesis_evidence (
|
||||
synthesis_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
take_page_id INTEGER NOT NULL,
|
||||
take_row_num INTEGER NOT NULL,
|
||||
citation_index INTEGER NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (synthesis_page_id, take_page_id, take_row_num),
|
||||
FOREIGN KEY (take_page_id, take_row_num)
|
||||
REFERENCES takes(page_id, row_num) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_synthesis_evidence_take
|
||||
ON synthesis_evidence(take_page_id, take_row_num);
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
has_bypass BOOLEAN;
|
||||
BEGIN
|
||||
SELECT rolbypassrls INTO has_bypass FROM pg_roles WHERE rolname = current_user;
|
||||
IF has_bypass THEN
|
||||
ALTER TABLE takes ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE synthesis_evidence ENABLE ROW LEVEL SECURITY;
|
||||
END IF;
|
||||
END $$;
|
||||
`,
|
||||
sqlFor: {
|
||||
// PGLite: same DDL minus the RLS DO-block (no rolbypassrls). Same HNSW
|
||||
// index syntax — pgvector 0.7+ supports it. Same FK semantics.
|
||||
pglite: `
|
||||
CREATE TABLE IF NOT EXISTS takes (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
row_num INTEGER NOT NULL,
|
||||
claim TEXT NOT NULL,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('fact','take','bet','hunch')),
|
||||
holder TEXT NOT NULL,
|
||||
weight REAL NOT NULL DEFAULT 0.5 CHECK (weight >= 0 AND weight <= 1),
|
||||
since_date TEXT,
|
||||
until_date TEXT,
|
||||
source TEXT,
|
||||
superseded_by INTEGER,
|
||||
active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
resolved_at TIMESTAMPTZ,
|
||||
resolved_outcome BOOLEAN,
|
||||
resolved_value REAL,
|
||||
resolved_unit TEXT,
|
||||
resolved_source TEXT,
|
||||
resolved_by TEXT,
|
||||
embedding VECTOR(1536),
|
||||
embedded_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT takes_page_row_key UNIQUE (page_id, row_num)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_takes_page ON takes(page_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_takes_kind_active ON takes(kind) WHERE active;
|
||||
CREATE INDEX IF NOT EXISTS idx_takes_holder_active ON takes(holder) WHERE active;
|
||||
CREATE INDEX IF NOT EXISTS idx_takes_weight_active ON takes(weight DESC) WHERE active;
|
||||
CREATE INDEX IF NOT EXISTS idx_takes_resolved_at ON takes(resolved_at) WHERE resolved_at IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_takes_embedding_hnsw ON takes
|
||||
USING hnsw (embedding vector_cosine_ops)
|
||||
WHERE active AND embedding IS NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS synthesis_evidence (
|
||||
synthesis_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
take_page_id INTEGER NOT NULL,
|
||||
take_row_num INTEGER NOT NULL,
|
||||
citation_index INTEGER NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (synthesis_page_id, take_page_id, take_row_num),
|
||||
FOREIGN KEY (take_page_id, take_row_num)
|
||||
REFERENCES takes(page_id, row_num) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_synthesis_evidence_take
|
||||
ON synthesis_evidence(take_page_id, take_row_num);
|
||||
`,
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 38,
|
||||
name: 'access_tokens_permissions',
|
||||
// v0.28: per-token allow-list for takes visibility (Codex P0 #3 partial fix).
|
||||
// The complementary fix (chunker strips fenced takes content from page chunks
|
||||
// so query results don't bypass the allow-list) lives in src/core/chunkers/takes-strip.ts.
|
||||
// Default permissions = {takes_holders: ['world']} keeps non-world takes (hunches,
|
||||
// private opinions) hidden from MCP-bound tokens until the operator explicitly
|
||||
// grants access via `gbrain auth permissions <id> set-takes-holders`.
|
||||
sql: `
|
||||
ALTER TABLE access_tokens
|
||||
ADD COLUMN IF NOT EXISTS permissions JSONB
|
||||
NOT NULL DEFAULT '{"takes_holders":["world"]}'::jsonb;
|
||||
|
||||
-- Backfill existing tokens to the default. NOT NULL DEFAULT covers new rows;
|
||||
-- this UPDATE handles any pre-existing rows from before the column was added.
|
||||
UPDATE access_tokens
|
||||
SET permissions = '{"takes_holders":["world"]}'::jsonb
|
||||
WHERE permissions IS NULL OR permissions = '{}'::jsonb;
|
||||
`,
|
||||
},
|
||||
{
|
||||
version: 30,
|
||||
name: 'dream_verdicts_table',
|
||||
@@ -1307,6 +1461,234 @@ export const MIGRATIONS: Migration[] = [
|
||||
ON mcp_request_log(agent_name, created_at DESC);
|
||||
`,
|
||||
},
|
||||
{
|
||||
version: 34,
|
||||
name: 'destructive_guard_columns',
|
||||
// v0.26.5 — soft-delete + recovery window for sources AND pages.
|
||||
// Renumbered v33→v34 on master merge: master's v33 (admin_dashboard_columns_v0_26_3)
|
||||
// landed first in PR #586. v34 follows it.
|
||||
//
|
||||
// pages.deleted_at: `delete_page` op now sets deleted_at = now() instead of
|
||||
// hard-deleting. The autopilot purge phase hard-deletes rows where
|
||||
// deleted_at < now() - 72h. Search and `get_page` filter
|
||||
// `WHERE deleted_at IS NULL` by default; `include_deleted: true` opts in.
|
||||
//
|
||||
// sources.archived/archived_at/archive_expires_at: promoted from JSONB keys
|
||||
// to real columns. v0.26.0 + the cherry-picked PR #595 wrote these inside
|
||||
// `sources.config` JSONB. Real columns are faster to filter, avoid the
|
||||
// reserved-key footgun, and let the search visibility filter compile to a
|
||||
// column lookup. The 72h TTL is preserved by reading
|
||||
// `archive_expires_at = archived_at + INTERVAL '72 hours'`.
|
||||
//
|
||||
// Backfill: any row that previously stored `{"archived":true,"archived_at":"...","archive_expires_at":"..."}`
|
||||
// in config gets migrated to the new columns, then the keys are stripped
|
||||
// from JSONB so the JSONB shape stays canonical going forward.
|
||||
//
|
||||
// Engine-aware partial index: Postgres uses CREATE INDEX CONCURRENTLY (no
|
||||
// write-blocking lock); PGLite uses plain CREATE INDEX. Mirrors v14
|
||||
// (pages_updated_at_index) handler shape.
|
||||
sql: '',
|
||||
handler: async (engine) => {
|
||||
// 1. Add columns. ALTER TABLE ADD COLUMN IF NOT EXISTS is idempotent on
|
||||
// both engines.
|
||||
await engine.runMigration(34, `
|
||||
ALTER TABLE pages ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||
ALTER TABLE sources ADD COLUMN IF NOT EXISTS archived BOOLEAN NOT NULL DEFAULT false;
|
||||
ALTER TABLE sources ADD COLUMN IF NOT EXISTS archived_at TIMESTAMPTZ;
|
||||
ALTER TABLE sources ADD COLUMN IF NOT EXISTS archive_expires_at TIMESTAMPTZ;
|
||||
`);
|
||||
|
||||
// 2. Backfill from JSONB shape used by pre-v0.26.5 cherry-picks of PR #595.
|
||||
// Idempotent: subsequent re-runs find zero matching rows.
|
||||
await engine.runMigration(34, `
|
||||
UPDATE sources
|
||||
SET archived = true,
|
||||
archived_at = COALESCE((config->>'archived_at')::timestamptz, now()),
|
||||
archive_expires_at = COALESCE(
|
||||
(config->>'archive_expires_at')::timestamptz,
|
||||
COALESCE((config->>'archived_at')::timestamptz, now()) + INTERVAL '72 hours'
|
||||
)
|
||||
WHERE config ? 'archived'
|
||||
AND (config->>'archived')::boolean = true
|
||||
AND archived = false;
|
||||
`);
|
||||
await engine.runMigration(34, `
|
||||
UPDATE sources
|
||||
SET config = config - 'archived' - 'archived_at' - 'archive_expires_at'
|
||||
WHERE config ?| ARRAY['archived', 'archived_at', 'archive_expires_at'];
|
||||
`);
|
||||
|
||||
// 3. Partial index for the autopilot purge sweep. Postgres CONCURRENTLY
|
||||
// avoids the SHARE lock on `pages`; PGLite has no concurrent writers.
|
||||
if (engine.kind === 'postgres') {
|
||||
// Pre-drop any invalid index from a prior CONCURRENTLY failure (matches v14 pattern).
|
||||
await engine.runMigration(34, `
|
||||
DO $$ BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE c.relname = 'pages_deleted_at_purge_idx' AND NOT i.indisvalid
|
||||
) THEN
|
||||
EXECUTE 'DROP INDEX CONCURRENTLY IF EXISTS pages_deleted_at_purge_idx';
|
||||
END IF;
|
||||
END $$;
|
||||
`);
|
||||
await engine.runMigration(34, `
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS pages_deleted_at_purge_idx
|
||||
ON pages (deleted_at) WHERE deleted_at IS NOT NULL;
|
||||
`);
|
||||
} else {
|
||||
await engine.runMigration(34, `
|
||||
CREATE INDEX IF NOT EXISTS pages_deleted_at_purge_idx
|
||||
ON pages (deleted_at) WHERE deleted_at IS NOT NULL;
|
||||
`);
|
||||
}
|
||||
},
|
||||
// CONCURRENTLY on Postgres requires no surrounding transaction. PGLite ignores
|
||||
// this flag, so the index DDL runs in whatever wrapper applies.
|
||||
transaction: false,
|
||||
},
|
||||
{
|
||||
version: 35,
|
||||
name: 'auto_rls_event_trigger',
|
||||
sql: '', // engine-specific via sqlFor
|
||||
// v0.26.7 — Postgres event trigger that auto-enables RLS on every new public.*
|
||||
// table, plus one-time backfill on every existing public.* table without it.
|
||||
//
|
||||
// Problem: tables created outside gbrain migrations (Baku's face_detections,
|
||||
// manual SQL, other apps sharing the Supabase project) shipped without RLS.
|
||||
// doctor caught them after the fact; the gap window between create and next
|
||||
// doctor run was the silent vector.
|
||||
//
|
||||
// Fix has two halves:
|
||||
// 1. Event trigger — fires on ddl_command_end for CREATE TABLE,
|
||||
// CREATE TABLE AS, and SELECT INTO; runs ALTER TABLE ... ENABLE ROW
|
||||
// LEVEL SECURITY for any new public.* table. Supabase-recommended
|
||||
// approach (no dashboard toggle exists).
|
||||
// 2. One-time backfill — every existing public.* table whose RLS is off
|
||||
// and whose comment does NOT match the GBRAIN:RLS_EXEMPT contract
|
||||
// (same regex doctor.ts uses) gets RLS enabled.
|
||||
//
|
||||
// Posture choices (vs PR-as-shipped):
|
||||
// - ENABLE only, no FORCE — matches v24/v29/schema.sql. FORCE would lock
|
||||
// out non-BYPASSRLS apps from their own newly-created tables (the
|
||||
// trigger function inherits the caller's role, and the new table is
|
||||
// owned by that role). gbrain has BYPASSRLS so gbrain itself is unaffected.
|
||||
// - public-only schema scope — Supabase manages auth/storage/realtime/etc.
|
||||
// and runs its own RLS posture there; we must not disturb those schemas.
|
||||
// - No EXCEPTION wrap inside the trigger — ddl_command_end fires inside
|
||||
// the DDL transaction, so a failed ALTER aborts the offending CREATE
|
||||
// TABLE. That's a loud signal, not a silent gap. Wrapping would CREATE
|
||||
// the silent path this migration exists to close.
|
||||
// - No privilege pre-check — runMigrations rethrows on SQL failure and
|
||||
// gates config.version, so a non-superuser run already fails loud with
|
||||
// an actionable Postgres error.
|
||||
//
|
||||
// BREAKING CHANGE: the backfill is a one-time override of intentionally
|
||||
// RLS-off public tables that don't carry the GBRAIN:RLS_EXEMPT comment.
|
||||
// Operators with such tables MUST add the exempt comment BEFORE upgrading.
|
||||
//
|
||||
// PGLite: no-op — no RLS engine, no event triggers, single-tenant by design.
|
||||
sqlFor: {
|
||||
postgres: `
|
||||
-- Trigger function: fires post-DDL inside the CREATE TABLE transaction.
|
||||
-- A failure here aborts the CREATE TABLE so no public.* table is ever
|
||||
-- created without RLS. object_identity is pre-quoted by Postgres
|
||||
-- (e.g. "public"."My Table"), so %s is correct — %I would double-quote.
|
||||
CREATE OR REPLACE FUNCTION auto_enable_rls()
|
||||
RETURNS event_trigger AS $$
|
||||
DECLARE
|
||||
obj record;
|
||||
BEGIN
|
||||
FOR obj IN SELECT * FROM pg_event_trigger_ddl_commands()
|
||||
WHERE object_type = 'table'
|
||||
AND schema_name = 'public'
|
||||
LOOP
|
||||
EXECUTE format('ALTER TABLE %s ENABLE ROW LEVEL SECURITY', obj.object_identity);
|
||||
END LOOP;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- WHEN TAG covers all three table-creation syntaxes Postgres reports.
|
||||
-- CREATE TABLE / CREATE TABLE AS / SELECT INTO produce distinct command
|
||||
-- tags; covering only 'CREATE TABLE' would leave a syntax-shaped hole.
|
||||
DROP EVENT TRIGGER IF EXISTS auto_rls_on_create_table;
|
||||
CREATE EVENT TRIGGER auto_rls_on_create_table
|
||||
ON ddl_command_end
|
||||
WHEN TAG IN ('CREATE TABLE', 'CREATE TABLE AS', 'SELECT INTO')
|
||||
EXECUTE FUNCTION auto_enable_rls();
|
||||
|
||||
-- One-time backfill of every existing public.* base table without RLS.
|
||||
-- Honors the same GBRAIN:RLS_EXEMPT regex doctor.ts uses
|
||||
-- (^GBRAIN:RLS_EXEMPT\\s+reason=\\S.{3,}) so the two surfaces stay aligned.
|
||||
-- %I.%I quotes the schema and table names safely, including mixed-case.
|
||||
DO $$
|
||||
DECLARE
|
||||
has_bypass BOOLEAN;
|
||||
r record;
|
||||
BEGIN
|
||||
SELECT rolbypassrls INTO has_bypass FROM pg_roles WHERE rolname = current_user;
|
||||
IF NOT has_bypass THEN
|
||||
-- Same posture as v24: raise to abort the migration so the runner
|
||||
-- leaves config.version unbumped and retries on the next call.
|
||||
RAISE EXCEPTION 'v35 auto_rls_event_trigger backfill: role % does not have BYPASSRLS — cannot enable RLS safely. Re-run as postgres (or another BYPASSRLS role).', current_user;
|
||||
END IF;
|
||||
|
||||
FOR r IN
|
||||
SELECT n.nspname AS schema_name, c.relname AS table_name
|
||||
FROM pg_class c
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = 0
|
||||
WHERE n.nspname = 'public'
|
||||
AND c.relkind = 'r'
|
||||
AND c.relrowsecurity = false
|
||||
AND (d.description IS NULL OR d.description !~ '^GBRAIN:RLS_EXEMPT\\s+reason=\\S.{3,}')
|
||||
LOOP
|
||||
EXECUTE format('ALTER TABLE %I.%I ENABLE ROW LEVEL SECURITY', r.schema_name, r.table_name);
|
||||
RAISE NOTICE 'v35: backfilled RLS on %.%', r.schema_name, r.table_name;
|
||||
END LOOP;
|
||||
END $$;
|
||||
`,
|
||||
pglite: '', // PGLite has no RLS and no event trigger support
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 36,
|
||||
name: 'subagent_provider_neutral_persistence_v0_27',
|
||||
// v0.27 multi-provider subagent. Codex F-OV-1 / D11: the subagent_messages
|
||||
// and subagent_tool_executions tables stored Anthropic-shaped tool_use /
|
||||
// tool_result blocks as JSONB. When a worker resumes a job mid-loop and
|
||||
// the live model is OpenAI/DeepSeek/etc, the persisted shape becomes the
|
||||
// runtime contract — translation at read time is lossy.
|
||||
//
|
||||
// Fix: add schema_version + provider_id columns. schema_version=1 is the
|
||||
// legacy Anthropic-shape (existing rows). schema_version=2 is the
|
||||
// provider-neutral ChatBlock format documented in src/core/ai/gateway.ts
|
||||
// (text / tool-call / tool-result blocks with normalized field names).
|
||||
// Subagent.ts (commit 2) writes schema_version=2 going forward and reads
|
||||
// both shapes via a versioned mapper.
|
||||
//
|
||||
// Renumbered v34→v35→v36 across master merges: master's v34
|
||||
// (destructive_guard_columns, v0.26.5 soft-delete) and v35
|
||||
// (auto_rls_event_trigger, v0.26.8) landed first.
|
||||
//
|
||||
// No data migration. Existing in-flight jobs continue to replay against
|
||||
// their original shape; new jobs use v2. ADD COLUMN IF NOT EXISTS makes
|
||||
// the migration idempotent.
|
||||
sql: `
|
||||
ALTER TABLE subagent_messages
|
||||
ADD COLUMN IF NOT EXISTS schema_version INTEGER NOT NULL DEFAULT 1,
|
||||
ADD COLUMN IF NOT EXISTS provider_id TEXT;
|
||||
|
||||
ALTER TABLE subagent_tool_executions
|
||||
ADD COLUMN IF NOT EXISTS schema_version INTEGER NOT NULL DEFAULT 1,
|
||||
ADD COLUMN IF NOT EXISTS provider_id TEXT;
|
||||
|
||||
-- Lookup by provider for cost rollups + per-provider replay diagnostics.
|
||||
CREATE INDEX IF NOT EXISTS idx_subagent_messages_provider
|
||||
ON subagent_messages (job_id, provider_id);
|
||||
`,
|
||||
},
|
||||
];
|
||||
|
||||
export const LATEST_VERSION = MIGRATIONS.length > 0
|
||||
@@ -1421,6 +1803,32 @@ async function runMigrationSQL(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cheap probe: does this engine have schema migrations pending?
|
||||
*
|
||||
* Reads the `version` config row in a single round-trip (no schema replay,
|
||||
* no migration apply). Used by `connectEngine` to gate `initSchema()` so
|
||||
* short-lived CLI invocations on already-migrated brains don't pay the
|
||||
* full bootstrap-probe + SCHEMA_SQL replay + ledger-check cost on every
|
||||
* `gbrain stats` / `gbrain query` / `gbrain doctor`.
|
||||
*
|
||||
* Defensive: treats a getConfig failure (config table missing, query error)
|
||||
* as "yes pending" so the caller falls through to the full initSchema path.
|
||||
* Worst case on a wedged brain is one extra schema replay — same as before.
|
||||
*
|
||||
* Closes #651 in cooperation with the post-upgrade auto-apply hook (X1)
|
||||
* without the perf cost #652 would have introduced on every CLI call.
|
||||
*/
|
||||
export async function hasPendingMigrations(engine: BrainEngine): Promise<boolean> {
|
||||
try {
|
||||
const currentStr = await engine.getConfig('version');
|
||||
const current = parseInt(currentStr || '1', 10);
|
||||
return current < LATEST_VERSION;
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
export async function runMigrations(engine: BrainEngine): Promise<{ applied: number; current: number }> {
|
||||
const currentStr = await engine.getConfig('version');
|
||||
const current = parseInt(currentStr || '1', 10);
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
/**
|
||||
* Pure helpers for spawning the gbrain worker, optionally wrapped in tini.
|
||||
*
|
||||
* Background: zombie children spawned by the worker (shell jobs, embed
|
||||
* batches, sub-agents) need a SIGCHLD handler to be reaped. The cli.ts
|
||||
* SIGCHLD handler covers JS-spawned children that exit while the parent is
|
||||
* alive; tini wraps the worker process tree to also reap native-addon
|
||||
* descendants and orphans. Together the two layers compose with AlphaClaw's
|
||||
* container-level tini-as-PID-1.
|
||||
*
|
||||
* `detectTini()` is called once at supervisor / autopilot startup. The
|
||||
* resolved path is reused on every respawn — we do NOT shell out per spawn.
|
||||
* `buildSpawnInvocation()` is a pure function describing the (cmd, args)
|
||||
* tuple to pass to `child_process.spawn`. Tests call it directly without
|
||||
* any module mocking.
|
||||
*/
|
||||
|
||||
import { execFileSync } from 'child_process';
|
||||
|
||||
/**
|
||||
* Resolve the tini binary path, or return an empty string when not on PATH.
|
||||
* Resolved once at startup so we don't shell out on every respawn.
|
||||
*/
|
||||
export function detectTini(): string {
|
||||
try {
|
||||
// Pass `env: process.env` explicitly: Bun's execFileSync does NOT
|
||||
// inherit the current process env by default (Bun snapshots env at
|
||||
// startup). Without this, runtime mutations to PATH (including in
|
||||
// tests) are invisible to `which`.
|
||||
return execFileSync('which', ['tini'], {
|
||||
encoding: 'utf8',
|
||||
timeout: 2000,
|
||||
env: process.env,
|
||||
}).trim();
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the (cmd, args) tuple for spawning the gbrain worker, optionally
|
||||
* wrapped in tini. When `tiniPath` is non-empty, returns
|
||||
* { cmd: tiniPath, args: ['--', cliPath, ...args] }
|
||||
* which makes tini PID 1 of the spawned subtree. When empty, returns
|
||||
* { cmd: cliPath, args }
|
||||
* for a direct spawn. Pure function, no side effects.
|
||||
*/
|
||||
export function buildSpawnInvocation(
|
||||
tiniPath: string,
|
||||
cliPath: string,
|
||||
args: string[],
|
||||
): { cmd: string; args: string[] } {
|
||||
return tiniPath
|
||||
? { cmd: tiniPath, args: ['--', cliPath, ...args] }
|
||||
: { cmd: cliPath, args };
|
||||
}
|
||||
@@ -27,6 +27,7 @@
|
||||
*/
|
||||
|
||||
import { spawn, type ChildProcess } from 'child_process';
|
||||
import { detectTini, buildSpawnInvocation } from './spawn-helpers.ts';
|
||||
import {
|
||||
closeSync,
|
||||
existsSync,
|
||||
@@ -138,6 +139,8 @@ export class MinionSupervisor {
|
||||
private child: ChildProcess | null = null;
|
||||
private crashCount = 0;
|
||||
private lastStartTime = 0;
|
||||
/** Path to tini binary for zombie reaping, or empty string when absent. */
|
||||
private readonly tiniPath: string;
|
||||
private stopping = false;
|
||||
private inBackoff = false;
|
||||
private healthInFlight = false;
|
||||
@@ -151,6 +154,22 @@ export class MinionSupervisor {
|
||||
constructor(engine: BrainEngine, opts: Partial<SupervisorOpts> & { cliPath: string }) {
|
||||
this.engine = engine;
|
||||
this.opts = { ...DEFAULTS, ...opts };
|
||||
|
||||
// Detect tini for zombie reaping. Resolved once at construction so we
|
||||
// don't shell out on every respawn. Belt-and-suspenders with the
|
||||
// SIGCHLD handler in cli.ts — tini catches children spawned by native
|
||||
// addons that bypass the JS event loop.
|
||||
this.tiniPath = detectTini();
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only accessor for whether tini was detected at construction.
|
||||
* Used by `test/supervisor-tini.test.ts` to verify the wiring without
|
||||
* exposing the resolved path. Returns true when `worker_spawned` events
|
||||
* will include `tini: true` in their payload.
|
||||
*/
|
||||
get isTiniDetected(): boolean {
|
||||
return this.tiniPath !== '';
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -439,9 +458,17 @@ export class MinionSupervisor {
|
||||
|
||||
this.lastStartTime = Date.now();
|
||||
|
||||
// Wrap with tini when available — reaps zombie children that the
|
||||
// SIGCHLD handler in cli.ts might miss (native addons, edge cases).
|
||||
const { cmd: spawnCmd, args: spawnArgs } = buildSpawnInvocation(
|
||||
this.tiniPath,
|
||||
this.opts.cliPath,
|
||||
args,
|
||||
);
|
||||
|
||||
let child: ChildProcess;
|
||||
try {
|
||||
child = spawn(this.opts.cliPath, args, {
|
||||
child = spawn(spawnCmd, spawnArgs, {
|
||||
stdio: 'inherit',
|
||||
env,
|
||||
});
|
||||
@@ -459,7 +486,11 @@ export class MinionSupervisor {
|
||||
|
||||
this.child = child;
|
||||
|
||||
this.emit('worker_spawned', { pid: child.pid, cli_path: this.opts.cliPath });
|
||||
this.emit('worker_spawned', {
|
||||
pid: child.pid,
|
||||
cli_path: this.opts.cliPath,
|
||||
...(this.tiniPath ? { tini: true } : {}),
|
||||
});
|
||||
|
||||
// Async spawn errors (ENOENT, EACCES after the fork/exec). Node fires
|
||||
// 'error' first, then 'exit' with code=null. We log the error; the
|
||||
|
||||
@@ -422,6 +422,17 @@ export class MinionWorker extends EventEmitter {
|
||||
]);
|
||||
}
|
||||
|
||||
// The worker does NOT disconnect the engine: it doesn't own the
|
||||
// engine's lifecycle. The caller (CLI handler at src/commands/jobs.ts
|
||||
// case 'work', or a test fixture) is responsible for disconnect when
|
||||
// it has finished using the engine. Earlier wave's experiment of
|
||||
// calling engine.disconnect() here violated ownership and broke
|
||||
// every test that shared a single engine across multiple
|
||||
// worker.start() / worker.stop() cycles (PGLiteEngine kills its
|
||||
// single _db connection; PostgresEngine.disconnect was non-idempotent
|
||||
// and clobbered the global db singleton on the second call). The
|
||||
// pool-slot-release intent is now handled in the CLI handler which
|
||||
// does own the engine.
|
||||
console.log('Minion worker stopped.');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
/**
|
||||
* v0.28: Unified model configuration.
|
||||
*
|
||||
* One resolver replaces every hardcoded `claude-*-X` string + every per-phase
|
||||
* `dream.<phase>.model` config key. Hierarchy (highest precedence first):
|
||||
*
|
||||
* 1. CLI flag (--model)
|
||||
* 2. New-key config (e.g. models.dream.synthesize)
|
||||
* 3. Old-key config (deprecated dream.synthesize.model, dream.patterns.model)
|
||||
* — read with stderr deprecation warning, one-per-process
|
||||
* 4. Global default (models.default)
|
||||
* 5. Env var (process.env[envVar] or GBRAIN_MODEL)
|
||||
* 6. Hardcoded fallback (caller-supplied)
|
||||
*
|
||||
* Aliases (`opus`, `sonnet`, `haiku`, `gemini`, `gpt`) resolve at the end so any
|
||||
* tier can use a short name. Unknown alias passes through unchanged so users can
|
||||
* pass full provider IDs without registering aliases.
|
||||
*
|
||||
* Per Codex P1 #11: deprecated keys are honored but stderr-warn once per process
|
||||
* AND lose to new-key config when both are set.
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
|
||||
export interface ResolveModelOpts {
|
||||
/** CLI flag value (e.g. `--model opus` → 'opus'). Highest precedence. */
|
||||
cliFlag?: string;
|
||||
/** New-key config name (e.g. 'models.dream.synthesize'). */
|
||||
configKey?: string;
|
||||
/** Deprecated old-key config name (e.g. 'dream.synthesize.model'). */
|
||||
deprecatedConfigKey?: string;
|
||||
/** Env var to consult after global default. Defaults to `GBRAIN_MODEL`. */
|
||||
envVar?: string;
|
||||
/** Hardcoded last-resort fallback. */
|
||||
fallback: string;
|
||||
}
|
||||
|
||||
/** Default aliases shipped in code. Users override via `models.aliases.<name>` config. */
|
||||
export const DEFAULT_ALIASES: Record<string, string> = {
|
||||
opus: 'claude-opus-4-7',
|
||||
sonnet: 'claude-sonnet-4-6',
|
||||
haiku: 'claude-haiku-4-5-20251001',
|
||||
gemini: 'gemini-3-pro',
|
||||
gpt: 'gpt-5',
|
||||
};
|
||||
|
||||
// Module-level set of deprecated config keys we've already warned about.
|
||||
// Reset on process restart; one warning per (key, process) per Codex P1 #11.
|
||||
const _deprecationWarningsEmitted = new Set<string>();
|
||||
|
||||
function emitDeprecationWarning(oldKey: string, newKey: string, ignored: boolean): void {
|
||||
if (_deprecationWarningsEmitted.has(oldKey)) return;
|
||||
_deprecationWarningsEmitted.add(oldKey);
|
||||
if (ignored) {
|
||||
process.stderr.write(
|
||||
`[models] deprecated config "${oldKey}" ignored; "${newKey}" is set and wins. ` +
|
||||
`Remove "${oldKey}" from your config in v0.30.\n`,
|
||||
);
|
||||
} else {
|
||||
process.stderr.write(
|
||||
`[models] deprecated config "${oldKey}" honored; rename to "${newKey}" before v0.30.\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a model name through the 6-tier precedence chain. Async because it
|
||||
* reads config from the engine. Pass `engine: null` for callsites that don't
|
||||
* have an engine (rare; usually CLI bootstrap before connect).
|
||||
*/
|
||||
export async function resolveModel(
|
||||
engine: BrainEngine | null,
|
||||
opts: ResolveModelOpts,
|
||||
): Promise<string> {
|
||||
const envVar = opts.envVar ?? 'GBRAIN_MODEL';
|
||||
|
||||
// 1. CLI flag wins
|
||||
if (opts.cliFlag && opts.cliFlag.trim()) {
|
||||
return await resolveAlias(engine, opts.cliFlag.trim());
|
||||
}
|
||||
|
||||
if (engine) {
|
||||
// 2. New-key config
|
||||
if (opts.configKey) {
|
||||
const v = await engine.getConfig(opts.configKey);
|
||||
if (v && v.trim()) {
|
||||
// If a deprecated key is also set, warn that it's being ignored.
|
||||
if (opts.deprecatedConfigKey) {
|
||||
const old = await engine.getConfig(opts.deprecatedConfigKey);
|
||||
if (old && old.trim()) {
|
||||
emitDeprecationWarning(opts.deprecatedConfigKey, opts.configKey, /*ignored=*/ true);
|
||||
}
|
||||
}
|
||||
return await resolveAlias(engine, v.trim());
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Old-key (deprecated) config
|
||||
if (opts.deprecatedConfigKey) {
|
||||
const v = await engine.getConfig(opts.deprecatedConfigKey);
|
||||
if (v && v.trim()) {
|
||||
emitDeprecationWarning(opts.deprecatedConfigKey, opts.configKey ?? '<no replacement>', /*ignored=*/ false);
|
||||
return await resolveAlias(engine, v.trim());
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Global default
|
||||
const def = await engine.getConfig('models.default');
|
||||
if (def && def.trim()) {
|
||||
return await resolveAlias(engine, def.trim());
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Env var
|
||||
const env = process.env[envVar];
|
||||
if (env && env.trim()) {
|
||||
return await resolveAlias(engine, env.trim());
|
||||
}
|
||||
|
||||
// 6. Hardcoded fallback
|
||||
return await resolveAlias(engine, opts.fallback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a name (possibly an alias) to its full provider model id. Order:
|
||||
* 1. User-defined alias via `models.aliases.<name>` config
|
||||
* 2. DEFAULT_ALIASES map
|
||||
* 3. Pass-through (treat as already-full model id)
|
||||
*
|
||||
* Cycles in user-defined aliases are broken at depth 2 — if `opus` aliases
|
||||
* to `super-opus` which aliases to `opus`, we return `super-opus` and stop.
|
||||
*/
|
||||
export async function resolveAlias(
|
||||
engine: BrainEngine | null,
|
||||
name: string,
|
||||
depth = 0,
|
||||
): Promise<string> {
|
||||
if (depth > 2) return name; // cycle break
|
||||
if (engine) {
|
||||
const userAlias = await engine.getConfig(`models.aliases.${name}`);
|
||||
if (userAlias && userAlias.trim() && userAlias.trim() !== name) {
|
||||
return await resolveAlias(engine, userAlias.trim(), depth + 1);
|
||||
}
|
||||
}
|
||||
if (name in DEFAULT_ALIASES) {
|
||||
const next = DEFAULT_ALIASES[name];
|
||||
if (next && next !== name) return await resolveAlias(engine, next, depth + 1);
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
/** Test-only helper: clear the deprecation-warning memo so tests re-emit. */
|
||||
export function _resetDeprecationWarningsForTest(): void {
|
||||
_deprecationWarningsEmitted.clear();
|
||||
}
|
||||
@@ -321,7 +321,7 @@ export function renderResolverMarkdown(composed: ComposedResolver): string {
|
||||
lines.push('# GBrain Skill Resolver (aggregated)');
|
||||
lines.push('');
|
||||
lines.push('Auto-generated by `gbrain mounts add|remove|sync`. Do not edit by hand.');
|
||||
lines.push('Host agents (your OpenClaw / Claude Code) should prefer this file over');
|
||||
lines.push('Host agents (your OpenClaw / Claude Code install) should prefer this file over');
|
||||
lines.push('the repo-checked-in `skills/RESOLVER.md` when it exists.');
|
||||
lines.push('');
|
||||
lines.push('See `docs/architecture/brains-and-sources.md` for the mental model.');
|
||||
|
||||
+143
-36
@@ -22,14 +22,15 @@ import type {
|
||||
import type { OAuthServerProvider, AuthorizationParams } from '@modelcontextprotocol/sdk/server/auth/provider.js';
|
||||
import type { OAuthRegisteredClientsStore } from '@modelcontextprotocol/sdk/server/auth/clients.js';
|
||||
import type { AuthInfo } from '@modelcontextprotocol/sdk/server/auth/types.js';
|
||||
import { hashToken, generateToken } from './utils.ts';
|
||||
import { hashToken, generateToken, isUndefinedColumnError } from './utils.ts';
|
||||
import { hasScope, assertAllowedScopes, parseScopeString, InvalidScopeError } from './scope.ts';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Raw SQL query function — works with both PGLite and postgres tagged templates */
|
||||
type SqlQuery = (strings: TemplateStringsArray, ...values: unknown[]) => Promise<Record<string, unknown>[]>;
|
||||
export type SqlQuery = (strings: TemplateStringsArray, ...values: unknown[]) => Promise<Record<string, unknown>[]>;
|
||||
|
||||
/**
|
||||
* Convert a JS array to a PostgreSQL array literal for PGLite compat.
|
||||
@@ -111,6 +112,16 @@ interface GBrainOAuthProviderOptions {
|
||||
tokenTtl?: number;
|
||||
/** Default refresh token TTL in seconds (default: 30 days) */
|
||||
refreshTtl?: number;
|
||||
/**
|
||||
* Disable Dynamic Client Registration (RFC 7591) while keeping the rest of
|
||||
* the OAuth surface intact. When true, `clientsStore.registerClient` is not
|
||||
* surfaced to the SDK router, so POST `/register` returns 404 even though
|
||||
* the underlying provider can still register clients programmatically via
|
||||
* `registerClientManual`. Replaces the previous monkey-patching pattern in
|
||||
* serve-http.ts (cleanup, not a security fix — DCR was never reachable
|
||||
* before mcpAuthRouter ran).
|
||||
*/
|
||||
dcrDisabled?: boolean;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -153,6 +164,12 @@ class GBrainClientsStore implements OAuthRegisteredClientsStore {
|
||||
validateRedirectUri(String(uri));
|
||||
}
|
||||
|
||||
// v0.28: ALLOWED_SCOPES allowlist. RFC 6749 §5.2 invalid_scope. The DCR
|
||||
// path is reachable by any unauthenticated network caller when --enable-dcr
|
||||
// is on, so this is the security-relevant gate (manual CLI registration
|
||||
// is operator-trusted).
|
||||
assertAllowedScopes(parseScopeString(client.scope));
|
||||
|
||||
const clientId = generateToken('gbrain_cl_');
|
||||
const clientSecret = generateToken('gbrain_cs_');
|
||||
const secretHash = hashToken(clientSecret);
|
||||
@@ -185,17 +202,29 @@ class GBrainClientsStore implements OAuthRegisteredClientsStore {
|
||||
export class GBrainOAuthProvider implements OAuthServerProvider {
|
||||
private sql: SqlQuery;
|
||||
private _clientsStore: GBrainClientsStore;
|
||||
private readonly dcrDisabled: boolean;
|
||||
private tokenTtl: number;
|
||||
private refreshTtl: number;
|
||||
|
||||
constructor(options: GBrainOAuthProviderOptions) {
|
||||
this.sql = options.sql;
|
||||
this._clientsStore = new GBrainClientsStore(this.sql);
|
||||
this.dcrDisabled = options.dcrDisabled === true;
|
||||
this.tokenTtl = options.tokenTtl || 3600;
|
||||
this.refreshTtl = options.refreshTtl || 30 * 24 * 3600;
|
||||
}
|
||||
|
||||
get clientsStore(): OAuthRegisteredClientsStore {
|
||||
if (this.dcrDisabled) {
|
||||
// Surface getClient only — without registerClient the SDK's mcpAuthRouter
|
||||
// does not wire up the /register DCR endpoint. Replaces the prior
|
||||
// monkey-patch in serve-http.ts; the outcome is identical (DCR off-by-
|
||||
// default), but the API expresses intent on the constructor instead of
|
||||
// requiring callers to mutate `_clientsStore` after construction.
|
||||
return {
|
||||
getClient: this._clientsStore.getClient.bind(this._clientsStore),
|
||||
} as OAuthRegisteredClientsStore;
|
||||
}
|
||||
return this._clientsStore;
|
||||
}
|
||||
|
||||
@@ -230,13 +259,18 @@ export class GBrainOAuthProvider implements OAuthServerProvider {
|
||||
}
|
||||
|
||||
async challengeForAuthorizationCode(
|
||||
_client: OAuthClientInformationFull,
|
||||
client: OAuthClientInformationFull,
|
||||
authorizationCode: string,
|
||||
): Promise<string> {
|
||||
const codeHash = hashToken(authorizationCode);
|
||||
// F1 hardening: bind client_id atomically so a wrong client cannot read
|
||||
// another client's PKCE challenge. Pre-fix the SELECT didn't filter on
|
||||
// client_id at all.
|
||||
const rows = await this.sql`
|
||||
SELECT code_challenge FROM oauth_codes
|
||||
WHERE code_hash = ${codeHash} AND expires_at > ${Math.floor(Date.now() / 1000)}
|
||||
WHERE code_hash = ${codeHash}
|
||||
AND client_id = ${client.client_id}
|
||||
AND expires_at > ${Math.floor(Date.now() / 1000)}
|
||||
`;
|
||||
if (rows.length === 0) throw new Error('Authorization code not found or expired');
|
||||
return rows[0].code_challenge as string;
|
||||
@@ -246,27 +280,44 @@ export class GBrainOAuthProvider implements OAuthServerProvider {
|
||||
client: OAuthClientInformationFull,
|
||||
authorizationCode: string,
|
||||
_codeVerifier?: string,
|
||||
_redirectUri?: string,
|
||||
redirectUri?: string,
|
||||
resource?: URL,
|
||||
): Promise<OAuthTokens> {
|
||||
const codeHash = hashToken(authorizationCode);
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
|
||||
// Atomic single-use: DELETE...RETURNING in one statement closes the
|
||||
// TOCTOU window. RFC 6749 §10.5 requires auth codes be single-use; the
|
||||
// earlier SELECT-then-DELETE pattern let two concurrent token requests
|
||||
// both pass the SELECT before either ran the DELETE, issuing two valid
|
||||
// token pairs from one code. With RETURNING, the second request gets
|
||||
// zero rows back and fails cleanly. See CSO finding #2.
|
||||
const rows = await this.sql`
|
||||
DELETE FROM oauth_codes
|
||||
WHERE code_hash = ${codeHash} AND expires_at > ${now}
|
||||
RETURNING client_id, scopes, resource
|
||||
`;
|
||||
// F1 + F7c hardening: bind client_id AND redirect_uri atomically into the
|
||||
// DELETE WHERE clause. RFC 6749 §10.5 requires auth codes be single-use;
|
||||
// RFC 6749 §4.1.3 requires the token endpoint validate redirect_uri
|
||||
// matches the value sent at /authorize. The previous SELECT-then-compare
|
||||
// pattern (a) burned the code on the wrong-client path so the legitimate
|
||||
// client could not retry, and (b) ignored redirect_uri on exchange
|
||||
// entirely. With RETURNING, the second request — or any wrong-client /
|
||||
// wrong-redirect-uri attempt — gets zero rows back and fails cleanly.
|
||||
// The legitimate client's code stays available for one valid redemption.
|
||||
//
|
||||
// Use `redirectUri !== undefined` rather than truthy — an attacker
|
||||
// submitting `redirect_uri=""` (empty string) at /token would otherwise
|
||||
// hit the falsy branch and bypass the binding entirely.
|
||||
const rows = redirectUri !== undefined
|
||||
? await this.sql`
|
||||
DELETE FROM oauth_codes
|
||||
WHERE code_hash = ${codeHash}
|
||||
AND client_id = ${client.client_id}
|
||||
AND redirect_uri = ${redirectUri}
|
||||
AND expires_at > ${now}
|
||||
RETURNING client_id, scopes, resource
|
||||
`
|
||||
: await this.sql`
|
||||
DELETE FROM oauth_codes
|
||||
WHERE code_hash = ${codeHash}
|
||||
AND client_id = ${client.client_id}
|
||||
AND expires_at > ${now}
|
||||
RETURNING client_id, scopes, resource
|
||||
`;
|
||||
if (rows.length === 0) throw new Error('Authorization code not found or expired');
|
||||
|
||||
const codeRow = rows[0];
|
||||
if (codeRow.client_id !== client.client_id) throw new Error('Client mismatch');
|
||||
|
||||
// Issue tokens
|
||||
const scopes = (codeRow.scopes as string[]) || [];
|
||||
@@ -286,26 +337,48 @@ export class GBrainOAuthProvider implements OAuthServerProvider {
|
||||
const tokenHash = hashToken(refreshToken);
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
|
||||
// Atomic rotation: DELETE...RETURNING closes the TOCTOU window. RFC 6749
|
||||
// §10.4 detection of stolen refresh tokens depends on second-use failure;
|
||||
// the earlier SELECT-then-DELETE pattern let attacker + victim both
|
||||
// succeed, defeating that signal. See CSO finding #3.
|
||||
// F2 hardening: bind client_id atomically into the DELETE WHERE clause.
|
||||
// RFC 6749 §10.4 detection of stolen refresh tokens depends on second-use
|
||||
// failure. The previous SELECT-then-DELETE pattern + post-hoc client
|
||||
// compare let an attacker who guessed/stole a refresh token burn it on
|
||||
// the wrong-client path, defeating the stolen-token signal for the
|
||||
// legitimate client. With the predicate in the DELETE, wrong-client
|
||||
// attempts get zero rows back; the legitimate client retains the row
|
||||
// for one valid rotation.
|
||||
const rows = await this.sql`
|
||||
DELETE FROM oauth_tokens
|
||||
WHERE token_hash = ${tokenHash} AND token_type = 'refresh'
|
||||
WHERE token_hash = ${tokenHash}
|
||||
AND token_type = 'refresh'
|
||||
AND client_id = ${client.client_id}
|
||||
RETURNING client_id, scopes, expires_at
|
||||
`;
|
||||
if (rows.length === 0) throw new Error('Refresh token not found');
|
||||
|
||||
const row = rows[0];
|
||||
if (row.client_id !== client.client_id) throw new Error('Client mismatch');
|
||||
// NULL expires_at is treated as expired (fail-closed). Schema permits NULL
|
||||
// even though issueTokens always sets it, so a corrupt or hand-modified row
|
||||
// can't ride past validation.
|
||||
const expiresAt = coerceTimestamp(row.expires_at);
|
||||
if (expiresAt === undefined || expiresAt < now) throw new Error('Refresh token expired');
|
||||
|
||||
const tokenScopes = scopes || (row.scopes as string[]) || [];
|
||||
// F3 hardening: requested scopes on refresh MUST be a subset of the
|
||||
// original grant on this refresh token's row. RFC 6749 §6: "the scope of
|
||||
// the access token … MUST NOT include any scope not originally granted by
|
||||
// the resource owner." Scope is checked against the row's scopes (the
|
||||
// grant), NOT against the client's currently-allowed scopes (which can
|
||||
// expand later). Omitted scope (`undefined`) inherits the original grant
|
||||
// verbatim and stays distinct from an explicit empty array.
|
||||
//
|
||||
// v0.28: hasScope replaces exact-string-match so an `admin` grant CAN
|
||||
// refresh down to `sources_admin` (admin implies all). Without this,
|
||||
// gstack /setup-gbrain Path 4 — which mints a sources_admin-scoped
|
||||
// refresh — would fail when the brain admin's bootstrap token was
|
||||
// issued at the `admin` tier.
|
||||
const grantedScopes = (row.scopes as string[]) || [];
|
||||
if (scopes && scopes.some(s => !hasScope(grantedScopes, s))) {
|
||||
throw new Error('Requested scope exceeds refresh token grant');
|
||||
}
|
||||
const tokenScopes = scopes ?? grantedScopes;
|
||||
return this.issueTokens(client.client_id, tokenScopes, resource, true);
|
||||
}
|
||||
|
||||
@@ -378,11 +451,21 @@ export class GBrainOAuthProvider implements OAuthServerProvider {
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
async revokeToken(
|
||||
_client: OAuthClientInformationFull,
|
||||
client: OAuthClientInformationFull,
|
||||
request: OAuthTokenRevocationRequest,
|
||||
): Promise<void> {
|
||||
const tokenHash = hashToken(request.token);
|
||||
await this.sql`DELETE FROM oauth_tokens WHERE token_hash = ${tokenHash}`;
|
||||
// F4 hardening: bind client_id so a client can only revoke its own
|
||||
// tokens. RFC 7009 §2.1: "The authorization server first validates the
|
||||
// client credentials … and then verifies whether the token was issued
|
||||
// to the client making the revocation request." Pre-fix, any
|
||||
// authenticated client that knew (or guessed) another client's token
|
||||
// hash could revoke it.
|
||||
await this.sql`
|
||||
DELETE FROM oauth_tokens
|
||||
WHERE token_hash = ${tokenHash}
|
||||
AND client_id = ${client.client_id}
|
||||
`;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
@@ -397,13 +480,19 @@ export class GBrainOAuthProvider implements OAuthServerProvider {
|
||||
const client = await this._clientsStore.getClient(clientId);
|
||||
if (!client) throw new Error('Client not found');
|
||||
|
||||
// Check if client has been revoked (soft-deleted)
|
||||
// Check if client has been revoked (soft-deleted). The deleted_at column
|
||||
// is recent — pre-migration brains don't have it, so the probe must
|
||||
// tolerate that one specific failure mode without swallowing real errors
|
||||
// (lock timeouts, network blips, auth failures).
|
||||
try {
|
||||
const [revoked] = await this.sql`SELECT deleted_at FROM oauth_clients WHERE client_id = ${clientId} AND deleted_at IS NOT NULL`;
|
||||
if (revoked) throw new Error('Client has been revoked');
|
||||
} catch (e) {
|
||||
// deleted_at column may not exist on PGLite/older schemas — skip check
|
||||
// F5 hardening: surface anything that ISN'T a missing-column error.
|
||||
// Bare `catch {}` masked DB outages as "client not revoked" — fail-open
|
||||
// posture in a security-sensitive code path.
|
||||
if (e instanceof Error && e.message === 'Client has been revoked') throw e;
|
||||
if (!isUndefinedColumnError(e, 'deleted_at')) throw e;
|
||||
}
|
||||
|
||||
// Check grant type first (before verifying secret)
|
||||
@@ -416,10 +505,14 @@ export class GBrainOAuthProvider implements OAuthServerProvider {
|
||||
const secretHash = hashToken(clientSecret);
|
||||
if (client.client_secret !== secretHash) throw new Error('Invalid client secret');
|
||||
|
||||
// Determine scopes
|
||||
const allowedScopes = (client.scope || '').split(' ').filter(Boolean);
|
||||
const requestedScopes = requestedScope ? requestedScope.split(' ').filter(Boolean) : allowedScopes;
|
||||
const grantedScopes = requestedScopes.filter(s => allowedScopes.includes(s));
|
||||
// Determine scopes. v0.28 swaps exact-string-match for hasScope so a
|
||||
// client whose grant is `admin` can mint tokens that include implied
|
||||
// scopes like `sources_admin` (admin implies all). Tokens are still
|
||||
// capped by what the client was registered for — this only changes how
|
||||
// the cap is computed.
|
||||
const allowedScopes = parseScopeString(client.scope);
|
||||
const requestedScopes = requestedScope ? parseScopeString(requestedScope) : allowedScopes;
|
||||
const grantedScopes = requestedScopes.filter(s => hasScope(allowedScopes, s));
|
||||
|
||||
// Per-client TTL override (stored in oauth_clients.token_ttl)
|
||||
// Column may not exist on PGLite/older schemas — graceful fallback
|
||||
@@ -427,7 +520,11 @@ export class GBrainOAuthProvider implements OAuthServerProvider {
|
||||
try {
|
||||
const ttlRows = await this.sql`SELECT token_ttl FROM oauth_clients WHERE client_id = ${clientId}`;
|
||||
if (ttlRows.length > 0 && ttlRows[0].token_ttl) clientTtl = Number(ttlRows[0].token_ttl);
|
||||
} catch { /* token_ttl column doesn't exist — use server default */ }
|
||||
} catch (e) {
|
||||
// F5 hardening: same posture as the deleted_at probe above. Only the
|
||||
// "column doesn't exist" path is a non-fatal fall-through.
|
||||
if (!isUndefinedColumnError(e, 'token_ttl')) throw e;
|
||||
}
|
||||
|
||||
// Client credentials: access token only, NO refresh token (RFC 6749 4.4.3)
|
||||
return this.issueTokens(clientId, grantedScopes, undefined, false, clientTtl);
|
||||
@@ -439,13 +536,17 @@ export class GBrainOAuthProvider implements OAuthServerProvider {
|
||||
|
||||
async sweepExpiredTokens(): Promise<number> {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
// F6 hardening: postgres.js and PGLite expose deleted-row count on
|
||||
// different shapes; `(result as any).count` returned 0 on at least one
|
||||
// engine even when rows were deleted, and codes were never counted at
|
||||
// all. RETURNING 1 + array length is portable across both engines.
|
||||
const result = await this.sql`
|
||||
DELETE FROM oauth_tokens WHERE expires_at < ${now}
|
||||
DELETE FROM oauth_tokens WHERE expires_at < ${now} RETURNING 1
|
||||
`;
|
||||
const deletedCodes = await this.sql`
|
||||
DELETE FROM oauth_codes WHERE expires_at < ${now}
|
||||
DELETE FROM oauth_codes WHERE expires_at < ${now} RETURNING 1
|
||||
`;
|
||||
return (result as any).count || 0;
|
||||
return result.length + deletedCodes.length;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
@@ -458,6 +559,12 @@ export class GBrainOAuthProvider implements OAuthServerProvider {
|
||||
scopes: string,
|
||||
redirectUris: string[] = [],
|
||||
): Promise<{ clientId: string; clientSecret: string }> {
|
||||
// v0.28: ALLOWED_SCOPES allowlist. Reject `--scopes "read flying-unicorn"`
|
||||
// at registration so meaningless scope strings can't pile up in the DB.
|
||||
// Pre-allowlist clients keep working (allowlist is registration-time;
|
||||
// existing rows aren't re-validated).
|
||||
assertAllowedScopes(parseScopeString(scopes));
|
||||
|
||||
const clientId = generateToken('gbrain_cl_');
|
||||
const clientSecret = generateToken('gbrain_cs_');
|
||||
const secretHash = hashToken(clientSecret);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user