mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 08:53:22 +00:00
Compare commits
23
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
401defcf76 | ||
|
|
c5165f5492 | ||
|
|
1ab9ed6097 | ||
|
|
28653261fa | ||
|
|
4c0b56e93f | ||
|
|
136b14fa61 | ||
|
|
f0825018dd | ||
|
|
1055e10c23 | ||
|
|
d01a921e01 | ||
|
|
3c032d79ec | ||
|
|
c2ae4dbfc5 | ||
|
|
736e8de1ec | ||
|
|
4fc1246606 | ||
|
|
579722d9dc | ||
|
|
90e22c22e2 | ||
|
|
18f5ba56cf | ||
|
|
80b3909702 | ||
|
|
527b87bd1e | ||
|
|
83e55ffcdb | ||
|
|
17c3c43783 | ||
|
|
0fb0c83d24 | ||
|
|
ed900c870e | ||
|
|
e96f054cf0 |
@@ -44,7 +44,10 @@ jobs:
|
||||
tier2:
|
||||
name: Tier 2 (LLM Skills)
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
# Runs on every push/PR now (promoted from schedule-only in v0.19.0).
|
||||
# Tier 1 must pass first; Tier 2 uses OPENAI_API_KEY + ANTHROPIC_API_KEY
|
||||
# from repo/org secrets. Nightly + manual triggers still supported via
|
||||
# the workflow-level `on:` list.
|
||||
needs: tier1
|
||||
services:
|
||||
postgres:
|
||||
|
||||
@@ -37,6 +37,6 @@ jobs:
|
||||
- run: bun install
|
||||
- name: Pre-test gates (shard 1 only — they're not test files)
|
||||
if: matrix.shard == 1
|
||||
run: scripts/check-jsonb-pattern.sh && scripts/check-progress-to-stdout.sh && scripts/check-wasm-embedded.sh && bun run typecheck
|
||||
run: bun run verify
|
||||
- name: Run test shard ${{ matrix.shard }}/4
|
||||
run: scripts/test-shard.sh ${{ matrix.shard }} 4
|
||||
|
||||
+20
@@ -11,10 +11,30 @@ bin/
|
||||
.gstack/
|
||||
supabase/.temp/
|
||||
.claude/skills/
|
||||
# admin/dist/ is the React SPA bundle. CLAUDE.md says it's committed for
|
||||
# self-contained binaries (the bun --compile path embeds it via
|
||||
# `import path from 'admin/dist/index.html' with { type: 'file' }`).
|
||||
# Build via: cd admin && bun install && bun run build.
|
||||
admin/node_modules/
|
||||
.idea
|
||||
eval/reports/
|
||||
eval/data/world-v1/world.html
|
||||
|
||||
# BrainBench amara-life-v1 Opus cache (regenerate via eval:generate-amara-life)
|
||||
eval/data/amara-life-v1/_cache/
|
||||
|
||||
# claw-test E2E build cache (shim + scratch outputs)
|
||||
test/.cache/
|
||||
|
||||
.claude/
|
||||
export/
|
||||
|
||||
# Conductor workspace-local agent artifacts: plans, todos, run-unit-parallel
|
||||
# failure logs and per-shard test output. v0.26.4 (run-unit-parallel.sh)
|
||||
# writes .context/test-failures.log + .context/test-summary.txt +
|
||||
# .context/test-shards/. Workspace-local by design — never committed.
|
||||
.context/
|
||||
|
||||
# Tier 3 PGLite snapshot fixture (built on demand by build:pglite-snapshot)
|
||||
test/fixtures/pglite-snapshot.tar
|
||||
test/fixtures/pglite-snapshot.version
|
||||
|
||||
@@ -18,7 +18,13 @@ start here.
|
||||
1. `./AGENTS.md` (this file) — install + operating protocol.
|
||||
2. [`./CLAUDE.md`](./CLAUDE.md) — architecture reference, key files, trust boundaries,
|
||||
test layout.
|
||||
3. [`./skills/RESOLVER.md`](./skills/RESOLVER.md) — skill dispatcher. Read before any task.
|
||||
3. [`./docs/architecture/brains-and-sources.md`](./docs/architecture/brains-and-sources.md)
|
||||
— the two-axis mental model (brain = which DB, source = which repo in the DB). Every
|
||||
query routes on both axes. Read before writing anything that touches brain ops.
|
||||
4. [`./skills/conventions/brain-routing.md`](./skills/conventions/brain-routing.md) —
|
||||
agent-facing decision table: when to switch brain, when to switch source, how
|
||||
cross-brain federation works (latent-space only; the agent decides).
|
||||
5. [`./skills/RESOLVER.md`](./skills/RESOLVER.md) — skill dispatcher. Read before any task.
|
||||
|
||||
## Trust boundary (critical)
|
||||
|
||||
@@ -37,15 +43,27 @@ writing or reviewing an operation, consult `src/core/operations.ts` for the cont
|
||||
[`docs/guides/minions-fix.md`](./docs/guides/minions-fix.md), `gbrain doctor --fix`.
|
||||
- **Migrate:** [`docs/UPGRADING_DOWNSTREAM_AGENTS.md`](./docs/UPGRADING_DOWNSTREAM_AGENTS.md),
|
||||
[`skills/migrations/`](./skills/migrations/), `gbrain apply-migrations`.
|
||||
- **Eval retrieval changes:** capture is off by default. To benchmark a
|
||||
retrieval change against real captured queries, set
|
||||
`GBRAIN_CONTRIBUTOR_MODE=1`, then `gbrain eval export --since 7d > base.ndjson`
|
||||
and `gbrain eval replay --against base.ndjson`. Full guide:
|
||||
[`docs/eval-bench.md`](./docs/eval-bench.md).
|
||||
- **Everything else:** [`./llms.txt`](./llms.txt) is the full documentation map.
|
||||
[`./llms-full.txt`](./llms-full.txt) is the same map with core docs inlined for
|
||||
single-fetch ingestion.
|
||||
|
||||
## Before shipping
|
||||
|
||||
Run `bun test` plus the E2E lifecycle described in `./CLAUDE.md` (spin up the test
|
||||
Postgres container, run `bun run test:e2e`, tear it down). Ship via the `/ship` skill,
|
||||
not by hand.
|
||||
Easiest path: `bun run ci:local` runs the full CI gate inside Docker (gitleaks,
|
||||
unit tests with `DATABASE_URL` unset, then all 29 E2E files sequentially against a
|
||||
fresh pgvector container) and tears down. Use `bun run ci:local:diff` for the
|
||||
diff-aware subset during fast iteration on a focused branch. Requires Docker
|
||||
(Docker Desktop / OrbStack / Colima) and `gitleaks` (`brew install gitleaks`).
|
||||
|
||||
Manual path: `bun test` plus the E2E lifecycle described in `./CLAUDE.md` (spin
|
||||
up the test Postgres container, run `bun run test:e2e`, tear it down).
|
||||
|
||||
Ship via the `/ship` skill, not by hand.
|
||||
|
||||
## Privacy
|
||||
|
||||
|
||||
+1317
-9
File diff suppressed because it is too large
Load Diff
@@ -7,6 +7,24 @@ suggests Supabase for 1000+ files. GStack teaches agents how to code. GBrain tea
|
||||
agents everything else: brain ops, signal detection, content ingestion, enrichment,
|
||||
cron scheduling, reports, identity, and access control.
|
||||
|
||||
## Two organizational axes (read this first)
|
||||
|
||||
GBrain knowledge is organized along two orthogonal axes. Users AND agents must
|
||||
understand both, or queries misroute silently.
|
||||
|
||||
- **Brain** — WHICH DATABASE. Your personal brain is `host`. You can mount
|
||||
additional brains (team-published, each with their own DB and access policy)
|
||||
via `gbrain mounts add` (v0.19+). Routing: `--brain`, `GBRAIN_BRAIN_ID`,
|
||||
`.gbrain-mount` dotfile.
|
||||
- **Source** — WHICH REPO INSIDE THE DATABASE. A brain can hold many sources
|
||||
(wiki, gstack, openclaw, essays). Slugs scope per source. Routing:
|
||||
`--source`, `GBRAIN_SOURCE`, `.gbrain-source` dotfile.
|
||||
|
||||
Both axes follow the same 6-tier resolution pattern. Read
|
||||
`docs/architecture/brains-and-sources.md` for topology diagrams (personal, team
|
||||
mount, CEO-class with multiple team brains) and
|
||||
`skills/conventions/brain-routing.md` for the agent-facing decision table.
|
||||
|
||||
## Architecture
|
||||
|
||||
Contract-first: `src/core/operations.ts` defines ~41 shared operations (adds `find_orphans` in v0.12.3). CLI and MCP
|
||||
@@ -22,7 +40,7 @@ strict behavior when unset.
|
||||
|
||||
## Key files
|
||||
|
||||
- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Also exports upload validators: `validateUploadPath`, `validatePageSlug`, `validateFilename`. `OperationContext.remote` flags untrusted callers.
|
||||
- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Also exports upload validators: `validateUploadPath`, `validatePageSlug`, `validateFilename`, plus `matchesSlugAllowList(slug, prefixes)` (v0.23 glob matcher: `<prefix>/*` matches recursive children; bare `<prefix>` matches exact only). `OperationContext.remote` flags untrusted callers; `OperationContext.allowedSlugPrefixes` (v0.23) is the trusted-workspace allow-list set by the dream cycle. `put_page` enforces: when `viaSubagent` and `allowedSlugPrefixes` is set, slug must match the allow-list; else the legacy `wiki/agents/<id>/...` namespace check applies. Auto-link enabled for trusted-workspace writes (skipped only when `remote=true && !trustedWorkspace`). As of v0.26.0, every `Operation` also carries `scope?: 'read' | 'write' | 'admin'` + `localOnly?: boolean`. All ops are annotated; `sync_brain`, `file_upload`, `file_list`, and `file_url` are `admin + localOnly` (rejected over HTTP). `OperationContext.auth?: AuthInfo` is threaded through HTTP dispatch for scope enforcement in `serve-http.ts` before the op runs.
|
||||
- `src/core/engine.ts` — Pluggable engine interface (BrainEngine). `clampSearchLimit(limit, default, cap)` takes an explicit cap so per-operation caps can be tighter than `MAX_SEARCH_LIMIT`. Exports `LinkBatchInput` / `TimelineBatchInput` for the v0.12.1 bulk-insert API (`addLinksBatch` / `addTimelineEntriesBatch`). As of v0.13.1, `BrainEngine` has a `readonly kind: 'postgres' | 'pglite'` discriminator so migrations (`src/core/migrate.ts`) and other consumers can branch on engine without `instanceof` + dynamic imports.
|
||||
- `src/core/engine-factory.ts` — Engine factory with dynamic imports (`'pglite'` | `'postgres'`)
|
||||
- `src/core/pglite-engine.ts` — PGLite (embedded Postgres 17.5 via WASM) implementation, all 40 BrainEngine methods. `addLinksBatch` / `addTimelineEntriesBatch` use multi-row `unnest()` with manual `$N` placeholders. As of v0.13.1, `connect()` wraps `PGlite.create()` in a try/catch that emits an actionable error naming the macOS 26.3 WASM bug (#223) and pointing at `gbrain doctor`; the lock is released on failure so the next process can retry cleanly. v0.22.0: `searchKeyword` and `searchKeywordChunks` multiply `ts_rank` by the source-factor CASE expression at the chunk-grain level; `searchVector` becomes a two-stage CTE — inner CTE keeps `ORDER BY cc.embedding <=> vec` so HNSW stays usable, outer SELECT re-ranks by `raw_score * source_factor`. Inner LIMIT scales with offset to preserve pagination contract. As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL — probes for the specific forward-referenced state the embedded schema blob needs (`pages.source_id`, `links.link_source`, `links.origin_page_id`, `content_chunks.symbol_name`, `content_chunks.language`, `sources` FK target table) and adds only what's missing. Closes the upgrade-wedge bug class that bit users 10+ times across 6 schema versions over 2 years (#239/#243/#266/#357/#366/#374/#375/#378/#395/#396). No-op on fresh installs and modern brains.
|
||||
@@ -49,7 +67,16 @@ strict behavior when unset.
|
||||
- `src/core/search/eval.ts` — Retrieval eval harness: P@k, R@k, MRR, nDCG@k metrics + runEval() orchestrator
|
||||
- `src/core/search/source-boost.ts` (v0.22.0) — Source-type boost map keyed by slug prefix. `DEFAULT_SOURCE_BOOSTS` (originals/ 1.5, concepts/ 1.3, writing/ 1.4, people/companies/deals/ 1.2, daily/ 0.8, media/x/ 0.7, wintermute/chat/ 0.5) and `DEFAULT_HARD_EXCLUDES` (test/, archive/, attachments/, .raw/). `parseSourceBoostEnv` / `parseHardExcludesEnv` parse comma-separated `prefix:factor` pairs from `GBRAIN_SOURCE_BOOST` / `GBRAIN_SEARCH_EXCLUDE` env vars. `resolveBoostMap` and `resolveHardExcludes` merge defaults + env + caller `SearchOpts.exclude_slug_prefixes`/`include_slug_prefixes`.
|
||||
- `src/core/search/sql-ranking.ts` (v0.22.0) — Pure SQL string builders. `buildSourceFactorCase(slugColumn, boostMap, detail)` emits a CASE expression with longest-prefix-match wins (returns literal `'1.0'` when `detail === 'high'` for temporal-bypass parity with COMPILED_TRUTH_BOOST). `buildHardExcludeClause(slugColumn, prefixes)` emits `NOT (col LIKE 'p1%' OR col LIKE 'p2%')` — OR-chain wrapped in NOT, NOT `NOT LIKE ALL/ANY` (those quantifiers don't express set-exclusion). LIKE meta-character escape covers all three of `%`, `_`, AND `\` (backslash matters because it's Postgres LIKE's default escape char). Single-quote doubling on SQL string literals so injection-style inputs are inert text.
|
||||
- `src/commands/eval.ts` — `gbrain eval` command: single-run table + A/B config comparison
|
||||
- `src/commands/eval.ts` — `gbrain eval` command: single-run table + A/B config comparison. v0.25.0 adds sub-subcommand dispatch on `args[0]` so `gbrain eval export` + `gbrain eval prune` + `gbrain eval replay` route into session-capture handlers; bare `gbrain eval --qrels …` fall-through preserves the legacy IR-metrics flow.
|
||||
- `src/commands/eval-export.ts` (v0.25.0) — streams `eval_candidates` rows as NDJSON to stdout with `schema_version: 1` prefix on every line. EPIPE-safe, progress heartbeats on stderr, stable id-desc tiebreaker so `--since` windows never dupe/miss rows.
|
||||
- `src/commands/eval-prune.ts` (v0.25.0) — explicit retention cleanup. Requires `--older-than DUR`. `--dry-run` reports would-delete count.
|
||||
- `src/commands/eval-replay.ts` (v0.25.0) — contributor-facing replay tool. Reads NDJSON from `gbrain eval export`, re-runs each captured `query` / `search` op against the current brain, computes set-Jaccard@k between captured + current `retrieved_slugs`, top-1 stability rate, and latency Δ. Stable JSON shape (`schema_version: 1`) for CI gating; human mode prints a regression table. Pure Bun, zero new deps. The dev-loop half of BrainBench-Real that closes the gap between "data captured" and "data used to gate a PR." See `docs/eval-bench.md` for the workflow.
|
||||
- `docs/eval-bench.md` (v0.25.0) — contributor guide for using captured data to benchmark retrieval changes before merging. Linked from CONTRIBUTING.md under "Running real-world eval benchmarks (touching retrieval code)".
|
||||
- `src/core/eval-capture.ts` (v0.25.0) — op-layer capture wrapper called from `src/core/operations.ts` `query` + `search` handlers. Catches MCP + CLI + subagent tool-bridge from one site. Fire-and-forget; failures route to `engine.logEvalCaptureFailure` so `gbrain doctor` sees drops cross-process. **Capture is off by default** — `isEvalCaptureEnabled` resolution: explicit `config.eval.capture` (true/false) wins, else `process.env.GBRAIN_CONTRIBUTOR_MODE === '1'`, else off. Production users get a quiet brain; contributors set `export GBRAIN_CONTRIBUTOR_MODE=1` in `.zshrc` to enable the dev loop. PII scrubber gate is independent and defaults to true regardless of CONTRIBUTOR_MODE.
|
||||
- `src/core/eval-capture-scrub.ts` (v0.25.0) — zero-deps PII scrubber: emails, phones, SSN, Luhn-verified credit cards, JWT-shaped tokens, bearer tokens.
|
||||
- `src/core/search/hybrid.ts` — Cathedral II `Promise<SearchResult[]>` return shape unchanged in v0.25.0. Adds `onMeta?: (m: HybridSearchMeta) => void` callback so op-layer capture can record what hybridSearch actually did. Existing callers leave it undefined.
|
||||
- `docs/eval-capture.md` (v0.25.0) — stable NDJSON schema reference for gbrain-evals consumers.
|
||||
- `test/public-exports.test.ts` (v0.25.0 / R2) — runtime contract test. Imports each of the 17 public subpaths via package name and pins a canary symbol per module. Paired with `scripts/check-exports-count.sh`.
|
||||
- `src/core/embedding.ts` — OpenAI text-embedding-3-large, batch, retry, backoff
|
||||
- `src/core/check-resolvable.ts` — Resolver validation: reachability, MECE overlap, DRY checks, structured fix objects. v0.14.1: `CROSS_CUTTING_PATTERNS.conventions` is an array (notability gate accepts both `conventions/quality.md` and `_brain-filing-rules.md`). New `extractDelegationTargets()` parses `> **Convention:**`, `> **Filing rule:**`, and inline backtick references. DRY suppression is proximity-based via `DRY_PROXIMITY_LINES = 40`.
|
||||
- `src/core/repo-root.ts` — Shared `findRepoRoot(startDir?)` (v0.16.4): walks up from `startDir` (default `process.cwd()`) looking for `skills/RESOLVER.md`. Zero-dependency module imported by both `doctor.ts` and `check-resolvable.ts`. Parameterized `startDir` makes tests hermetic.
|
||||
@@ -57,9 +84,12 @@ strict behavior when unset.
|
||||
- `src/core/resolver-filenames.ts` (v0.19) — central list of accepted routing filenames (`RESOLVER.md`, `AGENTS.md`). Shared by `findRepoRoot`, `check-resolvable`, and skillpack install so every code path walks the same fallback chain.
|
||||
- `src/commands/skillify.ts` + `src/core/skillify/{generator,templates}.ts` (v0.19) — `gbrain skillify scaffold <name>` creates all stubs for a new skill in one command: SKILL.md, script, tests, routing-eval.jsonl, resolver entry, filing-rules pointer. `gbrain skillify check <script>` runs the 10-step checklist (LLM evals, routing evals, check-resolvable gate, filing audit) against a candidate skill before it lands.
|
||||
- `src/commands/skillify-check.ts` (v0.19) — `gbrain skillpack-check` agent-readable health report. Exit 0/1/2 for CI pipeline gating; JSON for debugging. Wraps `check-resolvable --json`, `doctor --json`, and migration ledger into one payload so agents can decide whether a human action is required.
|
||||
- `src/commands/skillpack.ts` + `src/core/skillpack/{bundle,installer}.ts` (v0.19) — `gbrain skillpack install` drops gbrain's curated 25-skill bundle into a host workspace, managed-block style. Never clobbers local edits; tracks a skill manifest so subsequent `install --update` diffs cleanly. Bundle builder (`skillpack/bundle.ts`) packages the set from `skills/` into a versioned payload.
|
||||
- `src/commands/book-mirror.ts` (v0.25.1) — `gbrain book-mirror --chapters-dir <path> --slug <slug> [flags]`. Flagship of the v0.25.1 skills wave. Submits N read-only subagent jobs (one per chapter; `allowed_tools: ['get_page', 'search']`), waits for all via `waitForCompletion`, reads each child's `job.result`, assembles two-column markdown CLI-side, writes a single operator-trust `put_page` to `media/books/<slug>-personalized.md`. Codex HIGH-1 fix applied: trust narrowing happens at the tool-allowlist layer (subagents can't call put_page) instead of allowedSlugPrefixes — untrusted EPUB content cannot prompt-inject any people page. Cost-estimate prompt before launching; refuses to spend in non-TTY without `--yes`. Per-chapter idempotency keys (`book-mirror:<slug>:ch-<N>`) for retry-friendly re-runs. Partial-failure handling: assembles with completed chapters and a `## Failed chapters` section listing retries. Test surface: `test/book-mirror.test.ts` (9 cases — CLI registration + source invariants).
|
||||
- `src/commands/skillpack.ts` + `src/core/skillpack/{bundle,installer}.ts` (v0.19) — `gbrain skillpack install` drops gbrain's curated 25-skill bundle into a host workspace, managed-block style. Never clobbers local edits; tracks a skill manifest so subsequent `install --update` diffs cleanly. Bundle builder (`skillpack/bundle.ts`) packages the set from `skills/` into a versioned payload. **v0.24.0:** managed block embeds a `<!-- gbrain:skillpack:manifest cumulative-slugs="..." version="..." -->` receipt inside the fence. Per-skill installs accumulate via `union(prior_receipt, this_call)`; `install --all` is the only path that prunes (drops slugs no longer in the bundle). Rows inside the fence whose slug is in neither the new cumulative set nor the bundle survive as user-added with a stderr `[skillpack] unknown row in managed block: "<slug>" — Investigate: ...` warning. Pre-v0.24 fences upgrade silently on first install (extracted slugs become the prior cumulative set). **v0.25.1:** `gbrain skillpack uninstall <name>` lands as a real CLI subcommand. Inverse of install with symmetric data-loss posture: D8 refuses if the slug isn't in the cumulative-slugs receipt (won't nuke a hand-added row); D11 content-hash guard refuses if any installed file diverges from the bundle (you've edited it locally) unless `--overwrite-local` is passed. `applyUninstall` enforces an atomic-refusal contract: pre-scans ALL files for divergence; refuses BEFORE any unlink fires if anything is blocked. The bug fix landed via `test/skillpack-uninstall.test.ts`'s D11 case — the test was written with the contract in mind, the original implementation interleaved hash-check + unlink, and the lie surfaced immediately.
|
||||
- `src/core/archive-crawler-config.ts` (v0.25.1) — D12 + codex HIGH-4 safety gate for the `archive-crawler` skill. Refuses to run unless `archive-crawler.scan_paths:` is explicitly set in the brain repo's `gbrain.yml`. Mirrors the storage-config.ts parsing pattern (sibling file; separate concern from storage tiering). `loadArchiveCrawlerConfig(repoPath)` throws `ArchiveCrawlerConfigError(missing_section | empty_scan_paths | invalid_path | parse_error)`. `normalizeAndValidateArchiveCrawlerConfig` rejects relative paths and `..` traversal; `~` is expanded; trailing-slash normalized for unambiguous prefix matching. `isPathAllowed(candidate, config)` is the runtime per-file gate (scan_paths prefix-match with directory-boundary correctness; deny_paths overrides). Tests in `test/archive-crawler-config.test.ts` (19 cases).
|
||||
- `test/helpers/cli-pty-runner.ts` (v0.25.1) — generic real-PTY harness ported from gstack and trimmed to ~470 lines. Uses pure `Bun.spawn({terminal:})` (Bun 1.3.10+; engines.bun pin in package.json). Generic primitives only — no plan-mode orchestrators. Exports: `launchPty`, `resolveBinary`, `stripAnsi`, `parseNumberedOptions`, `optionsSignature`, `isNumberedOptionListVisible`, `isTrustDialogVisible`. Self-tests in `test/cli-pty-runner.test.ts` (24 cases).
|
||||
- `src/core/skill-manifest.ts` (v0.19) — parser for `skill-manifest.json` records. Used by skillpack installer to detect drift between the shipped bundle and the user's local edits, so updates merge instead of overwriting.
|
||||
- `src/commands/routing-eval.ts` + `src/core/routing-eval.ts` (v0.19) — `gbrain routing-eval` catches user phrasings that route to the wrong skill. Reads `skills/<name>/routing-eval.jsonl` fixtures (`{intent, expected_skill, ambiguous_with?}`). Structural layer runs in `check-resolvable` by default (zero API cost); `--llm` opts into a Haiku tie-break layer for CI. False positives surface before users hit them.
|
||||
- `src/commands/routing-eval.ts` + `src/core/routing-eval.ts` (v0.19) — `gbrain routing-eval` catches user phrasings that route to the wrong skill. Reads `skills/<name>/routing-eval.jsonl` fixtures (`{intent, expected_skill, ambiguous_with?}`). Structural layer runs in `check-resolvable` by default (zero API cost). The `--llm` flag is accepted as a placeholder for a future LLM tie-break layer; in v0.24.0 it emits a stderr notice and runs structural only. False positives surface before users hit them.
|
||||
- `src/core/filing-audit.ts` + `skills/_brain-filing-rules.json` (v0.19) — Check 6 of `check-resolvable`. Parses new `writes_pages:` / `writes_to:` frontmatter on skills and audits their filing claims against the filing-rules JSON. Warning-only in v0.19, upgrades to error in v0.20.
|
||||
- `src/core/dry-fix.ts` — `gbrain doctor --fix` engine. `autoFixDryViolations(fixes, {dryRun})` rewrites inlined rules to `> **Convention:** see [path](path).` callouts via three shape-aware expanders (bullet / blockquote / paragraph). Five guards: working-tree-dirty (`getWorkingTreeStatus()` returns 3-state `'clean' | 'dirty' | 'not_a_repo'`), no-git-backup, inside-code-fence, already-delegated (40-line proximity, consistent with detector), ambiguous-multi-match, block-is-callout. `execFileSync` array args (no shell — no injection surface). EOF newline preserved.
|
||||
- `src/core/backoff.ts` — Adaptive load-aware throttling: CPU/memory checks, exponential backoff, active hours multiplier
|
||||
@@ -87,34 +117,48 @@ strict behavior when unset.
|
||||
- `src/core/minions/wait-for-completion.ts` (v0.15) — poll-until-terminal helper for CLI callers. `TimeoutError` does NOT cancel the job; `AbortSignal` exits without throwing. Default `pollMs`: 1000 on Postgres, 250 on PGLite inline.
|
||||
- `src/core/minions/transcript.ts` (v0.15) — renders `subagent_messages` + `subagent_tool_executions` to markdown. Tool rows splice under their owning assistant `tool_use` by `tool_use_id`. UTF-8-safe truncation; unknown block types fall through to fenced JSON.
|
||||
- `src/core/minions/plugin-loader.ts` (v0.15) — `GBRAIN_PLUGIN_PATH` discovery. Absolute paths only, left-wins collision, `gbrain.plugin.json` with `plugin_version: "gbrain-plugin-v1"`, plugins ship DEFS only (no new tools), `allowed_tools:` validated at load time against the derived registry.
|
||||
- `src/core/minions/tools/brain-allowlist.ts` (v0.15) — derives subagent tool registry from `src/core/operations.ts`. 11-name allow-list: `query`, `search`, `get_page`, `list_pages`, `file_list`, `file_url`, `get_backlinks`, `traverse_graph`, `resolve_slugs`, `get_ingest_log`, `put_page`. `put_page` schema is namespace-wrapped per subagent (`^wiki/agents/<subagentId>/.+`); the `put_page` op's server-side check is the authoritative gate via `ctx.viaSubagent` fail-closed.
|
||||
- `src/core/minions/tools/brain-allowlist.ts` (v0.15, extended v0.23) — derives subagent tool registry from `src/core/operations.ts`. 11-name allow-list. By default `put_page` schema is namespace-wrapped per subagent (`^wiki/agents/<subagentId>/.+`). **v0.23 trusted-workspace path:** when `BuildBrainToolsOpts.allowedSlugPrefixes` is set, the put_page schema instead describes the prefix list to the model and the OperationContext is threaded with `allowedSlugPrefixes`. Trust comes from `PROTECTED_JOB_NAMES` gating subagent submission — MCP cannot reach this field. Only cycle.ts (synthesize/patterns) and direct CLI submitters set it.
|
||||
- `src/mcp/tool-defs.ts` (v0.15) — extracted `buildToolDefs(ops)` helper. MCP server + subagent tool registry both call it; byte-for-byte equivalence pinned by `test/mcp-tool-defs.test.ts`.
|
||||
- `src/core/minions/attachments.ts` — Attachment validation (path traversal, null byte, oversize, base64, duplicate detection)
|
||||
- `src/commands/agent.ts` (v0.16) — `gbrain agent run <prompt> [flags]` CLI. Submits `subagent` (or N children + 1 aggregator) under `{allowProtectedSubmit: true}`. Single-entry `--fanout-manifest` short-circuits. Children get `on_child_fail: 'continue'` + `max_stalled: 3`. `--follow` is the default on TTY; streams logs + polls `waitForCompletion` in parallel. Ctrl-C detaches, does not cancel.
|
||||
- `src/commands/agent-logs.ts` (v0.16) — `gbrain agent logs <job> [--follow] [--since]`. Merges JSONL heartbeat audit + `subagent_messages` into a chronological timeline. `parseSince` accepts ISO-8601 or relative (`5m`, `1h`, `2d`). Transcript tail renders only for terminal jobs.
|
||||
- `src/commands/jobs.ts` — `gbrain jobs` CLI subcommands + `gbrain jobs work` daemon. v0.13.1 surfaces the full `MinionJobInput` retry/backoff/timeout/idempotency surface as first-class CLI flags on `jobs submit`: `--max-stalled`, `--backoff-type fixed|exponential`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key`. `jobs smoke --sigkill-rescue` is the opt-in regression guard for #219. v0.16 wires `registerBuiltinHandlers` to always register `subagent` + `subagent_aggregator` (no env flag — `ANTHROPIC_API_KEY` is the natural cost gate, trust is via `PROTECTED_JOB_NAMES`) and loads `GBRAIN_PLUGIN_PATH` plugins at worker startup with a loud startup-line per plugin. `shell` handler still gated by `GBRAIN_ALLOW_SHELL_JOBS=1` (RCE surface, separate concern). v0.22.10 (#521): the `autopilot-cycle` handler now forwards `job.data.phases` to `runCycle` (was previously discarded — caller-supplied phase selection silently became a full cycle). Phases are validated against `ALL_PHASES` from `src/core/cycle.ts`; invalid names are filtered out and an empty/missing array falls back to the default 6-phase cycle.
|
||||
- `src/commands/jobs.ts` — `gbrain jobs` CLI subcommands + `gbrain jobs work` daemon. v0.13.1 surfaces the full `MinionJobInput` retry/backoff/timeout/idempotency surface as first-class CLI flags on `jobs submit`: `--max-stalled`, `--backoff-type fixed|exponential`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key`. `jobs smoke --sigkill-rescue` is the opt-in regression guard for #219. v0.16 wires `registerBuiltinHandlers` to always register `subagent` + `subagent_aggregator` (no env flag — `ANTHROPIC_API_KEY` is the natural cost gate, trust is via `PROTECTED_JOB_NAMES`) and loads `GBRAIN_PLUGIN_PATH` plugins at worker startup with a loud startup-line per plugin. `shell` handler still gated by `GBRAIN_ALLOW_SHELL_JOBS=1` (RCE surface, separate concern). v0.22.10 (#521): the `autopilot-cycle` handler now forwards `job.data.phases` to `runCycle` (was previously discarded — caller-supplied phase selection silently became a full cycle). Phases are validated against `ALL_PHASES` from `src/core/cycle.ts`; invalid names are filtered out and an empty/missing array falls back to the default 6-phase cycle. v0.22.13 (PR #490 CODEX-1+CODEX-4): `sync` handler now resolves `sourceId` at entry by looking up `sources.local_path` (mirrors `cycle.ts:480`'s autopilot fix from PR #475) so multi-source brains read the per-source `last_commit` anchor instead of the global config key. Concurrency routed through the shared `autoConcurrency()` policy in `src/core/sync-concurrency.ts` instead of the prior hardcoded `4`; PGLite stays serial. `noEmbed` default is `true` (embed is a separate job — submit `gbrain embed --stale` after sync, or rely on the autopilot cycle's embed phase).
|
||||
- `src/commands/features.ts` — `gbrain features --json --auto-fix`: usage scan + feature adoption salesman
|
||||
- `src/commands/autopilot.ts` — `gbrain autopilot --install`: self-maintaining brain daemon (sync+extract+embed)
|
||||
- `src/mcp/server.ts` — MCP stdio server (generated from operations). v0.22.7: tool-call handler delegates to `dispatchToolCall` from `src/mcp/dispatch.ts` so stdio + HTTP transports share one validation, context-build, and error-format path.
|
||||
- `src/mcp/dispatch.ts` (v0.22.7) — Shared tool-call dispatch consumed by both stdio (`server.ts`) and HTTP (`http-transport.ts`). Exports `dispatchToolCall(engine, name, params, opts)`, `buildOperationContext(engine, params, opts)`, and `validateParams(op, params)`. Single source of truth for `(ctx, params)` handler arg order and the 5-field `OperationContext` shape (engine + config + logger + dryRun + remote). Defaults to `remote: true` (untrusted); local CLI callers pass `remote: false`. Closed F1 (reversed handler args) + F2 (incomplete OperationContext) + F3 (no param validation) drift bugs in the original v0.22.5 HTTP transport.
|
||||
- `src/mcp/rate-limit.ts` (v0.22.7) — Bounded-LRU token-bucket limiter for `gbrain serve --http`. `buildDefaultLimiters()` returns the two-bucket pipeline used by http-transport: pre-auth IP (default 30/60s, fires BEFORE the DB lookup so brute-force load against `access_tokens` is actually capped) + post-auth token-id (default 60/60s). Tracks `lastTouchedMs` separately from `lastRefillMs` so an exhausted key can't be reset by hammering past the TTL. LRU cap (default 10K keys) bounds memory under attacker-controlled key growth; TTL prune at 2× window evicts abandoned buckets.
|
||||
- `src/mcp/http-transport.ts` (v0.22.7, rewrite) — `gbrain serve --http` HTTP transport. Postgres-only — fails fast at startup on PGLite (the `access_tokens` table only exists on Postgres). Bearer auth against SHA-256 hashes in `access_tokens`. CORS default-deny via `GBRAIN_HTTP_CORS_ORIGIN` allowlist. Body cap stream-counted (1 MiB default via `GBRAIN_HTTP_MAX_BODY_BYTES`) so chunked transfers without Content-Length still hit the cap. `last_used_at` SQL-level debounce (one UPDATE per token per 60s). Per-request audit row in `mcp_request_log` with token_name + operation + status + latency. Optional `GBRAIN_HTTP_TRUST_PROXY=1` honors `X-Forwarded-For` — only safe when bound to a private interface AND the proxy strips client-supplied XFF (otherwise enables IP spoofing past the pre-auth rate limit). `/health` does `SELECT 1` against Postgres and returns 503 + `status:unhealthy` when the DB is unreachable so orchestration doesn't see green pods while clients get misleading 401s. Replaces the standalone OAuth wrapper that was vulnerable to unauthenticated client registration.
|
||||
- `src/commands/auth.ts` — Token management for the HTTP transport. `gbrain auth create/list/revoke/test`. As of v0.22.7 wired into the main CLI (`src/cli.ts`); also runs standalone via `bun run src/commands/auth.ts ...` for environments without a compiled binary. Tokens stored as SHA-256 hashes in `access_tokens` (Postgres-only).
|
||||
- `src/mcp/dispatch.ts` (v0.22.7) — Shared tool-call dispatch consumed by both stdio (`server.ts`) and HTTP transports. Exports `dispatchToolCall(engine, name, params, opts)`, `buildOperationContext(engine, params, opts)`, and `validateParams(op, params)`. Single source of truth for `(ctx, params)` handler arg order and the 5-field `OperationContext` shape (engine + config + logger + dryRun + remote). Defaults to `remote: true` (untrusted); local CLI callers pass `remote: false`. Closed F1/F2/F3 drift bugs in the original v0.22.5 HTTP transport.
|
||||
- `src/mcp/rate-limit.ts` (v0.22.7) — Bounded-LRU token-bucket limiter. `buildDefaultLimiters()` returns the two-bucket pipeline: pre-auth IP (30/60s, fires BEFORE the DB lookup so brute-force load against `access_tokens` is actually capped) + post-auth token-id (60/60s). Tracks `lastTouchedMs` separately from `lastRefillMs` so an exhausted key can't be reset by hammering past the TTL. LRU cap bounds memory under attacker-controlled key growth.
|
||||
- `src/commands/serve-http.ts` (v0.26.0) — Express 5 HTTP MCP server with OAuth 2.1, admin dashboard, and SSE live activity feed. Started via `gbrain serve --http [--port N] [--token-ttl N] [--enable-dcr] [--public-url URL]`. Supersedes the v0.22.7 `src/mcp/http-transport.ts` simple bearer-auth path. Combines MCP SDK's `mcpAuthRouter` (authorize / token / register / revoke endpoints), a custom `client_credentials` handler (SDK's token endpoint throws `UnsupportedGrantTypeError` for CC; the custom handler runs BEFORE the router and falls through for `auth_code` / `refresh_token`), `requireBearerAuth` middleware for `/mcp` with scope enforcement before op dispatch, `localOnly` rejection, and `express-rate-limit` at 50 req / 15 min on `/token`. Serves the built admin SPA from `admin/dist/` with SPA fallback. `/admin/events` SSE endpoint broadcasts every MCP request to connected admin browsers. `cookie-parser` middleware wired (Express 5 has no built-in). Startup logging prints port, engine, configured issuer URL (honors `--public-url`), registered-client count, DCR status, and admin bootstrap token.
|
||||
- `src/core/oauth-provider.ts` (v0.26.0) — `GBrainOAuthProvider` implementing the MCP SDK's `OAuthServerProvider` + `OAuthRegisteredClientsStore` interfaces. Backed by raw SQL (works on both PGLite and Postgres — OAuth is infrastructure, not a BrainEngine concern). Full OAuth 2.1 spec: `authorize` + `exchangeAuthorizationCode` with PKCE (for ChatGPT), `client_credentials` (for Perplexity / Claude), `refresh_token` with rotation, `revokeToken`, `registerClient` (DCR path validates redirect_uri must be `https://` or loopback per RFC 6749 §3.1.2.1). All tokens + client secrets SHA-256 hashed before storage. Auth codes single-use with 10-minute TTL via atomic `DELETE...RETURNING` (closes RFC 6749 §10.5 TOCTOU race). Refresh rotation also `DELETE...RETURNING` (closes §10.4 stolen-token detection bypass). `pgArray()` escapes commas/quotes/braces in elements so a comma-bearing redirect_uri can't smuggle a second array element. Legacy `access_tokens` fallback in `verifyAccessToken` grandfathers pre-v0.26 bearer tokens as `read+write+admin`. `sweepExpiredTokens()` runs on startup wrapped in try/catch. **v0.26.2:** module-private `coerceTimestamp()` boundary helper at the top of the file normalizes postgres-driver-as-string BIGINT columns to JS numbers at every read site (5 call sites: `getClient` L112+L113 for DCR `/register` RFC 7591 §3.2.1 numeric timestamps, `exchangeRefreshToken` L274 + `verifyAccessToken` L296+L303 for the SDK's `typeof === 'number'` bearerAuth check). Throws on non-finite input (NaN/Infinity) so corrupt rows fail loud at the boundary instead of riding through as `expiresAt: NaN`; returns undefined for SQL NULL so callers decide NULL semantics explicitly (refresh + access token paths treat NULL as expired). Helper intentionally NOT promoted to `src/core/utils.ts` — codex review flagged repo-wide BIGINT precision-loss risk for a generic helper.
|
||||
- `admin/` (v0.26.0) — React 19 + Vite + TypeScript admin SPA embedded in the binary via `admin/dist/` served by `serve-http.ts`. 7 screens: Login (bootstrap token → session cookie), Dashboard (metrics + SSE feed + token health), Agents (sortable table + sparklines + Register button), Register (modal with scope checkboxes + grant type selector), Credentials reveal (full-screen modal with Copy + Download JSON + yellow one-time-only warning), Request Log (filterable paginated), Agent Detail drawer (Details / Activity / Config Export tabs + Revoke). Design tokens: `#0a0a0f` bg, Inter for UI, JetBrains Mono for data, 4-32px spacing scale, rounded pill badges. HTTP-only SameSite=Strict cookie auth. 65KB gzip. Build: `cd admin && bun install && bun run build`; output at `admin/dist/` is committed for self-contained binaries.
|
||||
- `src/commands/auth.ts` — Token management. `gbrain auth create/list/revoke/test` for legacy bearer tokens (v0.22.7 wired as a first-class CLI subcommand) plus `gbrain auth register-client` (v0.26.0) and `gbrain auth revoke-client <client_id>` (v0.26.2) for OAuth 2.1 client lifecycle. `revoke-client` runs an atomic `DELETE...RETURNING` on `oauth_clients`; FK `ON DELETE CASCADE` on `oauth_tokens.client_id` and `oauth_codes.client_id` purges every active token + authorization code in a single transaction. `process.exit(1)` on no-such-client (idempotent — re-running on the same id produces the same exit-1 message). Legacy tokens stored as SHA-256 hashes in `access_tokens`; OAuth clients in `oauth_clients`. As of v0.26.0, legacy tokens grandfather to `read+write+admin` scopes on the OAuth HTTP server, so pre-v0.26 deployments keep working with no migration.
|
||||
- `src/commands/upgrade.ts` — Self-update CLI. `runPostUpgrade()` enumerates migrations from the TS registry (src/commands/migrations/index.ts) and tail-calls `runApplyMigrations(['--yes', '--non-interactive'])` so the mechanical side of every outstanding migration runs unconditionally.
|
||||
- `src/commands/migrations/` — TS migration registry (compiled into the binary; no filesystem walk of `skills/migrations/*.md` needed at runtime). `index.ts` lists migrations in semver order. `v0_11_0.ts` = Minions adoption orchestrator (8 phases). `v0_12_0.ts` = Knowledge Graph auto-wire orchestrator (5 phases: schema → config check → backfill links → backfill timeline → verify). `phaseASchema` has a 600s timeout (bumped from 60s in v0.12.1 for duplicate-heavy brains). `v0_12_2.ts` = JSONB double-encode repair orchestrator (4 phases: schema → repair-jsonb → verify → record). `v0_14_0.ts` = shell-jobs + autopilot cooperative (2 phases: schema ALTER minion_jobs.max_stalled SET DEFAULT 3 — superseded by v0.14.3's schema-level DEFAULT 5 + UPDATE backfill; pending-host-work ping for skills/migrations/v0.14.0.md). All orchestrators are idempotent and resumable from `partial` status. As of v0.14.2 (Bug 3), the RUNNER owns all ledger writes — orchestrators return `OrchestratorResult` and `apply-migrations.ts` persists a canonical `{version, status, phases}` shape after return. Orchestrators no longer call `appendCompletedMigration` directly. `statusForVersion` prefers `complete` over `partial` (never regresses). 3 consecutive partials → wedged → `--force-retry <version>` writes a `'retry'` reset marker. v0.14.3 (fix wave) ships schema-only migrations v14 (`pages_updated_at_index`) + v15 (`minion_jobs_max_stalled_default_5` with UPDATE backfill) via the `MIGRATIONS` array in `src/core/migrate.ts` — no orchestrator phases needed.
|
||||
- `src/commands/repair-jsonb.ts` — `gbrain repair-jsonb [--dry-run] [--json]`: rewrites `jsonb_typeof='string'` rows in place across 5 affected columns (pages.frontmatter, raw_data.data, ingest_log.pages_updated, files.metadata, page_versions.frontmatter). Fixes v0.12.0 double-encode bug on Postgres; PGLite no-ops. Idempotent.
|
||||
- `src/commands/orphans.ts` — `gbrain orphans [--json] [--count] [--include-pseudo]`: surfaces pages with zero inbound wikilinks, grouped by domain. Auto-generated/raw/pseudo pages filtered by default. Also exposed as `find_orphans` MCP operation. Shipped in v0.12.3 (contributed by @knee5).
|
||||
- `src/commands/integrity.ts` — `gbrain integrity check|auto|review|extract`: bare-tweet detection, dead-link detection, three-bucket repair (auto-repair / review-queue / skip). `scanIntegrity()` is the shared library function called from `gbrain doctor` (sampled at limit=500) and `cmdCheck` (full scan). v0.22.8: batch-load fast path on Postgres uses `SELECT DISTINCT ON (slug)` in a single SQL query to fix the PgBouncer round-trip timeout (60s → ~6s) while preserving `engine.getAllSlugs()`'s `Set<string>` semantics on multi-source brains. Gated by `engine.kind === 'postgres'` at the call site so PGLite never enters batch; fallback `catch` logs at `GBRAIN_DEBUG=1` so real Postgres errors are diagnosable.
|
||||
- `src/commands/doctor.ts` — `gbrain doctor [--json] [--fast] [--fix] [--dry-run] [--index-audit]`: health checks. v0.12.3 added `jsonb_integrity` + `markdown_body_completeness` reliability checks. v0.14.1: `--fix` delegates inlined cross-cutting rules to `> **Convention:** see [path](path).` callouts (pipes DRY violations into `src/core/dry-fix.ts`); `--fix --dry-run` previews without writing. v0.14.2: `schema_version` check fails loudly when `version=0` (migrations never ran — the #218 `bun install -g` signature) and routes users to `gbrain apply-migrations --yes`; new opt-in `--index-audit` flag (Postgres-only) reports zero-scan indexes from `pg_stat_user_indexes` (informational only, no auto-drop). v0.15.2: every DB check is wrapped in a progress phase; `markdown_body_completeness` runs under a 1s heartbeat timer so 10+ min scans are observable on 50K-page brains. v0.19.1 added `queue_health` (Postgres-only) with two subchecks: stalled-forever active jobs (started_at > 1h) and waiting-depth-per-name > threshold (default 10, override via `GBRAIN_QUEUE_WAITING_THRESHOLD`). Worker-heartbeat subcheck intentionally deferred to follow-up B7 because it needs a `minion_workers` table to produce ground-truth signal. Fix hints point at `gbrain repair-jsonb`, `gbrain sync --force`, `gbrain apply-migrations`, and `gbrain jobs get/cancel <id>`. v0.22.12 (#500): `sync_failures` check shows `[CODE=N, ...]` breakdown for both unacked entries (warn) and acked-historical entries (ok), surfacing systemic failure modes (`SLUG_MISMATCH=2685`) instead of a bare count.
|
||||
- `src/core/migrate.ts` — schema-migration runner. Owns the `MIGRATIONS` array (source of truth for schema DDL). v0.14.2 extended the `Migration` interface with `sqlFor?: { postgres?, pglite? }` (engine-specific SQL overrides `sql`) and `transaction?: boolean` (set to false for `CREATE INDEX CONCURRENTLY`, which Postgres refuses inside a transaction; ignored on PGLite since it has no concurrent writers). Migration v14 (fix wave) uses a handler branching on `engine.kind` to run CONCURRENTLY on Postgres (with a pre-drop of any invalid remnant via `pg_index.indisvalid`) and plain `CREATE INDEX` on PGLite. v15 bumps `minion_jobs.max_stalled` default 1→5 and backfills existing non-terminal rows. v0.22.6.1: migration v24 (`rls_backfill_missing_tables`) uses `sqlFor: { pglite: '' }` to no-op on PGLite — PGLite has no RLS engine and is single-tenant by definition, and the v24 ALTERs target subagent tables that don't exist in pglite-schema.ts. Closes #395 (contributed by @jdcastro2).
|
||||
- `src/core/migrate.ts` — schema-migration runner. Owns the `MIGRATIONS` array (source of truth for schema DDL). v0.14.2 extended the `Migration` interface with `sqlFor?: { postgres?, pglite? }` (engine-specific SQL overrides `sql`) and `transaction?: boolean` (set to false for `CREATE INDEX CONCURRENTLY`, which Postgres refuses inside a transaction; ignored on PGLite since it has no concurrent writers). Migration v14 (fix wave) uses a handler branching on `engine.kind` to run CONCURRENTLY on Postgres (with a pre-drop of any invalid remnant via `pg_index.indisvalid`) and plain `CREATE INDEX` on PGLite. v15 bumps `minion_jobs.max_stalled` default 1→5 and backfills existing non-terminal rows. v0.22.6.1: migration v24 (`rls_backfill_missing_tables`) uses `sqlFor: { pglite: '' }` to no-op on PGLite — PGLite has no RLS engine and is single-tenant by definition, and the v24 ALTERs target subagent tables that don't exist in pglite-schema.ts. Closes #395 (contributed by @jdcastro2). **v30 (v0.23):** creates `dream_verdicts (file_path TEXT, content_hash TEXT, worth_processing BOOL, reasons JSONB, judged_at TIMESTAMPTZ, PK(file_path, content_hash))`. RLS-enabled when running as a BYPASSRLS role. The synthesize phase reads/writes this table to avoid re-judging on backfill re-runs.
|
||||
- `src/core/progress.ts` — Shared bulk-action progress reporter. Writes to stderr. Modes: `auto` (TTY: `\r`-rewriting; non-TTY: plain lines), `human`, `json` (JSONL), `quiet`. Rate-gated by `minIntervalMs` and `minItems`. `startHeartbeat(reporter, note)` helper for single long queries. `child()` composes phase paths. Singleton SIGINT/SIGTERM coordinator emits `abort` events for every live phase. EPIPE defense on both sync throws and stream `'error'` events. Zero dependencies. Introduced in v0.15.2.
|
||||
- `src/core/cli-options.ts` — Global CLI flag parser. `parseGlobalFlags(argv)` returns `{cliOpts, rest}` with `--quiet` / `--progress-json` / `--progress-interval=<ms>` stripped. `getCliOptions()` / `setCliOptions()` expose a module-level singleton so commands reach the resolved flags without parameter threading. `cliOptsToProgressOptions()` maps to reporter options. `childGlobalFlags()` returns the flag suffix to append to `execSync('gbrain ...')` calls in migration orchestrators. `OperationContext.cliOpts` extends shared-op dispatch for MCP callers.
|
||||
- `src/core/cycle.ts` — v0.17 brain maintenance cycle primitive. `runCycle(engine: BrainEngine | null, opts: CycleOpts): Promise<CycleReport>` composes 6 phases in semantically-driven order (lint → backlinks → sync → extract → embed → orphans). Three callers: `gbrain dream` CLI, `gbrain autopilot` daemon's inline path, and the Minions `autopilot-cycle` handler (`src/commands/jobs.ts`). One source of truth for what the brain does overnight. Coordination via `gbrain_cycle_locks` DB table (TTL-based; works through PgBouncer transaction pooling, unlike session-scoped `pg_try_advisory_lock`) + `~/.gbrain/cycle.lock` file lock with PID-liveness for PGLite / engine=null mode. `CycleReport.schema_version: "1"` is the stable agent-consumable shape. `PhaseResult.error: { class, code, message, hint?, docs_url? }` is Stripe-API-tier structured failure info. `yieldBetweenPhases` hook awaited between every phase — Minions handler uses this to renew its job lock and prevent v0.14 stall-death regression. Engine nullable: filesystem phases (lint, backlinks) run without DB; DB phases skip with `status: "skipped", reason: "no_database"`. Lock-skip: read-only phase selections (`--phase orphans`) bypass the cycle lock. v0.22.1 (#403): `CycleOpts.signal?: AbortSignal` propagates the worker's abort signal; `checkAborted()` fires between every phase and throws if the signal is aborted (cooperative — can't interrupt a phase mid-execution). v0.22.1 (#417): `runPhaseSync` returns `pagesAffected` via `SyncPhaseResult`; `runCycle` captures it and threads to `runPhaseExtract` as the 4th arg, enabling incremental extract on the cycle path. v0.22.1 (Codex F2): `runPhaseSync` takes `willRunExtractPhase: boolean` and sets `noExtract: phases.includes('extract')` so `gbrain dream --phase sync` doesn't silently lose extraction. v0.22.5 (#475): new `resolveSourceForDir(engine, brainDir)` helper queries `SELECT id FROM sources WHERE local_path = $1 LIMIT 1`; `runPhaseSync` threads result as `sourceId` to `performSync()` so sync reads the per-source `sources.last_commit` anchor instead of the drift-prone global `config.sync.last_commit` key. Bare try/catch lets pre-v0.18 brains fall through to the global key. Closes the prod hang where every autopilot cycle ran a 30-min full reimport because the global anchor commit had been GC'd from git history.
|
||||
- `src/commands/dream.ts` — v0.17 `gbrain dream` CLI. ~80-line thin alias over `runCycle`. brainDir resolution requires explicit `--dir` OR `sync.repo_path` config (no more walk-up-cwd-for-.git footgun). Flags: `--dry-run`, `--json`, `--phase <name>`, `--pull`, `--dir <path>`. Exit code 1 on status=failed (partial/warn not fatal — don't page on warnings).
|
||||
- `src/core/db-lock.ts` (v0.22.13) — generic `tryAcquireDbLock(engine, lockId, ttlMinutes)` over the existing `gbrain_cycle_locks` table. Parameterized lock id so different scopes can nest cleanly: `gbrain-cycle` for the broad cycle (held by `cycle.ts`) and `gbrain-sync` (`SYNC_LOCK_ID` constant) for `performSync`'s narrower writer window. Same UPSERT-with-TTL semantics as the prior cycle-only helper, just generalized. Survives PgBouncer transaction pooling (unlike session-scoped `pg_try_advisory_lock`); crashed holders auto-release once their TTL expires.
|
||||
- `src/core/sync-concurrency.ts` (v0.22.13) — single source of truth for the parallel-sync policy. Exports `autoConcurrency(engine, fileCount, override?)` (PGLite always serial; explicit override clamped to >=1; auto path returns `DEFAULT_PARALLEL_WORKERS=4` when `fileCount > AUTO_CONCURRENCY_FILE_THRESHOLD=100`), `shouldRunParallel(workers, fileCount, explicit)` (Q1: explicit `--workers` bypasses the >50-file floor), and `parseWorkers(s)` (rejects `'0'`, `'-3'`, `'foo'`, `'1.5'`, trailing chars — replaces the prior parseInt-with-no-validation in both `sync.ts` and `import.ts`). Used by `performSync`, `performFullSync`, `runImport`, and the Minion `sync` handler so the three sites can no longer drift.
|
||||
- `src/commands/sync.ts` — `gbrain sync` CLI + the `performSync` / `performFullSync` library entrypoints (consumed by the autopilot cycle and the Minion sync handler). v0.22.13 (PR #490): `performSync` wraps its body in a `gbrain-sync` writer lock so two concurrent syncs (manual + autopilot, two terminals, two Conductor workspaces) cannot both write `last_commit` and let the last writer win. Head-drift gate after the import phase re-checks `git rev-parse HEAD`; if HEAD moved (someone ran `git checkout` / `git pull` mid-sync), the bookmark refuses to advance. Vanished files now record a failedFiles entry instead of silent-skip — the silent-skip-then-advance pathology that survived prior hardening passes is dead. Worker engines wrap in try/finally so disconnect always fires (panic-path leak fix). Both PGLite-detection sites use `engine.kind === 'pglite'`. CLI accepts `--workers N` (alias `--concurrency N`), validated via `parseWorkers`. Explicit `--workers` bypasses the auto-path file-count floor; auto path defers to `autoConcurrency()`. Banner moved to stderr.
|
||||
- `src/core/cycle.ts` — v0.17 brain maintenance cycle primitive (extended to **8 phases in v0.23**). `runCycle(engine: BrainEngine | null, opts: CycleOpts): Promise<CycleReport>` composes phases in semantically-driven order: **lint → backlinks → sync → synthesize → extract → patterns → embed → orphans**. v0.23's `synthesize` phase runs after sync (cross-references see fresh brain) and before extract (auto-link materializes its writes); `patterns` runs after extract so it reads a fresh graph (codex finding #7 — subagent put_page sets `ctx.remote=true` and skips auto-link/timeline by default; extract is the canonical materialization). Three callers: `gbrain dream` CLI, `gbrain autopilot` daemon's inline path, and the Minions `autopilot-cycle` handler. Coordination via `gbrain_cycle_locks` DB table + `~/.gbrain/cycle.lock` file lock with PID-liveness for PGLite. `CycleReport.schema_version: "1"` is stable; totals additively grew in v0.23 (`transcripts_processed`, `synth_pages_written`, `patterns_written`). `yieldBetweenPhases` runs between phases. **v0.23 added `yieldDuringPhase`** for in-phase keepalive — synthesize/patterns call it during long waits to renew the cycle-lock TTL. Engine nullable; lock-skip on read-only phase selections. v0.22.1 (#403): `CycleOpts.signal?: AbortSignal` propagates the worker's abort signal; `checkAborted()` fires between every phase. v0.22.1 (#417): `runPhaseSync` returns `pagesAffected` via `SyncPhaseResult`; `runCycle` captures it and threads to `runPhaseExtract` as the 4th arg. v0.22.1 (Codex F2): `runPhaseSync` takes `willRunExtractPhase: boolean` and sets `noExtract: phases.includes('extract')` so `gbrain dream --phase sync` doesn't silently lose extraction. v0.22.5 (#475): `resolveSourceForDir(engine, brainDir)` threads `sourceId` to `performSync()` so sync reads the per-source `sources.last_commit` anchor instead of the drift-prone global `config.sync.last_commit` key.
|
||||
- `src/core/cycle/synthesize.ts` (v0.23) — Synthesize phase: conversation-transcript-to-brain pipeline. Reads from `dream.synthesize.session_corpus_dir`, runs cheap Haiku verdict (cached in `dream_verdicts`), then fans out one Sonnet subagent per worth-processing transcript with `allowed_slug_prefixes` (sourced from `skills/_brain-filing-rules.json` `dream_synthesize_paths.globs`). Orchestrator collects slugs from `subagent_tool_executions` (NOT `pages.updated_at` — codex finding #2) and reverse-renders DB → markdown via `serializeMarkdown`. Cooldown via `dream.synthesize.last_completion_ts`, written ONLY on success. Idempotency key `dream:synth:<file_path>:<content_hash>`. Auto-commit deferred to v1.1 (codex #5). `--dry-run` runs Haiku, skips Sonnet (codex #8). Subagent never gets fs-write access. **v0.23.2:** `renderPageToMarkdown` (now exported) stamps `dream_generated: true` and `dream_cycle_date` into every reverse-write's frontmatter; `writeSummaryPage` does the same on the dream-cycle summary index. The marker is the explicit identity surface checked by `isDreamOutput` in `transcript-discovery.ts` — replaces the v0.23.1 content-prefix heuristic that could miss real output (`serializeMarkdown` doesn't embed slugs in body) and false-positive on user transcripts citing brain pages. `judgeSignificance` and `JudgeClient` are exported; `judgeSignificance` accepts a `verdictModel` parameter (default `claude-haiku-4-5-20251001`) loaded from `dream.synthesize.verdict_model` via `loadSynthConfig`.
|
||||
- `src/core/cycle/patterns.ts` (v0.23) — Patterns phase: cross-session theme detection over reflections within `dream.patterns.lookback_days` (default 30). Names a pattern only when ≥`dream.patterns.min_evidence` (default 3) reflections support it. Single Sonnet subagent; same allow-list path as synthesize. Runs AFTER `extract` so the graph is fresh.
|
||||
- `src/core/cycle/transcript-discovery.ts` (v0.23) — Pure filesystem walk for synthesize. `discoverTranscripts(opts)` filters `.txt` files by date range, min_chars, and word-boundary regex `excludePatterns` (Q-3: `medical` matches "medical advice" but NOT "comedical"; power users may pass full regex). `readSingleTranscript(path)` is the `gbrain dream --input <file>` ad-hoc path. **v0.23.2 self-consumption guard:** `DREAM_OUTPUT_MARKER_RE` (anchored at frontmatter open `---\n`, optional BOM + CRLF tolerance, scans first 2000 chars for `dream_generated: true` with case-insensitive value and word boundary on `true`) drives `isDreamOutput(content, bypass=false)`. Both `discoverTranscripts` and `readSingleTranscript` skip matching files and emit a `[dream] skipped <basename>: dream_generated marker` stderr log (no more silent skips). `bypassGuard?: boolean` on `DiscoverOpts` and `readSingleTranscript`'s opts disables the guard for the explicit `--unsafe-bypass-dream-guard` escape hatch only — never auto-applied for `--input`. Replaces v0.23.1's `DREAM_OUTPUT_SLUGS` content-prefix list.
|
||||
- `src/commands/dream.ts` — v0.17 `gbrain dream` CLI; ~80-line thin alias over `runCycle`. brainDir resolution requires explicit `--dir` OR `sync.repo_path` config. Flags: `--dry-run`, `--json`, `--phase <name>`, `--pull`, `--dir <path>`. **v0.23 added** `--input <file>` (ad-hoc transcript, implies `--phase synthesize`), `--date YYYY-MM-DD`, `--from <d> --to <d>` (backfill range). Conflict detection: `--input` + `--date` exits 2. ISO date validation. `--dry-run` runs Haiku significance verdict but skips Sonnet synthesis (codex finding #8 — NOT zero LLM calls). Exit code 1 on status=failed. **v0.23.2 added** `--unsafe-bypass-dream-guard` (long-form intentional, plumbed through `runCycle.synthBypassDreamGuard` → `SynthesizePhaseOpts.bypassDreamGuard` → `discoverTranscripts({bypassGuard})` and `readSingleTranscript({bypassGuard})`). Loud stderr warning fires at synthesize-phase entry when set. Never auto-applied for `--input` so any caller can't silently re-trigger the loop bug.
|
||||
- `src/commands/friction.ts` + `src/core/friction.ts` (v0.23) — `gbrain friction {log,render,list,summary}` reporter. Append-only JSONL under `$GBRAIN_HOME/friction/<run-id>.jsonl`. Schema is a flat extension of `StructuredAgentError` (D20). Render groups by severity → phase, defaults to `--redact` for md output (strips `$HOME`/`$CWD` to placeholders so reports paste safely in PRs). Run-id resolves from `--run-id` > `$GBRAIN_FRICTION_RUN_ID` > `standalone.jsonl`. Skills the claw-test exercises gain a `_friction-protocol.md` callout so agents know when to log friction.
|
||||
- `src/commands/claw-test.ts` + `src/core/claw-test/` (v0.23) — `gbrain claw-test [--scenario <name>] [--live --agent openclaw]`. End-to-end "fresh user" friction harness. Two modes: scripted (CI gate, agent-free) and live (real openclaw subprocess, $1–2 in tokens). Sets `GBRAIN_HOME=<tempdir>` for hermeticity and captures gbrain's `--progress-json` events from each child's stderr to verify expected phases ran (`import.files`, `extract.links_fs`, `doctor.db_checks`). Phases for scripted mode: setup → install_brain (`gbrain init --pglite`) → import (`--no-embed`) → query → extract → verify (`gbrain doctor --json`, asserts `status: 'ok'`) → render. Live mode hands `BRIEF.md` from `test/fixtures/claw-test-scenarios/<name>/` to the agent runner. v1 ships with the OpenClaw runner only (`src/core/claw-test/runners/openclaw.ts`, invokes `openclaw agent --local --agent <name> --message <brief>`); hermes runner deferred to v1.1. Transcript capture (`transcript-capture.ts`) uses `fs.createWriteStream` with `'drain'`-event backpressure — D17 fix for the 256KB-burst child-stall scenario. v0.18 upgrade scenario seeded via `seed-pglite.ts` SQL replay.
|
||||
- `skills/_friction-protocol.md` (v0.23) — shared cross-cutting convention skill (like `_brain-filing-rules.md`). Tells agents when to call `gbrain friction log` and how to choose a severity. Routes to friction CLI from any skill the claw-test exercises.
|
||||
- `scripts/check-progress-to-stdout.sh` — CI guard against regressing to `\r`-on-stdout progress. Wired into `bun run test` via `scripts/check-progress-to-stdout.sh && bun test` in package.json.
|
||||
- `docs/progress-events.md` — Canonical JSON event schema reference. Stable from v0.15.2, additive only.
|
||||
- `src/core/markdown.ts` — Frontmatter parsing + body splitter. `splitBody` requires an explicit timeline sentinel (`<!-- timeline -->`, `--- timeline ---`, or `---` immediately before `## Timeline`/`## History`). Plain `---` in body text is a markdown horizontal rule, not a separator. `inferType` auto-types `/wiki/analysis/` → analysis, `/wiki/guides/` → guide, `/wiki/hardware/` → hardware, `/wiki/architecture/` → architecture, `/writing/` → writing (plus the existing people/companies/deals/etc heuristics).
|
||||
- `scripts/check-jsonb-pattern.sh` — CI grep guard. Fails the build if anyone reintroduces (a) the `${JSON.stringify(x)}::jsonb` interpolation pattern (postgres.js v3 double-encodes it), or (b) `max_stalled INTEGER NOT NULL DEFAULT 1` in any schema source file (v0.15.1 #219 regression guard — must be DEFAULT 5 to preserve SIGKILL-rescue). Wired into `bun test`.
|
||||
- `docker-compose.ci.yml` + `scripts/ci-local.sh` (v0.23.1) — Local CI gate. `bun run ci:local` spins up `pgvector/pgvector:pg16` + `oven/bun:1` with named volumes (`gbrain-ci-pg-data`, `gbrain-ci-node-modules`, `gbrain-ci-bun-cache`), runs gitleaks on host, smoke-tests `scripts/run-e2e.sh` argv handling, runs unit tests with `DATABASE_URL` unset (matches GH Actions structure), then runs all 29 E2E files sequentially. `--diff` swaps in the diff-aware selector; `--no-pull` skips upstream pulls; `--clean` nukes named volumes. Postgres host port defaults to 5434 (avoids 5432 manual `gbrain-test-pg` and 5433 sibling-project conflict); override with `GBRAIN_CI_PG_PORT=NNNN`. Stronger gate than current PR CI's 2-file Tier 1 set — closes the "push-and-wait" feedback loop pre-push.
|
||||
- `scripts/select-e2e.ts` + `scripts/e2e-test-map.ts` (v0.23.1) — Diff-aware E2E test selector. Reads three git sources (committed `origin/master...HEAD`, working-tree `HEAD`, and `git ls-files --others --exclude-standard` for untracked, NOT-gitignored files), classifies as EMPTY / DOC_ONLY / SRC. Fail-closed by design: EMPTY → all 29 files (clean branch shouldn't run nothing), DOC_ONLY (every path matches the README/CLAUDE/AGENTS/CHANGELOG/TODOS allowlist) → empty stdout, SRC → escape-hatch paths (schema, package.json, skills/) trigger all; otherwise the hand-tuned `E2E_TEST_MAP` glob → tests narrows; an unmapped src/ change still emits ALL files, never silently nothing. Pure-function exports (`selectTests`, `classify`, `matchGlob`) so it's trivial to test and fork. `bun run ci:select-e2e` prints the current selection on stdout, pipe-friendly. `test/select-e2e.test.ts` covers all 4 branches plus 3 codex regression guards (skills/, untracked files, unmapped src/) — 24 cases.
|
||||
- `scripts/run-e2e.sh` (v0.23.1 update) — Sequential E2E runner. Now accepts an optional argv-driven file list (used by `ci:local:diff` to pipe in selector output) and a `--dry-run-list` flag that prints the resolved file list and exits (used by `ci-local.sh`'s startup smoke-test). Falls back to `test/e2e/*.test.ts` when invoked with no args.
|
||||
- `scripts/llms-config.ts` + `scripts/build-llms.ts` — Generator for `llms.txt` (llmstxt.org-spec web index) + `llms-full.txt` (inlined single-fetch bundle). Curated config drives both. Run `bun run build:llms` after adding a new doc. `LLMS_REPO_BASE` env var lets forks regenerate with their own URL base. `FULL_SIZE_BUDGET` (600KB) caps the inline bundle; generator WARNs if exceeded. Committed output is not analogous to `schema-embedded.ts` (no runtime consumer); we commit for GitHub browsing and fork-safe fetching.
|
||||
- `AGENTS.md` — Local-clone entry point for non-Claude agents (Codex, Cursor, OpenClaw, Aider). Mirrors `CLAUDE.md` intent via relative links. Claude Code keeps using `CLAUDE.md`.
|
||||
- `docs/UPGRADING_DOWNSTREAM_AGENTS.md` — Patches for downstream agent skill forks to apply when upgrading. Each release appends a new section. v0.10.3 includes diffs for brain-ops, meeting-ingestion, signal-detector, enrich.
|
||||
@@ -200,6 +244,14 @@ Key commands added for Minions (job queue):
|
||||
- `gbrain jobs smoke [--sigkill-rescue]` — health smoke test. `--sigkill-rescue` is the v0.13.1 regression guard for #219: simulates a killed worker and asserts the stalled job is requeued instead of dead-lettered on first stall.
|
||||
- `gbrain jobs work [--queue Q] [--concurrency N]` — start worker daemon (Postgres only)
|
||||
|
||||
Key commands added in v0.25.0:
|
||||
- `gbrain eval export [--since DUR] [--limit N] [--tool query|search]` — stream captured `eval_candidates` rows as NDJSON to stdout. Every line starts with `"schema_version": 1` per the stable contract in `docs/eval-capture.md`. EPIPE-safe, progress heartbeats on stderr, deterministic ordering. Primary consumer is the sibling `gbrain-evals` repo for BrainBench-Real replay.
|
||||
- `gbrain eval prune --older-than DUR [--dry-run]` — explicit retention cleanup for `eval_candidates`. Requires `--older-than` (never deletes without a window). Duration strings: 30d, 7d, 1h, 90m, 3600s.
|
||||
- `gbrain eval replay --against FILE.ndjson [--limit N] [--top-regressions K] [--json] [--verbose]` — contributor-facing dev loop. Reads a captured NDJSON snapshot, re-runs each `query` / `search` op against the current brain, computes mean set-Jaccard@k between captured + current `retrieved_slugs`, top-1 stability rate, and latency Δ. JSON mode (`schema_version: 1`) for CI gating; human mode prints a regression table sorted worst-first. Closes the gap between "data captured" and "data used to gate a PR." See `docs/eval-bench.md` for the workflow.
|
||||
- `gbrain doctor` gains an `eval_capture` check: reads `eval_capture_failures` for the last 24h, groups by reason, warns when non-zero. Cross-process visibility (doctor runs in a separate process from MCP). Pre-v31 brains get `Skipped (table unavailable)` — non-fatal.
|
||||
- Config addition: `eval: { capture?: boolean, scrub_pii?: boolean }` in `~/.gbrain/config.json`. **File-plane only** — `gbrain config set` writes the DB plane and does NOT control capture.
|
||||
- **`GBRAIN_CONTRIBUTOR_MODE=1` env var** is the contributor-facing toggle. Capture is **off by default** as of v0.25.0; production users get a quiet brain. Resolution order: explicit `eval.capture` config wins both directions, then env var, then off. Documented in README.md, CONTRIBUTING.md, and `docs/eval-bench.md`.
|
||||
|
||||
Key commands added in v0.12.2:
|
||||
- `gbrain repair-jsonb [--dry-run] [--json]` — repair double-encoded JSONB rows left over from v0.12.0-and-earlier Postgres writes. Idempotent; PGLite no-ops. The `v0_12_2` migration runs this automatically on `gbrain upgrade`.
|
||||
|
||||
@@ -214,14 +266,80 @@ Key commands added in v0.14.2:
|
||||
- `GBRAIN_POOL_SIZE` env var — honored by both the singleton pool (`src/core/db.ts`) and the parallel-import worker pool (`src/commands/import.ts`). Default is 10; lower to 2 for Supabase transaction pooler to avoid MaxClients crashes during `gbrain upgrade` subprocess spawns. Read at call time via `resolvePoolSize()`.
|
||||
- `gbrain doctor` gains two new checks: `sync_failures` (surfaces unacknowledged parse failures with exact paths + fix hints) and `brain_score` (renders the 5-component breakdown when score < 100: embed coverage / 35, link density / 25, timeline coverage / 15, orphans / 15, dead links / 10 — sum equals total).
|
||||
|
||||
Key commands added in v0.26.0 (OAuth 2.1 + HTTP server + admin dashboard):
|
||||
- `gbrain serve --http [--port 3131] [--token-ttl 3600] [--enable-dcr]` — HTTP MCP server with OAuth 2.1, admin dashboard at `/admin`, SSE activity feed at `/admin/events`, health check at `/health`. Prints admin bootstrap token on first start. Alongside (not replacing) stdio `gbrain serve`.
|
||||
- **OAuth client registration** — three paths:
|
||||
1. CLI: `gbrain auth register-client <name> --grant-types <types> --scopes <scopes>` (wired into `src/commands/auth.ts` as a thin wrapper over `GBrainOAuthProvider.registerClientManual`). Default grant types: `client_credentials`. Default scopes: `read`.
|
||||
2. Admin dashboard: Register client modal → credential reveal with Copy + Download JSON.
|
||||
3. SDK: `oauthProvider.registerClientManual(name, grantTypes, scopes, redirectUris)` for programmatic wrappers.
|
||||
`--enable-dcr` on `serve --http` opens the `/register` endpoint for RFC 7591 self-service registration (off by default).
|
||||
- `gbrain auth create|list|revoke|test` — legacy bearer tokens still work and grandfather to `read+write+admin` scopes on the OAuth server. `auth` is wired as a first-class `gbrain` subcommand in v0.26.0 (previously only invokable via `bun run src/commands/auth.ts`). No migration required to keep pre-v0.26 clients working.
|
||||
|
||||
Key commands added in v0.14.3 (fix wave):
|
||||
- `gbrain doctor --index-audit` — opt-in Postgres-only check reporting zero-scan indexes from `pg_stat_user_indexes`. Informational only; never auto-drops.
|
||||
- `gbrain doctor` schema_version check fails loudly when `version=0` — catches `bun install -g github:...` postinstall failures (#218) and routes users to `gbrain apply-migrations --yes`.
|
||||
- `gbrain jobs submit` gains `--max-stalled`, `--backoff-type`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key` — exposing existing `MinionJobInput` fields as first-class CLI flags.
|
||||
- `gbrain jobs smoke --sigkill-rescue` — opt-in regression smoke case simulating a killed worker; asserts the v0.14.3 schema default (`max_stalled=5`) actually rescues on first stall.
|
||||
|
||||
Key commands added in v0.22.13 (PR #490):
|
||||
- `gbrain sync --workers N` (alias `--concurrency N`) — parallelize the import phase using per-worker Postgres engines (small pool of 2 each) with an atomic queue index. Auto-concurrency: defaults to 4 workers when the diff exceeds 100 files. Smaller diffs stay serial. Explicit `--workers` always wins (even on a 30-file diff). PGLite forces serial regardless. Validation rejects `0`, negatives, non-integers loud (replaces the prior silent fall-through to auto-concurrency).
|
||||
- `gbrain import --workers N` — same `parseWorkers()` validation as sync; same try/finally worker-engine cleanup. Behavior surface unchanged.
|
||||
|
||||
Key commands added in v0.22.16 (claw-test friction loop):
|
||||
- `gbrain claw-test [--scenario fresh-install|upgrade-from-v0.18] [--keep-tempdir]` — scripted-mode CI gate that runs the full canonical first-day flow against a fresh tempdir. Asserts every expected `--progress-json` phase fired and doctor's `status === 'ok'`. ~30s, no API keys.
|
||||
- `gbrain claw-test --live --agent openclaw` — friction-discovery mode. Spawns real openclaw, hands it `BRIEF.md`, captures stdin/stdout/stderr to `<run>/transcript.jsonl`, lets the agent log friction via the friction CLI. Run on demand; ~5–10 min and ~$1–2 in tokens.
|
||||
- `gbrain claw-test --list-agents` — reports which agent runners are registered + their detection state (binary path or unavailable reason).
|
||||
- `gbrain friction log --severity {confused|error|blocker|nit} --phase <name> --message <text> [--hint ...] [--kind {friction|delight}] [--run-id ...]` — append a friction or delight entry to the active run JSONL.
|
||||
- `gbrain friction render --run-id <id> [--json] [--transcripts] [--no-redact]` — markdown report grouped by severity + phase; `--redact` is the default for md output (strips `$HOME`/`$CWD` placeholders so reports paste safely in PRs/issues).
|
||||
- `gbrain friction list [--json]` — recent run-ids with friction/delight counts; interrupted runs marked `(interrupted)`.
|
||||
- `gbrain friction summary --run-id <id> [--json]` — two-column friction + delight summary.
|
||||
- `GBRAIN_HOME` env override is now honored uniformly across every gbrain write site (config, audit, friction, sync-failures, import checkpoint, integrity log, integrations heartbeat, migration rollback, etc.) — `gbrainPath(...)` from `src/core/config.ts` is the canonical helper. Read-side host-fingerprint detection (`~/.claude`/`~/.openclaw` etc.) intentionally NOT confined in v1; that's a v1.1 follow-up.
|
||||
|
||||
## Testing
|
||||
|
||||
### Test command tiers (v0.26.4 — parallel fast loop)
|
||||
|
||||
Five tiers of test commands, each with a clear scope:
|
||||
|
||||
| Command | What it runs | Wallclock | When to use |
|
||||
|---|---|---|---|
|
||||
| `bun run test` | Parallel unit-test fast loop. 8-shard fan-out via `scripts/run-unit-parallel.sh`, then a serial pass over `*.serial.test.ts`. Excludes `*.slow.test.ts` and `test/e2e/*`. No pre-checks, no typecheck. | ~85s on a Mac dev box (3650+ tests) | Inner edit loop. Default. |
|
||||
| `bun run verify` | CI's authoritative pre-test gate set: `check:privacy && check:jsonb && check:progress && check:wasm && bun run typecheck`. The 4 checks `.github/workflows/test.yml` runs on shard 1 + typecheck. Single source of truth — CI literally calls `bun run verify`. | ~12s (wasm-compile dominates) | Before pushing; before `/ship`. |
|
||||
| `bun run test:full` | `verify && bun run test && bun run test:slow && [smart e2e]`. The local equivalent of "everything CI runs." Smart e2e: runs e2e only when `DATABASE_URL` is set; else loud skip notice to stderr. | ~3-5min depending on slow + e2e | Pre-merge sanity, before opening a PR. |
|
||||
| `bun run test:slow` | Just the `*.slow.test.ts` set (intentional cold-path correctness checks). | seconds-to-minutes | When touching slow-path code. |
|
||||
| `bun run test:serial` | Just the `*.serial.test.ts` set (cross-file-contention quarantine; runs at `--max-concurrency=1`). | ~1s per quarantined file | Debugging a specific quarantined file. |
|
||||
| `bun run test:e2e` | Real Postgres E2E. Requires Docker + `DATABASE_URL`. Sequential (template-DB parallelization is a v0.27+ TODO). | ~5-10min | Pre-ship; nightly. |
|
||||
| `bun run check:all` | All 7 historical pre-checks (privacy + jsonb + progress + no-legacy-getconnection + trailing-newline + wasm + exports-count). Superset of `verify`. | ~10s | Local-only sweep. The 4 not in `verify` are nice-to-haves. |
|
||||
|
||||
### CI vs local: intentionally divergent file sets
|
||||
|
||||
- **CI matrix** (`.github/workflows/test.yml`) runs `scripts/test-shard.sh` 4-way, which uses FNV-1a hash bucketing and INCLUDES `*.slow.test.ts`. CI is the ground truth for "did everything pass."
|
||||
- **Local fast loop** (`scripts/run-unit-shard.sh` via the parallel wrapper) uses round-robin-by-index sharding and EXCLUDES `*.slow.test.ts` AND `*.serial.test.ts`. Local trades coverage for inner-loop speed; CI catches what local skips.
|
||||
|
||||
This divergence is intentional. Don't try to make them equal — the two scripts deliberately solve different problems. The regression test at `test/scripts/run-unit-shard.test.ts` pins what the local fast loop should and shouldn't include.
|
||||
|
||||
### Failure-first logging
|
||||
|
||||
When `bun run test` finds any failure, the wrapper:
|
||||
|
||||
1. Writes failure blocks (each prefixed with `--- shard N: <test name> ---`) to `.context/test-failures.log` (workspace-local, gitignored). On systems without a writable `.context/`, falls back to `/tmp/gbrain-test-failures.log`.
|
||||
2. Prints a loud stderr banner with the absolute log path, plus the last 30 lines of the failure log inlined. Banner survives `| head` / `| tail` / agent-side log truncation.
|
||||
3. Writes a one-line-per-shard summary to `.context/test-summary.txt` (`shard N/M: pass=X fail=Y skip=Z rc=W`).
|
||||
4. Exits non-zero. Empty failure log + non-zero exit = infrastructure problem (wedged shard, killed child); the banner says so.
|
||||
|
||||
If a shard wedges (per-shard `GBRAIN_TEST_SHARD_TIMEOUT` cap, default 600s), the wrapper writes `--- shard N: WEDGED after ${SHARD_TIMEOUT}s ---` to the failure log, includes the last 50 lines of the shard log, and proceeds with other shards' results.
|
||||
|
||||
### File taxonomy
|
||||
|
||||
- `*.test.ts` → fast loop (parallel 8-shard fan-out).
|
||||
- `*.slow.test.ts` → run via `bun run test:slow` only (intentional cold-path tests; would dominate the fast loop's wallclock).
|
||||
- `*.serial.test.ts` → run via `bun run test:serial` after the parallel pass completes; uses `--max-concurrency=1`. Quarantine for tests that share file-wide state and race when run alongside other files in the same `bun test` process. Currently: `test/brain-registry.serial.test.ts`, `test/reconcile-links.serial.test.ts`. **Do not put the parallelism back on a serial file unless you've fixed the contention root cause** (it just re-introduces the flake).
|
||||
- `test/e2e/*.test.ts` → real-Postgres E2E. Skipped when `DATABASE_URL` is unset.
|
||||
|
||||
The intra-file parallelism project (turn `bun test` into `bun test --concurrent` after sweeping shared-state contention sites — ~58 PGLite + ~40 env-mutation + ~2 mock.module sites) is filed as a P0 TODO for a follow-up release. v0.26.4 ships file-level parallelism only.
|
||||
|
||||
### Inventory (legacy)
|
||||
|
||||
`bun test` runs all tests. After the v0.12.1 release: ~75 unit test files + 8 E2E test files (1412 unit pass, 119 E2E when `DATABASE_URL` is set — skip gracefully otherwise). Unit tests run
|
||||
without a database. E2E tests skip gracefully when `DATABASE_URL` is not set.
|
||||
|
||||
@@ -275,10 +393,13 @@ parity), `test/cli.test.ts` (CLI structure), `test/config.test.ts` (config redac
|
||||
`test/orphans.test.ts` (v0.12.3 orphans command: detection, pseudo filtering, text/json/count outputs, MCP op),
|
||||
`test/postgres-engine.test.ts` (v0.12.3 statement_timeout scoping: `sql.begin` + `SET LOCAL` shape, source-level grep guardrail against reintroduced bare `SET statement_timeout`),
|
||||
`test/sync.test.ts` (sync logic + v0.12.3 regression guard asserting top-level `engine.transaction` is not called),
|
||||
`test/sync-concurrency.test.ts` (v0.22.13 PR #490: 17 cases covering `autoConcurrency()` thresholds + PGLite-forces-serial + explicit-override clamping, `shouldRunParallel()` Q1 explicit-bypasses-floor contract, and `parseWorkers()` validation that rejects `'0'`/`'-3'`/`'foo'`/`'1.5'`/trailing chars),
|
||||
`test/sync-parallel.test.ts` (v0.22.13 PR #490: PGLite-routed coverage of the bookmark gate under concurrency request, head-drift gate, vanished-file failure capture, PGLite-stays-serial, and the `gbrain-sync` writer-lock contract — 7 cases),
|
||||
`test/sync-failures.test.ts` (v0.22.12: 28 cases pinning `classifyErrorCode` regex coverage for all 12 codes against literal production message strings from `markdown.ts:159-244` and `import-file.ts:199, 347, 352, 401`; `summarizeFailuresByCode` sort + pre-classified-honor; `recordSyncFailures` code-field persistence; `acknowledgeSyncFailures` AcknowledgeResult shape + backfill on pre-v0.22.12 entries),
|
||||
`test/doctor.test.ts` (doctor command + v0.12.3 assertions that `jsonb_integrity` scans the four v0.12.0 write sites and `markdown_body_completeness` is present),
|
||||
`test/utils.test.ts` (shared SQL utilities + `tryParseEmbedding` null-return and single-warn semantics),
|
||||
`test/build-llms.test.ts` (llms.txt/llms-full.txt generator: path resolution, idempotence, spec shape, regen-drift guard, content contract, AGENTS.md install-path mirror, size-budget enforcement — 7 cases),
|
||||
`test/oauth.test.ts` (v0.26.0 OAuth 2.1 provider — 27 cases: register, getClient, `client_credentials` grant exchange, `authorization_code` flow with PKCE challenge / verifier, refresh token rotation, `verifyAccessToken` with both OAuth + legacy `access_tokens` fallback, `revokeToken`, `sweepExpiredTokens`, and a contract test asserting `scope` + `localOnly` annotations are set correctly on all 30 operations; **v0.26.2** adds 5 `coerceTimestamp` unit cases (null/undefined/string/number/throw-on-NaN), NULL-`expires_at`-as-expired contract tests for both refresh + access token paths, and a cascade-delete contract test asserting `revoke-client` purges `oauth_tokens` + `oauth_codes` rows via FK CASCADE),
|
||||
`test/check-resolvable-cli.test.ts` (v0.19 CLI wrapper: exit codes, JSON envelope shape, AGENTS.md fallback chain),
|
||||
`test/regression-v0_16_4.test.ts` (findRepoRoot regression guard — hermetic startDir parameterization),
|
||||
`test/filing-audit.test.ts` (v0.19 Check 6: `writes_pages` / `writes_to` frontmatter, filing-rules JSON validation),
|
||||
@@ -305,6 +426,8 @@ E2E tests (`test/e2e/`): Run against real Postgres+pgvector. Require `DATABASE_U
|
||||
- `test/e2e/engine-parity.test.ts` (v0.22.0) — Postgres ↔ PGLite top-result and result-set parity for `searchKeyword` + `searchVector`. Codex flagged that Postgres ranks pages then picks best chunk while PGLite returns chunks directly — without parity coverage the source-boost fix could pass on PGLite and fail on Postgres. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/postgres-bootstrap.test.ts` (v0.22.6.1) — exercises `PostgresEngine.initSchema()` directly against a fresh real Postgres database. Asserts the bootstrap path is no-op on fresh installs and that SCHEMA_SQL replays cleanly through the engine path (not via the standalone `db.initSchema` from `src/core/db.ts`, which would have produced false-positive coverage). Codex caught the E2E-shape gap during plan review.
|
||||
- `test/e2e/http-transport.test.ts` (v0.22.7) — 8 cases against real Postgres covering `gbrain serve --http` end-to-end: bearer auth round-trip, `last_used_at` SQL-level debounce semantics, `mcp_request_log` row insertion on success and auth_failed paths, `/health` DB-down → 503 (DB-probing health check), and the F1+F2+F3 dispatch round-trip with a real operation. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/serve-http-oauth.test.ts` (v0.26.0, expanded v0.26.2) — real-Postgres E2E against `gbrain serve --http` with full OAuth 2.1. Spawns a subprocess server, registers a client via the CLI, mints `client_credentials` tokens, exercises the `/mcp` JSON-RPC pipeline. **v0.26.2 adds:** real DCR `/register` HTTP-level response-shape test (asserts `typeof body.client_id_issued_at === 'number'` over the wire — RFC 7591 §3.2.1 spec compliance, not just internal-store shape); real CLI subprocess test for `revoke-client` (registers → mints token → revokes via `execSync` → asserts token rejected at `/mcp` → asserts re-run exits 1); server fixture flips on `--enable-dcr` so `/register` is reachable. **bun execSync env-inheritance fix:** bun's `execSync` does NOT inherit env mutations done via `process.env.X = ...`, only OS-level env from before bun started. helpers.ts loads `.env.testing` and sets `DATABASE_URL` via `process.env` mutation, which is invisible to subprocesses unless `env: { ...process.env }` is passed explicitly — every subprocess call in this file passes `env: { ...process.env }` for that reason. Reference fix for the next maintainer hitting the same failure mode in sibling sync/cycle/dream/claw-test E2Es. `afterAll` cleanup is guarded on `clientId` (won't throw if `beforeAll` failed before registration); cleanup errors surface to stderr without throwing so real test failures aren't masked. Tracks DCR-registered clients alongside the manual one. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/sync-parallel.test.ts` (v0.22.13 PR #490) — DATABASE_URL-gated. T2: 60-file Postgres sync at concurrency=4 imports all + no connection leak (probes `pg_stat_activity` before/after to confirm worker engines disconnected). P4: 120-file serial-vs-parallel benchmark prints `SYNC_PARALLEL_BENCH N files | serial=Xms | parallel(4)=Yms | speedup=Zx` for CHANGELOG quoting. Asserts parallel ≤ serial × 1.5 (CI-noise tolerant; not a strict speedup gate).
|
||||
- Tier 2 (`skills.test.ts`) requires OpenClaw + API keys, runs nightly in CI
|
||||
- If `.env.testing` doesn't exist in this directory, check sibling worktrees for one:
|
||||
`find ../ -maxdepth 2 -name .env.testing -print -quit` and copy it here if found.
|
||||
@@ -415,6 +538,40 @@ For single long-running queries, use `startHeartbeat(reporter, note)` with a
|
||||
try/finally to guarantee cleanup. Never call `process.stdout.write('\r...')`
|
||||
in bulk paths, the CI guard will fail the build.
|
||||
|
||||
## Capturing test output (NEVER pipe through `tail` / `head`)
|
||||
|
||||
**Iron rule:** when running `bun test`, `bun run test:e2e`, `bun run typecheck`,
|
||||
or any other test/check command, redirect to a file FIRST, then `tail` the file
|
||||
separately:
|
||||
|
||||
```bash
|
||||
# RIGHT — full output preserved, real exit code visible
|
||||
bun test > /tmp/ship_units.txt 2>&1
|
||||
echo "EXIT=$?"
|
||||
tail -50 /tmp/ship_units.txt
|
||||
grep -E '(fail\)|✗|error:' /tmp/ship_units.txt | head -30
|
||||
```
|
||||
|
||||
```bash
|
||||
# WRONG — exit code is `tail`'s (always 0), failures truncated, ship gates fail open
|
||||
bun test 2>&1 | tail -10
|
||||
```
|
||||
|
||||
The pipe form silently breaks /ship Step T1 (test failure ownership triage) and
|
||||
the test verification gate (Step 16) because:
|
||||
- `$?` after a pipe is the LAST command's exit code (`tail` → 0), not bun's
|
||||
- bun prints failure details before the summary line, so `tail -N` drops them
|
||||
- Step T1 needs the full failure list to classify in-branch vs pre-existing
|
||||
|
||||
This bit us during v0.26.2 ship: `bun test 2>&1 | tail -10` reported "3911 pass / 23 fail"
|
||||
but no failure details survived, forcing a 23-minute re-run to triage.
|
||||
|
||||
Apply the same pattern to any long-running command whose exit code matters:
|
||||
`bun run typecheck`, `bun run ci:local`, migration runs, eval suites, etc.
|
||||
For background tasks (`run_in_background: true`), the harness captures the exit
|
||||
file separately — use it via the bg task's `<id>.exit` file, not the streamed
|
||||
output.
|
||||
|
||||
## Build
|
||||
|
||||
`bun build --compile --outfile bin/gbrain src/cli.ts`
|
||||
@@ -474,13 +631,45 @@ will detect drift and re-bump on the next run.
|
||||
|
||||
## Pre-ship requirements
|
||||
|
||||
Before shipping (/ship) or reviewing (/review), always run the full test suite:
|
||||
Before shipping (/ship) or reviewing (/review), always run the full test suite.
|
||||
Two equivalent paths:
|
||||
|
||||
**Path A — local CI gate (recommended, v0.23.1+):**
|
||||
- `bun run ci:local` runs the entire stack inside Docker: gitleaks (host), unit
|
||||
tests with `DATABASE_URL` unset, and all 29 E2E files sequentially against a
|
||||
fresh pgvector container. Stronger than PR CI's 2-file Tier 1 set; closer to
|
||||
what nightly Tier 1 catches. Spins up + tears down postgres automatically via
|
||||
`docker-compose.ci.yml`. Override the host port with
|
||||
`GBRAIN_CI_PG_PORT=5435 bun run ci:local` if 5434 collides.
|
||||
- `bun run ci:local:diff` runs only the E2E files matched by the diff selector
|
||||
(`scripts/select-e2e.ts`), falling back to all 29 on unmapped src/ paths or
|
||||
schema/skills/package.json changes. Fast iteration during a focused branch.
|
||||
|
||||
**Path B — manual lifecycle (still supported):**
|
||||
- `bun test` — unit tests (no database required)
|
||||
- Follow the "E2E test DB lifecycle" steps above to spin up the test DB,
|
||||
run `bun run test:e2e`, then tear it down.
|
||||
|
||||
Both must pass. Do not ship with failing E2E tests. Do not skip E2E tests.
|
||||
|
||||
**Always run typecheck before pushing.** `bun test` (the bun runner)
|
||||
skips TypeScript type checking — it only enforces runtime behavior.
|
||||
Three ways to actually gate on types:
|
||||
|
||||
1. `bun run test` (npm script in `package.json`) — includes `bun run typecheck`
|
||||
plus the four shell pre-checks (`check-jsonb-pattern.sh`,
|
||||
`check-progress-to-stdout.sh`, `check-trailing-newline.sh`,
|
||||
`check-wasm-embedded.sh`) before the runner. Use this mid-branch.
|
||||
2. `bun run typecheck` — `tsc --noEmit` standalone. Fast (~5s on this repo).
|
||||
3. `bun run ci:local` — the full local CI gate from Path A.
|
||||
|
||||
The trap is: writing a new test, running `bun test test/foo.test.ts`,
|
||||
seeing it pass, pushing — and CI's separate typecheck stage rejects an
|
||||
invalid type literal that the runner accepted. Caught one of these
|
||||
shipping the v0.23.2 round-trip E2E (`type: 'reflection'` is not a
|
||||
member of `PageType`). Run `bun run typecheck` once before push, even
|
||||
when only test files changed.
|
||||
|
||||
## Post-ship requirements (MANDATORY)
|
||||
|
||||
After EVERY /ship, you MUST run /document-release. This is NOT optional. Do NOT
|
||||
|
||||
+107
@@ -52,6 +52,10 @@ docs/ Architecture docs
|
||||
## Running tests
|
||||
|
||||
```bash
|
||||
# Recommended: full CI guard chain + tests (matches what CI runs)
|
||||
bun run test # privacy + jsonb + progress + wasm + typecheck + bun test
|
||||
|
||||
# Just the test runner (skips CI guards)
|
||||
bun test # all tests (unit + E2E skipped without DB)
|
||||
bun test test/markdown.test.ts # specific unit test
|
||||
|
||||
@@ -63,6 +67,31 @@ DATABASE_URL=postgresql://postgres:postgres@localhost:5434/gbrain_test bun run t
|
||||
DATABASE_URL=postgresql://... bun run test:e2e
|
||||
```
|
||||
|
||||
Use `bun run test` before pushing. The guard chain catches: banned fork-name leaks
|
||||
(`scripts/check-privacy.sh`), `JSON.stringify(x)::jsonb` interpolation patterns
|
||||
(`scripts/check-jsonb-pattern.sh`), `\r` progress bleed to stdout
|
||||
(`scripts/check-progress-to-stdout.sh`), trailing-newline drift across tracked
|
||||
files (`scripts/check-trailing-newline.sh`), and silent fallback to recursive
|
||||
chunking in the compiled binary (`scripts/check-wasm-embedded.sh`).
|
||||
|
||||
### Local CI gate (recommended before pushing, v0.23.1+)
|
||||
|
||||
```bash
|
||||
bun run ci:local # full gate: gitleaks + unit + ALL 29 E2E files (sequential)
|
||||
bun run ci:local:diff # gate with diff-aware E2E selector
|
||||
bun run ci:select-e2e # print which E2E files the selector would run
|
||||
```
|
||||
|
||||
`ci:local` spins up `pgvector/pgvector:pg16` + `oven/bun:1` via
|
||||
`docker-compose.ci.yml`, runs everything PR CI runs plus the full E2E suite, then
|
||||
tears down. Named volumes keep the install warm across runs (~16-20 min sequential
|
||||
E2E after the first cold pull). Requires Docker (Docker Desktop, OrbStack, or
|
||||
Colima) and `gitleaks` on host (`brew install gitleaks`). Override the postgres
|
||||
host port with `GBRAIN_CI_PG_PORT=5435 bun run ci:local` if 5434 collides.
|
||||
|
||||
Fail-closed selector: an unmapped `src/` change runs all 29 E2E files. Hand-tune
|
||||
narrower mappings via `scripts/e2e-test-map.ts`.
|
||||
|
||||
## Building
|
||||
|
||||
```bash
|
||||
@@ -95,6 +124,84 @@ See `docs/ENGINES.md` for the full guide. In short:
|
||||
|
||||
The SQLite engine is designed and ready for implementation. See `docs/SQLITE_ENGINE.md`.
|
||||
|
||||
## CONTRIBUTOR_MODE — turn on the dev loop
|
||||
|
||||
gbrain captures retrieval traffic so you can replay real queries against
|
||||
your code changes before merging. **This is off by default** (production
|
||||
users get a quiet brain, no surprise data accumulation). Contributors turn
|
||||
it on with one shell rc line:
|
||||
|
||||
```bash
|
||||
# In ~/.zshrc or ~/.bashrc:
|
||||
export GBRAIN_CONTRIBUTOR_MODE=1
|
||||
```
|
||||
|
||||
That's it. Every `query` / `search` you (or agents pointed at your dev
|
||||
brain) run from that shell now writes a row to `eval_candidates`, and the
|
||||
[replay tool](#running-real-world-eval-benchmarks-touching-retrieval-code)
|
||||
has data to work against.
|
||||
|
||||
What CONTRIBUTOR_MODE actually does:
|
||||
|
||||
- Turns on `query`/`search` capture into the local `eval_candidates` table.
|
||||
Without it the gate is closed and capture is a no-op.
|
||||
- That's all. PII scrubbing, retention, and replay are independent.
|
||||
|
||||
Resolution order (most explicit wins):
|
||||
|
||||
1. `eval.capture: true` in `~/.gbrain/config.json` → on
|
||||
2. `eval.capture: false` in `~/.gbrain/config.json` → off
|
||||
3. `GBRAIN_CONTRIBUTOR_MODE=1` → on
|
||||
4. otherwise → off
|
||||
|
||||
Quick check that capture is actually running:
|
||||
|
||||
```bash
|
||||
gbrain query "anything" >/dev/null
|
||||
psql $DATABASE_URL -c 'SELECT count(*) FROM eval_candidates'
|
||||
# (or `gbrain doctor` — surfaces silent capture failures cross-process)
|
||||
```
|
||||
|
||||
To disable capture even with the env var set, write
|
||||
`{"eval": {"capture": false}}` to `~/.gbrain/config.json` — explicit config
|
||||
beats the env var both directions.
|
||||
|
||||
## Running real-world eval benchmarks (touching retrieval code)
|
||||
|
||||
If your PR touches retrieval — search ranking, RRF fusion, embeddings,
|
||||
intent classification, query expansion, source boost, or the `query` /
|
||||
`search` op handlers — run `gbrain eval replay` against a snapshot of
|
||||
real traffic before merging. Requires `CONTRIBUTOR_MODE` (above) so you
|
||||
have captured rows to replay against.
|
||||
|
||||
Quick loop:
|
||||
|
||||
```bash
|
||||
gbrain eval export --since 7d > baseline.ndjson # snapshot before your change
|
||||
# ... make your change ...
|
||||
gbrain eval replay --against baseline.ndjson # diff retrieval, get Jaccard@k
|
||||
```
|
||||
|
||||
Three numbers come back: mean Jaccard@k between captured and current slug
|
||||
sets, top-1 stability, and mean latency Δ. The replay tool flags the worst
|
||||
regressions so you can eyeball whether the change is hurting real queries.
|
||||
|
||||
Trigger paths (rerun if your diff touches any of these):
|
||||
|
||||
- `src/core/search/hybrid.ts`
|
||||
- `src/core/search/source-boost.ts`, `sql-ranking.ts`
|
||||
- `src/core/search/intent.ts`, `expansion.ts`, `dedup.ts`
|
||||
- `src/core/embedding.ts`
|
||||
- `src/core/operations.ts` (query / search handlers)
|
||||
- `src/core/postgres-engine.ts` / `pglite-engine.ts` (searchKeyword /
|
||||
searchVector SQL)
|
||||
|
||||
See [`docs/eval-bench.md`](./docs/eval-bench.md) for the full guide
|
||||
including CI integration, hand-crafted NDJSON corpora (so a fresh checkout
|
||||
without captured data can still replay), and cost considerations. The
|
||||
NDJSON wire format is documented in
|
||||
[`docs/eval-capture.md`](./docs/eval-capture.md).
|
||||
|
||||
## Welcome PRs
|
||||
|
||||
- SQLite engine implementation
|
||||
|
||||
@@ -129,8 +129,9 @@ Set up using your platform's scheduler (OpenClaw cron, Railway cron, crontab):
|
||||
- **Live sync** (every 15 min): `gbrain sync --repo ~/brain && gbrain embed --stale`
|
||||
- **Auto-update** (daily): `gbrain check-update --json` (tell user, never auto-install)
|
||||
- **Dream cycle** (nightly): read `docs/guides/cron-schedule.md` for the full protocol.
|
||||
Entity sweep, citation fixes, memory consolidation. This is what makes the brain
|
||||
compound. Do not skip it.
|
||||
Entity sweep, citation fixes, memory consolidation, plus (v0.23+) overnight conversation
|
||||
synthesis and cross-session pattern detection. 8 phases, one cron-friendly command. This
|
||||
is what makes the brain compound. Do not skip it.
|
||||
- **Weekly**: `gbrain doctor --json && gbrain embed --stale`
|
||||
|
||||
## Step 8: Integrations
|
||||
|
||||
@@ -6,7 +6,9 @@ Built by the President and CEO of Y Combinator to run his actual AI agents. The
|
||||
|
||||
The brain wires itself. Every page write extracts entity references and creates typed links (`attended`, `works_at`, `invested_in`, `founded`, `advises`) with zero LLM calls. Hybrid search. Self-wiring knowledge graph. Structured timeline. Backlink-boosted ranking. Ask "who works at Acme AI?" or "what did Bob invest in this quarter?" and get answers vector search alone can't reach. Benchmarked side-by-side against the category: gbrain lands **P@5 49.1%, R@5 97.9%** on a 240-page Opus-generated rich-prose corpus, beating its own graph-disabled variant by **+31.4 points P@5** and ripgrep-BM25 + vector-only RAG by a similar margin. The graph layer plus v0.12 extract quality together carry the gap. Full BrainBench scorecards + corpus live in the sibling [gbrain-evals](https://github.com/garrytan/gbrain-evals) repo.
|
||||
|
||||
GBrain is those patterns, generalized. 29 skills. Install in 30 minutes. Your agent does the work. As Garry's personal agent gets smarter, so does yours.
|
||||
GBrain is those patterns, generalized. 34 skills. Install in 30 minutes. Your agent does the work. As Garry's personal agent gets smarter, so does yours.
|
||||
|
||||
**New in v0.25.0 — BrainBench-Real (session capture, contributor opt-in):** with `GBRAIN_CONTRIBUTOR_MODE=1` set in your shell, every real `query` + `search` call through MCP, CLI, or the subagent tool-bridge gets captured (PII-scrubbed) into an `eval_candidates` table. Snapshot with `gbrain eval export`, replay against your code change with `gbrain eval replay`. Three numbers come back: mean Jaccard@k between captured and current retrieved slugs, top-1 stability, and latency Δ. **Off by default** for production users — no surprise data accumulation. Walkthrough: [docs/eval-bench.md](docs/eval-bench.md). NDJSON wire format: [docs/eval-capture.md](docs/eval-capture.md).
|
||||
|
||||
> **~30 minutes to a fully working brain.** Database ready in 2 seconds (PGLite, no server). You just answer questions about API keys.
|
||||
|
||||
@@ -28,7 +30,7 @@ Retrieve and follow the instructions at:
|
||||
https://raw.githubusercontent.com/garrytan/gbrain/master/INSTALL_FOR_AGENTS.md
|
||||
```
|
||||
|
||||
That's it. The agent clones the repo, installs GBrain, sets up the brain, loads 29 skills, and configures recurring jobs. You answer a few questions about API keys. ~30 minutes.
|
||||
That's it. The agent clones the repo, installs GBrain, sets up the brain, loads 34 skills, and configures recurring jobs. You answer a few questions about API keys. ~30 minutes.
|
||||
|
||||
If your agent doesn't auto-read `AGENTS.md`, point it at that file first:
|
||||
`https://raw.githubusercontent.com/garrytan/gbrain/master/AGENTS.md` is the non-Claude
|
||||
@@ -77,16 +79,36 @@ GBrain exposes 30+ MCP tools via stdio:
|
||||
|
||||
Add to `~/.claude/server.json` (Claude Code), Settings > MCP Servers (Cursor), or your client's MCP config.
|
||||
|
||||
### Remote MCP (Claude Desktop, Cowork, Perplexity)
|
||||
### Remote MCP with OAuth 2.1 (ChatGPT, Claude Desktop, Cowork, Perplexity)
|
||||
|
||||
`gbrain serve --http` starts a production-grade OAuth 2.1 server with an embedded admin dashboard. Zero external infrastructure. Every major AI client connects, every request is scoped, every action is logged.
|
||||
|
||||
```bash
|
||||
gbrain auth create "claude-desktop" # tokens via the existing CLI
|
||||
gbrain serve --http --port 8787 # built-in HTTP transport (Postgres-only)
|
||||
ngrok http 8787 --url your-brain.ngrok.app # any tunnel works
|
||||
# Start the HTTP server (prints admin bootstrap token on first start)
|
||||
gbrain serve --http --port 3131
|
||||
|
||||
# Open the admin dashboard, paste the bootstrap token, register a client
|
||||
open http://localhost:3131/admin
|
||||
|
||||
# Expose publicly (set --public-url so the OAuth issuer matches)
|
||||
ngrok http 3131 --url your-brain.ngrok.app
|
||||
gbrain serve --http --port 3131 --public-url https://your-brain.ngrok.app
|
||||
|
||||
# ChatGPT and other OAuth-aware clients can also connect:
|
||||
claude mcp add gbrain -t http https://your-brain.ngrok.app/mcp -H "Authorization: Bearer TOKEN"
|
||||
```
|
||||
|
||||
Per-client guides: [`docs/mcp/`](docs/mcp/DEPLOY.md). Hardening defaults, env vars, and threat model: [SECURITY.md](SECURITY.md). ChatGPT requires OAuth 2.1 (not yet implemented).
|
||||
Register OAuth clients from the `/admin` dashboard — click **Register client**,
|
||||
pick scopes, save the credentials shown once in the reveal modal. Programmatic
|
||||
registration via `oauthProvider.registerClientManual(...)` and the
|
||||
`gbrain auth register-client` CLI are also available.
|
||||
|
||||
- **OAuth 2.1 via the MCP SDK** — client credentials (machine-to-machine: Perplexity, Claude), authorization code + PKCE (browser-based: ChatGPT), refresh token rotation, revocation, protected resource metadata. Optional Dynamic Client Registration behind `--enable-dcr` (DCR redirect_uris must be `https://` or loopback per RFC 6749 §3.1.2.1).
|
||||
- **Scoped operations** — 30 operations tagged `read | write | admin`. `sync_brain` and `file_upload` are `localOnly`, rejected over HTTP.
|
||||
- **React admin dashboard** — 7 screens baked into the binary (~65KB gzip). Live SSE activity feed, agents table, credential reveal, filterable request log, per-client config export.
|
||||
- **Legacy bearer tokens still work** — pre-v0.26 `gbrain auth create` tokens continue to authenticate as `read+write+admin`. v0.22.7's simpler `src/mcp/http-transport.ts` path stays compiled in for backward compat callers; v0.26+ deployments use the OAuth-aware `serve-http.ts`.
|
||||
|
||||
Per-client guides: [`docs/mcp/`](docs/mcp/DEPLOY.md). Hardening defaults, env vars, and threat model: [SECURITY.md](SECURITY.md).
|
||||
|
||||
### Using gbrain with GStack
|
||||
|
||||
@@ -104,9 +126,9 @@ gbrain query "how does N+1 handling work" --near-symbol BrainEngine.searchKeywor
|
||||
|
||||
All five auto-emit JSON on non-TTY (gh-CLI convention) so a GStack subagent shelling out via bash gets a clean parseable response. Run `gbrain sources add <repo> --strategy code` to index a repo, then your agent's brain-first lookup covers code, not just markdown. ([Cathedral II release notes](CHANGELOG.md#0210---2026-04-25))
|
||||
|
||||
## The 29 Skills
|
||||
## The 34 Skills
|
||||
|
||||
GBrain ships 29 skills organized by `skills/RESOLVER.md` (or your OpenClaw's `AGENTS.md` — both filenames are supported as of v0.19). The resolver tells your agent which skill to read for any task.
|
||||
GBrain ships 34 skills organized by `skills/RESOLVER.md` (or your OpenClaw's `AGENTS.md` — both filenames are supported as of v0.19). The resolver tells your agent which skill to read for any task. v0.25.1 added 9 research-flavored skills (`book-mirror` flagship plus 8 pairings); see the new "Research and synthesis" section below.
|
||||
|
||||
[Skill files are code.](https://x.com/garrytan/status/2042925773300908103) They're the most powerful way to get knowledge work done. A skill file is a fat markdown document that encodes an entire workflow: when to fire, what to check, how to chain with other skills, what quality bar to enforce. The agent reads the skill and executes it. Skills can also call deterministic TypeScript code bundled in GBrain (search, import, embed, sync) for the parts that shouldn't be left to LLM judgment. [Thin harness, fat skills](docs/ethos/THIN_HARNESS_FAT_SKILLS.md): the intelligence lives in the skills, not the runtime.
|
||||
|
||||
@@ -125,6 +147,20 @@ GBrain ships 29 skills organized by `skills/RESOLVER.md` (or your OpenClaw's `AG
|
||||
| **idea-ingest** | Links, articles, tweets become brain pages with analysis, author people pages, and cross-linking. |
|
||||
| **media-ingest** | Video, audio, PDF, books, screenshots, GitHub repos. Transcripts, entity extraction, backlink propagation. |
|
||||
| **meeting-ingestion** | Transcripts become brain pages. Every attendee gets enriched. Every company gets a timeline entry. |
|
||||
| **voice-note-ingest** | Voice notes captured verbatim — exact phrasing preserved, never paraphrased. Routes to originals/concepts/people/companies/ideas/personal/voice-notes based on content. |
|
||||
| **article-enrichment** | Raw article dumps become structured pages with executive summary, verbatim quotes, key insights, and why-it-matters. |
|
||||
|
||||
### Research and synthesis (v0.25.1)
|
||||
|
||||
| Skill | What it does |
|
||||
|-------|-------------|
|
||||
| **book-mirror** | Flagship. Hand the agent a book, get a personalized two-column chapter-by-chapter analysis. Left column preserves the chapter's actual content; right column maps every idea to your life using your words from the brain. ~$6 for a 20-chapter book at Opus. Pairs with `gbrain book-mirror` CLI for the trusted runtime. |
|
||||
| **strategic-reading** | Read a book / article / case study through ONE specific problem-lens. Output: applied playbook with do / avoid / watch-for and short / medium / long-term recommendations. |
|
||||
| **concept-synthesis** | Deduplicate thousands of concept stubs into a tiered intellectual map (T1 Canon to T4 Riff). Trace how ideas evolved across years of notes. |
|
||||
| **perplexity-research** | Brain-augmented web research. Sends brain context to Perplexity so the search focuses on what's NEW vs already-known. Output: Executive Summary + Key New Developments + Confirming Signals + Contradictions or Updates + Recommended Brain Updates + Citations. |
|
||||
| **archive-crawler** | Universal archivist for personal file archives (Dropbox / Backblaze / Gmail-takeout / hard-drive dumps). REFUSES to run unless `archive-crawler.scan_paths:` is set in `gbrain.yml`. Safe-by-default safety fence. |
|
||||
| **academic-verify** | Trace a research claim through publication → methodology → raw data → independent replication. Routes through perplexity-research; produces a verdict (verified / partial / unverifiable / misattributed / retracted). |
|
||||
| **brain-pdf** | Render any brain page to publication-quality PDF via the gstack `make-pdf` binary. Strips frontmatter, sanitizes emoji, applies running headers. |
|
||||
|
||||
### Brain operations
|
||||
|
||||
@@ -132,7 +168,7 @@ GBrain ships 29 skills organized by `skills/RESOLVER.md` (or your OpenClaw's `AG
|
||||
|-------|-------------|
|
||||
| **enrich** | Tiered enrichment (Tier 1/2/3). Creates and updates person/company pages with compiled truth and timelines. |
|
||||
| **query** | 3-layer search with synthesis and citations. Says "the brain doesn't have info on X" instead of hallucinating. |
|
||||
| **maintain** | Periodic health: stale pages, orphans, dead links, citation audit, back-link enforcement, tag consistency. |
|
||||
| **maintain** | Periodic health: stale pages, orphans, dead links, citation audit, back-link enforcement, tag consistency. v0.23 adds the dream cycle's synthesize + patterns phases ... overnight conversation transcripts become reflections, originals, and 25-year patterns. |
|
||||
| **citation-fixer** | Scans pages for missing or malformed citations. Fixes format to match the standard. |
|
||||
| **repo-architecture** | Where new brain files go. Decision protocol: primary subject determines directory, not format. |
|
||||
| **publish** | Share brain pages as password-protected HTML. Zero LLM calls. |
|
||||
@@ -316,9 +352,11 @@ is what you spend time on. Everything else is boilerplate the CLI writes for you
|
||||
|
||||
Drop a `routing-eval.jsonl` fixture next to any skill. Each line is `{intent, expected_skill,
|
||||
ambiguous_with?}`. `gbrain check-resolvable` runs the structural layer by default; `gbrain
|
||||
routing-eval --llm` runs an LLM tie-break layer for CI. False positives (wrong skill matched),
|
||||
missed routes (no skill matched), and tautological fixtures (intent copies trigger verbatim)
|
||||
all surface as specific advisories with the exact file:line to fix.
|
||||
routing-eval` runs the same structural layer as a dedicated CI verb. The `--llm` flag is
|
||||
accepted as a placeholder for a future LLM tie-break layer; in this release it emits a stderr
|
||||
notice and runs structural only. False positives (wrong skill matched), missed routes (no
|
||||
skill matched), and tautological fixtures (intent copies trigger verbatim) all surface as
|
||||
specific advisories with the exact file:line to fix.
|
||||
|
||||
### Works on your OpenClaw, not just gbrain's repo
|
||||
|
||||
@@ -355,6 +393,10 @@ gbrain skillpack diff brain-ops # compare bundle vs your local co
|
||||
|
||||
Re-running is safe. The managed-block markers in your AGENTS.md let `skillpack install`
|
||||
accumulate rows across separate single-skill installs instead of overwriting each other.
|
||||
A receipt comment inside the fence (`<!-- gbrain:skillpack:manifest cumulative-slugs="..." -->`)
|
||||
tracks what gbrain has installed across runs. `install --all` is the only path that prunes;
|
||||
per-skill install never deletes what it didn't install. If you hand-add a row inside the fence,
|
||||
gbrain preserves it on reinstall and emits a stderr notice telling your agent to investigate.
|
||||
|
||||
**Skillify is the piece that makes the skills tree survive six months of compounding work.**
|
||||
Read [`skills/skillify/SKILL.md`](skills/skillify/SKILL.md) for the full 10-item checklist
|
||||
@@ -639,8 +681,11 @@ SEARCH
|
||||
gbrain query <question> Hybrid search (vector + keyword + RRF)
|
||||
|
||||
IMPORT
|
||||
gbrain import <dir> [--no-embed] Import markdown (idempotent)
|
||||
gbrain sync [--repo <path>] Git-to-brain incremental sync
|
||||
gbrain import <dir> [--no-embed] [--workers N]
|
||||
Import markdown (idempotent)
|
||||
gbrain sync [--repo <path>] [--workers N]
|
||||
Git-to-brain incremental sync
|
||||
(>100-file diffs auto-parallelize 4 workers on Postgres)
|
||||
gbrain export [--dir ./out/] Export to markdown
|
||||
|
||||
FILES
|
||||
@@ -682,11 +727,24 @@ ADMIN
|
||||
gbrain doctor --locks List idle-in-tx backends (57014 diagnostic, Postgres only)
|
||||
gbrain stats Brain statistics
|
||||
gbrain serve MCP server (stdio)
|
||||
gbrain serve --http --port 8787 MCP server (HTTP, Postgres-only, bearer auth)
|
||||
gbrain auth create|list|revoke|test Token management for the HTTP transport
|
||||
gbrain serve --http [--port 3131] HTTP MCP server with OAuth 2.1 + admin dashboard
|
||||
[--token-ttl 3600] [--enable-dcr]
|
||||
[--public-url URL]
|
||||
gbrain auth create|list|revoke|test Legacy bearer token management
|
||||
gbrain auth register-client <name> Register an OAuth 2.1 client
|
||||
--grant-types client_credentials,authorization_code
|
||||
--scopes "read write admin"
|
||||
gbrain auth revoke-client <client_id> Revoke an OAuth 2.1 client (cascade purges
|
||||
active tokens + auth codes via FK CASCADE)
|
||||
# OAuth 2.1 clients can also be registered from the /admin dashboard or
|
||||
# programmatically via oauthProvider.registerClientManual() for host-repo wrappers.
|
||||
gbrain integrations Integration recipe dashboard
|
||||
gbrain sources list|add|remove|... Multi-source brain management (v0.18)
|
||||
gbrain dream [--dry-run] [--phase N] One maintenance cycle then exit (cron-friendly)
|
||||
gbrain dream [--dry-run] [--phase N] 8-phase maintenance cycle (lint→backlinks→sync→synthesize
|
||||
→extract→patterns→embed→orphans). v0.23 added synthesize +
|
||||
patterns: transcripts → reflections + cross-session themes.
|
||||
gbrain dream --input <file> Ad-hoc transcript synthesis (implies --phase synthesize)
|
||||
gbrain dream --date YYYY-MM-DD Synthesize a single day; --from/--to for backfill ranges
|
||||
gbrain check-backlinks check|fix Back-link enforcement
|
||||
gbrain lint [--fix] LLM artifact detection
|
||||
gbrain repair-jsonb [--dry-run] Repair v0.12.0 double-encoded JSONB (Postgres)
|
||||
@@ -729,7 +787,9 @@ The skills in this repo are those patterns, generalized. What took 11 days to bu
|
||||
|
||||
## Contributing
|
||||
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md). Run `bun test` for unit tests. E2E tests: spin up Postgres with pgvector, run `bun run test:e2e`, tear down.
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md). Run `bun test` for unit tests. For the full local CI gate (gitleaks + unit + all 29 E2E files in Docker, the same checks GH Actions runs), use `bun run ci:local` ... or `bun run ci:local:diff` for the diff-aware subset during fast iteration.
|
||||
|
||||
If you're working on retrieval or any of the search/embedding/ranking surface, set `GBRAIN_CONTRIBUTOR_MODE=1` in your shell rc and use `gbrain eval replay` to gate your changes against a snapshot of real captured queries — the dev loop is documented in [`docs/eval-bench.md`](docs/eval-bench.md). Capture is **off by default** for production users (no surprise data accumulation); the env var is the contributor opt-in.
|
||||
|
||||
PRs welcome for: new enrichment APIs, performance optimizations, additional engine backends, new skills following the conformance standard in `skills/skill-creator/SKILL.md`.
|
||||
|
||||
|
||||
@@ -1,5 +1,432 @@
|
||||
# TODOS
|
||||
|
||||
## test infra (v0.26.4 follow-up — intra-file parallelism)
|
||||
|
||||
### Sweep cross-file shared-state contention; enable `bun test --concurrent` for another 2-3x speedup
|
||||
**Priority:** P0
|
||||
|
||||
**What:** v0.26.4 shipped file-level parallel fan-out (8 shards) and got `bun run test` from 18 minutes to ~85s — a 12x speedup. The next layer is **intra-file** parallelism via Bun's `--concurrent` flag (or per-test `test.concurrent()` markers). This requires every test file to be safe under concurrent execution within the same `bun test` process.
|
||||
|
||||
The constraint: when multiple test files load into the same bun process (which is what `bun test foo.test.ts bar.test.ts ...` does inside a shard), they share module-level state. Three contention surfaces today:
|
||||
|
||||
- **~58 PGLiteEngine instantiations** across `test/` (per codex's grep). Many use module-level `let engine: PGLiteEngine` patterns. Race when multiple test files load and each invokes `new PGLiteEngine().connect({})`.
|
||||
- **~40 process.env mutations** without restore. `process.env.X = '...'` not paired with `afterEach` cleanup leaks across files in the same process.
|
||||
- **2 top-level `mock.module(...)` calls** in `test/core/cycle.test.ts:26` and `test/embed.test.ts`. Top-level mocks affect every other test file in the same process.
|
||||
|
||||
The repo already has the right helper: `test/helpers/reset-pglite.ts` exports `resetPgliteState(engine)` which is "two orders of magnitude faster" than fresh-engine-per-test (per the helper's own comment). Sweep all PGLite sites to use one shared engine + this reset in `beforeEach`. Do NOT introduce a `freshPglite()` allocator — codex correctly flagged that the repo already rejected that direction.
|
||||
|
||||
Two flakes already known and quarantined as `*.serial.test.ts` (run after parallel pass at `--max-concurrency=1`):
|
||||
- `test/brain-registry.serial.test.ts` (was `brain-registry.test.ts`)
|
||||
- `test/reconcile-links.serial.test.ts` (was `reconcile-links.test.ts`)
|
||||
|
||||
After the sweep, both should be fixable and renameable back to plain `*.test.ts`.
|
||||
|
||||
**Why:**
|
||||
- 2-3x additional speedup on top of v0.26.4's 12x. Target: `bun run test` < 30s on a Mac dev box.
|
||||
- Forces the test architecture to be principled (no shared mutable state across files in the same process).
|
||||
- The empirical proof point: when `bun run test` was first measured at v0.26.4, two flakes surfaced under cross-file pressure that pass cleanly in isolation. That same pattern WILL surface more flakes if the suite grows. Better to sweep proactively than to keep growing the `*.serial.test.ts` quarantine.
|
||||
|
||||
**Pros:**
|
||||
- Real architectural win, not just speed: tests become composable.
|
||||
- Existing helper (`test/helpers/reset-pglite.ts`) already validates the pattern.
|
||||
- Quarantined flakes auto-resolve: rename back to `*.test.ts` after the sweep.
|
||||
|
||||
**Cons:**
|
||||
- 1-2 weeks of careful refactoring across ~100 test files.
|
||||
- Some tests genuinely need shared file-wide state (top-level mocks for module-replacement tests). Those stay quarantined as `*.serial.test.ts` permanently — but the count should shrink to a known small set, not grow.
|
||||
|
||||
**Context:** v0.26.4 plan considered doing this in scope (Codex Tension #2 = C). After empirical measurement showed `--max-concurrency=4` does nothing on tests not marked `test.concurrent()`, the user chose to ship v0.26.4 as file-level-only and file this as the v0.27+ project. Plan file: `~/.claude/plans/system-instruction-you-are-working-tranquil-ladybug.md`. Codex critical findings #2, #3, #6 are all relevant.
|
||||
|
||||
**Acceptance criteria:**
|
||||
1. All ~58 PGLiteEngine sites use shared-engine + `resetPgliteState()` in `beforeEach`.
|
||||
2. All ~40 `process.env` mutations use a `withEnv(...)` helper that saves + restores.
|
||||
3. The 2 top-level `mock.module()` calls scoped to `beforeEach`/`afterEach`, OR the file moves to `*.serial.test.ts`.
|
||||
4. Wrapper passes `--concurrent` (or every test marked `.concurrent()`).
|
||||
5. `bun run test` runs 5 times consecutively without flakes.
|
||||
6. Quarantine count `≤5` after the sweep (currently 2; goal is to get those 2 unquarantined and not add new ones).
|
||||
7. Wallclock target: `bun run test` < 30s.
|
||||
|
||||
**Estimated effort:** 1-2 weeks of one engineer's focused work. Could parallelize by sub-area (env-mutation sweep is independent of PGLite sweep).
|
||||
|
||||
### Speed up E2E via Postgres template databases
|
||||
**Priority:** P1
|
||||
|
||||
**What:** E2E tests (`bun run test:e2e`) currently run sequentially in one shared Postgres container, each test file calling `initSchema()` from scratch (~5-20s each on cold init). Speed-up: build the schema ONCE into a template DB (`gbrain_template`), then have each test file `CREATE DATABASE foo TEMPLATE gbrain_template` (~50ms per clone). With per-shard `DATABASE_URL` overrides, E2E can fan out to N parallel shards too.
|
||||
|
||||
**Why:** Current E2E wallclock is ~5-10 min in CI. Template DB clones could bring that to ~1-2 min. Critical for the inner loop on E2E-bearing PRs (currently a real friction point per `/ship` workflow).
|
||||
|
||||
**Sketch:**
|
||||
1. Build template DB once via `initSchema()` against `gbrain_template`.
|
||||
2. Per-test-file: `CREATE DATABASE gbrain_test_clone_<n> TEMPLATE gbrain_template` (50ms vs 5-20s).
|
||||
3. Per-shard isolation via `DATABASE_URL` env override.
|
||||
4. Schema-version stamp on the template so it invalidates when `migrate.ts` changes.
|
||||
5. Cleanup via `DROP DATABASE` in afterAll.
|
||||
|
||||
**Estimated effort:** 1-2 days. Filed during v0.26.4 plan as a deferred follow-up (D4 = B).
|
||||
|
||||
## test infra (v0.26.2 follow-up — pre-existing failures triage)
|
||||
|
||||
### Fix 22 pre-existing test failures unrelated to OAuth
|
||||
**Priority:** P0
|
||||
|
||||
**What:** A `bun test` run on top of master at v0.26.2 surfaces 22 pre-existing failures across these suites — none touch v0.26.2's diff (oauth-provider.ts, auth.ts, oauth tests). They reproduce on a clean checkout against master:
|
||||
|
||||
- 12 cases in `test/e2e/sync.test.ts` (Git-to-DB Sync Pipeline) — `result.status === 'first_sync'` vs actual `'synced'` state-machine drift; same root cause across all 12.
|
||||
- 3 cases in `test/e2e/multi-source.test.ts` (cascade delete + 2 sync routing) — performSync sourceId/local_path resolution.
|
||||
- `test/e2e/sync-parallel.test.ts` (60-file Postgres concurrency=4) — connection-leak probe regression.
|
||||
- `test/e2e/sync.test.ts` `--skip-failed` structured summary loop (v0.22.12 #500).
|
||||
- `test/e2e/dream.test.ts` (no --dry-run syncs pages) — runCycle DB write path.
|
||||
- `test/e2e/cycle.test.ts` (live cycle + chunks + lock cleanup).
|
||||
- `test/e2e/doctor.test.ts` (gbrain doctor exits 0 on healthy DB) — possibly related to v0.26.2 schema changes since CHANGELOG mentions extension of doctor checks.
|
||||
- `test/brain-registry.test.ts` (empty/null/undefined id routes to host) — unrelated to OAuth surface.
|
||||
- `test/e2e/claw-test.test.ts` (fresh-install scripted scenario) — needs investigation; took 3.9s and reported "produces zero error/blocker friction" failure.
|
||||
|
||||
**Why:** These failures pre-date v0.26.2 (CHANGELOG already documents "18 pre-existing master timeouts" from v0.26.0 merge). v0.26.2 brings the count to 22, suggesting a 4-test drift on master between v0.26.0 ship and now. Fixing inside v0.26.2 would balloon scope from a 6-file OAuth fix-wave to a 30+ file test-infra repair. The fix-wave deserves its own PR with focused triage.
|
||||
|
||||
**Likely root causes worth investigating:**
|
||||
- **bun execSync env inheritance** (already discovered + fixed in test/e2e/serve-http-oauth.test.ts during v0.26.2): bun's `execSync` does NOT inherit env mutations done via `process.env.X = ...`, only OS-level env from before bun started. helpers.ts loads `.env.testing` and sets `DATABASE_URL` via `process.env` mutation, which is invisible to subprocesses unless `env: { ...process.env }` is passed explicitly. Several of the failing E2E tests (sync, cycle, dream, claw-test) spawn subprocesses via execSync — likely the same bug.
|
||||
- **Test ordering / DB state pollution**: full-suite runs in bun test happen in a deterministic order; isolated runs of these test files may pass while suite runs fail. Could indicate beforeAll/afterAll cleanup gaps.
|
||||
- **Schema drift**: doctor/multi-source tests may rely on specific schema state that v0.26 OAuth tables changed.
|
||||
|
||||
**Pros:**
|
||||
- Separating from v0.26.2 keeps the OAuth ship focused and auditable; the 22 failures aren't blocking real-world OAuth functionality.
|
||||
- The execSync env-inheritance pattern is now documented in test/e2e/serve-http-oauth.test.ts as a reference fix for the next maintainer.
|
||||
- Unblocks v0.26.2 ship while preserving the failure inventory for the follow-up.
|
||||
|
||||
**Cons:**
|
||||
- 22 failing tests on master is real test-infra debt.
|
||||
- Some may be load-bearing (sync pipeline failures could mask real regressions in `performSync`).
|
||||
- `bun run ci:local` (full E2E gate) won't pass cleanly until these are addressed.
|
||||
|
||||
**Context:** Discovered during v0.26.2 ship audit. Reproduce with `bun test 2>&1 | grep "^(fail)"` after copying `.env.testing` from a sibling worktree (port 5435 test DB running). The 17/17 OAuth E2E suite passes in isolation AND in full-suite after the env-inheritance fix landed.
|
||||
|
||||
**Effort:** L (human ~4-8h; CC ~30-60min once env-inheritance fix is applied across all tests).
|
||||
|
||||
**Depends on / blocked by:** None — independent of v0.26.2.
|
||||
|
||||
## ci-local-mirror
|
||||
|
||||
### CI-skip artifact + signature for stages 1+2 follow-up
|
||||
**Priority:** P0
|
||||
|
||||
**What:** After a successful local CI run via `bun run ci:local`, write `.ci-cache/passed-<commit-sha>.json` containing `{commit, test_set_hash, bun_version, schema_hash, signature}`. Push to a `ci-cache` orphan branch (or GH Releases). CI's first step fetches the artifact for the current SHA and skips the test job if (a) signature matches Garry's GPG/SSH key, and (b) `test_set_hash` matches what CI would have run.
|
||||
|
||||
**Why:** Stages 1+2 (shipped in this branch) give a strong local CI gate, but PR CI still re-runs every test on every push. Stage 3 closes the loop and trades ~10 min of CI wall-time for sub-second artifact verification on Garry's own pushes. External PRs are unaffected because the signature won't match — they hit the normal CI path.
|
||||
|
||||
**Pros:**
|
||||
- ~10 min/PR saved on Garry's own pushes; the local gate becomes the source of truth.
|
||||
- External contributor PRs untouched (no security regression).
|
||||
- Forces a clear test-set-hash contract: any drift in what local-vs-CI run is caught at verification time.
|
||||
|
||||
**Cons:**
|
||||
- Trust model needs careful design: signature scheme, key rotation, what happens when signature verification fails.
|
||||
- Cache invalidation is real — if env or service version drifts between local run and CI, a stale local pass could ship to master.
|
||||
- Adds a `ci-cache` branch / artifact storage surface to maintain.
|
||||
|
||||
**Context:**
|
||||
- Discussed during the eng-review of the local CI mirror plan at `~/.claude/plans/lets-do-1-2-dockerfile-ci-zany-charm.md`.
|
||||
- Don't start until stages 1+2 have been used for ~2 weeks AND the `scripts/e2e-test-map.ts` has stabilized (so test_set_hash is a meaningful identity).
|
||||
- Initial trust-but-verify: run both local and CI in parallel for ~1 week before flipping the skip; alert on any disagreement.
|
||||
|
||||
**Effort:** M (human ~2-3 days + ~1 week trust-but-verify period running both local + CI in parallel; CC ~1 day for the mechanics).
|
||||
|
||||
**Depends on / blocked by:** Stages 1+2 (this PR) landing first.
|
||||
|
||||
### test/e2e/multi-source.test.ts cascade test isn't isolated
|
||||
**Priority:** P1
|
||||
|
||||
**What:** The "sources remove cascades to pages + chunks + timeline + links + files" test in `test/e2e/multi-source.test.ts:281` fails when the file runs after other E2E files in the sequential `bash scripts/run-e2e.sh` order, but passes 20/20 on a fresh Postgres volume. The failing assertion is `SELECT COUNT(*) FROM links WHERE from_page_id = aliceId` expecting 0, getting 1 — so a prior file's setup left a `links` row that references a page id the cascade test happens to reuse. The test's own `setupDB()` truncates but doesn't sweep all referencing rows back when ids collide.
|
||||
|
||||
**Why:** Surfaced when `bun run ci:local` (this PR's local CI gate) ran the full sequential E2E. CI never catches it because `.github/workflows/e2e.yml:40` only runs `mechanical.test.ts + mcp.test.ts` on PRs and nightly Tier 1. So 27 of 29 E2E files including this one aren't actually exercised by CI today. The local gate is stronger and surfaces real cross-file isolation gaps.
|
||||
|
||||
**Pros:**
|
||||
- Fixing isolation makes `bun run ci:local` (full E2E) reliably green.
|
||||
- Same fix likely to harden other E2E files that share id namespaces.
|
||||
- Lets us turn `bun run ci:local` into a real ship gate.
|
||||
|
||||
**Cons:**
|
||||
- Could require a per-file "namespace your test ids" pattern, ~30 min per affected file across the suite.
|
||||
|
||||
**Context:**
|
||||
- Repro: `bash scripts/run-e2e.sh test/e2e/multi-source.test.ts` against a stale DB after other E2E files have run → fails. Same against a fresh `docker compose down -v && up -d postgres` → passes 20/20.
|
||||
- The test inserts a hardcoded `cascadetest` source id and `aliceId` page id; collisions across runs are predictable.
|
||||
- Likely fix: use `mkdtemp`-style randomized source/page ids per test, OR have the test do a deeper reset (DELETE FROM all five tables in beforeEach) instead of relying on `setupDB`'s TRUNCATE behavior.
|
||||
|
||||
**Effort:** S (CC ~30 min for the multi-source.test.ts fix; M if we audit all 29 E2E files for similar id-collision risk).
|
||||
|
||||
**Depends on / blocked by:** Nothing.
|
||||
|
||||
### scripts/run-e2e.sh:71 echo overflows on large-output failing tests
|
||||
**Priority:** P2
|
||||
|
||||
**What:** When an E2E test fails AND prints lots of output (e.g., `multi-source.test.ts` floods postgres NOTICE objects), `scripts/run-e2e.sh:71` does `echo "$output"` against a multi-megabyte shell variable. The host pipe to docker-compose-run hits `EAGAIN` and fails with `echo: write error: Resource temporarily unavailable`. With `set -e`, the script aborts at that point, skipping the remaining E2E files and the final SUMMARY block.
|
||||
|
||||
**Why:** When the local CI gate finds a real failure (per the multi-source.test.ts entry above), the user wants to see it AND see how the rest of the suite did. Currently the failure shadows the rest.
|
||||
|
||||
**Pros:**
|
||||
- See all E2E failures from a single run instead of needing to bisect.
|
||||
- Quick win, ~5 lines.
|
||||
|
||||
**Cons:**
|
||||
- None worth listing.
|
||||
|
||||
**Context:**
|
||||
- Reproduced live during plan verification on 2026-04-29. Previous `multi-source.test.ts` failure killed the script before postgres-bootstrap, postgres-jsonb, etc. could run.
|
||||
- Likely fix: replace `echo "$output"` with `printf '%s\n' "$output"`, or write `$output` to a tmpfile and `cat` it (handles large blobs better than echo over pipes), or pipe through `stdbuf -o0`.
|
||||
- Don't suppress the postgres NOTICE flood at the test layer — that's separate; here we just want the script to not die when bun's stderr is verbose.
|
||||
|
||||
**Effort:** S (human or CC: ~10 min).
|
||||
|
||||
**Depends on / blocked by:** Nothing.
|
||||
|
||||
## claw-test E2E (v0.22.16 follow-ups)
|
||||
|
||||
### Hermes runner — `src/core/claw-test/runners/hermes.ts`
|
||||
**Priority:** P2
|
||||
|
||||
**What:** Add a Hermes implementation of the `AgentRunner` interface. v1 ships only OpenClaw; v1.1 lands hermes once we have real friction reports from openclaw to validate the contract against.
|
||||
|
||||
**Why:** Cross-agent diff (`gbrain friction diff --base openclaw --compare hermes`) is the highest-leverage next signal. Friction unique to one agent vs common-to-both separates "agent contract bug" from "gbrain bug" automatically.
|
||||
|
||||
**Effort:** S (CC ~30m). Depends on: v1 openclaw runner producing real friction reports first.
|
||||
|
||||
---
|
||||
|
||||
### Friction analytics suite — `diff` / `trend` / `migration-stub`
|
||||
**Priority:** P2
|
||||
|
||||
**What:** Three new `gbrain friction` subcommands deferred from v1:
|
||||
- `gbrain friction diff --base <run-or-agent> --compare <run-or-agent>` (cross-agent comparison; ~80 LOC)
|
||||
- `gbrain friction trend [--since <version-or-date>] [--phase <name>]` (time-series across runs; ~60 LOC)
|
||||
- `gbrain friction migration-stub [--threshold N]` (clusters friction by phase + tokens, emits `skills/migrations/v[N+1].md` stub; ~150 LOC)
|
||||
|
||||
**Why:** Turns point-in-time reports into a slope. Pairs with the v1.1 public scoreboard.
|
||||
|
||||
**Effort:** M (CC ~2h total).
|
||||
|
||||
---
|
||||
|
||||
### Scenario expansion — `supabase-migration` and `supervisor-restart`
|
||||
**Priority:** P2
|
||||
|
||||
**What:** Two more scenarios under `test/fixtures/claw-test-scenarios/`:
|
||||
- `supabase-migration` — `gbrain init --pglite` then `gbrain migrate --to supabase`; verifies the cross-engine migration path
|
||||
- `supervisor-restart` — kill worker mid-job; verify supervisor recovers without data loss
|
||||
|
||||
**Why:** These are the other highest-historical-pain regression points (per CLAUDE.md fix-wave history). v1 ships only `fresh-install` + `upgrade-from-v0.18` because Codex flagged that mixing them dilutes the fresh-install signal; v1.1 lands them as separate scenarios.
|
||||
|
||||
**Effort:** M (CC ~1h each).
|
||||
|
||||
---
|
||||
|
||||
### Real v0.18 SQL dump for upgrade scenario
|
||||
**Priority:** P2
|
||||
|
||||
**What:** The `upgrade-from-v0.18` scenario ships scaffolded — `seed/dump.sql` is missing. The harness gracefully no-ops the seed phase when absent, so the scenario currently behaves like fresh-install. v1.1: generate a real v0.18-shape PGLite dump per the procedure documented in `test/fixtures/claw-test-scenarios/upgrade-from-v0.18/seed/README.md`.
|
||||
|
||||
**Why:** Without a real seed, the scenario doesn't actually exercise the migration chain forward-walk. That's the whole point of the upgrade scenario — proves issue #239/#243/#266/#357 class regressions stay fixed.
|
||||
|
||||
**Effort:** S (CC ~30m once a v0.18 checkout is handy). Depends on: ability to run a v0.18 gbrain build.
|
||||
|
||||
---
|
||||
|
||||
### Public scoreboard — `gbrain-evals.io/friction`
|
||||
**Priority:** P3
|
||||
|
||||
**What:** Sibling-repo PR in `garrytan/gbrain-evals` that renders friction JSONL into a public dashboard. Friction count per version per agent, line charts over time. v1's JSONL already includes `gbrain_version` + `agent` tags so the scoreboard is a thin layer on top.
|
||||
|
||||
**Why:** Marketing surface. Proves install quality is improving release-over-release. The friction loop becomes visible to the world, not just maintainers.
|
||||
|
||||
**Effort:** M. Depends on: a working live mode and ≥10 real friction reports.
|
||||
|
||||
---
|
||||
|
||||
### PTY-mode transcript capture
|
||||
**Priority:** P3
|
||||
|
||||
**What:** `transcript-capture.ts` currently uses plain `child_process.spawn` pipes. Some agents only emit ANSI colors / progress UI on a TTY. v1.1 adds a PTY mode (likely via `node-pty`) so live-mode transcripts capture the full agent UX.
|
||||
|
||||
**Why:** Faithful transcripts make the friction → reasoning link more useful. v1 accepts that some agent UI is lost.
|
||||
|
||||
**Effort:** S (CC ~30m). Mostly a ~30 LOC swap inside `spawnWithCapture`.
|
||||
|
||||
---
|
||||
|
||||
### Read-side host-isolation (`$GBRAIN_HOST_HOME`)
|
||||
**Priority:** P3
|
||||
|
||||
**What:** v0.22.16 confined every `~/.gbrain` write site to honor `$GBRAIN_HOME`. But `src/commands/init.ts:299-313` still reads real `~/.claude` / `~/.openclaw` / `~/.codex` / `~/.factory` / `~/.kiro` for module fingerprinting (host detection). Even with write-isolation, a claw-test running on a developer's box discovers their real installed mods. v1.1: add a separate `$GBRAIN_HOST_HOME` override for the read-side detection so the claw-test can run truly hermetic.
|
||||
|
||||
**Why:** v1's hermeticity contract is "writes are isolated, reads are not." v1.1 closes the read-side gap.
|
||||
|
||||
**Effort:** S (CC ~30m).
|
||||
|
||||
---
|
||||
|
||||
### Routing-callout sweep — annotate skills the claw-test exercises
|
||||
**Priority:** P3
|
||||
|
||||
**What:** `skills/_friction-protocol.md` is a cross-cutting convention. v1.1: sweep the 4–6 skills the claw-test actually exercises (setup, brain-ops, query, ingest, smoke-test, the migrations the test covers) and add a `> **Convention:** see [skills/_friction-protocol.md](_friction-protocol.md).` callout via the existing `src/core/dry-fix.ts` shape so DRY auto-fix doesn't fight it.
|
||||
|
||||
**Why:** Right now agents only call `gbrain friction log` if they find the protocol skill on their own. The callouts route them there proactively from any harness-exercised skill.
|
||||
|
||||
**Effort:** S (CC ~15m).
|
||||
|
||||
---
|
||||
|
||||
## minions / worker (v0.22.14 follow-ups)
|
||||
|
||||
### v0.22.15 — Embed cooperative-abort (HIGHEST PRIORITY — daily pain)
|
||||
**Priority:** P0
|
||||
|
||||
**What:** Plumb `signal: AbortSignal` through `runPhaseEmbed` →
|
||||
`src/commands/embed.ts` → `embedBatch` in `src/core/embedding.ts`. Check
|
||||
`signal?.aborted` between OpenAI batch calls (every ~100 texts, ~2s
|
||||
real-time) and between slugs in the per-slug loop.
|
||||
|
||||
**Why:** Embed phase ignores `signal.aborted` between batches today. Job
|
||||
wall-clock timeout fires → handler keeps running → cycle's finally block
|
||||
unreachable → `gbrain_cycle_locks` row stays held indefinitely. Every
|
||||
subsequent autopilot cron cycle sees `cycle_already_running` → skips. Lock
|
||||
TTL is 30 min; new cycles give up before that. Doctor reports UNHEALTHY.
|
||||
|
||||
**The chain in production:** ~5min cron submits cycle → 22K stale pages →
|
||||
embed phase takes 10–15 min → 600s timeout fires → job dead-lettered → embed
|
||||
keeps running → lock held → all subsequent cycles skip. Garry hits this
|
||||
DAILY on his production brain.
|
||||
|
||||
**Pros:** Closes the daily wedge. Makes timeouts actually effective. Lets
|
||||
operators bump worker timeouts confidently knowing abort actually stops
|
||||
work.
|
||||
|
||||
**Cons:** Touching the embed hot path; small risk of botching the abort
|
||||
checks. Mitigation: between-batch granularity (~2s), not per-text (too fine)
|
||||
or per-slug (too coarse for 500+ chunk slugs).
|
||||
|
||||
**Context:** PR #503 (v0.22.14) catches the SYMPTOM (worker stalled, queue
|
||||
piling up) via self-health-monitoring. This PR catches the CAUSE for one
|
||||
specific failure class. Both fixes are needed; they're complementary, not
|
||||
duplicative.
|
||||
|
||||
**Files to touch:**
|
||||
- `src/core/cycle.ts:579` — `runPhaseEmbed(engine, dryRun)` → add
|
||||
`signal?: AbortSignal` arg
|
||||
- `src/core/cycle.ts:803` — pass `opts.signal` through
|
||||
- `src/commands/embed.ts:~363` — accept signal, check between slugs
|
||||
- `src/core/embedding.ts:51-56` — `embedBatch(texts, onProgress?, signal?)`,
|
||||
check between for-loop iterations of `BATCH_SIZE` slices
|
||||
|
||||
**Tests required:**
|
||||
1. embedBatch checks signal between OpenAI calls; aborts within one batch (~2s)
|
||||
2. Per-slug loop in `embed.ts` checks signal between slugs
|
||||
3. End-to-end: cycle handler with embed phase + signal aborted mid-flight →
|
||||
finally runs → `gbrain_cycle_locks` row deleted
|
||||
4. Regression: 1K+ chunks scenario — embed does NOT block lock release when
|
||||
timeout fires
|
||||
|
||||
**Effort:** M (human: ~3 hr / CC: ~30 min).
|
||||
|
||||
**Depends on / blocked by:** Nothing. v0.22.14 ships first.
|
||||
|
||||
### v0.23+ — Bare-worker engine reconnect parity with supervisor
|
||||
**Priority:** P2
|
||||
|
||||
**What:** Extract the supervisor's reconnect-then-fail pattern into
|
||||
`MinionWorker` so bare workers can retry transient DB blips before exiting.
|
||||
Today the supervisor calls `engine.reconnect()` after 3 consecutive DB health
|
||||
failures (#406); the bare worker just emits `'unhealthy'` and the CLI calls
|
||||
`process.exit(1)`.
|
||||
|
||||
**Why:** Bare-worker behavior is more disruptive than supervised behavior on
|
||||
transient PgBouncer blips. A bare worker restarts the entire process; a
|
||||
supervised worker just reconnects the pool. Operationally the supervisor
|
||||
approach is gentler (no in-flight job loss, no PM restart latency).
|
||||
|
||||
**Pros:** Unifies bare and supervised behavior. Reduces process churn on
|
||||
transient network blips.
|
||||
|
||||
**Cons:** More code in MinionWorker; risk of reconnect masking a real
|
||||
problem. Mitigation: cap retry attempts, fall through to `'unhealthy'`
|
||||
emission after the cap.
|
||||
|
||||
**Context:** Filed during v0.22.14 plan-eng-review. The asymmetry is
|
||||
documented in v0.22.14 CHANGELOG as deliberate; this TODO captures the
|
||||
"unify someday" intent.
|
||||
|
||||
**Effort:** S (human: ~2 hr / CC: ~20 min).
|
||||
|
||||
**Depends on / blocked by:** Nothing.
|
||||
|
||||
### v0.23+ — `minion_workers` heartbeat table for queue_health doctor (B7)
|
||||
**Priority:** P3
|
||||
|
||||
**What:** Add a `minion_workers` table (`worker_id` PK, `hostname`,
|
||||
`last_heartbeat`, `queue`, `concurrency`, `started_at`) so the existing
|
||||
`queue_health` doctor check (Postgres path) can detect dead workers via
|
||||
heartbeat staleness instead of relying on the indirect `lock_until` proxy.
|
||||
|
||||
**Why:** v0.19.1 added `queue_health` checks for stalled-active jobs and
|
||||
waiting-depth threshold. The worker-heartbeat subcheck was deferred (B7)
|
||||
because the `lock_until`-on-active-jobs proxy can't distinguish "worker
|
||||
exited cleanly" from "worker idle" — a check that cries wolf erodes trust
|
||||
in every doctor check. With a real heartbeat row, doctor can say "no worker
|
||||
seen in N intervals" with confidence.
|
||||
|
||||
**Pros:** Doctor's `queue_health` becomes ground-truth. Detects "worker
|
||||
container died but cron didn't restart it" scenario.
|
||||
|
||||
**Cons:** New table, schema migration, every health-tick UPSERTs. Costs
|
||||
a write per worker per minute (default).
|
||||
|
||||
**Context:** Filed during v0.22.14 plan-eng-review. PR #503's self-health
|
||||
monitoring is the worker-side liveness; this would be the queue-side
|
||||
ground-truth.
|
||||
|
||||
**Effort:** M (human: ~1 day / CC: ~1 hr).
|
||||
|
||||
**Depends on / blocked by:** Schema migration system; nothing else.
|
||||
|
||||
## sync (v0.22.13 follow-up — PR #490 review)
|
||||
|
||||
### D-PR490-1 — Plumb resolved `database_url` through `SyncOpts`
|
||||
**Priority:** P3
|
||||
|
||||
**What:** Add `database_url?: string` (or a richer `resolvedConnection` shape) to
|
||||
`SyncOpts` and have the caller (`runSync`, the cycle handler, the jobs handler)
|
||||
populate it from the active engine instead of having `performSync` /
|
||||
`performFullSync` / `import.ts` each call `loadConfig()` separately. Today every
|
||||
sync run hits the config file three times.
|
||||
|
||||
**Why:** v0.18 multi-source brains can in principle run different sources against
|
||||
different `database_url` endpoints (or different per-source overrides via
|
||||
`sources.config_jsonb`). Right now `loadConfig()` returns the global config, and
|
||||
that always matches the engine in practice — but the convention papers over a
|
||||
real divergence the moment someone wants per-source connection settings. Folding
|
||||
the resolution into `SyncOpts` makes the worker-engine creation in `sync.ts` and
|
||||
`import.ts` deterministic from `SyncOpts` alone.
|
||||
|
||||
**Pros:**
|
||||
- Removes 3 redundant `loadConfig()` calls per sync.
|
||||
- Makes `performSync` / `performFullSync` side-effect-free with respect to the
|
||||
on-disk config file.
|
||||
- Sets up for per-source `database_url` overrides without further refactor.
|
||||
- Makes the v0.22.13 belt-and-suspenders fallback (PR #490 Q3) cleaner — no
|
||||
more `!config?.database_url` short-circuit inside the parallel branch.
|
||||
|
||||
**Cons:**
|
||||
- API-shape change to `SyncOpts` (mild; not externally exported).
|
||||
- Touching three callers (`runSync`, jobs handler, `cycle.ts` `runPhaseSync`).
|
||||
- Only worth doing when paired with a per-source override story; otherwise
|
||||
it's just plumbing.
|
||||
|
||||
**Context:** Surfaced during the PR #490 plan-eng-review (parallel sync).
|
||||
Deferred because it isn't on the v0.22.13 critical path. The same pattern would
|
||||
benefit the cycle handler and the autopilot daemon. See the plan-eng-review
|
||||
decisions log: A4 = "Defer; file as TODO."
|
||||
|
||||
**Depends on / blocked by:** Nothing structural. Best paired with the v0.18
|
||||
per-source `config_jsonb` work if/when that lands.
|
||||
|
||||
## sync error-code classification (PR #501 follow-ups)
|
||||
|
||||
### Plumb structured `ParseValidationCode` through `ImportResult`
|
||||
@@ -220,6 +647,18 @@ keeping both skills' triggers intact for chaining.
|
||||
|
||||
**Depends on / blocked by:** Nothing — UNION-on-read path keeps unresolved edges surfaced even without this.
|
||||
|
||||
## P3 — Dev experience: test suite parallelism on fast multi-core machines
|
||||
|
||||
**Context:** `bun test` on M-series Macs spawns ~1 worker per core. `test/dream.test.ts` (5 describe blocks, 11 tests) and `test/orphans.test.ts` create a fresh PGLite engine in `beforeEach` that runs ~20 schema migrations per test. Under parallel load, WASM-instance contention causes ~18 `beforeEach` timeouts at 5–9s.
|
||||
|
||||
**Evidence:** CI (ubuntu-latest, fewer cores) is green on every PR. Running the suspect files in isolation (`bun test test/dream.test.ts test/orphans.test.ts`) is also green. Reproduces only on fast multi-core local machines running the full 136-file parallel suite.
|
||||
|
||||
**Fix:** move engine creation from `beforeEach` to `beforeAll` per describe block; add a data-reset helper (delete-all-rows-in-relevant-tables) between tests. ~80 LOC change across two test files.
|
||||
|
||||
**Priority:** P3 because production CI is unaffected. Hits local dev iteration speed on fast Macs.
|
||||
|
||||
**Found:** 2026-04-24 during v0.19.0 production-readiness review.
|
||||
|
||||
## Completed
|
||||
|
||||
### ~~Checks 5 + 6 for check-resolvable~~
|
||||
@@ -352,6 +791,21 @@ iteration's residuals.
|
||||
|
||||
## P0
|
||||
|
||||
### PGLite test-runner concurrency flake (~27 false failures in full `bun test`)
|
||||
**What:** Fix the concurrent-PGLite-init flake that surfaces ~27 `error: PGLite not connected. Call connect() first.` failures when `bun test` runs all 174 unit-test files together. Each failing file passes in isolation; failures only appear under full-suite parallelism.
|
||||
|
||||
**Why:** The failures are masking real signal. /ship and any solo dev running `bun test` has to manually triage 27 results every time. Today they're all in `test/cathedral-ii-pglite.test.ts`, `test/cathedral-ii-brainbench.test.ts` (Layer 5/6/7/8 + parent_scope_coverage + call_graph_recall), `test/sync.test.ts` (4 dry-run cases), `test/reindex-code.test.ts` (Layer 13 E2). All exist on master and date back to v0.12.3-v0.21.0 — pre-existing, not caused by any one branch.
|
||||
|
||||
**Context:** Confirmed pre-existing on master via `git diff origin/master...HEAD --stat -- <failing files>` returning empty. Tests pass cleanly in 1-3-file batches. Wall clock for the full suite is 596s. Likely root causes: (a) PGLite has a singleton or shared OPFS-like state that races under parallel `PGlite.create()` calls, (b) `test/cathedral-ii-pglite.test.ts` "fresh-install schema" tests assume exclusive PGLite access, (c) bun test concurrency exceeds what PGLite's WASM init can handle.
|
||||
|
||||
**Pros:** Green suite signal. Faster shipping. Stops eroding trust in `bun test`.
|
||||
|
||||
**Cons:** Likely needs PGLite engine-per-test isolation (each test gets its own dedicated engine instance via tmpdir) or a `bun test --concurrency=N` cap. Both touch test infra used by 50+ files.
|
||||
|
||||
**Effort:** M (human: 1 day to root-cause + implement / CC: ~2-3 hours via /investigate).
|
||||
|
||||
**Discovered:** v0.25.0 ship, 2026-04-25.
|
||||
|
||||
### Fix `bun build --compile` WASM embedding for PGLite
|
||||
**What:** Submit PR to oven-sh/bun fixing WASM file embedding in `bun build --compile` (issue oven-sh/bun#15032).
|
||||
|
||||
@@ -365,19 +819,6 @@ iteration's residuals.
|
||||
|
||||
**Depends on:** PGLite engine shipping (to have a real use case for the PR).
|
||||
|
||||
### ChatGPT MCP support (OAuth 2.1)
|
||||
**What:** Add OAuth 2.1 with Dynamic Client Registration to the self-hosted MCP server so ChatGPT can connect.
|
||||
|
||||
**Why:** ChatGPT requires OAuth 2.1 for MCP connectors. Bearer token auth is NOT supported. This is the only major AI client that can't use GBrain remotely.
|
||||
|
||||
**Pros:** Completes the "every AI client" promise. ChatGPT has the largest user base.
|
||||
|
||||
**Cons:** OAuth 2.1 is a significant implementation: authorization endpoint, token endpoint, PKCE flow, dynamic client registration. Estimated CC: ~3-4 hours.
|
||||
|
||||
**Context:** Discovered during DX review (2026-04-10). All other clients (Claude Desktop/Code/Cowork, Perplexity) work with bearer tokens. The Edge Function deployment was removed in v0.8.0. OAuth needs to be added to the self-hosted HTTP MCP server (or `gbrain serve --http` when implemented).
|
||||
|
||||
**Depends on:** `gbrain serve --http` (not yet implemented).
|
||||
|
||||
### Runtime MCP access control
|
||||
**What:** Add sender identity checking to MCP operations. Brain ops return filtered data based on access tier (Full/Work/Family/None).
|
||||
|
||||
@@ -391,11 +832,47 @@ iteration's residuals.
|
||||
|
||||
**Depends on:** v0.10.0 GStackBrain skill layer (shipped).
|
||||
|
||||
## P1 (new from v0.25.0 — eval-capture adversarial review)
|
||||
|
||||
### v0.25.0 eval-capture follow-ups (6 surgical hardenings)
|
||||
**Priority:** P1
|
||||
|
||||
**What:** Six targeted hardenings on the v0.25.0 eval-capture surface, all surfaced by the /ship adversarial review and triaged out of the v0.25.0 PR to keep scope tight:
|
||||
|
||||
1. `gbrain eval prune --dry-run`: replace the `listEvalCandidates(limit:100k) + filter` count with a real `engine.countEvalCandidatesBefore(date)` method. Today the warning at `eval-prune.ts:107-109` honestly tells the user the count may be undercounted, but a brain with > 100k rows + old data could still confuse a careful operator. New `BrainEngine` method on both engines, ~30 LOC, lifts the floor count to a true count.
|
||||
2. PII scrubber CC false-positive rate: 16-digit Luhn-valid order IDs / invoice numbers get redacted as `[REDACTED]`. Either require a contextual prefix (`card`, `cc`, `credit`) within N chars, or document the tradeoff explicitly in `docs/eval-capture.md`. The two approaches differ in coverage so list them as alternatives.
|
||||
3. `eval_capture_failures.reason` enum: `'scrubber_exception'` is dead telemetry — no realistic path emits it (the scrubber is regex-only and never throws). Either remove the value from the schema CHECK + enum, OR wrap `scrubPii` in a try-catch inside `buildEvalCandidateInput` so the value is actually reachable.
|
||||
4. `id DESC` tiebreaker docs: CLAUDE.md says "stable id-desc tiebreaker so `--since` windows never dupe/miss rows". This is true within a single call but doesn't prevent dupe/miss across overlapping windows when LIMIT < total. Either add a real `id`-cursor (`WHERE id < $cursor`) for export, or scope the doc claim to "within a single export call".
|
||||
5. Public-exports canaries: 6 of 17 subpaths (`gbrain` root, `/minions`, `/engine-factory`, `/transcription`, `/backoff`, `/extract`) have `canary: []` — the test only checks the import resolves, so a barrel module accidentally losing its named exports would still pass. Pin one stable canary symbol per subpath.
|
||||
6. `EXPECTED_COUNT` duplication: `scripts/check-exports-count.sh` and `test/public-exports.test.ts` both hardcode `17`. Drift risk. Make one read the other (or both compute from `package.json`).
|
||||
|
||||
**Why:** All 6 are real (some informational, some footgun-class) but each is small and surgical. Bundling into one v0.25.1 follow-up PR keeps the v0.25.0 ship clean and lets the fixes land with their own dedicated tests + CHANGELOG entry.
|
||||
|
||||
**Effort:** S total (human: ~half day / CC: ~1.5 hours).
|
||||
|
||||
**Discovered:** v0.25.0 ship adversarial review, 2026-04-25.
|
||||
|
||||
## P1 (new from v0.7.0)
|
||||
|
||||
### ~~Constrained health_check DSL for third-party recipes~~
|
||||
**Completed:** v0.9.3 (2026-04-12). Typed DSL with 4 check types (`http`, `env_exists`, `command`, `any_of`). All 7 first-party recipes migrated. String health checks accepted with deprecation warning + metachar validation for non-embedded recipes.
|
||||
|
||||
## P1 (new from v0.18.0 — test flakiness)
|
||||
|
||||
### beforeAll hook timeouts under parallel test runner
|
||||
**What:** 17 tests across 9 files (dream, orphans, brain-allowlist, extract-db, multi-source-integration, core/cycle, migrations-v0_12_2, migrations-v0_13_1, oauth) fail with `beforeEach/afterEach hook timed out for this test` at the 7-10 second threshold when run via `bun run test` (parallel). Every test passes in isolation (`bun test path/to/file.test.ts` → 0 fail). Root cause is PGLite schema init racing under concurrent test files.
|
||||
|
||||
**Why:** `bun run test` is the pre-ship gate and reports these as failures, forcing manual triage on every /ship. The tests themselves are correct — the runner is stressing PGLite boot. Bumping the hook timeout or running E2E-like tests with `--bail` or serial execution would clear the 18 false positives.
|
||||
|
||||
**Fix options:**
|
||||
1. Bump per-test hook timeout to 30s in `bunfig.toml` (quick fix, low risk)
|
||||
2. Move PGLite-init-heavy tests to `test/e2e/` so they run serially via `scripts/run-e2e.sh` (follows existing pattern)
|
||||
3. Share a module-scoped PGLite instance across describe blocks within a file (biggest win — most fixture setup is identical)
|
||||
|
||||
**Effort:** 30 min for option 1, ~2 hours for option 3.
|
||||
|
||||
**Context:** Noticed during /ship merge wave on `garrytan/mcp-key-mgmt` (2026-04-16 branch merge of v0.18.0). Failure set stayed exactly 17-18 tests across multiple /ship runs, confirming deterministic flakes rather than real regressions. Blocking workaround: run the specific test file to verify after any suite change.
|
||||
|
||||
## P1 (new from v0.11.0 — Minions)
|
||||
|
||||
### Per-queue rate limiting for Minions
|
||||
@@ -626,6 +1103,9 @@ iteration's residuals.
|
||||
|
||||
## Completed
|
||||
|
||||
### ChatGPT MCP support (OAuth 2.1)
|
||||
**Completed:** v0.26.0 (2026-04-25) — `gbrain serve --http` ships full OAuth 2.1 via MCP SDK's `mcpAuthRouter` + `OAuthServerProvider`. Authorization code flow with PKCE unblocks ChatGPT. Client credentials flow unblocks Perplexity/Claude. Dynamic Client Registration available behind `--enable-dcr` flag (off by default). See `docs/mcp/CHATGPT.md` for connector setup. Closed the P0 that had been blocking the "every AI client" promise since v0.6.
|
||||
|
||||
### Implement AWS Signature V4 for S3 storage backend
|
||||
**Completed:** v0.6.0 (2026-04-10) — replaced with @aws-sdk/client-s3 for proper SigV4 signing.
|
||||
|
||||
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
# Design System — GBrain Admin Dashboard
|
||||
|
||||
## Product Context
|
||||
- **What this is:** Admin dashboard for GBrain MCP server — manage OAuth agents, API keys, monitor requests
|
||||
- **Who it's for:** GBrain operators managing multi-agent access to their brain
|
||||
- **Space/industry:** Developer infrastructure (peers: Supabase dashboard, Vercel, Railway)
|
||||
- **Project type:** Dense utilitarian admin panel — Steve Krug "Don't Make Me Think"
|
||||
|
||||
## Aesthetic Direction
|
||||
- **Direction:** Industrial/Utilitarian — function-first, data-dense, zero decoration
|
||||
- **Decoration level:** None — every pixel earns its place with information
|
||||
- **Mood:** Ops dashboard for someone who builds. Not a marketing site. Not a consumer app. A cockpit.
|
||||
- **Reference:** Supabase dashboard (dark + dense), Linear (restrained), Grafana (data-forward)
|
||||
|
||||
## Alignment
|
||||
- **Text alignment:** Left-align everything. No centered text in tables, cards, forms, or labels.
|
||||
- **Headings:** Left-aligned
|
||||
- **Table data:** Left-aligned (including numbers — contextual readability over columnar alignment)
|
||||
- **Form labels:** Left-aligned above inputs
|
||||
- **Buttons in forms:** Right-aligned (action flows left-to-right: Cancel → Submit)
|
||||
- **Modal titles:** Left-aligned
|
||||
- **Page titles:** Left-aligned
|
||||
- **Only exception:** Empty states and the login page lock icon can center for visual weight
|
||||
|
||||
## Typography
|
||||
- **Display/Headings:** Inter (Semibold 600) — clean, neutral, disappears into the content
|
||||
- **Body/UI:** Inter (Regular 400 / Medium 500)
|
||||
- **Data/Tables/Code:** JetBrains Mono (Regular 400 / Medium 500) — monospace for anything the user might copy, any ID, any token, any technical value
|
||||
- **Loading:** Google Fonts. `display=swap`.
|
||||
- **Scale:**
|
||||
- Page title: 24px / Inter Semibold
|
||||
- Section title: 14px / Inter Semibold, uppercase, letter-spacing 0.5px
|
||||
- Table header: 12px / Inter Medium, uppercase, letter-spacing 1px, muted color
|
||||
- Body: 14px / Inter Regular
|
||||
- Small/Caption: 13px
|
||||
- Micro: 12px (badges, timestamps)
|
||||
- Code/Data: 13px / JetBrains Mono
|
||||
|
||||
## Color
|
||||
- **Approach:** Monochrome base + semantic color only. No primary brand color. Color means something.
|
||||
- **Background:**
|
||||
- Base: #0a0a0f (near-black with blue undertone)
|
||||
- Surface/cards: #12121a
|
||||
- Hover: #1a1a2a
|
||||
- Input/code blocks: #0f0f1a
|
||||
- **Borders:** #1e1e2e (default), #3a3a5a (hover/active)
|
||||
- **Text:**
|
||||
- Primary: #e0e0e0
|
||||
- Secondary: #888888
|
||||
- Muted: #555555
|
||||
- Link: #88aaff
|
||||
- **Semantic (badges only):**
|
||||
- Success/active: #34a853
|
||||
- Error/danger: #ff6b6b
|
||||
- Warning: #f5a623
|
||||
- Read scope: #3b82f6
|
||||
- Write scope: #f59e0b
|
||||
- Admin scope: #ef4444
|
||||
- **No accent color.** The data IS the interface. Badges carry all the color.
|
||||
|
||||
## Spacing
|
||||
- **Base unit:** 4px
|
||||
- **Density:** Dense — this is an ops tool, not a landing page
|
||||
- **Scale:** 4px, 8px, 12px, 16px, 20px, 24px, 32px, 48px
|
||||
- **Table row padding:** 10px 16px
|
||||
- **Card padding:** 24px
|
||||
- **Modal padding:** 24px
|
||||
- **Section gaps:** 24px between sections, 12px between related elements
|
||||
|
||||
## Layout
|
||||
- **Sidebar:** Fixed left, 200px wide, dark (#0a0a0f)
|
||||
- **Main content:** Fluid, max-width none (fills available space)
|
||||
- **Grid:** Single column for tables (full width), 2-column for stats cards
|
||||
- **Border radius:**
|
||||
- Cards/panels: 16px
|
||||
- Buttons/inputs: 8px
|
||||
- Badges: 9999px (pill)
|
||||
- Tables: 0 (sharp edges — data is rectangular)
|
||||
|
||||
## Components
|
||||
|
||||
### Tables
|
||||
- Full-width, no outer border
|
||||
- Header row: uppercase, letter-spaced, muted color, no background
|
||||
- Data rows: subtle hover (#1a1a2a), pointer cursor when clickable
|
||||
- All text left-aligned
|
||||
- Monospace for IDs, tokens, latency values
|
||||
|
||||
### Badges
|
||||
- Pill shape (border-radius: 9999px)
|
||||
- Padding: 2px 8px
|
||||
- Font: 12px
|
||||
- Scoped to semantic meaning: `success`, `danger`, `read`, `write`, `admin`
|
||||
|
||||
### Buttons
|
||||
- Primary: white text on #3a3a5a, hover brightens
|
||||
- Secondary: muted text on transparent, border #1e1e2e
|
||||
- Danger: white text on #ff6b6b background
|
||||
- Size: 13px font, 6px 14px padding
|
||||
|
||||
### Modals
|
||||
- Overlay: rgba(0,0,0,0.7)
|
||||
- Card: #12121a, border #1e1e2e, border-radius 16px, max-width 480px
|
||||
- Title: 18px Semibold, left-aligned
|
||||
- Close: top-right ✕ button
|
||||
|
||||
### Drawers
|
||||
- Right-side panel, 400px wide
|
||||
- Slide in from right
|
||||
- Dark overlay behind
|
||||
- Close button top-right
|
||||
- Sections separated by section titles (uppercase, muted)
|
||||
|
||||
### Tabs
|
||||
- Inline horizontal, wrapping allowed
|
||||
- Active: white text, bottom border
|
||||
- Inactive: muted text, no border
|
||||
- No background color on tabs
|
||||
|
||||
### Code blocks
|
||||
- Background: rgba(0,0,0,0.3)
|
||||
- Border-radius: 8px
|
||||
- Padding: 10px 14px
|
||||
- Font: JetBrains Mono 12px
|
||||
- Copy button: right-aligned, subtle
|
||||
|
||||
### Empty states
|
||||
- Centered text (only exception to left-align rule)
|
||||
- Muted color
|
||||
- Suggest next action
|
||||
|
||||
## Motion
|
||||
- **Approach:** Minimal — transitions for hover states only
|
||||
- **Duration:** 150ms for hovers, 200ms for drawer slide
|
||||
- **No loading spinners** — show stale data until fresh arrives
|
||||
- **SSE live feed:** Real-time, no animation on new entries (just prepend)
|
||||
|
||||
## Anti-Patterns (do NOT do these)
|
||||
- ❌ Center-aligned table data
|
||||
- ❌ Center-aligned headings or labels (except empty states)
|
||||
- ❌ Gradient backgrounds
|
||||
- ❌ Shadows (the dark theme IS the depth model)
|
||||
- ❌ Rounded table corners
|
||||
- ❌ Icons as navigation (use text labels)
|
||||
- ❌ Loading skeletons (show real data or nothing)
|
||||
- ❌ Confirmation toasts (action → result is immediate and visible)
|
||||
- ❌ Color for decoration (every color means something)
|
||||
|
||||
## Decisions Log
|
||||
| Date | Decision | Rationale |
|
||||
|------|----------|-----------|
|
||||
| 2026-05-01 | Dark theme only | Ops dashboard. No light mode needed. |
|
||||
| 2026-05-01 | Steve Krug lens | Zero happy talk, mindless choices, scannable tables, billboard-speed comprehension. |
|
||||
| 2026-05-01 | JetBrains Mono for data | Anything copyable or technical should be monospace. |
|
||||
| 2026-05-03 | Left-align everything | Garry preference. Centered text is a design crutch. Left-align forces hierarchy through typography weight and spacing, not position. |
|
||||
| 2026-05-03 | Incorporate GStack design DNA | Same family: Inter + JetBrains Mono, dark base, semantic-only color. Diverges on accent (GStack: amber; GBrain: none — data is the color). |
|
||||
| 2026-05-03 | Per-client config export tabs | Claude Code, ChatGPT, Claude.ai, Cursor, Perplexity, JSON. Every agent has a copy-paste setup path. |
|
||||
| 2026-05-03 | Magic link auth | Login page tells you to ask your agent. No pasting hex strings into forms. |
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"configVersion": 1,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "gbrain-admin",
|
||||
"dependencies": {
|
||||
"react": "^19.1.0",
|
||||
"react-dom": "^19.1.0",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.1.2",
|
||||
"@types/react-dom": "^19.1.2",
|
||||
"@vitejs/plugin-react": "^4.4.1",
|
||||
"typescript": "^5.8.3",
|
||||
"vite": "^6.3.3",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@babel/code-frame": ["@babel/code-frame@7.29.0", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw=="],
|
||||
|
||||
"@babel/compat-data": ["@babel/compat-data@7.29.0", "", {}, "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg=="],
|
||||
|
||||
"@babel/core": ["@babel/core@7.29.0", "", { "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", "@babel/helper-compilation-targets": "^7.28.6", "@babel/helper-module-transforms": "^7.28.6", "@babel/helpers": "^7.28.6", "@babel/parser": "^7.29.0", "@babel/template": "^7.28.6", "@babel/traverse": "^7.29.0", "@babel/types": "^7.29.0", "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", "json5": "^2.2.3", "semver": "^6.3.1" } }, "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA=="],
|
||||
|
||||
"@babel/generator": ["@babel/generator@7.29.1", "", { "dependencies": { "@babel/parser": "^7.29.0", "@babel/types": "^7.29.0", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" } }, "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw=="],
|
||||
|
||||
"@babel/helper-compilation-targets": ["@babel/helper-compilation-targets@7.28.6", "", { "dependencies": { "@babel/compat-data": "^7.28.6", "@babel/helper-validator-option": "^7.27.1", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" } }, "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA=="],
|
||||
|
||||
"@babel/helper-globals": ["@babel/helper-globals@7.28.0", "", {}, "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw=="],
|
||||
|
||||
"@babel/helper-module-imports": ["@babel/helper-module-imports@7.28.6", "", { "dependencies": { "@babel/traverse": "^7.28.6", "@babel/types": "^7.28.6" } }, "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw=="],
|
||||
|
||||
"@babel/helper-module-transforms": ["@babel/helper-module-transforms@7.28.6", "", { "dependencies": { "@babel/helper-module-imports": "^7.28.6", "@babel/helper-validator-identifier": "^7.28.5", "@babel/traverse": "^7.28.6" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA=="],
|
||||
|
||||
"@babel/helper-plugin-utils": ["@babel/helper-plugin-utils@7.28.6", "", {}, "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug=="],
|
||||
|
||||
"@babel/helper-string-parser": ["@babel/helper-string-parser@7.27.1", "", {}, "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="],
|
||||
|
||||
"@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="],
|
||||
|
||||
"@babel/helper-validator-option": ["@babel/helper-validator-option@7.27.1", "", {}, "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg=="],
|
||||
|
||||
"@babel/helpers": ["@babel/helpers@7.29.2", "", { "dependencies": { "@babel/template": "^7.28.6", "@babel/types": "^7.29.0" } }, "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw=="],
|
||||
|
||||
"@babel/parser": ["@babel/parser@7.29.2", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA=="],
|
||||
|
||||
"@babel/plugin-transform-react-jsx-self": ["@babel/plugin-transform-react-jsx-self@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw=="],
|
||||
|
||||
"@babel/plugin-transform-react-jsx-source": ["@babel/plugin-transform-react-jsx-source@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw=="],
|
||||
|
||||
"@babel/template": ["@babel/template@7.28.6", "", { "dependencies": { "@babel/code-frame": "^7.28.6", "@babel/parser": "^7.28.6", "@babel/types": "^7.28.6" } }, "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ=="],
|
||||
|
||||
"@babel/traverse": ["@babel/traverse@7.29.0", "", { "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", "@babel/helper-globals": "^7.28.0", "@babel/parser": "^7.29.0", "@babel/template": "^7.28.6", "@babel/types": "^7.29.0", "debug": "^4.3.1" } }, "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA=="],
|
||||
|
||||
"@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="],
|
||||
|
||||
"@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="],
|
||||
|
||||
"@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="],
|
||||
|
||||
"@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.12", "", { "os": "android", "cpu": "arm64" }, "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg=="],
|
||||
|
||||
"@esbuild/android-x64": ["@esbuild/android-x64@0.25.12", "", { "os": "android", "cpu": "x64" }, "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg=="],
|
||||
|
||||
"@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg=="],
|
||||
|
||||
"@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA=="],
|
||||
|
||||
"@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.12", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg=="],
|
||||
|
||||
"@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.12", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ=="],
|
||||
|
||||
"@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.12", "", { "os": "linux", "cpu": "arm" }, "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw=="],
|
||||
|
||||
"@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ=="],
|
||||
|
||||
"@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.12", "", { "os": "linux", "cpu": "ia32" }, "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA=="],
|
||||
|
||||
"@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng=="],
|
||||
|
||||
"@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw=="],
|
||||
|
||||
"@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.12", "", { "os": "linux", "cpu": "ppc64" }, "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA=="],
|
||||
|
||||
"@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w=="],
|
||||
|
||||
"@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.12", "", { "os": "linux", "cpu": "s390x" }, "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg=="],
|
||||
|
||||
"@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.12", "", { "os": "linux", "cpu": "x64" }, "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw=="],
|
||||
|
||||
"@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg=="],
|
||||
|
||||
"@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.12", "", { "os": "none", "cpu": "x64" }, "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ=="],
|
||||
|
||||
"@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.12", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A=="],
|
||||
|
||||
"@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.12", "", { "os": "openbsd", "cpu": "x64" }, "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw=="],
|
||||
|
||||
"@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg=="],
|
||||
|
||||
"@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.12", "", { "os": "sunos", "cpu": "x64" }, "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w=="],
|
||||
|
||||
"@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg=="],
|
||||
|
||||
"@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.12", "", { "os": "win32", "cpu": "ia32" }, "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ=="],
|
||||
|
||||
"@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="],
|
||||
|
||||
"@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="],
|
||||
|
||||
"@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="],
|
||||
|
||||
"@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="],
|
||||
|
||||
"@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="],
|
||||
|
||||
"@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="],
|
||||
|
||||
"@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-beta.27", "", {}, "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA=="],
|
||||
|
||||
"@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.60.1", "", { "os": "android", "cpu": "arm" }, "sha512-d6FinEBLdIiK+1uACUttJKfgZREXrF0Qc2SmLII7W2AD8FfiZ9Wjd+rD/iRuf5s5dWrr1GgwXCvPqOuDquOowA=="],
|
||||
|
||||
"@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.60.1", "", { "os": "android", "cpu": "arm64" }, "sha512-YjG/EwIDvvYI1YvYbHvDz/BYHtkY4ygUIXHnTdLhG+hKIQFBiosfWiACWortsKPKU/+dUwQQCKQM3qrDe8c9BA=="],
|
||||
|
||||
"@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.60.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-mjCpF7GmkRtSJwon+Rq1N8+pI+8l7w5g9Z3vWj4T7abguC4Czwi3Yu/pFaLvA3TTeMVjnu3ctigusqWUfjZzvw=="],
|
||||
|
||||
"@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.60.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-haZ7hJ1JT4e9hqkoT9R/19XW2QKqjfJVv+i5AGg57S+nLk9lQnJ1F/eZloRO3o9Scy9CM3wQ9l+dkXtcBgN5Ew=="],
|
||||
|
||||
"@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.60.1", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-czw90wpQq3ZsAVBlinZjAYTKduOjTywlG7fEeWKUA7oCmpA8xdTkxZZlwNJKWqILlq0wehoZcJYfBvOyhPTQ6w=="],
|
||||
|
||||
"@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.60.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-KVB2rqsxTHuBtfOeySEyzEOB7ltlB/ux38iu2rBQzkjbwRVlkhAGIEDiiYnO2kFOkJp+Z7pUXKyrRRFuFUKt+g=="],
|
||||
|
||||
"@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.60.1", "", { "os": "linux", "cpu": "arm" }, "sha512-L+34Qqil+v5uC0zEubW7uByo78WOCIrBvci69E7sFASRl0X7b/MB6Cqd1lky/CtcSVTydWa2WZwFuWexjS5o6g=="],
|
||||
|
||||
"@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.60.1", "", { "os": "linux", "cpu": "arm" }, "sha512-n83O8rt4v34hgFzlkb1ycniJh7IR5RCIqt6mz1VRJD6pmhRi0CXdmfnLu9dIUS6buzh60IvACM842Ffb3xd6Gg=="],
|
||||
|
||||
"@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.60.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-Nql7sTeAzhTAja3QXeAI48+/+GjBJ+QmAH13snn0AJSNL50JsDqotyudHyMbO2RbJkskbMbFJfIJKWA6R1LCJQ=="],
|
||||
|
||||
"@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.60.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-+pUymDhd0ys9GcKZPPWlFiZ67sTWV5UU6zOJat02M1+PiuSGDziyRuI/pPue3hoUwm2uGfxdL+trT6Z9rxnlMA=="],
|
||||
|
||||
"@rollup/rollup-linux-loong64-gnu": ["@rollup/rollup-linux-loong64-gnu@4.60.1", "", { "os": "linux", "cpu": "none" }, "sha512-VSvgvQeIcsEvY4bKDHEDWcpW4Yw7BtlKG1GUT4FzBUlEKQK0rWHYBqQt6Fm2taXS+1bXvJT6kICu5ZwqKCnvlQ=="],
|
||||
|
||||
"@rollup/rollup-linux-loong64-musl": ["@rollup/rollup-linux-loong64-musl@4.60.1", "", { "os": "linux", "cpu": "none" }, "sha512-4LqhUomJqwe641gsPp6xLfhqWMbQV04KtPp7/dIp0nzPxAkNY1AbwL5W0MQpcalLYk07vaW9Kp1PBhdpZYYcEw=="],
|
||||
|
||||
"@rollup/rollup-linux-ppc64-gnu": ["@rollup/rollup-linux-ppc64-gnu@4.60.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-tLQQ9aPvkBxOc/EUT6j3pyeMD6Hb8QF2BTBnCQWP/uu1lhc9AIrIjKnLYMEroIz/JvtGYgI9dF3AxHZNaEH0rw=="],
|
||||
|
||||
"@rollup/rollup-linux-ppc64-musl": ["@rollup/rollup-linux-ppc64-musl@4.60.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-RMxFhJwc9fSXP6PqmAz4cbv3kAyvD1etJFjTx4ONqFP9DkTkXsAMU4v3Vyc5BgzC+anz7nS/9tp4obsKfqkDHg=="],
|
||||
|
||||
"@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.60.1", "", { "os": "linux", "cpu": "none" }, "sha512-QKgFl+Yc1eEk6MmOBfRHYF6lTxiiiV3/z/BRrbSiW2I7AFTXoBFvdMEyglohPj//2mZS4hDOqeB0H1ACh3sBbg=="],
|
||||
|
||||
"@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.60.1", "", { "os": "linux", "cpu": "none" }, "sha512-RAjXjP/8c6ZtzatZcA1RaQr6O1TRhzC+adn8YZDnChliZHviqIjmvFwHcxi4JKPSDAt6Uhf/7vqcBzQJy0PDJg=="],
|
||||
|
||||
"@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.60.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-wcuocpaOlaL1COBYiA89O6yfjlp3RwKDeTIA0hM7OpmhR1Bjo9j31G1uQVpDlTvwxGn2nQs65fBFL5UFd76FcQ=="],
|
||||
|
||||
"@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.60.1", "", { "os": "linux", "cpu": "x64" }, "sha512-77PpsFQUCOiZR9+LQEFg9GClyfkNXj1MP6wRnzYs0EeWbPcHs02AXu4xuUbM1zhwn3wqaizle3AEYg5aeoohhg=="],
|
||||
|
||||
"@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.60.1", "", { "os": "linux", "cpu": "x64" }, "sha512-5cIATbk5vynAjqqmyBjlciMJl1+R/CwX9oLk/EyiFXDWd95KpHdrOJT//rnUl4cUcskrd0jCCw3wpZnhIHdD9w=="],
|
||||
|
||||
"@rollup/rollup-openbsd-x64": ["@rollup/rollup-openbsd-x64@4.60.1", "", { "os": "openbsd", "cpu": "x64" }, "sha512-cl0w09WsCi17mcmWqqglez9Gk8isgeWvoUZ3WiJFYSR3zjBQc2J5/ihSjpl+VLjPqjQ/1hJRcqBfLjssREQILw=="],
|
||||
|
||||
"@rollup/rollup-openharmony-arm64": ["@rollup/rollup-openharmony-arm64@4.60.1", "", { "os": "none", "cpu": "arm64" }, "sha512-4Cv23ZrONRbNtbZa37mLSueXUCtN7MXccChtKpUnQNgF010rjrjfHx3QxkS2PI7LqGT5xXyYs1a7LbzAwT0iCA=="],
|
||||
|
||||
"@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.60.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-i1okWYkA4FJICtr7KpYzFpRTHgy5jdDbZiWfvny21iIKky5YExiDXP+zbXzm3dUcFpkEeYNHgQ5fuG236JPq0g=="],
|
||||
|
||||
"@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.60.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-u09m3CuwLzShA0EYKMNiFgcjjzwqtUMLmuCJLeZWjjOYA3IT2Di09KaxGBTP9xVztWyIWjVdsB2E9goMjZvTQg=="],
|
||||
|
||||
"@rollup/rollup-win32-x64-gnu": ["@rollup/rollup-win32-x64-gnu@4.60.1", "", { "os": "win32", "cpu": "x64" }, "sha512-k+600V9Zl1CM7eZxJgMyTUzmrmhB/0XZnF4pRypKAlAgxmedUA+1v9R+XOFv56W4SlHEzfeMtzujLJD22Uz5zg=="],
|
||||
|
||||
"@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.60.1", "", { "os": "win32", "cpu": "x64" }, "sha512-lWMnixq/QzxyhTV6NjQJ4SFo1J6PvOX8vUx5Wb4bBPsEb+8xZ89Bz6kOXpfXj9ak9AHTQVQzlgzBEc1SyM27xQ=="],
|
||||
|
||||
"@types/babel__core": ["@types/babel__core@7.20.5", "", { "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", "@types/babel__generator": "*", "@types/babel__template": "*", "@types/babel__traverse": "*" } }, "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA=="],
|
||||
|
||||
"@types/babel__generator": ["@types/babel__generator@7.27.0", "", { "dependencies": { "@babel/types": "^7.0.0" } }, "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg=="],
|
||||
|
||||
"@types/babel__template": ["@types/babel__template@7.4.4", "", { "dependencies": { "@babel/parser": "^7.1.0", "@babel/types": "^7.0.0" } }, "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A=="],
|
||||
|
||||
"@types/babel__traverse": ["@types/babel__traverse@7.28.0", "", { "dependencies": { "@babel/types": "^7.28.2" } }, "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q=="],
|
||||
|
||||
"@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
|
||||
|
||||
"@types/react": ["@types/react@19.2.14", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w=="],
|
||||
|
||||
"@types/react-dom": ["@types/react-dom@19.2.3", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ=="],
|
||||
|
||||
"@vitejs/plugin-react": ["@vitejs/plugin-react@4.7.0", "", { "dependencies": { "@babel/core": "^7.28.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-beta.27", "@types/babel__core": "^7.20.5", "react-refresh": "^0.17.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA=="],
|
||||
|
||||
"baseline-browser-mapping": ["baseline-browser-mapping@2.10.18", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-VSnGQAOLtP5mib/DPyg2/t+Tlv65NTBz83BJBJvmLVHHuKJVaDOBvJJykiT5TR++em5nfAySPccDZDa4oSrn8A=="],
|
||||
|
||||
"browserslist": ["browserslist@4.28.2", "", { "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", "electron-to-chromium": "^1.5.328", "node-releases": "^2.0.36", "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg=="],
|
||||
|
||||
"caniuse-lite": ["caniuse-lite@1.0.30001788", "", {}, "sha512-6q8HFp+lOQtcf7wBK+uEenxymVWkGKkjFpCvw5W25cmMwEDU45p1xQFBQv8JDlMMry7eNxyBaR+qxgmTUZkIRQ=="],
|
||||
|
||||
"convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="],
|
||||
|
||||
"csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="],
|
||||
|
||||
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
||||
|
||||
"electron-to-chromium": ["electron-to-chromium@1.5.336", "", {}, "sha512-AbH9q9J455r/nLmdNZes0G0ZKcRX73FicwowalLs6ijwOmCJSRRrLX63lcAlzy9ux3dWK1w1+1nsBJEWN11hcQ=="],
|
||||
|
||||
"esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="],
|
||||
|
||||
"escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="],
|
||||
|
||||
"fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="],
|
||||
|
||||
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
|
||||
|
||||
"gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="],
|
||||
|
||||
"js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="],
|
||||
|
||||
"jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="],
|
||||
|
||||
"json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="],
|
||||
|
||||
"lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="],
|
||||
|
||||
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
|
||||
|
||||
"nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
|
||||
|
||||
"node-releases": ["node-releases@2.0.37", "", {}, "sha512-1h5gKZCF+pO/o3Iqt5Jp7wc9rH3eJJ0+nh/CIoiRwjRxde/hAHyLPXYN4V3CqKAbiZPSeJFSWHmJsbkicta0Eg=="],
|
||||
|
||||
"picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="],
|
||||
|
||||
"picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="],
|
||||
|
||||
"postcss": ["postcss@8.5.9", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-7a70Nsot+EMX9fFU3064K/kdHWZqGVY+BADLyXc8Dfv+mTLLVl6JzJpPaCZ2kQL9gIJvKXSLMHhqdRRjwQeFtw=="],
|
||||
|
||||
"react": ["react@19.2.5", "", {}, "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA=="],
|
||||
|
||||
"react-dom": ["react-dom@19.2.5", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.5" } }, "sha512-J5bAZz+DXMMwW/wV3xzKke59Af6CHY7G4uYLN1OvBcKEsWOs4pQExj86BBKamxl/Ik5bx9whOrvBlSDfWzgSag=="],
|
||||
|
||||
"react-refresh": ["react-refresh@0.17.0", "", {}, "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ=="],
|
||||
|
||||
"rollup": ["rollup@4.60.1", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.60.1", "@rollup/rollup-android-arm64": "4.60.1", "@rollup/rollup-darwin-arm64": "4.60.1", "@rollup/rollup-darwin-x64": "4.60.1", "@rollup/rollup-freebsd-arm64": "4.60.1", "@rollup/rollup-freebsd-x64": "4.60.1", "@rollup/rollup-linux-arm-gnueabihf": "4.60.1", "@rollup/rollup-linux-arm-musleabihf": "4.60.1", "@rollup/rollup-linux-arm64-gnu": "4.60.1", "@rollup/rollup-linux-arm64-musl": "4.60.1", "@rollup/rollup-linux-loong64-gnu": "4.60.1", "@rollup/rollup-linux-loong64-musl": "4.60.1", "@rollup/rollup-linux-ppc64-gnu": "4.60.1", "@rollup/rollup-linux-ppc64-musl": "4.60.1", "@rollup/rollup-linux-riscv64-gnu": "4.60.1", "@rollup/rollup-linux-riscv64-musl": "4.60.1", "@rollup/rollup-linux-s390x-gnu": "4.60.1", "@rollup/rollup-linux-x64-gnu": "4.60.1", "@rollup/rollup-linux-x64-musl": "4.60.1", "@rollup/rollup-openbsd-x64": "4.60.1", "@rollup/rollup-openharmony-arm64": "4.60.1", "@rollup/rollup-win32-arm64-msvc": "4.60.1", "@rollup/rollup-win32-ia32-msvc": "4.60.1", "@rollup/rollup-win32-x64-gnu": "4.60.1", "@rollup/rollup-win32-x64-msvc": "4.60.1", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-VmtB2rFU/GroZ4oL8+ZqXgSA38O6GR8KSIvWmEFv63pQ0G6KaBH9s07PO8XTXP4vI+3UJUEypOfjkGfmSBBR0w=="],
|
||||
|
||||
"scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="],
|
||||
|
||||
"semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="],
|
||||
|
||||
"source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
|
||||
|
||||
"tinyglobby": ["tinyglobby@0.2.16", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg=="],
|
||||
|
||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
"update-browserslist-db": ["update-browserslist-db@1.2.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w=="],
|
||||
|
||||
"vite": ["vite@6.4.2", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", "picomatch": "^4.0.2", "postcss": "^8.5.3", "rollup": "^4.34.9", "tinyglobby": "^0.2.13" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "jiti": ">=1.21.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ=="],
|
||||
|
||||
"yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="],
|
||||
}
|
||||
}
|
||||
Vendored
+1
File diff suppressed because one or more lines are too long
Vendored
+56
File diff suppressed because one or more lines are too long
Vendored
+16
@@ -0,0 +1,16 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>GBrain Admin</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
|
||||
<script type="module" crossorigin src="/admin/assets/index-DWYc55rS.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/admin/assets/index-BOifXQpQ.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,15 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>GBrain Admin</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "gbrain-admin",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"react": "^19.1.0",
|
||||
"react-dom": "^19.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.1.2",
|
||||
"@types/react-dom": "^19.1.2",
|
||||
"@vitejs/plugin-react": "^4.4.1",
|
||||
"vite": "^6.3.3",
|
||||
"typescript": "^5.8.3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { LoginPage } from './pages/Login';
|
||||
import { DashboardPage } from './pages/Dashboard';
|
||||
import { AgentsPage } from './pages/Agents';
|
||||
import { RequestLogPage } from './pages/RequestLog';
|
||||
import { api } from './api';
|
||||
|
||||
type Page = 'login' | 'dashboard' | 'agents' | 'log';
|
||||
|
||||
function getPage(): Page {
|
||||
const hash = window.location.hash.replace('#', '') || 'dashboard';
|
||||
if (['login', 'dashboard', 'agents', 'log'].includes(hash)) return hash as Page;
|
||||
return 'dashboard';
|
||||
}
|
||||
|
||||
export function App() {
|
||||
const [page, setPage] = useState<Page>(getPage);
|
||||
|
||||
useEffect(() => {
|
||||
const onHash = () => setPage(getPage());
|
||||
window.addEventListener('hashchange', onHash);
|
||||
return () => window.removeEventListener('hashchange', onHash);
|
||||
}, []);
|
||||
|
||||
const navigate = (p: Page) => {
|
||||
window.location.hash = p;
|
||||
setPage(p);
|
||||
};
|
||||
|
||||
if (page === 'login') {
|
||||
return <LoginPage onLogin={() => navigate('dashboard')} />;
|
||||
}
|
||||
|
||||
const handleSignOutEverywhere = async () => {
|
||||
if (!confirm('Sign out every active admin session, including other browsers and tabs? Each one will need to re-authenticate via a fresh magic link.')) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await api.signOutEverywhere();
|
||||
} catch {
|
||||
// Even if the call fails, push to login — cookie is likely already invalid.
|
||||
}
|
||||
navigate('login');
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="app">
|
||||
<nav className="sidebar">
|
||||
<div className="sidebar-logo">GBrain</div>
|
||||
<div className="sidebar-nav">
|
||||
<a className={`nav-item ${page === 'dashboard' ? 'active' : ''}`}
|
||||
onClick={() => navigate('dashboard')}>Dashboard</a>
|
||||
<a className={`nav-item ${page === 'agents' ? 'active' : ''}`}
|
||||
onClick={() => navigate('agents')}>Agents</a>
|
||||
<a className={`nav-item ${page === 'log' ? 'active' : ''}`}
|
||||
onClick={() => navigate('log')}>Request Log</a>
|
||||
</div>
|
||||
<div style={{ marginTop: 'auto', padding: '16px 12px', borderTop: '1px solid var(--border)' }}>
|
||||
<button
|
||||
onClick={handleSignOutEverywhere}
|
||||
style={{
|
||||
background: 'transparent',
|
||||
border: '1px solid var(--border)',
|
||||
color: 'var(--text-secondary)',
|
||||
padding: '6px 10px',
|
||||
borderRadius: 6,
|
||||
fontSize: 12,
|
||||
cursor: 'pointer',
|
||||
width: '100%',
|
||||
}}
|
||||
title="Revoke every active admin session — every browser, every tab"
|
||||
>
|
||||
Sign out everywhere
|
||||
</button>
|
||||
</div>
|
||||
</nav>
|
||||
<main className="main">
|
||||
{page === 'dashboard' && <DashboardPage />}
|
||||
{page === 'agents' && <AgentsPage />}
|
||||
{page === 'log' && <RequestLogPage />}
|
||||
</main>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
const BASE = '';
|
||||
|
||||
// v0.26.3 trust model (D11 + D12): the admin UI does NOT cache the
|
||||
// bootstrap token in browser JS state. On 401, redirect to login —
|
||||
// no auto-reauth via saved token, no localStorage/sessionStorage read.
|
||||
// The HttpOnly cookie set by /admin/login is the only session credential.
|
||||
async function apiFetch(path: string, options?: RequestInit) {
|
||||
const res = await fetch(`${BASE}${path}`, {
|
||||
...options,
|
||||
credentials: 'same-origin',
|
||||
headers: { 'Content-Type': 'application/json', ...options?.headers },
|
||||
});
|
||||
if (res.status === 401) {
|
||||
// No token cache to retry from. Redirect to login.
|
||||
window.location.hash = '#login';
|
||||
throw new Error('Unauthorized');
|
||||
}
|
||||
if (!res.ok) {
|
||||
const body = await res.json().catch(() => ({}));
|
||||
throw new Error(body.error || `HTTP ${res.status}`);
|
||||
}
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export const api = {
|
||||
login: (token: string) => apiFetch('/admin/login', { method: 'POST', body: JSON.stringify({ token }) }),
|
||||
signOutEverywhere: () => apiFetch('/admin/api/sign-out-everywhere', { method: 'POST' }),
|
||||
stats: () => apiFetch('/admin/api/stats'),
|
||||
health: () => apiFetch('/admin/api/health-indicators'),
|
||||
agents: () => apiFetch('/admin/api/agents'),
|
||||
requests: (page = 1, qs = '') => apiFetch(`/admin/api/requests?page=${page}${qs}`),
|
||||
apiKeys: () => apiFetch('/admin/api/api-keys'),
|
||||
createApiKey: (name: string) => apiFetch('/admin/api/api-keys', { method: 'POST', body: JSON.stringify({ name }) }),
|
||||
revokeApiKey: (name: string) => apiFetch('/admin/api/api-keys/revoke', { method: 'POST', body: JSON.stringify({ name }) }),
|
||||
updateClientTtl: (clientId: string, tokenTtl: number | null) => apiFetch('/admin/api/update-client-ttl', { method: 'POST', body: JSON.stringify({ clientId, tokenTtl }) }),
|
||||
revokeClient: (clientId: string) => apiFetch('/admin/api/revoke-client', { method: 'POST', body: JSON.stringify({ clientId }) }),
|
||||
};
|
||||
@@ -0,0 +1,356 @@
|
||||
:root {
|
||||
--bg-primary: #0a0a0f;
|
||||
--bg-secondary: #14141f;
|
||||
--bg-tertiary: #1e1e2e;
|
||||
--text-primary: #e0e0e0;
|
||||
--text-secondary: #888;
|
||||
--text-muted: #555;
|
||||
--accent: #3b82f6;
|
||||
--success: #22c55e;
|
||||
--warning: #f59e0b;
|
||||
--error: #ef4444;
|
||||
--font-mono: 'JetBrains Mono', monospace;
|
||||
--font-sans: 'Inter', system-ui, sans-serif;
|
||||
}
|
||||
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
|
||||
body {
|
||||
font-family: var(--font-sans);
|
||||
background: var(--bg-primary);
|
||||
color: var(--text-primary);
|
||||
font-size: 14px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
/* Layout */
|
||||
.app { display: flex; min-height: 100vh; }
|
||||
|
||||
.sidebar {
|
||||
width: 200px;
|
||||
background: var(--bg-secondary);
|
||||
border-right: 1px solid #1e1e2e;
|
||||
padding: 16px 0;
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.sidebar-logo {
|
||||
font-size: 18px;
|
||||
font-weight: 600;
|
||||
padding: 0 16px 24px;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.sidebar-nav { display: flex; flex-direction: column; gap: 2px; }
|
||||
|
||||
.nav-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 8px 16px;
|
||||
color: var(--text-secondary);
|
||||
text-decoration: none;
|
||||
font-size: 13px;
|
||||
cursor: pointer;
|
||||
border-left: 3px solid transparent;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
|
||||
.nav-item:hover { background: var(--bg-tertiary); color: var(--text-primary); }
|
||||
.nav-item.active {
|
||||
border-left-color: var(--accent);
|
||||
background: var(--bg-tertiary);
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.main { flex: 1; padding: 24px 32px; overflow-y: auto; }
|
||||
|
||||
.page-title {
|
||||
font-size: 24px;
|
||||
font-weight: 600;
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
|
||||
/* Metrics bar */
|
||||
.metrics { display: flex; gap: 16px; margin-bottom: 24px; }
|
||||
.metric {
|
||||
background: var(--bg-secondary);
|
||||
padding: 16px 20px;
|
||||
border-radius: 6px;
|
||||
min-width: 140px;
|
||||
}
|
||||
.metric-value {
|
||||
font-family: var(--font-mono);
|
||||
font-size: 28px;
|
||||
font-weight: 500;
|
||||
}
|
||||
.metric-label { font-size: 12px; color: var(--text-secondary); margin-top: 4px; }
|
||||
|
||||
/* Tables */
|
||||
table { width: 100%; border-collapse: collapse; }
|
||||
th {
|
||||
text-align: left;
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
color: var(--text-muted);
|
||||
padding: 8px 12px;
|
||||
font-weight: 500;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
td {
|
||||
padding: 10px 12px;
|
||||
font-size: 13px;
|
||||
border-top: 1px solid #1a1a2a;
|
||||
}
|
||||
tr:hover td { background: var(--bg-tertiary); }
|
||||
|
||||
/* Badges */
|
||||
.badge {
|
||||
display: inline-block;
|
||||
padding: 2px 8px;
|
||||
border-radius: 10px;
|
||||
font-size: 11px;
|
||||
font-weight: 500;
|
||||
}
|
||||
.badge-read { background: rgba(59,130,246,0.15); color: var(--accent); }
|
||||
.badge-write { background: rgba(245,158,11,0.15); color: var(--warning); }
|
||||
.badge-admin { background: rgba(239,68,68,0.15); color: var(--error); }
|
||||
.badge-success { background: rgba(34,197,94,0.15); color: var(--success); }
|
||||
.badge-error { background: rgba(239,68,68,0.15); color: var(--error); }
|
||||
|
||||
/* Status dots */
|
||||
.status-dot {
|
||||
display: inline-block;
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
margin-right: 6px;
|
||||
}
|
||||
.status-active { background: var(--success); }
|
||||
.status-warning { background: var(--warning); }
|
||||
.status-inactive { background: var(--text-muted); }
|
||||
|
||||
/* Buttons */
|
||||
.btn {
|
||||
padding: 8px 16px;
|
||||
border-radius: 6px;
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
border: none;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
.btn-primary { background: var(--accent); color: white; }
|
||||
.btn-primary:hover { background: #2563eb; }
|
||||
.btn-secondary { background: transparent; color: var(--text-secondary); border: 1px solid #333; }
|
||||
.btn-secondary:hover { border-color: var(--text-secondary); color: var(--text-primary); }
|
||||
.btn-danger { background: transparent; color: var(--error); border: 1px solid var(--error); }
|
||||
.btn-danger:hover { background: rgba(239,68,68,0.1); }
|
||||
|
||||
/* Forms */
|
||||
input, select {
|
||||
background: var(--bg-primary);
|
||||
border: 1px solid #333;
|
||||
color: var(--text-primary);
|
||||
padding: 8px 12px;
|
||||
border-radius: 6px;
|
||||
font-size: 13px;
|
||||
font-family: var(--font-sans);
|
||||
width: 100%;
|
||||
}
|
||||
input:focus, select:focus {
|
||||
outline: none;
|
||||
border-color: var(--accent);
|
||||
box-shadow: 0 0 0 2px rgba(59,130,246,0.2);
|
||||
}
|
||||
input::placeholder { color: var(--text-muted); }
|
||||
label { display: block; font-size: 13px; font-weight: 500; margin-bottom: 6px; }
|
||||
|
||||
/* Modal */
|
||||
.modal-overlay {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0,0,0,0.7);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 100;
|
||||
}
|
||||
.modal {
|
||||
background: var(--bg-secondary);
|
||||
border-radius: 8px;
|
||||
padding: 24px;
|
||||
min-width: 420px;
|
||||
max-width: 520px;
|
||||
}
|
||||
.modal-title { font-size: 18px; font-weight: 600; margin-bottom: 20px; }
|
||||
|
||||
/* Drawer */
|
||||
.drawer-overlay {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0,0,0,0.5);
|
||||
z-index: 90;
|
||||
}
|
||||
.drawer {
|
||||
position: fixed;
|
||||
right: 0;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
width: 420px;
|
||||
background: var(--bg-secondary);
|
||||
border-left: 1px solid var(--accent);
|
||||
padding: 24px;
|
||||
z-index: 91;
|
||||
overflow-y: auto;
|
||||
}
|
||||
.drawer-close {
|
||||
position: absolute;
|
||||
top: 16px;
|
||||
right: 16px;
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--text-muted);
|
||||
font-size: 18px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* Section headers */
|
||||
.section-title {
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
color: var(--text-muted);
|
||||
letter-spacing: 0.5px;
|
||||
margin: 20px 0 12px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
/* Health panel */
|
||||
.health-panel {
|
||||
background: var(--bg-secondary);
|
||||
border-radius: 6px;
|
||||
padding: 16px;
|
||||
}
|
||||
.health-row {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
padding: 6px 0;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
/* Code block */
|
||||
.code-block {
|
||||
background: var(--bg-primary);
|
||||
border-radius: 6px;
|
||||
padding: 12px;
|
||||
font-family: var(--font-mono);
|
||||
font-size: 12px;
|
||||
overflow-x: auto;
|
||||
position: relative;
|
||||
}
|
||||
.code-block .copy-btn {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 8px;
|
||||
background: var(--accent);
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 4px 10px;
|
||||
border-radius: 4px;
|
||||
font-size: 11px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* Activity feed */
|
||||
.feed { max-height: 400px; overflow-y: auto; }
|
||||
.feed-empty {
|
||||
color: var(--text-muted);
|
||||
text-align: center;
|
||||
padding: 32px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
/* Sparkline */
|
||||
.sparkline { display: inline-block; vertical-align: middle; }
|
||||
|
||||
/* Filter bar */
|
||||
.filter-bar { display: flex; gap: 12px; margin-bottom: 16px; align-items: center; }
|
||||
.filter-bar select { width: auto; min-width: 140px; }
|
||||
|
||||
/* Pagination */
|
||||
.pagination {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 12px 0;
|
||||
font-size: 13px;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
.pagination button {
|
||||
background: var(--bg-secondary);
|
||||
border: 1px solid #333;
|
||||
color: var(--text-primary);
|
||||
padding: 6px 12px;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
font-size: 12px;
|
||||
}
|
||||
.pagination button:disabled { opacity: 0.3; cursor: default; }
|
||||
|
||||
/* Warning bar */
|
||||
.warning-bar {
|
||||
background: rgba(245,158,11,0.15);
|
||||
border: 1px solid var(--warning);
|
||||
color: var(--warning);
|
||||
padding: 10px 16px;
|
||||
border-radius: 6px;
|
||||
font-size: 13px;
|
||||
margin: 12px 0;
|
||||
}
|
||||
|
||||
/* Checkbox */
|
||||
.checkbox-group { display: flex; gap: 16px; flex-wrap: wrap; }
|
||||
.checkbox-label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-size: 13px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* Tabs */
|
||||
.tabs { display: flex; gap: 0; margin-bottom: 12px; }
|
||||
.tab {
|
||||
padding: 6px 12px;
|
||||
font-size: 13px;
|
||||
color: var(--text-secondary);
|
||||
cursor: pointer;
|
||||
border-bottom: 2px solid transparent;
|
||||
}
|
||||
.tab.active { color: var(--accent); border-bottom-color: var(--accent); }
|
||||
|
||||
/* Login page */
|
||||
.login-page {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
min-height: 100vh;
|
||||
background: var(--bg-primary);
|
||||
}
|
||||
.login-box { text-align: left; width: 340px; }
|
||||
.login-logo { font-size: 32px; font-weight: 600; margin-bottom: 32px; }
|
||||
.login-hint { color: var(--text-muted); font-size: 12px; margin-top: 12px; }
|
||||
.login-error { color: var(--error); font-size: 13px; margin-top: 8px; }
|
||||
|
||||
/* Monospace data */
|
||||
.mono { font-family: var(--font-mono); font-size: 12px; }
|
||||
|
||||
/* Responsive */
|
||||
@media (max-width: 768px) {
|
||||
.sidebar { display: none; }
|
||||
.main { padding: 16px; }
|
||||
.metrics { flex-wrap: wrap; }
|
||||
.drawer { width: 100%; }
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import React from 'react';
|
||||
import ReactDOM from 'react-dom/client';
|
||||
import { App } from './App';
|
||||
import './index.css';
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||
<React.StrictMode>
|
||||
<App />
|
||||
</React.StrictMode>,
|
||||
);
|
||||
@@ -0,0 +1,627 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { api } from '../api';
|
||||
|
||||
function timeAgo(date: Date): string {
|
||||
const s = Math.floor((Date.now() - date.getTime()) / 1000);
|
||||
if (s < 60) return 'just now';
|
||||
if (s < 3600) return `${Math.floor(s / 60)}m ago`;
|
||||
if (s < 86400) return `${Math.floor(s / 3600)}h ago`;
|
||||
return `${Math.floor(s / 86400)}d ago`;
|
||||
}
|
||||
|
||||
interface Agent {
|
||||
id: string;
|
||||
name: string;
|
||||
auth_type: 'oauth' | 'api_key';
|
||||
client_id?: string; // compat
|
||||
client_name?: string; // compat
|
||||
grant_types: string[];
|
||||
scope: string;
|
||||
created_at: string;
|
||||
last_used_at: string | null;
|
||||
total_requests: number;
|
||||
requests_today: number;
|
||||
token_ttl: number | null;
|
||||
status: 'active' | 'revoked';
|
||||
}
|
||||
|
||||
interface ApiKey {
|
||||
id: string;
|
||||
name: string;
|
||||
created_at: string;
|
||||
last_used_at: string | null;
|
||||
status: 'active' | 'revoked';
|
||||
}
|
||||
|
||||
export function AgentsPage() {
|
||||
const [agents, setAgents] = useState<Agent[]>([]);
|
||||
const [hideRevoked, setHideRevoked] = useState(true);
|
||||
const [showRegister, setShowRegister] = useState(false);
|
||||
const [showCredentials, setShowCredentials] = useState<{ clientId: string; clientSecret: string; name: string } | null>(null);
|
||||
const [showApiKeyCreate, setShowApiKeyCreate] = useState(false);
|
||||
const [showApiKeyToken, setShowApiKeyToken] = useState<{ name: string; token: string } | null>(null);
|
||||
const [selectedAgent, setSelectedAgent] = useState<Agent | null>(null);
|
||||
|
||||
useEffect(() => { loadAgents(); }, []);
|
||||
|
||||
const loadAgents = () => { api.agents().then(setAgents).catch(() => {}); };
|
||||
|
||||
return (
|
||||
<>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', marginBottom: 24 }}>
|
||||
<h1 className="page-title" style={{ marginBottom: 0 }}>Agents</h1>
|
||||
<div style={{ display: 'flex', gap: 8, alignItems: 'center' }}>
|
||||
<label style={{ fontSize: 13, color: 'var(--text-secondary)', display: 'flex', alignItems: 'center', gap: 6, cursor: 'pointer' }}>
|
||||
<input type="checkbox" checked={hideRevoked} onChange={e => setHideRevoked(e.target.checked)} /> Hide revoked
|
||||
</label>
|
||||
<button className="btn btn-secondary" onClick={() => setShowApiKeyCreate(true)}>+ API Key</button>
|
||||
<button className="btn btn-primary" onClick={() => setShowRegister(true)}>+ OAuth Client</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{(() => {
|
||||
// Filter once and reuse, so the empty-state guard sees the same
|
||||
// rows the table renders. Pre-fix: agents.length === 0 used the
|
||||
// unfiltered array, so an all-revoked dataset with hideRevoked=on
|
||||
// showed a header-only table with no placeholder.
|
||||
const visibleAgents = agents.filter(a => !hideRevoked || a.status !== 'revoked');
|
||||
if (agents.length === 0) {
|
||||
return (
|
||||
<div style={{ textAlign: 'center', padding: 48, color: 'var(--text-muted)' }}>
|
||||
No agents registered. Register your first agent to get started.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (visibleAgents.length === 0) {
|
||||
return (
|
||||
<div style={{ textAlign: 'center', padding: 48, color: 'var(--text-muted)' }}>
|
||||
All agents are revoked. Uncheck "Hide revoked" to view them.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Type</th>
|
||||
<th>Scopes</th>
|
||||
<th>Status</th>
|
||||
<th>Requests</th>
|
||||
<th>Last Used</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{visibleAgents.map(a => (
|
||||
<tr key={a.id} onClick={() => setSelectedAgent(a)}
|
||||
style={{ cursor: 'pointer' }}>
|
||||
<td style={{ fontWeight: 500 }}>{a.name || a.client_name}</td>
|
||||
<td>
|
||||
<span className={`badge ${a.auth_type === 'oauth' ? 'badge-read' : 'badge-write'}`} style={{ fontSize: 11 }}>
|
||||
{a.auth_type === 'oauth' ? 'OAuth' : 'API Key'}
|
||||
</span>
|
||||
</td>
|
||||
<td>
|
||||
{(a.scope || '').split(' ').filter(Boolean).map(s => (
|
||||
<span key={s} className={`badge badge-${s}`} style={{ marginRight: 4 }}>{s}</span>
|
||||
))}
|
||||
</td>
|
||||
<td>
|
||||
<span className={`badge ${a.status === 'active' ? 'badge-success' : 'badge-danger'}`}>{a.status}</span>
|
||||
</td>
|
||||
<td>
|
||||
<span style={{ fontWeight: 500 }}>{a.requests_today || 0}</span>
|
||||
<span style={{ color: 'var(--text-muted)', fontSize: 12 }}> / {a.total_requests || 0}</span>
|
||||
</td>
|
||||
<td style={{ color: 'var(--text-secondary)' }}>
|
||||
{a.last_used_at ? timeAgo(new Date(a.last_used_at)) : 'Never'}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
<div style={{ color: 'var(--text-muted)', fontSize: 13, marginTop: 12 }}>
|
||||
{agents.filter(a => a.status === 'active').length} active / {agents.length} total
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
})()}
|
||||
|
||||
{showRegister && (
|
||||
<RegisterModal
|
||||
onClose={() => setShowRegister(false)}
|
||||
onRegistered={(creds) => { setShowRegister(false); setShowCredentials(creds); loadAgents(); }}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showCredentials && (
|
||||
<CredentialsModal
|
||||
credentials={showCredentials}
|
||||
onClose={() => setShowCredentials(null)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{selectedAgent && (
|
||||
<AgentDrawer agent={selectedAgent} onClose={() => setSelectedAgent(null)} onRevoked={loadAgents} />
|
||||
)}
|
||||
|
||||
{showApiKeyCreate && (
|
||||
<ApiKeyCreateModal
|
||||
onClose={() => setShowApiKeyCreate(false)}
|
||||
onCreated={(result) => { setShowApiKeyCreate(false); setShowApiKeyToken(result); loadAgents(); }}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showApiKeyToken && (
|
||||
<ApiKeyTokenModal token={showApiKeyToken} onClose={() => setShowApiKeyToken(null)} />
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function ApiKeyCreateModal({ onClose, onCreated }: {
|
||||
onClose: () => void;
|
||||
onCreated: (result: { name: string; token: string }) => void;
|
||||
}) {
|
||||
const [name, setName] = useState('');
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState('');
|
||||
|
||||
const handleSubmit = async (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
if (!name.trim()) { setError('Name required'); return; }
|
||||
setLoading(true);
|
||||
try {
|
||||
const data = await api.createApiKey(name.trim());
|
||||
onCreated({ name: data.name, token: data.token });
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Failed');
|
||||
} finally { setLoading(false); }
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="modal-overlay" onClick={onClose}>
|
||||
<form className="modal" onClick={e => e.stopPropagation()} onSubmit={handleSubmit}>
|
||||
<div className="modal-title">Create API Key</div>
|
||||
<p style={{ color: 'var(--text-secondary)', fontSize: 13, marginBottom: 16 }}>
|
||||
API keys use simple bearer token auth. They grant full read+write+admin access.
|
||||
For scoped access, use OAuth clients instead.
|
||||
</p>
|
||||
<div style={{ marginBottom: 16 }}>
|
||||
<label>Key Name</label>
|
||||
<input placeholder="e.g. claude-code-local" value={name} onChange={e => setName(e.target.value)} autoFocus />
|
||||
</div>
|
||||
{error && <div style={{ color: 'var(--error)', fontSize: 13, marginBottom: 12 }}>{error}</div>}
|
||||
<div style={{ display: 'flex', gap: 12, justifyContent: 'flex-end' }}>
|
||||
<button type="button" className="btn btn-secondary" onClick={onClose}>Cancel</button>
|
||||
<button type="submit" className="btn btn-primary" disabled={loading}>
|
||||
{loading ? 'Creating...' : 'Create Key'}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ApiKeyTokenModal({ token, onClose }: {
|
||||
token: { name: string; token: string };
|
||||
onClose: () => void;
|
||||
}) {
|
||||
const copy = (text: string) => navigator.clipboard.writeText(text);
|
||||
|
||||
return (
|
||||
<div className="modal-overlay">
|
||||
<div className="modal" style={{ maxWidth: 560 }}>
|
||||
<div style={{ textAlign: 'center', marginBottom: 16 }}>
|
||||
<div style={{ fontSize: 36, color: 'var(--success)', marginBottom: 8 }}>✓</div>
|
||||
<div style={{ fontSize: 20, fontWeight: 600 }}>API Key Created</div>
|
||||
</div>
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<label style={{ fontSize: 12 }}>Name</label>
|
||||
<div className="code-block"><span>{token.name}</span></div>
|
||||
</div>
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<label style={{ fontSize: 12 }}>Bearer Token</label>
|
||||
<div className="code-block">
|
||||
<span>{token.token}</span>
|
||||
<button className="copy-btn" onClick={() => copy(token.token)}>Copy</button>
|
||||
</div>
|
||||
</div>
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<label style={{ fontSize: 12 }}>Usage</label>
|
||||
<div className="code-block">
|
||||
<pre style={{ whiteSpace: 'pre-wrap', margin: 0, fontSize: 12 }}>{`Authorization: Bearer ${token.token}`}</pre>
|
||||
<button className="copy-btn" onClick={() => copy(`Authorization: Bearer ${token.token}`)}>Copy</button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="warning-bar">Save this token now. It will not be shown again.</div>
|
||||
<div style={{ display: 'flex', gap: 12, justifyContent: 'flex-end', marginTop: 20 }}>
|
||||
<button className="btn btn-primary" onClick={onClose}>Done</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function RegisterModal({ onClose, onRegistered }: {
|
||||
onClose: () => void;
|
||||
onRegistered: (creds: { clientId: string; clientSecret: string; name: string }) => void;
|
||||
}) {
|
||||
const [name, setName] = useState('');
|
||||
const [scopes, setScopes] = useState({ read: true, write: false, admin: false });
|
||||
const [ttl, setTtl] = useState('86400'); // 24h default
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState('');
|
||||
|
||||
const ttlOptions = [
|
||||
{ label: '1 hour', value: '3600' },
|
||||
{ label: '24 hours', value: '86400' },
|
||||
{ label: '7 days', value: '604800' },
|
||||
{ label: '30 days', value: '2592000' },
|
||||
{ label: '1 year', value: '31536000' },
|
||||
{ label: 'No expiry', value: '0' },
|
||||
];
|
||||
|
||||
const handleSubmit = async (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
if (!name.trim()) { setError('Name required'); return; }
|
||||
setLoading(true);
|
||||
setError('');
|
||||
try {
|
||||
// Use the CLI registration endpoint (POST to admin API)
|
||||
const selectedScopes = Object.entries(scopes).filter(([, v]) => v).map(([k]) => k).join(' ');
|
||||
const res = await fetch('/admin/api/register-client', {
|
||||
method: 'POST',
|
||||
credentials: 'same-origin',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ name: name.trim(), scopes: selectedScopes, tokenTtl: ttl === '0' ? 315360000 : Number(ttl) }),
|
||||
});
|
||||
if (!res.ok) throw new Error('Registration failed');
|
||||
const data = await res.json();
|
||||
onRegistered({ clientId: data.clientId, clientSecret: data.clientSecret, name: name.trim() });
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Registration failed');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="modal-overlay" onClick={onClose}>
|
||||
<form className="modal" onClick={e => e.stopPropagation()} onSubmit={handleSubmit}>
|
||||
<div className="modal-title">Register Agent</div>
|
||||
<div style={{ marginBottom: 16 }}>
|
||||
<label>Agent Name</label>
|
||||
<input placeholder="e.g. perplexity-production" value={name} onChange={e => setName(e.target.value)} autoFocus />
|
||||
</div>
|
||||
<div style={{ marginBottom: 16 }}>
|
||||
<label>Scopes</label>
|
||||
<div className="checkbox-group">
|
||||
{(['read', 'write', 'admin'] as const).map(s => (
|
||||
<label key={s} className="checkbox-label">
|
||||
<input type="checkbox" checked={scopes[s]} onChange={e => setScopes(p => ({ ...p, [s]: e.target.checked }))} />
|
||||
{s}
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
<div style={{ marginBottom: 20 }}>
|
||||
<label>Token Lifetime</label>
|
||||
<select value={ttl} onChange={e => setTtl(e.target.value)}
|
||||
style={{ width: '100%', background: 'var(--bg-secondary)', color: 'var(--text-primary)', border: '1px solid var(--border)', borderRadius: 6, padding: '6px 10px', fontSize: 14 }}>
|
||||
{ttlOptions.map(o => <option key={o.value} value={o.value}>{o.label}</option>)}
|
||||
</select>
|
||||
</div>
|
||||
{error && <div style={{ color: 'var(--error)', fontSize: 13, marginBottom: 12 }}>{error}</div>}
|
||||
<div style={{ display: 'flex', gap: 12, justifyContent: 'flex-end' }}>
|
||||
<button type="button" className="btn btn-secondary" onClick={onClose}>Cancel</button>
|
||||
<button type="submit" className="btn btn-primary" disabled={loading}>
|
||||
{loading ? 'Registering...' : 'Register'}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function CredentialsModal({ credentials, onClose }: {
|
||||
credentials: { clientId: string; clientSecret: string; name: string };
|
||||
onClose: () => void;
|
||||
}) {
|
||||
const copy = (text: string) => navigator.clipboard.writeText(text);
|
||||
const downloadJson = () => {
|
||||
const blob = new Blob([JSON.stringify(credentials, null, 2)], { type: 'application/json' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url; a.download = `${credentials.name}-credentials.json`; a.click();
|
||||
URL.revokeObjectURL(url);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="modal-overlay">
|
||||
<div className="modal" style={{ maxWidth: 560 }}>
|
||||
<div style={{ textAlign: 'center', marginBottom: 16 }}>
|
||||
<div style={{ fontSize: 36, color: 'var(--success)', marginBottom: 8 }}>✓</div>
|
||||
<div style={{ fontSize: 20, fontWeight: 600 }}>Agent Registered</div>
|
||||
</div>
|
||||
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<label style={{ fontSize: 12 }}>Client ID</label>
|
||||
<div className="code-block">
|
||||
<span>{credentials.clientId}</span>
|
||||
<button className="copy-btn" onClick={() => copy(credentials.clientId)}>Copy</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<label style={{ fontSize: 12 }}>Client Secret</label>
|
||||
<div className="code-block">
|
||||
<span>{credentials.clientSecret}</span>
|
||||
<button className="copy-btn" onClick={() => copy(credentials.clientSecret)}>Copy</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="warning-bar">
|
||||
Save this secret now. It will not be shown again.
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'flex', gap: 12, justifyContent: 'flex-end', marginTop: 20 }}>
|
||||
<button className="btn btn-secondary" onClick={downloadJson}>Download as JSON</button>
|
||||
<button className="btn btn-primary" onClick={onClose}>Done</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: () => void; onRevoked: () => void }) {
|
||||
const [tab, setTab] = useState<'claude-code' | 'chatgpt' | 'claude-cowork' | 'perplexity' | 'cursor' | 'json'>('claude-code');
|
||||
const copy = (text: string) => navigator.clipboard.writeText(text);
|
||||
const serverUrl = window.location.origin;
|
||||
|
||||
const cid = agent.id || agent.client_id || '';
|
||||
const isOAuth = agent.auth_type === 'oauth';
|
||||
const agentName = agent.name || agent.client_name || 'unknown';
|
||||
|
||||
// For API keys, we can't show the actual token (it was shown once at creation).
|
||||
// For OAuth, we show the client_id and tell them to use their secret.
|
||||
|
||||
const configSnippets: Record<string, string> = {
|
||||
'claude-code': isOAuth
|
||||
? [
|
||||
`The user wants to connect GBrain MCP to your context. Here's how.`,
|
||||
``,
|
||||
`# Step 1 — read the client secret from the operator without putting it in shell history.`,
|
||||
`# read -s suppresses the echo; export keeps it scoped to this shell session.`,
|
||||
`read -rs -p "Client Secret: " GBRAIN_CS && echo`,
|
||||
``,
|
||||
`# Step 2 — mint a token and register GBrain in one go.`,
|
||||
`claude mcp add --transport http gbrain ${serverUrl}/mcp \\`,
|
||||
` --header "Authorization: Bearer $(curl -s -X POST ${serverUrl}/token \\`,
|
||||
` -d 'grant_type=client_credentials' \\`,
|
||||
` -d 'client_id=${cid}' \\`,
|
||||
` --data-urlencode "client_secret=$GBRAIN_CS" \\`,
|
||||
` -d 'scope=${agent.scope || 'read write'}' | jq -r .access_token)"`,
|
||||
``,
|
||||
`# Step 3 — clear the secret from this shell.`,
|
||||
`unset GBRAIN_CS`,
|
||||
``,
|
||||
`The token will last ${agent.token_ttl ? (agent.token_ttl >= 86400 ? Math.floor(agent.token_ttl / 86400) + ' days' : Math.floor(agent.token_ttl / 3600) + ' hours') : '1 hour (default)'}.`,
|
||||
``,
|
||||
`─── Fallback: 2-step curl + paste ───`,
|
||||
``,
|
||||
`If your shell doesn't support read -s, mint the token first, then paste:`,
|
||||
``,
|
||||
`curl -s -X POST ${serverUrl}/token \\`,
|
||||
` -d 'grant_type=client_credentials' \\`,
|
||||
` -d 'client_id=${cid}' \\`,
|
||||
` -d 'client_secret=PASTE_YOUR_CLIENT_SECRET_HERE' \\`,
|
||||
` -d 'scope=${agent.scope || 'read write'}' | jq -r .access_token`,
|
||||
``,
|
||||
`claude mcp add --transport http gbrain ${serverUrl}/mcp \\`,
|
||||
` --header "Authorization: Bearer PASTE_TOKEN_FROM_ABOVE"`,
|
||||
``,
|
||||
`# Then run: history -d $((HISTCMD-1)) # zsh: print -s -- '' && fc -p`,
|
||||
].join('\n')
|
||||
: [
|
||||
`The user wants to connect GBrain MCP to your context. Here's how.`,
|
||||
``,
|
||||
`Run this command in terminal to register GBrain:`,
|
||||
``,
|
||||
`claude mcp add --transport http gbrain ${serverUrl}/mcp \\`,
|
||||
` --header "Authorization: Bearer PASTE_YOUR_API_KEY_HERE"`,
|
||||
``,
|
||||
`Replace PASTE_YOUR_API_KEY_HERE with the API key shown when "${agentName}" was created.`,
|
||||
`API keys never expire.`,
|
||||
].join('\n'),
|
||||
|
||||
'chatgpt': [
|
||||
`The user wants to connect GBrain MCP to ChatGPT.`,
|
||||
``,
|
||||
`1. Open ChatGPT > Settings > Tools & Integrations > Add MCP Server`,
|
||||
`2. Enter this URL — ChatGPT will auto-discover the OAuth configuration:`,
|
||||
``,
|
||||
` ${serverUrl}/.well-known/oauth-authorization-server`,
|
||||
``,
|
||||
`3. When prompted for credentials:`,
|
||||
` Client ID: ${cid}`,
|
||||
` Client Secret: (the secret from agent registration)`,
|
||||
` Grant Type: client_credentials`,
|
||||
` Scope: ${agent.scope || 'read write'}`,
|
||||
].join('\n'),
|
||||
|
||||
'claude-cowork': [
|
||||
`The user wants to connect GBrain MCP to Claude.ai.`,
|
||||
``,
|
||||
`1. Open claude.ai > Settings > Connected Apps > Add MCP Server`,
|
||||
`2. Server URL: ${serverUrl}/mcp`,
|
||||
`3. When prompted for auth:`,
|
||||
` Token endpoint: ${serverUrl}/token`,
|
||||
` Client ID: ${cid}`,
|
||||
` Client Secret: (the secret from agent registration)`,
|
||||
` Scope: ${agent.scope || 'read write'}`,
|
||||
``,
|
||||
`Discovery URL: ${serverUrl}/.well-known/oauth-authorization-server`,
|
||||
].join('\n'),
|
||||
|
||||
cursor: isOAuth
|
||||
? [
|
||||
`The user wants to connect GBrain MCP to Cursor.`,
|
||||
``,
|
||||
`Cursor supports OAuth for remote MCP. Add to .cursor/mcp.json:`,
|
||||
``,
|
||||
`{`,
|
||||
` "mcpServers": {`,
|
||||
` "gbrain": {`,
|
||||
` "url": "${serverUrl}/mcp",`,
|
||||
` "transport": "sse"`,
|
||||
` }`,
|
||||
` }`,
|
||||
`}`,
|
||||
``,
|
||||
`Cursor will auto-discover OAuth via:`,
|
||||
`${serverUrl}/.well-known/oauth-authorization-server`,
|
||||
``,
|
||||
`When prompted: Client ID ${cid}, use the secret from registration.`,
|
||||
].join('\n')
|
||||
: [
|
||||
`The user wants to connect GBrain MCP to Cursor.`,
|
||||
``,
|
||||
`Add to .cursor/mcp.json:`,
|
||||
``,
|
||||
`{`,
|
||||
` "mcpServers": {`,
|
||||
` "gbrain": {`,
|
||||
` "url": "${serverUrl}/mcp",`,
|
||||
` "transport": "sse",`,
|
||||
` "headers": {`,
|
||||
` "Authorization": "Bearer PASTE_YOUR_API_KEY_HERE"`,
|
||||
` }`,
|
||||
` }`,
|
||||
` }`,
|
||||
`}`,
|
||||
``,
|
||||
`Replace PASTE_YOUR_API_KEY_HERE with the API key shown when "${agentName}" was created.`,
|
||||
].join('\n'),
|
||||
|
||||
perplexity: [
|
||||
`The user wants to connect GBrain MCP to Perplexity.`,
|
||||
``,
|
||||
`1. Go to Settings > Connectors > Add MCP`,
|
||||
`2. Server URL: ${serverUrl}/mcp`,
|
||||
`3. Client ID: ${cid}`,
|
||||
`4. Client Secret: (the secret from agent registration)`,
|
||||
].join('\n'),
|
||||
|
||||
json: JSON.stringify({
|
||||
server_url: serverUrl + '/mcp',
|
||||
token_url: serverUrl + '/token',
|
||||
discovery_url: serverUrl + '/.well-known/oauth-authorization-server',
|
||||
client_id: cid,
|
||||
client_name: agentName,
|
||||
auth_type: agent.auth_type,
|
||||
scope: agent.scope,
|
||||
}, null, 2),
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="drawer-overlay" onClick={onClose} />
|
||||
<div className="drawer">
|
||||
<button className="drawer-close" onClick={onClose}>✕</button>
|
||||
<div style={{ fontSize: 18, fontWeight: 600, marginBottom: 4 }}>{agent.name || agent.client_name}</div>
|
||||
<span className={`badge ${agent.status === 'active' ? 'badge-success' : 'badge-danger'}`}>{agent.status}</span>
|
||||
|
||||
<div className="section-title">Details</div>
|
||||
<div style={{ display: 'grid', gridTemplateColumns: '100px 1fr', gap: '6px 12px', fontSize: 13 }}>
|
||||
<span style={{ color: 'var(--text-secondary)' }}>Client ID</span>
|
||||
<span className="mono">{(agent.id || agent.id || agent.client_id || '').substring(0, 24)}...</span>
|
||||
<span style={{ color: 'var(--text-secondary)' }}>Scopes</span>
|
||||
<span>{(agent.scope || '').split(' ').filter(Boolean).map(s => (
|
||||
<span key={s} className={`badge badge-${s}`} style={{ marginRight: 4 }}>{s}</span>
|
||||
))}</span>
|
||||
<span style={{ color: 'var(--text-secondary)' }}>Registered</span>
|
||||
<span>{new Date(agent.created_at).toLocaleDateString()}</span>
|
||||
<span style={{ color: 'var(--text-secondary)' }}>Token TTL</span>
|
||||
<span>{agent.token_ttl ? (agent.token_ttl >= 31536000 ? 'No expiry' : agent.token_ttl >= 86400 ? `${Math.floor(agent.token_ttl / 86400)}d` : agent.token_ttl >= 3600 ? `${Math.floor(agent.token_ttl / 3600)}h` : `${agent.token_ttl}s`) : '1h (default)'}</span>
|
||||
</div>
|
||||
|
||||
{/*
|
||||
Config Export visible for both auth_type=oauth AND auth_type=api_key.
|
||||
Claude Code + Cursor + JSON tabs render real snippets regardless
|
||||
(commit 15's snippets are auth-type-aware for those two clients;
|
||||
JSON is just structured metadata). ChatGPT, Claude.ai, and
|
||||
Perplexity tabs render an "OAuth client required" message on
|
||||
api_key agents — those MCP clients only speak OAuth 2.0
|
||||
client_credentials, not raw bearer tokens.
|
||||
|
||||
Pre-fix (Wintermute commit 16): the entire Config Export
|
||||
section was hidden for api_key agents, dropping the working
|
||||
Claude Code + Cursor snippets along with the broken ones.
|
||||
(D5=C in the eng review.)
|
||||
*/}
|
||||
<div className="section-title">Config Export</div>
|
||||
<div className="tabs" style={{ flexWrap: 'wrap' }}>
|
||||
<div className={`tab ${tab === 'claude-code' ? 'active' : ''}`} onClick={() => setTab('claude-code')}>Claude Code</div>
|
||||
<div className={`tab ${tab === 'chatgpt' ? 'active' : ''}`} onClick={() => setTab('chatgpt')}>ChatGPT</div>
|
||||
<div className={`tab ${tab === 'claude-cowork' ? 'active' : ''}`} onClick={() => setTab('claude-cowork')}>Claude.ai</div>
|
||||
<div className={`tab ${tab === 'cursor' ? 'active' : ''}`} onClick={() => setTab('cursor')}>Cursor</div>
|
||||
<div className={`tab ${tab === 'perplexity' ? 'active' : ''}`} onClick={() => setTab('perplexity')}>Perplexity</div>
|
||||
<div className={`tab ${tab === 'json' ? 'active' : ''}`} onClick={() => setTab('json')}>JSON</div>
|
||||
</div>
|
||||
{(() => {
|
||||
const oauthOnlyTabs = new Set(['chatgpt', 'claude-cowork', 'perplexity']);
|
||||
if (!isOAuth && oauthOnlyTabs.has(tab)) {
|
||||
const clientName = { chatgpt: 'ChatGPT', 'claude-cowork': 'Claude.ai', perplexity: 'Perplexity' }[tab] || tab;
|
||||
return (
|
||||
<div style={{
|
||||
background: 'rgba(255, 200, 100, 0.08)',
|
||||
border: '1px solid rgba(255, 200, 100, 0.2)',
|
||||
borderRadius: 8,
|
||||
padding: '14px 16px',
|
||||
marginTop: 12,
|
||||
fontSize: 13,
|
||||
lineHeight: 1.6,
|
||||
color: 'var(--text-secondary)',
|
||||
}}>
|
||||
<div style={{ fontWeight: 600, color: 'var(--text-primary)', marginBottom: 6 }}>
|
||||
{clientName} requires an OAuth client
|
||||
</div>
|
||||
{clientName} only supports OAuth 2.0 (client_credentials). API keys use raw bearer tokens, which {clientName} does not accept. Register a separate OAuth client and use that to connect this AI.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<div className="code-block">
|
||||
<pre style={{ whiteSpace: 'pre-wrap', margin: 0 }}>{configSnippets[tab]}</pre>
|
||||
<button className="copy-btn" onClick={() => copy(configSnippets[tab])}>Copy</button>
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
|
||||
<div style={{ marginTop: 32 }}>
|
||||
{agent.status === 'active' && (
|
||||
<button className="btn btn-danger" onClick={async () => {
|
||||
if (!confirm(`Revoke ${agent.name || agent.client_name}? All active tokens will be invalidated.`)) return;
|
||||
try {
|
||||
if (agent.auth_type === 'oauth') {
|
||||
await api.revokeClient(agent.id || agent.client_id || '');
|
||||
} else {
|
||||
await api.revokeApiKey(agent.name || '');
|
||||
}
|
||||
onRevoked();
|
||||
onClose();
|
||||
} catch (e) {
|
||||
alert('Revoke failed: ' + (e instanceof Error ? e.message : 'unknown error'));
|
||||
}
|
||||
}}>Revoke Agent</button>
|
||||
)}
|
||||
{agent.status === 'revoked' && (
|
||||
<span style={{ color: 'var(--text-muted)', fontSize: 13 }}>This agent has been revoked.</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import React, { useState, useEffect, useRef } from 'react';
|
||||
import { api } from '../api';
|
||||
|
||||
interface FeedEvent {
|
||||
agent: string;
|
||||
operation: string;
|
||||
scopes: string;
|
||||
latency_ms: number;
|
||||
status: string;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export function DashboardPage() {
|
||||
const [stats, setStats] = useState({ connected_agents: 0, requests_today: 0, active_tokens: 0 });
|
||||
const [health, setHealth] = useState({ expiring_soon: 0, error_rate: '0%' });
|
||||
const [events, setEvents] = useState<FeedEvent[]>([]);
|
||||
const [sseStatus, setSseStatus] = useState<'connecting' | 'connected' | 'disconnected'>('connecting');
|
||||
const eventSourceRef = useRef<EventSource | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
api.stats().then(setStats).catch(() => {});
|
||||
api.health().then(setHealth).catch(() => {});
|
||||
|
||||
const es = new EventSource('/admin/events');
|
||||
eventSourceRef.current = es;
|
||||
es.onopen = () => setSseStatus('connected');
|
||||
es.onmessage = (e) => {
|
||||
try {
|
||||
const event = JSON.parse(e.data) as FeedEvent;
|
||||
setEvents(prev => [event, ...prev].slice(0, 50));
|
||||
} catch {}
|
||||
};
|
||||
es.onerror = () => {
|
||||
setSseStatus('disconnected');
|
||||
setTimeout(() => {
|
||||
setSseStatus('connecting');
|
||||
es.close();
|
||||
// Reconnect handled by browser EventSource auto-retry
|
||||
}, 3000);
|
||||
};
|
||||
|
||||
const interval = setInterval(() => {
|
||||
api.stats().then(setStats).catch(() => {});
|
||||
api.health().then(setHealth).catch(() => {});
|
||||
}, 30000);
|
||||
|
||||
return () => { es.close(); clearInterval(interval); };
|
||||
}, []);
|
||||
|
||||
const timeAgo = (ts: string) => {
|
||||
const diff = Date.now() - new Date(ts).getTime();
|
||||
if (diff < 60000) return `${Math.floor(diff / 1000)}s ago`;
|
||||
if (diff < 3600000) return `${Math.floor(diff / 60000)} min ago`;
|
||||
return `${Math.floor(diff / 3600000)}h ago`;
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<h1 className="page-title">Dashboard</h1>
|
||||
|
||||
<div style={{ display: 'flex', gap: 24 }}>
|
||||
<div style={{ flex: 1 }}>
|
||||
<div className="metrics">
|
||||
<div className="metric">
|
||||
<div className="metric-value">{stats.connected_agents}</div>
|
||||
<div className="metric-label">Connected Agents</div>
|
||||
</div>
|
||||
<div className="metric">
|
||||
<div className="metric-value">{stats.requests_today}</div>
|
||||
<div className="metric-label">Requests Today</div>
|
||||
</div>
|
||||
<div className="metric">
|
||||
<div className="metric-value">{stats.active_tokens}</div>
|
||||
<div className="metric-label">Active Tokens</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h2 className="section-title">
|
||||
Live Activity
|
||||
<span style={{ marginLeft: 8, fontSize: 10, color: sseStatus === 'connected' ? 'var(--success)' : sseStatus === 'connecting' ? 'var(--warning)' : 'var(--error)' }}>
|
||||
{sseStatus === 'connected' ? '● connected' : sseStatus === 'connecting' ? '● connecting...' : '● disconnected'}
|
||||
</span>
|
||||
</h2>
|
||||
|
||||
<div className="feed">
|
||||
{events.length === 0 ? (
|
||||
<div className="feed-empty">
|
||||
{sseStatus === 'connected' ? 'No requests yet. Agents will appear when they connect.' : 'Connecting...'}
|
||||
</div>
|
||||
) : (
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Agent</th>
|
||||
<th>Operation</th>
|
||||
<th>Scopes</th>
|
||||
<th>Latency</th>
|
||||
<th>Status</th>
|
||||
<th>Time</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{events.map((e, i) => (
|
||||
<tr key={i}>
|
||||
<td className="mono">{e.agent}</td>
|
||||
<td className="mono">{e.operation}</td>
|
||||
<td>{e.scopes.split(',').map(s => (
|
||||
<span key={s} className={`badge badge-${s.trim()}`} style={{ marginRight: 4 }}>{s.trim()}</span>
|
||||
))}</td>
|
||||
<td className="mono">{e.latency_ms} ms</td>
|
||||
<td><span className={`badge badge-${e.status}`}>{e.status}</span></td>
|
||||
<td style={{ color: 'var(--text-secondary)' }}>{timeAgo(e.timestamp)}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={{ width: 220 }}>
|
||||
<h2 className="section-title">Token Health</h2>
|
||||
<div className="health-panel">
|
||||
<div className="health-row">
|
||||
<span style={{ color: 'var(--warning)' }}>Expiring Soon</span>
|
||||
<span className="mono">{health.expiring_soon}</span>
|
||||
</div>
|
||||
<div className="health-row">
|
||||
<span style={{ color: 'var(--error)' }}>Error Rate</span>
|
||||
<span className="mono">{health.error_rate}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import React, { useState } from 'react';
|
||||
import { api } from '../api';
|
||||
|
||||
// v0.26.3 trust model (D11 + D12):
|
||||
// - The bootstrap token is NEVER stored in browser JS state. No
|
||||
// localStorage, no sessionStorage, no React state beyond the form
|
||||
// submit cycle. After successful POST /admin/login the operator's
|
||||
// token only lives in the HttpOnly cookie that the server set.
|
||||
// - Magic-link URLs use single-use server-issued nonces, not the
|
||||
// bootstrap token itself (see /admin/api/issue-magic-link). The
|
||||
// bootstrap token never appears in a URL.
|
||||
// - Closing the tab ends the session client-side. Reopening the
|
||||
// dashboard 401s and shows this page again. Operator asks the agent
|
||||
// for a fresh magic link or pastes the bootstrap token from the
|
||||
// server's terminal scrollback.
|
||||
export function LoginPage({ onLogin }: { onLogin: () => void }) {
|
||||
const [token, setToken] = useState('');
|
||||
const [error, setError] = useState('');
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
const handleSubmit = async (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
setError('');
|
||||
setLoading(true);
|
||||
try {
|
||||
await api.login(token);
|
||||
// Don't persist the token. The HttpOnly cookie is the only
|
||||
// session credential after this point.
|
||||
setToken('');
|
||||
onLogin();
|
||||
} catch (err) {
|
||||
setError('Invalid token.');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="login-page">
|
||||
<div className="login-box">
|
||||
<div className="login-logo">GBrain</div>
|
||||
|
||||
<div style={{
|
||||
background: 'rgba(136, 170, 255, 0.08)',
|
||||
border: '1px solid rgba(136, 170, 255, 0.2)',
|
||||
borderRadius: 8,
|
||||
padding: '14px 16px',
|
||||
marginBottom: 20,
|
||||
fontSize: 13,
|
||||
lineHeight: 1.5,
|
||||
color: 'var(--text-secondary)',
|
||||
}}>
|
||||
<div style={{ fontWeight: 600, color: 'var(--text-primary)', marginBottom: 6 }}>
|
||||
🔒 This is a protected dashboard
|
||||
</div>
|
||||
Ask your AI agent for the admin login link:
|
||||
<div style={{
|
||||
background: 'rgba(0,0,0,0.3)',
|
||||
borderRadius: 6,
|
||||
padding: '8px 12px',
|
||||
marginTop: 8,
|
||||
fontFamily: 'var(--font-mono)',
|
||||
fontSize: 12,
|
||||
color: '#88aaff',
|
||||
wordBreak: 'break-all',
|
||||
}}>
|
||||
"Give me the GBrain admin login link"
|
||||
</div>
|
||||
<div style={{ marginTop: 8, fontSize: 12, color: 'var(--text-muted)' }}>
|
||||
Each link is single-use. Your agent generates a fresh one each time.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<details style={{ marginBottom: 16 }}>
|
||||
<summary style={{ cursor: 'pointer', fontSize: 13, color: 'var(--text-muted)' }}>
|
||||
Or paste bootstrap token manually
|
||||
</summary>
|
||||
<form onSubmit={handleSubmit} style={{ marginTop: 12 }}>
|
||||
<div style={{ marginBottom: 12 }}>
|
||||
<input
|
||||
type="password"
|
||||
placeholder="Admin Token"
|
||||
value={token}
|
||||
onChange={e => setToken(e.target.value)}
|
||||
/>
|
||||
</div>
|
||||
<button className="btn btn-primary" style={{ width: '100%' }} disabled={loading}>
|
||||
{loading ? 'Authenticating...' : 'Submit'}
|
||||
</button>
|
||||
{error && <div className="login-error">{error}</div>}
|
||||
</form>
|
||||
</details>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { api } from '../api';
|
||||
|
||||
interface LogEntry {
|
||||
id: number;
|
||||
token_name: string;
|
||||
agent_name: string;
|
||||
operation: string;
|
||||
latency_ms: number;
|
||||
status: string;
|
||||
params: Record<string, unknown> | null;
|
||||
error_message: string | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export function RequestLogPage() {
|
||||
const [data, setData] = useState<{ rows: LogEntry[]; total: number; page: number; pages: number }>({
|
||||
rows: [], total: 0, page: 1, pages: 1,
|
||||
});
|
||||
const [page, setPage] = useState(1);
|
||||
const [agentFilter, setAgentFilter] = useState('all');
|
||||
const [expandedRow, setExpandedRow] = useState<number | null>(null);
|
||||
|
||||
useEffect(() => { loadPage(page); }, [page, agentFilter]);
|
||||
|
||||
const loadPage = (p: number) => {
|
||||
const qs = agentFilter !== 'all' ? `&agent=${encodeURIComponent(agentFilter)}` : '';
|
||||
api.requests(p, qs).then(setData).catch(() => {});
|
||||
};
|
||||
|
||||
const timeAgo = (ts: string) => {
|
||||
const diff = Date.now() - new Date(ts).getTime();
|
||||
if (diff < 60000) return `${Math.floor(diff / 1000)}s ago`;
|
||||
if (diff < 3600000) return `${Math.floor(diff / 60000)} min ago`;
|
||||
if (diff < 86400000) return `${Math.floor(diff / 3600000)}h ago`;
|
||||
return new Date(ts).toLocaleDateString();
|
||||
};
|
||||
|
||||
|
||||
|
||||
const formatParams = (params: Record<string, unknown> | null) => {
|
||||
if (!params) return null;
|
||||
const { query, slug, partial, limit, ...rest } = params as any;
|
||||
const parts: string[] = [];
|
||||
if (query) parts.push(`"${query}"`);
|
||||
if (slug) parts.push(slug);
|
||||
if (partial) parts.push(`~${partial}`);
|
||||
if (limit) parts.push(`limit=${limit}`);
|
||||
if (Object.keys(rest).length > 0) parts.push(`+${Object.keys(rest).length} params`);
|
||||
return parts.join(' ');
|
||||
};
|
||||
|
||||
// Collect unique agents for filter (use name for display, token_name for value)
|
||||
const agentMap = new Map<string, string>();
|
||||
data.rows.forEach(r => { if (r.token_name) agentMap.set(r.token_name, r.agent_name || r.token_name); });
|
||||
|
||||
return (
|
||||
<>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', marginBottom: 24 }}>
|
||||
<h1 className="page-title" style={{ marginBottom: 0 }}>Request Log</h1>
|
||||
<select value={agentFilter} onChange={e => { setAgentFilter(e.target.value); setPage(1); }}
|
||||
style={{ background: 'var(--bg-secondary)', color: 'var(--text-primary)', border: '1px solid var(--border)', borderRadius: 6, padding: '4px 8px', fontSize: 13 }}>
|
||||
<option value="all">All agents</option>
|
||||
{[...agentMap.entries()].map(([id, name]) => <option key={id} value={id}>{name}</option>)}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
{data.rows.length === 0 ? (
|
||||
<div style={{ textAlign: 'center', padding: 48, color: 'var(--text-muted)' }}>
|
||||
No requests yet.
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Time</th>
|
||||
<th>Agent</th>
|
||||
<th>Operation</th>
|
||||
<th>Params</th>
|
||||
<th>Latency</th>
|
||||
<th>Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{data.rows.map(r => (
|
||||
<React.Fragment key={r.id}>
|
||||
<tr onClick={() => setExpandedRow(expandedRow === r.id ? null : r.id)}
|
||||
style={{ cursor: 'pointer' }}>
|
||||
<td style={{ color: 'var(--text-secondary)', whiteSpace: 'nowrap' }}>{timeAgo(r.created_at)}</td>
|
||||
<td>
|
||||
<a style={{ color: 'var(--text-link, #88aaff)', cursor: 'pointer', textDecoration: 'none', fontWeight: 500 }}
|
||||
onClick={(e) => { e.stopPropagation(); setAgentFilter(r.token_name); setPage(1); }}>
|
||||
{r.agent_name || r.token_name}
|
||||
</a>
|
||||
</td>
|
||||
<td className="mono">{r.operation}</td>
|
||||
<td style={{ color: 'var(--text-secondary)', fontSize: 12, maxWidth: 200, overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>
|
||||
{formatParams(r.params)}
|
||||
</td>
|
||||
<td className="mono">{r.latency_ms}ms</td>
|
||||
<td><span className={`badge badge-${r.status}`}>{r.status}</span></td>
|
||||
</tr>
|
||||
{expandedRow === r.id && (
|
||||
<tr>
|
||||
<td colSpan={6} style={{ background: 'var(--bg-secondary, #0f0f1a)', padding: 16 }}>
|
||||
<div style={{ display: 'grid', gridTemplateColumns: '100px 1fr', gap: '6px 12px', fontSize: 13 }}>
|
||||
<span style={{ color: 'var(--text-muted)' }}>Time</span>
|
||||
<span>{new Date(r.created_at).toLocaleString()}</span>
|
||||
<span style={{ color: 'var(--text-muted)' }}>Agent</span>
|
||||
<span className="mono">{r.token_name}</span>
|
||||
<span style={{ color: 'var(--text-muted)' }}>Operation</span>
|
||||
<span className="mono">{r.operation}</span>
|
||||
<span style={{ color: 'var(--text-muted)' }}>Latency</span>
|
||||
<span>{r.latency_ms}ms</span>
|
||||
{r.params && (
|
||||
<>
|
||||
<span style={{ color: 'var(--text-muted)' }}>Params</span>
|
||||
<pre className="mono" style={{ margin: 0, whiteSpace: 'pre-wrap', fontSize: 12 }}>
|
||||
{JSON.stringify(r.params, null, 2)}
|
||||
</pre>
|
||||
</>
|
||||
)}
|
||||
{r.error_message && (
|
||||
<>
|
||||
<span style={{ color: 'var(--error, #ff6b6b)' }}>Error</span>
|
||||
<span style={{ color: 'var(--error, #ff6b6b)' }}>{r.error_message}</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
)}
|
||||
</React.Fragment>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<div className="pagination">
|
||||
<span>Page {data.page} of {data.pages} ({data.total} total)</span>
|
||||
<div style={{ display: 'flex', gap: 8 }}>
|
||||
<button disabled={data.page <= 1} onClick={() => setPage(p => p - 1)}>Previous</button>
|
||||
<button disabled={data.page >= data.pages} onClick={() => setPage(p => p + 1)}>Next</button>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
/// <reference types="vite/client" />
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"useDefineForClassFields": true,
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"skipLibCheck": true,
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"isolatedModules": true,
|
||||
"moduleDetection": "force",
|
||||
"noEmit": true,
|
||||
"jsx": "react-jsx",
|
||||
"strict": true
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { defineConfig } from 'vite';
|
||||
import react from '@vitejs/plugin-react';
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
base: '/admin/',
|
||||
build: {
|
||||
outDir: 'dist',
|
||||
emptyOutDir: true,
|
||||
},
|
||||
});
|
||||
@@ -9,7 +9,11 @@
|
||||
"@aws-sdk/client-s3": "^3.1028.0",
|
||||
"@dqbd/tiktoken": "^1.0.22",
|
||||
"@electric-sql/pglite": "0.4.3",
|
||||
"@modelcontextprotocol/sdk": "^1.0.0",
|
||||
"@modelcontextprotocol/sdk": "1.29.0",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cors": "^2.8.5",
|
||||
"express": "^5.1.0",
|
||||
"express-rate-limit": "^7.5.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
"marked": "^18.0.0",
|
||||
"openai": "^4.0.0",
|
||||
@@ -20,6 +24,10 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"@types/cookie-parser": "^1.4.7",
|
||||
"@types/cors": "^2.8.19",
|
||||
"@types/express": "^5.0.6",
|
||||
"bun-types": "^1.3.13",
|
||||
"typescript": "^5.6.0",
|
||||
},
|
||||
},
|
||||
@@ -218,12 +226,34 @@
|
||||
|
||||
"@smithy/uuid": ["@smithy/uuid@1.1.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-O/IEdcCUKkubz60tFbGA7ceITTAJsty+lBjNoorP4Z6XRqaFb/OjQjZODophEcuq68nKm6/0r+6/lLQ+XVpk8g=="],
|
||||
|
||||
"@types/body-parser": ["@types/body-parser@1.19.6", "", { "dependencies": { "@types/connect": "*", "@types/node": "*" } }, "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g=="],
|
||||
|
||||
"@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="],
|
||||
|
||||
"@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="],
|
||||
"@types/connect": ["@types/connect@3.4.38", "", { "dependencies": { "@types/node": "*" } }, "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug=="],
|
||||
|
||||
"@types/cookie-parser": ["@types/cookie-parser@1.4.10", "", { "peerDependencies": { "@types/express": "*" } }, "sha512-B4xqkqfZ8Wek+rCOeRxsjMS9OgvzebEzzLYw7NHYuvzb7IdxOkI0ZHGgeEBX4PUM7QGVvNSK60T3OvWj3YfBRg=="],
|
||||
|
||||
"@types/cors": ["@types/cors@2.8.19", "", { "dependencies": { "@types/node": "*" } }, "sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg=="],
|
||||
|
||||
"@types/express": ["@types/express@5.0.6", "", { "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", "@types/serve-static": "^2" } }, "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA=="],
|
||||
|
||||
"@types/express-serve-static-core": ["@types/express-serve-static-core@5.1.1", "", { "dependencies": { "@types/node": "*", "@types/qs": "*", "@types/range-parser": "*", "@types/send": "*" } }, "sha512-v4zIMr/cX7/d2BpAEX3KNKL/JrT1s43s96lLvvdTmza1oEvDudCqK9aF/djc/SWgy8Yh0h30TZx5VpzqFCxk5A=="],
|
||||
|
||||
"@types/http-errors": ["@types/http-errors@2.0.5", "", {}, "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg=="],
|
||||
|
||||
"@types/node": ["@types/node@25.5.2", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg=="],
|
||||
|
||||
"@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="],
|
||||
|
||||
"@types/qs": ["@types/qs@6.15.0", "", {}, "sha512-JawvT8iBVWpzTrz3EGw9BTQFg3BQNmwERdKE22vlTxawwtbyUSlMppvZYKLZzB5zgACXdXxbD3m1bXaMqP/9ow=="],
|
||||
|
||||
"@types/range-parser": ["@types/range-parser@1.2.7", "", {}, "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ=="],
|
||||
|
||||
"@types/send": ["@types/send@1.2.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ=="],
|
||||
|
||||
"@types/serve-static": ["@types/serve-static@2.2.0", "", { "dependencies": { "@types/http-errors": "*", "@types/node": "*" } }, "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ=="],
|
||||
|
||||
"abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="],
|
||||
|
||||
"accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="],
|
||||
@@ -242,7 +272,7 @@
|
||||
|
||||
"bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="],
|
||||
|
||||
"bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
|
||||
"bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="],
|
||||
|
||||
"bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="],
|
||||
|
||||
@@ -258,7 +288,9 @@
|
||||
|
||||
"cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="],
|
||||
|
||||
"cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="],
|
||||
"cookie-parser": ["cookie-parser@1.4.7", "", { "dependencies": { "cookie": "0.7.2", "cookie-signature": "1.0.6" } }, "sha512-nGUvgXnotP3BsjiLX2ypbQnWoGUPIIfHQNZkkC668ntrzGWEZVW70HDEB1qnNGMicPje6EttlIgzo51YSwNQGw=="],
|
||||
|
||||
"cookie-signature": ["cookie-signature@1.0.6", "", {}, "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="],
|
||||
|
||||
"cors": ["cors@2.8.6", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw=="],
|
||||
|
||||
@@ -298,7 +330,7 @@
|
||||
|
||||
"express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="],
|
||||
|
||||
"express-rate-limit": ["express-rate-limit@8.3.2", "", { "dependencies": { "ip-address": "10.1.0" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-77VmFeJkO0/rvimEDuUC5H30oqUC4EyOhyGccfqoLebB0oiEYfM7nwPrsDsBL1gsTpwfzX8SFy2MT3TDyRq+bg=="],
|
||||
"express-rate-limit": ["express-rate-limit@7.5.1", "", { "peerDependencies": { "express": ">= 4.11" } }, "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw=="],
|
||||
|
||||
"extend-shallow": ["extend-shallow@2.0.1", "", { "dependencies": { "is-extendable": "^0.1.0" } }, "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug=="],
|
||||
|
||||
@@ -466,7 +498,7 @@
|
||||
|
||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
"undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="],
|
||||
"undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
|
||||
|
||||
"unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="],
|
||||
|
||||
@@ -488,30 +520,36 @@
|
||||
|
||||
"zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="],
|
||||
|
||||
"@anthropic-ai/sdk/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="],
|
||||
|
||||
"@aws-crypto/sha1-browser/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
|
||||
|
||||
"@aws-crypto/sha256-browser/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
|
||||
|
||||
"@aws-crypto/util/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
|
||||
|
||||
"@types/node-fetch/@types/node": ["@types/node@25.5.2", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg=="],
|
||||
"@modelcontextprotocol/sdk/express-rate-limit": ["express-rate-limit@8.3.2", "", { "dependencies": { "ip-address": "10.1.0" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-77VmFeJkO0/rvimEDuUC5H30oqUC4EyOhyGccfqoLebB0oiEYfM7nwPrsDsBL1gsTpwfzX8SFy2MT3TDyRq+bg=="],
|
||||
|
||||
"bun-types/@types/node": ["@types/node@25.5.2", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg=="],
|
||||
"@types/bun/bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
|
||||
|
||||
"express/cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="],
|
||||
|
||||
"form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="],
|
||||
|
||||
"openai/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="],
|
||||
|
||||
"@anthropic-ai/sdk/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="],
|
||||
|
||||
"@aws-crypto/sha1-browser/@smithy/util-utf8/@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="],
|
||||
|
||||
"@aws-crypto/sha256-browser/@smithy/util-utf8/@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="],
|
||||
|
||||
"@aws-crypto/util/@smithy/util-utf8/@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="],
|
||||
|
||||
"@types/node-fetch/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
|
||||
|
||||
"bun-types/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
|
||||
|
||||
"form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="],
|
||||
|
||||
"openai/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="],
|
||||
|
||||
"@aws-crypto/sha1-browser/@smithy/util-utf8/@smithy/util-buffer-from/@smithy/is-array-buffer": ["@smithy/is-array-buffer@2.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA=="],
|
||||
|
||||
"@aws-crypto/sha256-browser/@smithy/util-utf8/@smithy/util-buffer-from/@smithy/is-array-buffer": ["@smithy/is-array-buffer@2.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA=="],
|
||||
|
||||
+6
-9
@@ -1,12 +1,9 @@
|
||||
[test]
|
||||
# PGLite initialization can be slow under parallel test execution.
|
||||
# Default 5s is too short when many test files boot PGLite instances at once.
|
||||
# 60s is the empirical ceiling we observed before the first file's beforeAll
|
||||
# completed on a loaded machine.
|
||||
# PGLite WASM cold start + initSchema() runs ~5–20s on loaded machines.
|
||||
# Default 5s is too short for those tests' beforeAll hooks. 60s is the
|
||||
# empirical ceiling we observed for the slowest cold-init paths.
|
||||
#
|
||||
# NOTE: this bunfig.toml `timeout` key is read by `bun test` but empirically
|
||||
# does NOT apply to beforeEach/afterEach hook timeouts under `bun run test`
|
||||
# chained behind `bun run typecheck`. The test script in package.json passes
|
||||
# `--timeout=60000` explicitly to cover both per-test and per-hook timeouts.
|
||||
# Leaving both in place as belt-and-suspenders.
|
||||
# v0.26.4: scripts/run-unit-parallel.sh and scripts/run-unit-shard.sh
|
||||
# also pass `--timeout=60000` explicitly so the ceiling is consistent
|
||||
# whether tests are invoked through the wrapper or directly via bun test.
|
||||
timeout = 60_000
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
# docker-compose.ci.yml
|
||||
#
|
||||
# Local CI gate with 4-way E2E sharding. Spins up 4 pgvector services + a bun
|
||||
# runner that bind-mounts the repo. Used by `bun run ci:local` and
|
||||
# `bun run ci:local:diff` (see scripts/ci-local.sh).
|
||||
#
|
||||
# All services are pulled as `image:` (no build) so `docker compose pull`
|
||||
# refreshes everything. The bun version floats with `oven/bun:1` to track CI's
|
||||
# `bun-version: latest`. Named volumes isolate the Linux container's deps from
|
||||
# the host's darwin-arm64 deps and keep bun + postgres data warm across runs.
|
||||
#
|
||||
# Why 4 postgres services: bun's E2E suite shares one DB across 36 files and
|
||||
# uses TRUNCATE CASCADE in setupDB(). Running files in parallel against ONE DB
|
||||
# races (file A's TRUNCATE clobbers file B's fixture import). 4 separate DBs
|
||||
# remove the race; we shard the file list 1/4..4/4 and run shards in parallel.
|
||||
# Within a shard, files still run sequentially. Total wall-time on a 16-core
|
||||
# host: ~6 min sequential -> ~1.5-2 min sharded.
|
||||
#
|
||||
# Postgres host ports default to 5434-5437 (avoid 5432 manual `gbrain-test-pg`
|
||||
# and 5433 sibling-project conflicts). Override BASE port with GBRAIN_CI_PG_PORT;
|
||||
# shards take BASE..BASE+3.
|
||||
|
||||
services:
|
||||
postgres-1:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- "${GBRAIN_CI_PG_PORT:-5434}:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d gbrain_test"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
volumes:
|
||||
- gbrain-ci-pg-data-1:/var/lib/postgresql/data
|
||||
|
||||
postgres-2:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- "${GBRAIN_CI_PG_PORT_2:-5435}:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d gbrain_test"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
volumes:
|
||||
- gbrain-ci-pg-data-2:/var/lib/postgresql/data
|
||||
|
||||
postgres-3:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- "${GBRAIN_CI_PG_PORT_3:-5436}:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d gbrain_test"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
volumes:
|
||||
- gbrain-ci-pg-data-3:/var/lib/postgresql/data
|
||||
|
||||
postgres-4:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- "${GBRAIN_CI_PG_PORT_4:-5437}:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d gbrain_test"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
volumes:
|
||||
- gbrain-ci-pg-data-4:/var/lib/postgresql/data
|
||||
|
||||
runner:
|
||||
image: oven/bun:1
|
||||
working_dir: /app
|
||||
depends_on:
|
||||
postgres-1:
|
||||
condition: service_healthy
|
||||
postgres-2:
|
||||
condition: service_healthy
|
||||
postgres-3:
|
||||
condition: service_healthy
|
||||
postgres-4:
|
||||
condition: service_healthy
|
||||
# No global DATABASE_URL — scripts/ci-local.sh sets per-shard URL via -e.
|
||||
# Unit phase explicitly unsets DATABASE_URL so test/e2e/* gracefully skip.
|
||||
volumes:
|
||||
- .:/app
|
||||
# Linux container's node_modules MUST be isolated from host darwin-arm64.
|
||||
# Without this, container `bun install` stomps host node_modules and
|
||||
# subsequent `bun test` on host fails with binary-incompat errors.
|
||||
- gbrain-ci-node-modules:/app/node_modules
|
||||
# Warm install cache across runs.
|
||||
- gbrain-ci-bun-cache:/root/.bun/install/cache
|
||||
|
||||
volumes:
|
||||
gbrain-ci-pg-data-1:
|
||||
gbrain-ci-pg-data-2:
|
||||
gbrain-ci-pg-data-3:
|
||||
gbrain-ci-pg-data-4:
|
||||
gbrain-ci-node-modules:
|
||||
gbrain-ci-bun-cache:
|
||||
@@ -0,0 +1,242 @@
|
||||
# Brains and Sources — the mental model
|
||||
|
||||
GBrain has two orthogonal axes for organizing knowledge. Users and agents both
|
||||
need to understand both of them, or queries misroute silently.
|
||||
|
||||
**TL;DR:**
|
||||
- A **brain** is a database. You can have many.
|
||||
- A **source** is a named repo of content *inside* a brain. One brain can hold many.
|
||||
- `--brain <id>` picks WHICH DATABASE.
|
||||
- `--source <id>` picks WHICH REPO WITHIN that database.
|
||||
- They're independent. You can target any combination.
|
||||
|
||||
---
|
||||
|
||||
## The two axes
|
||||
|
||||
### Brains (the DB axis)
|
||||
|
||||
A **brain** is one database — PGLite file, self-hosted Postgres, or Supabase.
|
||||
Each brain has:
|
||||
- Its own `pages` table, `chunks` table, `embeddings`, etc.
|
||||
- Its own OAuth surface if served over HTTP MCP (v0.19+, PR 2).
|
||||
- Its own separate lifecycle, backup, access control.
|
||||
|
||||
Brains are enumerated by:
|
||||
- **host** — your default brain, configured in `~/.gbrain/config.json`.
|
||||
- **mounts** — additional brains registered in `~/.gbrain/mounts.json` via
|
||||
`gbrain mounts add <id>` (v0.19+).
|
||||
|
||||
Routing: `--brain <id>`, `GBRAIN_BRAIN_ID`, `.gbrain-mount` dotfile, or
|
||||
longest-path match against registered mount paths. Falls back to `host`.
|
||||
|
||||
### Sources (the repo axis, v0.18.0+)
|
||||
|
||||
A **source** is a named content repo *inside* one brain. Every `pages` row
|
||||
carries a `source_id`. Slugs are unique per source, not globally.
|
||||
|
||||
Example: in one brain, the slug `topics/ai` can exist under `source=wiki`
|
||||
AND under `source=gstack` — they're different pages.
|
||||
|
||||
Routing: `--source <id>`, `GBRAIN_SOURCE`, `.gbrain-source` dotfile, or
|
||||
registered `local_path` match in the `sources` table.
|
||||
|
||||
### When does each axis move?
|
||||
|
||||
| You want to | Adjust |
|
||||
|---|---|
|
||||
| Work in a different repo within the same brain (wiki → gstack notes) | `--source` |
|
||||
| Query a team-published brain that isn't yours | `--brain` |
|
||||
| Isolate a topic so it never leaks into personal search | `--source` with `federated=false` |
|
||||
| Share a brain with teammates | `--brain` (mount the team brain) |
|
||||
| Add a new repo to your personal brain | `--source` via `gbrain sources add` |
|
||||
| Add a team brain | `--brain` via `gbrain mounts add` |
|
||||
|
||||
**Rule of thumb:** if the data owner changes, it's a brain boundary. If the
|
||||
data owner stays the same but the topic/repo changes, it's a source boundary.
|
||||
|
||||
---
|
||||
|
||||
## Topology: a single-person developer
|
||||
|
||||
Simplest case. One brain, one source.
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────┐
|
||||
│ host brain (~/.gbrain) │
|
||||
│ ├── source: default (federated=true) │
|
||||
│ │ └── all pages │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
`gbrain query "retry budgets"` finds everything. No `--brain`, no `--source`
|
||||
needed.
|
||||
|
||||
---
|
||||
|
||||
## Topology: a personal brain with multiple repos
|
||||
|
||||
You maintain several codebases or writing streams. Each is its own source
|
||||
inside one brain. Cross-source search is on by default so a query about
|
||||
"caching" returns hits from every repo.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ host brain (~/.gbrain) │
|
||||
│ ├── source: wiki (federated=true) │
|
||||
│ │ └── personal notes, people, companies │
|
||||
│ ├── source: gstack (federated=true) │
|
||||
│ │ └── gstack plans, learnings │
|
||||
│ ├── source: openclaw (federated=true) │
|
||||
│ │ └── openclaw docs, memos │
|
||||
│ └── source: essays (federated=false) │
|
||||
│ └── draft essays, isolated on purpose │
|
||||
└──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Inside `~/openclaw/` the `.gbrain-source` dotfile pins every command to
|
||||
`source=openclaw`. Inside `~/gstack/` the dotfile pins to `source=gstack`.
|
||||
Everything still targets one DB.
|
||||
|
||||
Use this topology when:
|
||||
- You own all the content.
|
||||
- You want cross-repo search to just work.
|
||||
- You don't need to share any of it with someone who isn't you.
|
||||
|
||||
---
|
||||
|
||||
## Topology: personal brain + one team brain
|
||||
|
||||
You're on a team that publishes a shared brain. Your personal brain stays
|
||||
as-is; you mount the team brain alongside it.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ host brain (~/.gbrain) — YOUR personal DB │
|
||||
│ ├── source: wiki │
|
||||
│ ├── source: gstack │
|
||||
│ └── ... │
|
||||
└──────────────────────────────────────────────┘
|
||||
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ mount: media-team │
|
||||
│ path: ~/team-brains/media │
|
||||
│ engine: postgres (team's Supabase) │
|
||||
│ └── sources: wiki, raw, enriched │
|
||||
└──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
`gbrain query "X"` (no flags) → runs against host (your personal brain).
|
||||
`gbrain query "X" --brain media-team` → runs against the team's DB.
|
||||
Inside `~/team-brains/media/` a `.gbrain-mount` dotfile pins brain to
|
||||
`media-team` automatically.
|
||||
|
||||
Use this topology when:
|
||||
- You're on a team and someone publishes a brain the team subscribes to.
|
||||
- You need data isolation between work and personal.
|
||||
- Different teams/orgs own different brains.
|
||||
|
||||
---
|
||||
|
||||
## Topology: a CEO-class user with multiple team memberships
|
||||
|
||||
You're senior enough to sit across multiple teams. You maintain your personal
|
||||
brain (with N sources inside) AND mount several work team brains. Each team
|
||||
brain is itself a multi-source brain in the v0.18.0 sense — organized
|
||||
internally however the team owner chose.
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ host brain — YOUR personal DB │
|
||||
│ ├── source: wiki │
|
||||
│ ├── source: essays │
|
||||
│ ├── source: gstack │
|
||||
│ └── source: openclaw │
|
||||
└──────────────────────────────────────────────┘
|
||||
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ mount: media-team (your media team's brain) │
|
||||
│ └── sources: wiki, pipeline, enriched │
|
||||
└──────────────────────────────────────────────┘
|
||||
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ mount: policy-team (your policy team's) │
|
||||
│ └── sources: wiki, research, letters │
|
||||
└──────────────────────────────────────────────┘
|
||||
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ mount: portfolio (another team's) │
|
||||
│ └── sources: companies, deals, diligence │
|
||||
└──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Inside each team's checkout, a `.gbrain-mount` dotfile pins the brain. Inside
|
||||
a specific subdirectory, a `.gbrain-source` dotfile pins the source. So `cd
|
||||
~/team-brains/policy/research && gbrain query "X"` targets
|
||||
`brain=policy-team, source=research` with zero flags.
|
||||
|
||||
Use this topology when:
|
||||
- You cross-cut multiple teams.
|
||||
- Each team owns its own brain with its own access policy.
|
||||
- You need latent-space federation (agent decides when to query across
|
||||
brains), not SQL federation.
|
||||
|
||||
Cross-brain queries are **not deterministic** in v0.19. The agent sees the
|
||||
brain list and re-queries as needed. That's the feature — it keeps debugging
|
||||
sane and access control clean.
|
||||
|
||||
---
|
||||
|
||||
## Resolution precedence (one page to remember)
|
||||
|
||||
```
|
||||
WHICH BRAIN (DB)? WHICH SOURCE (repo in DB)?
|
||||
1. --brain <id> 1. --source <id>
|
||||
2. GBRAIN_BRAIN_ID env 2. GBRAIN_SOURCE env
|
||||
3. .gbrain-mount dotfile 3. .gbrain-source dotfile
|
||||
4. longest-prefix mount path match 4. longest-prefix source path match
|
||||
5. (reserved: brains.default v2) 5. sources.default config
|
||||
6. fallback: 'host' 6. fallback: 'default'
|
||||
```
|
||||
|
||||
Both axes follow the same layered pattern on purpose. If you know one, you
|
||||
know the other.
|
||||
|
||||
---
|
||||
|
||||
## For agents reading this
|
||||
|
||||
- Default assumption when the user asks a question: start in the current
|
||||
brain (resolved via the precedence above). Don't jump brains without a
|
||||
reason.
|
||||
- If the user asks a question that crosses topic areas a team might own
|
||||
(e.g. "what did Team X decide last week?"), the right move is to *query
|
||||
the team's brain explicitly* rather than searching host with "team x".
|
||||
- Cross-brain federation is YOUR JOB, not the DB's. You have the brain list
|
||||
(`gbrain mounts list`). You decide when to fan out. You synthesize
|
||||
findings. You cite `brain:source:slug`.
|
||||
- When writing a page, respect the brain boundary. A fact about a team's
|
||||
work belongs in the team's brain, not in the user's personal brain. Ask
|
||||
before writing cross-brain.
|
||||
- See `skills/conventions/brain-routing.md` for the full decision table.
|
||||
|
||||
## For users reading this
|
||||
|
||||
- **Default path:** set up your personal brain (`gbrain init`), add a source
|
||||
per repo you care about (`gbrain sources add gstack --path ~/gstack`).
|
||||
You'll almost never need `--brain`.
|
||||
- **When a team publishes a brain:** `gbrain mounts add <team-id> --path
|
||||
<clone> --db-url <url>` and the `.gbrain-mount` dotfile in that checkout
|
||||
routes queries there automatically.
|
||||
- **When you are the CEO-class user with multiple team memberships:** mount
|
||||
each team brain. Trust the resolver — inside a team's directory the
|
||||
dotfile picks the brain, inside a subdirectory the dotfile picks the
|
||||
source. The flags are for when you want to query across the boundary
|
||||
deliberately.
|
||||
|
||||
## Further reading
|
||||
|
||||
- v0.18.0 CHANGELOG — introduced `sources` primitive.
|
||||
- v0.19.0 CHANGELOG (TBD after PR 0+1+2 ship) — introduces `mounts`.
|
||||
- `docs/mounts/publishing-a-team-brain.md` (PR 2) — how to be the brain
|
||||
publisher, not just the subscriber.
|
||||
@@ -0,0 +1,224 @@
|
||||
# Running real-world eval benchmarks against your gbrain changes
|
||||
|
||||
Audience: gbrain maintainers and contributors. If you're touching retrieval
|
||||
(search, ranking, embeddings, intent classification, query expansion, source
|
||||
boost, hybrid fusion), this is the doc.
|
||||
|
||||
For the **NDJSON wire format** consumed by gbrain-evals, see
|
||||
[`eval-capture.md`](./eval-capture.md). This doc is the human dev loop
|
||||
that lives on top of that format.
|
||||
|
||||
## Prerequisite: turn on contributor mode
|
||||
|
||||
Capture is **off by default** for production users (privacy-positive — no
|
||||
surprise data accumulation). Contributors flip it on with one line:
|
||||
|
||||
```bash
|
||||
# In ~/.zshrc or ~/.bashrc:
|
||||
export GBRAIN_CONTRIBUTOR_MODE=1
|
||||
```
|
||||
|
||||
Verify:
|
||||
|
||||
```bash
|
||||
gbrain query "anything" >/dev/null
|
||||
psql $DATABASE_URL -c 'SELECT count(*) FROM eval_candidates' # should be > 0
|
||||
```
|
||||
|
||||
To override (force on/off regardless of env var), edit `~/.gbrain/config.json`:
|
||||
|
||||
```json
|
||||
{"eval": {"capture": true}} // force on
|
||||
{"eval": {"capture": false}} // force off
|
||||
```
|
||||
|
||||
Explicit config beats the env var both directions.
|
||||
|
||||
## The 4-command loop
|
||||
|
||||
```bash
|
||||
# ① Capture: writes to eval_candidates whenever CONTRIBUTOR_MODE is set.
|
||||
# Inspect what's been collected:
|
||||
gbrain doctor # surfaces capture failures
|
||||
psql $DATABASE_URL -c 'SELECT count(*) FROM eval_candidates'
|
||||
|
||||
# ② Snapshot: freeze a baseline before your code change.
|
||||
gbrain eval export --since 7d > baseline.ndjson
|
||||
|
||||
# ③ Code change: do whatever you want — tune RRF_K, swap embed model, edit
|
||||
# hybrid.ts, add a new boost source, change the intent classifier.
|
||||
|
||||
# ④ Replay: re-run every captured query against the current build.
|
||||
gbrain eval replay --against baseline.ndjson
|
||||
```
|
||||
|
||||
Output:
|
||||
|
||||
```
|
||||
Replaying 247 captured queries…
|
||||
...25/247
|
||||
...50/247
|
||||
...
|
||||
Replayed 247 of 247 captured queries (0 skipped, 0 errored)
|
||||
Mean Jaccard@k: 0.927
|
||||
Top-1 stability: 91.5%
|
||||
Mean latency Δ: +14ms (current vs captured)
|
||||
|
||||
Top 5 regression(s):
|
||||
jaccard=0.20 captured=12 current=3 "find every reference to widget-co"
|
||||
jaccard=0.43 captured=14 current=8 "show me everything tagged for review"
|
||||
jaccard=0.50 captured=8 current=4 "what did alice say about the spec"
|
||||
...
|
||||
```
|
||||
|
||||
Three numbers tell you whether the change is safe to land:
|
||||
|
||||
| Metric | What it means | Healthy range |
|
||||
|---|---|---|
|
||||
| **Mean Jaccard@k** | Average overlap between captured retrieved slugs and current run's slugs. 1.0 = identical sets. | ≥0.85 for "neutral" changes. <0.7 means major retrieval shift. |
|
||||
| **Top-1 stability** | Fraction of queries whose #1 result didn't change. | ≥85% for tuning passes. <70% means top-of-funnel broke. |
|
||||
| **Mean latency Δ** | Current minus captured. Positive = slower now. | Within ±50ms of captured. >2× anywhere = regression alarm. |
|
||||
|
||||
## What it actually does
|
||||
|
||||
`gbrain eval replay` reads your NDJSON snapshot and, for each row:
|
||||
|
||||
1. Re-executes the same op (`searchKeyword` for `tool_name='search'`,
|
||||
`hybridSearch` for `tool_name='query'`) with the captured `detail` and
|
||||
`expand_enabled` values threaded back in.
|
||||
2. Captures the current `retrieved_slugs` (deduped, in result order).
|
||||
3. Computes set-Jaccard between captured and current slug sets.
|
||||
4. Records top-1 match (was the #1 result the same slug?).
|
||||
5. Records latency delta vs captured `latency_ms`.
|
||||
|
||||
It does NOT compute MRR or nDCG — those need ground-truth relevance labels,
|
||||
not a baseline comparison. For metric-against-truth eval, use
|
||||
`gbrain eval --qrels <path>` (the legacy IR-eval path, still supported). The
|
||||
replay tool answers a different question: "did my code change move
|
||||
retrieval, and which queries did it move most?"
|
||||
|
||||
## Best-effort by design
|
||||
|
||||
Replay is not pure. Three things can drift between capture and replay:
|
||||
|
||||
1. **Brain state** — your brain probably has more pages now than when the
|
||||
snapshot was taken. Unless you explicitly seed a fixed corpus, mean
|
||||
Jaccard will drop simply because new pages are eligible.
|
||||
2. **Embedding source** — if you changed `OPENAI_API_KEY` between capture
|
||||
and replay (or the embedding model rotated), vector-path results drift
|
||||
even with identical code.
|
||||
3. **Capture cap** — captured `retrieved_slugs` is a deduped set; it doesn't
|
||||
preserve internal ranking metadata. Two tools can return the same slug
|
||||
set with different scores — Jaccard will say 1.0, but a downstream
|
||||
consumer that orders by score may behave differently.
|
||||
|
||||
The metrics are **regression alarms on real queries**, not a hash check.
|
||||
Pair them with manual inspection of the top regressions.
|
||||
|
||||
## Cost
|
||||
|
||||
Every `query` row in the snapshot embeds the query string via OpenAI to run
|
||||
the vector half of `hybridSearch`. Cost is identical to a normal `gbrain
|
||||
query` invocation — text-embedding-3-large at OpenAI list price, batched
|
||||
inside a single replay row.
|
||||
|
||||
If you're iterating locally and don't want to pay per change, use
|
||||
`--limit 50` to cap rows replayed. The 50 most recent rows are usually
|
||||
enough to catch direction; expand for the final pre-merge run.
|
||||
|
||||
```bash
|
||||
# Iteration mode — 50 most recent queries
|
||||
gbrain eval replay --against baseline.ndjson --limit 50
|
||||
|
||||
# Pre-merge — full snapshot
|
||||
gbrain eval replay --against baseline.ndjson --top-regressions 20
|
||||
```
|
||||
|
||||
## CI integration
|
||||
|
||||
```bash
|
||||
gbrain eval replay --against baseline.ndjson --json > replay.json
|
||||
jq -e '.summary.mean_jaccard >= 0.85' replay.json || exit 1
|
||||
jq -e '.summary.top1_stability_rate >= 0.85' replay.json || exit 1
|
||||
```
|
||||
|
||||
Stable JSON shape (schema_version: 1):
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"summary": {
|
||||
"rows_total": 247,
|
||||
"rows_replayed": 247,
|
||||
"rows_skipped": 0,
|
||||
"rows_errored": 0,
|
||||
"mean_jaccard": 0.927,
|
||||
"top1_stability_rate": 0.915,
|
||||
"mean_latency_delta_ms": 14,
|
||||
"rows_over_2x_latency": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`--verbose` adds a `results: [...]` array with one entry per replayed row
|
||||
(useful for piping into jq or a notebook for deeper analysis).
|
||||
|
||||
## When to run this
|
||||
|
||||
Before merging anything that touches:
|
||||
|
||||
- `src/core/search/hybrid.ts` (RRF, fusion, dedup, two-pass retrieval)
|
||||
- `src/core/search/source-boost.ts` / `sql-ranking.ts` (per-source ranking)
|
||||
- `src/core/search/intent.ts` (auto-detail classification)
|
||||
- `src/core/search/expansion.ts` (Haiku query expansion)
|
||||
- `src/core/search/dedup.ts` (cross-page result collapse)
|
||||
- `src/core/embedding.ts` or any embedding model swap
|
||||
- `src/core/operations.ts` `query` or `search` op handlers (capture surface)
|
||||
- `src/core/postgres-engine.ts` / `pglite-engine.ts` `searchKeyword` /
|
||||
`searchVector` SQL
|
||||
|
||||
Skip for: schema-only migrations, doc changes, tests-only PRs, CLI ergonomics
|
||||
that don't touch retrieval.
|
||||
|
||||
## Building your own corpus
|
||||
|
||||
If you don't have captured traffic yet (fresh install, can't dogfood for a
|
||||
week before merging), you can hand-author an NDJSON file:
|
||||
|
||||
```jsonl
|
||||
{"schema_version":1,"id":1,"tool_name":"query","query":"who is alice","retrieved_slugs":["people/alice","people/alice-bio"],"expand_enabled":false,"detail":null,"latency_ms":0,"remote":false}
|
||||
{"schema_version":1,"id":2,"tool_name":"search","query":"acme deal","retrieved_slugs":["deals/acme-seed","companies/acme"],"latency_ms":0,"remote":false}
|
||||
```
|
||||
|
||||
Then run `gbrain eval replay --against handcrafted.ndjson` to confirm the
|
||||
authoritative slugs come back. This is the seam between the BrainBench-Real
|
||||
pipeline (replay against live captures) and the BrainBench fixed-fixture
|
||||
pipeline (`gbrain eval --qrels` with the sibling
|
||||
[gbrain-evals](https://github.com/garrytan/gbrain-evals) corpus).
|
||||
|
||||
## Off-switch
|
||||
|
||||
Two ways to disable capture:
|
||||
|
||||
```bash
|
||||
unset GBRAIN_CONTRIBUTOR_MODE # easy: just unset the env var
|
||||
```
|
||||
|
||||
Or force off regardless of the env var via `~/.gbrain/config.json`:
|
||||
|
||||
```json
|
||||
{"eval": {"capture": false}}
|
||||
```
|
||||
|
||||
Existing `eval_candidates` rows stay until you `gbrain eval prune
|
||||
--older-than 0d` (or just drop the table).
|
||||
|
||||
## Failure modes
|
||||
|
||||
| What you see | What it means |
|
||||
|---|---|
|
||||
| `Mean Jaccard@k: 0.4`, top regressions all in one source dir | Source boost or hard-exclude regression on that prefix |
|
||||
| `Top-1 stability: 30%`, mean Jaccard still high | RRF tuning shifted the rank order without changing the set — re-tune `rrfK` |
|
||||
| `Mean latency Δ: +500ms`, jaccard high | Vector path got slower; check embedding API or HNSW probes |
|
||||
| `rows_errored > 0` | One or more queries threw. Inspect first 3 in human output, or `--json` to see all `error_message` fields |
|
||||
| Many `skipped: empty query` | Capture ran on rows where someone passed empty `query` — check why those were captured |
|
||||
@@ -0,0 +1,160 @@
|
||||
# Eval capture — NDJSON schema reference
|
||||
|
||||
**Status:** stable from v0.21.0. Schema versioning via `schema_version`
|
||||
on every row; additive changes increment the minor version; removals
|
||||
are breaking-schema-v2.
|
||||
|
||||
**Audience:** downstream consumers (primarily the sibling
|
||||
[gbrain-evals](https://github.com/garrytan/gbrain-evals) repo) that
|
||||
replay captured real-world queries as a BrainBench-Real fixture.
|
||||
|
||||
## The pipeline
|
||||
|
||||
```
|
||||
MCP / CLI / subagent tool-bridge caller
|
||||
│
|
||||
▼
|
||||
src/core/operations.ts — query + search op handlers
|
||||
│
|
||||
│ (hybridSearch or searchKeyword)
|
||||
│
|
||||
▼
|
||||
{results, meta: HybridSearchMeta} ┌── captureEvalCandidate
|
||||
│ │ (fire-and-forget)
|
||||
▼ │
|
||||
return to caller ▼
|
||||
scrubPii(query) ←── src/core/eval-capture-scrub.ts
|
||||
│
|
||||
▼
|
||||
buildEvalCandidateInput
|
||||
│
|
||||
▼
|
||||
engine.logEvalCandidate
|
||||
│
|
||||
┌──────────────┴──────────────┐
|
||||
│ success │ fail
|
||||
▼ ▼
|
||||
INSERT into eval_candidates engine.logEvalCaptureFailure
|
||||
(reason: db_down | rls_reject |
|
||||
check_violation |
|
||||
scrubber_exception | other)
|
||||
```
|
||||
|
||||
## `gbrain eval export` — the consumer contract
|
||||
|
||||
```sh
|
||||
gbrain eval export [--since DUR] [--limit N] [--tool query|search]
|
||||
```
|
||||
|
||||
Emits NDJSON to **stdout**. One JSON object per `\n`-terminated line.
|
||||
stderr receives progress heartbeats. Every line starts with
|
||||
`"schema_version": 1` so a forward-compat parser can fail loudly on
|
||||
schema v2 instead of silently misparsing.
|
||||
|
||||
Typical usage from gbrain-evals:
|
||||
|
||||
```sh
|
||||
# Snapshot the last week of real traffic for replay
|
||||
gbrain eval export --since 7d > brainbench-real.ndjson
|
||||
```
|
||||
|
||||
```sh
|
||||
# Stream through jq for ad-hoc analysis
|
||||
gbrain eval export --tool query | jq -c 'select(.latency_ms > 500)'
|
||||
```
|
||||
|
||||
## Row schema (v1)
|
||||
|
||||
Every exported row has this shape. Field order in JSON output is not
|
||||
guaranteed; consumers MUST key by name, not position.
|
||||
|
||||
| Field | Type | Notes |
|
||||
|---|---|---|
|
||||
| `schema_version` | number | Always `1` on v1 rows. Forward-compat gate. |
|
||||
| `id` | number | Autoincrement primary key. Stable across exports. |
|
||||
| `tool_name` | `"query"` \| `"search"` | Which MCP operation captured this row. |
|
||||
| `query` | string | **Already PII-scrubbed** by `scrubPii` unless `eval.scrub_pii: false`. Emails / phones / SSN / Luhn-verified credit cards / JWTs / bearer tokens replaced with `[REDACTED]`. Max length 50KB (CHECK-enforced). |
|
||||
| `retrieved_slugs` | string[] | Deduplicated slugs that came back in `SearchResult[]`. |
|
||||
| `retrieved_chunk_ids` | number[] | Every chunk id in result order (duplicates preserved — one per hit). |
|
||||
| `source_ids` | string[] | Distinct `sources.id` values across the result set (v0.18 multi-source). Empty for pre-v0.18 rows that lacked the column. |
|
||||
| `expand_enabled` | boolean \| null | Whether the caller **requested** Haiku expansion. `null` for `search` (no expansion concept). |
|
||||
| `detail` | `"low"` \| `"medium"` \| `"high"` \| null | Detail level the caller **requested**. `null` when omitted. |
|
||||
| `detail_resolved` | `"low"` \| `"medium"` \| `"high"` \| null | What `hybridSearch` **actually used** after auto-detect. `null` when neither caller nor heuristic classified. |
|
||||
| `vector_enabled` | boolean | True iff vector search actually ran. `false` when `OPENAI_API_KEY` was missing or the embed call failed. **Replay MUST respect this** — rows with `false` only exercised the keyword path. |
|
||||
| `expansion_applied` | boolean | True iff Haiku expansion actually produced variants (not just "was requested"). |
|
||||
| `latency_ms` | number | Wall-clock duration of the op handler (includes capture itself — negligible since it's fire-and-forget). |
|
||||
| `remote` | boolean | `true` for MCP callers (untrusted), `false` for local CLI. Partitions "real agent traffic" from "operator probing." |
|
||||
| `job_id` | number \| null | `OperationContext.jobId` when the caller was a subagent tool-bridge. Null for MCP + CLI. |
|
||||
| `subagent_id` | number \| null | `OperationContext.subagentId` for subagent-owned runs. |
|
||||
| `created_at` | string (ISO 8601) | UTC timestamp of insert. |
|
||||
|
||||
## Ordering + determinism
|
||||
|
||||
`listEvalCandidates` orders by `created_at DESC, id DESC`. Same-
|
||||
millisecond inserts tie on `created_at`; `id DESC` is the stable
|
||||
tiebreaker. Replay tools can consume rows in order and assume:
|
||||
- no duplicate rows across calls with non-overlapping `--since` windows
|
||||
- no missed rows across calls that chain `--since` windows (window end
|
||||
of run 1 is the strict upper bound, not a soft cursor)
|
||||
|
||||
## Schema versioning promise
|
||||
|
||||
- **v1 (shipped v0.21.0)** — this document. All fields listed above.
|
||||
- **Additive changes** increment gbrain minor version (v0.25.0, v0.23.0
|
||||
…) and ship with new optional fields. Consumers keyed on known fields
|
||||
ignore unknown keys and keep working.
|
||||
- **Breaking changes** (rename, type change, removal) increment
|
||||
`schema_version` to 2. Consumers MUST branch on `schema_version` to
|
||||
stay compatible.
|
||||
|
||||
## `eval_capture_failures` — companion audit table
|
||||
|
||||
Not exported by `gbrain eval export`. Surfaced via `gbrain doctor`:
|
||||
|
||||
```sh
|
||||
gbrain doctor # warns when failures in last 24h > 0
|
||||
```
|
||||
|
||||
Reason enum (stable): `db_down` | `rls_reject` | `check_violation` |
|
||||
`scrubber_exception` | `other`. Cross-process visibility is the whole
|
||||
point — `gbrain doctor` runs in its own process and reads the table
|
||||
directly, so in-process counters wouldn't work.
|
||||
|
||||
## Config + CONTRIBUTOR_MODE
|
||||
|
||||
Capture is **off by default** as of v0.25.0 (was on for everyone in
|
||||
earlier drafts). Two paths to turn it on:
|
||||
|
||||
**Path A — env var (contributor opt-in, the common case):**
|
||||
|
||||
```bash
|
||||
export GBRAIN_CONTRIBUTOR_MODE=1 # in ~/.zshrc or ~/.bashrc
|
||||
```
|
||||
|
||||
**Path B — explicit config (`~/.gbrain/config.json`, file-plane only):**
|
||||
|
||||
```json
|
||||
{
|
||||
"engine": "postgres",
|
||||
"database_url": "...",
|
||||
"eval": {
|
||||
"capture": true,
|
||||
"scrub_pii": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Resolution order (most explicit wins):
|
||||
|
||||
1. `eval.capture: true` in config → on
|
||||
2. `eval.capture: false` in config → off (overrides CONTRIBUTOR_MODE=1)
|
||||
3. `GBRAIN_CONTRIBUTOR_MODE === '1'` → on
|
||||
4. otherwise → off
|
||||
|
||||
`scrub_pii` defaults to `true` independent of capture. Set
|
||||
`eval.scrub_pii: false` to preserve raw query text (only if you control
|
||||
the brain's distribution).
|
||||
|
||||
`gbrain config set eval.capture false` does **not** work — that
|
||||
command writes the DB-plane config, and the MCP server reads the
|
||||
file-plane. Edit the JSON directly or use the env var.
|
||||
@@ -73,7 +73,7 @@ hook resumes blocking malformed pages.
|
||||
|
||||
## For downstream agent forks
|
||||
|
||||
If your fork (Wintermute, Hermes, OpenClaw) wraps gbrain in a host repo
|
||||
If your OpenClaw wraps gbrain in a host repo
|
||||
that's not the brain repo itself, you may want a separate hook strategy:
|
||||
|
||||
- **Brain repo IS the host repo** (gbrain skills + brain pages in one repo):
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# Connect GBrain to ChatGPT
|
||||
|
||||
**Status (v0.26.0):** Unblocked. GBrain's `gbrain serve --http` ships OAuth 2.1
|
||||
with PKCE, which is the ChatGPT MCP connector's hard requirement. Before v1.0,
|
||||
this was a P0 TODO — the only major AI client that could not connect.
|
||||
|
||||
ChatGPT does not support bearer-token MCP servers. You must use the OAuth 2.1
|
||||
HTTP server.
|
||||
|
||||
## Setup
|
||||
|
||||
### 1. Start the HTTP server
|
||||
|
||||
```bash
|
||||
gbrain serve --http --port 3131
|
||||
```
|
||||
|
||||
Save the admin bootstrap token printed on stderr. Open
|
||||
`http://localhost:3131/admin` and paste it to access the dashboard.
|
||||
|
||||
### 2. Register a ChatGPT client
|
||||
|
||||
ChatGPT uses the authorization code flow with PKCE (browser-based OAuth).
|
||||
Register from the `/admin` dashboard:
|
||||
|
||||
1. Click **Register client**.
|
||||
2. Name: `chatgpt`.
|
||||
3. Grant type: `authorization_code`.
|
||||
4. Scopes: `read`, `write` (leave `admin` unchecked for ChatGPT).
|
||||
5. Redirect URI: ChatGPT's OAuth redirect (copy it from the ChatGPT
|
||||
connector setup screen — something like
|
||||
`https://chat.openai.com/connector_platform_oauth_redirect`).
|
||||
6. Hit **Register**. The credential-reveal modal shows the `client_id` once
|
||||
with Copy and Download JSON buttons. There is no client secret for
|
||||
PKCE-based public clients.
|
||||
|
||||
Host-repo wrappers can register programmatically:
|
||||
|
||||
```ts
|
||||
await oauthProvider.registerClientManual(
|
||||
'chatgpt',
|
||||
['authorization_code'],
|
||||
'read write',
|
||||
['https://chat.openai.com/connector_platform_oauth_redirect'],
|
||||
);
|
||||
```
|
||||
|
||||
### 3. Expose the server publicly
|
||||
|
||||
```bash
|
||||
brew install ngrok
|
||||
ngrok http 3131 --url your-brain.ngrok.app
|
||||
```
|
||||
|
||||
Your OAuth issuer URL becomes `https://your-brain.ngrok.app`. ChatGPT's
|
||||
connector auto-discovers the spec-compliant endpoint at
|
||||
`/.well-known/oauth-authorization-server`.
|
||||
|
||||
### 4. Add the connector in ChatGPT
|
||||
|
||||
1. Open ChatGPT > Settings > Connectors.
|
||||
2. Click **Add connector**.
|
||||
3. MCP server URL: `https://your-brain.ngrok.app/mcp`.
|
||||
4. Client ID: the `client_id` you saved in step 2.
|
||||
5. Click **Connect**. ChatGPT opens the OAuth consent page, you approve, and
|
||||
the connector is live.
|
||||
|
||||
Start a new conversation and ask ChatGPT to search your brain. The MCP tool
|
||||
calls show up in the admin dashboard's live SSE feed in real time.
|
||||
|
||||
## Scopes
|
||||
|
||||
ChatGPT clients can request any combination of `read`, `write`, `admin`. The
|
||||
scopes granted at consent time are enforced on every tool call. Four
|
||||
operations are `localOnly` and rejected over HTTP regardless of scope:
|
||||
`sync_brain`, `file_upload`, `file_list`, `file_url`. The HTTP server fails
|
||||
closed for any attempt to reach local filesystem surface area.
|
||||
|
||||
Recommended ChatGPT scope: `read write`. Leave `admin` for your local CLI
|
||||
and the admin dashboard.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"Invalid redirect_uri" during the ChatGPT connector OAuth handshake**
|
||||
The registered `redirect-uri` must match ChatGPT's exactly. If ChatGPT
|
||||
rejects your server, check the admin dashboard's **Agents** table for the
|
||||
client, confirm the redirect URI matches what the error page shows, and
|
||||
re-register with the correct URI.
|
||||
|
||||
**ChatGPT shows an MCP connection error after approval**
|
||||
Open `/admin`, watch the SSE feed, and try again. If no request arrives, the
|
||||
connector isn't reaching your ngrok URL. If a request arrives but fails,
|
||||
the Request Log tab shows the exact error.
|
||||
|
||||
**"Unsupported grant_type" on the token endpoint**
|
||||
ChatGPT uses `authorization_code`, which the MCP SDK supports natively.
|
||||
If you see this error, verify the client was registered with
|
||||
`--grant-types authorization_code` and not `client_credentials`.
|
||||
|
||||
## See also
|
||||
|
||||
- [DEPLOY.md](DEPLOY.md) — full OAuth 2.1 setup reference
|
||||
- [ALTERNATIVES.md](ALTERNATIVES.md) — tunnel options (ngrok, Tailscale, Fly)
|
||||
+131
-15
@@ -1,17 +1,21 @@
|
||||
# Deploy GBrain Remote MCP Server
|
||||
|
||||
> **v0.22.7+:** Use `gbrain serve --http` for remote access. It includes built-in
|
||||
> bearer token auth, default-deny CORS, two-bucket rate limiting, body cap, and
|
||||
> per-request audit log. **Postgres-only** (PGLite is local-only by design).
|
||||
> See [SECURITY.md](../../SECURITY.md) for env vars and tunable defaults.
|
||||
> **v0.26.0+:** `gbrain serve --http` ships full OAuth 2.1 (client credentials,
|
||||
> auth code + PKCE, refresh rotation, optional DCR), an embedded React admin
|
||||
> dashboard at `/admin`, scoped operations, and a live SSE activity feed.
|
||||
> Pre-v0.26 legacy bearer tokens still work — `verifyAccessToken` falls back
|
||||
> to the `access_tokens` table and grandfathers tokens to `read+write+admin`.
|
||||
> Postgres-only for the legacy fallback (the `access_tokens` table is Postgres-only);
|
||||
> OAuth tables work on both PGLite and Postgres. See [SECURITY.md](../../SECURITY.md)
|
||||
> for env vars and tunable defaults.
|
||||
|
||||
Access your brain from any device, any AI client. GBrain's MCP server runs locally
|
||||
via `gbrain serve` (stdio). For remote access, expose it via the built-in HTTP
|
||||
transport behind a public tunnel.
|
||||
Access your brain from any device, any AI client. GBrain ships two transports:
|
||||
`gbrain serve` (stdio) for local agents, and `gbrain serve --http` (v0.26.0+)
|
||||
for remote clients over OAuth 2.1.
|
||||
|
||||
## Two Paths
|
||||
## Three Paths
|
||||
|
||||
### Local (zero setup)
|
||||
### Local stdio (zero setup)
|
||||
|
||||
```bash
|
||||
gbrain serve
|
||||
@@ -20,7 +24,30 @@ gbrain serve
|
||||
Works with Claude Code, Cursor, Windsurf, and any MCP client that supports stdio.
|
||||
No server, no tunnel, no token needed. Works on both PGLite and Postgres engines.
|
||||
|
||||
### Remote (any device, any AI client) — Postgres only
|
||||
### Remote over OAuth 2.1 (recommended, v0.26.0+)
|
||||
|
||||
```bash
|
||||
gbrain serve --http --port 3131
|
||||
ngrok http 3131 --url your-brain.ngrok.app
|
||||
gbrain serve --http --port 3131 --public-url https://your-brain.ngrok.app
|
||||
```
|
||||
|
||||
Built-in HTTP transport with OAuth 2.1, scoped operations, an admin dashboard
|
||||
at `/admin`, and a live SSE activity feed. Zero external dependencies. This is
|
||||
the only path that works with ChatGPT (OAuth 2.1 + PKCE is required by the
|
||||
ChatGPT MCP connector). Pass `--public-url` whenever the server is reachable
|
||||
at anything other than `http://localhost:<port>` so the OAuth issuer in
|
||||
discovery metadata matches what clients hit (RFC 8414 §3.3).
|
||||
|
||||
Supported clients:
|
||||
- **ChatGPT** — requires OAuth 2.1 + PKCE. Works natively with `--http`.
|
||||
- **Claude Desktop / Cowork** — OAuth 2.1 or legacy bearer tokens.
|
||||
- **Perplexity** — OAuth 2.1 client credentials grant.
|
||||
- **Claude Code, Cursor, Windsurf** — can use OAuth or legacy bearer.
|
||||
|
||||
See the [OAuth 2.1 setup](#oauth-21-setup-v100) section below.
|
||||
|
||||
### Remote with legacy bearer tokens (pre-v0.26 deployments) — Postgres only
|
||||
|
||||
```
|
||||
Your AI client (Claude Desktop, Perplexity, etc.)
|
||||
@@ -36,7 +63,94 @@ This requires:
|
||||
3. A public tunnel (ngrok, Tailscale, or cloud host)
|
||||
4. A bearer token created via `gbrain auth create <name>`
|
||||
|
||||
## Remote Setup
|
||||
Pre-v1.0 tokens are grandfathered as `read+write+admin` scopes when you upgrade
|
||||
to the HTTP server, so no migration is required.
|
||||
|
||||
## OAuth 2.1 Setup (v0.26.0+)
|
||||
|
||||
### 1. Start the HTTP server
|
||||
|
||||
```bash
|
||||
gbrain serve --http --port 3131
|
||||
```
|
||||
|
||||
On first start, the server prints an **admin bootstrap token** to stderr:
|
||||
|
||||
```
|
||||
Admin bootstrap token: 3a1f9c...
|
||||
Open http://localhost:3131/admin and paste it to log in.
|
||||
```
|
||||
|
||||
Save this token. Open `http://localhost:3131/admin` and paste it to access the
|
||||
dashboard. The dashboard shows live activity, registered clients, request logs,
|
||||
and per-client config export.
|
||||
|
||||
### 2. Register OAuth clients
|
||||
|
||||
Register clients from the **`/admin` dashboard**:
|
||||
|
||||
1. Click **Register client**.
|
||||
2. Enter a name (e.g. `perplexity`, `chatgpt`).
|
||||
3. Pick scopes: `read`, `write`, `admin` (checkboxes).
|
||||
4. Pick grant type: `client_credentials` for machine-to-machine (Perplexity,
|
||||
Claude Desktop bearer mode) or `authorization_code` for browser-based
|
||||
clients with PKCE (ChatGPT).
|
||||
5. For `authorization_code` clients, paste the redirect URI.
|
||||
6. Hit **Register**. The credential-reveal modal shows the `client_id` (and
|
||||
`client_secret` for confidential clients) once. Copy or Download JSON
|
||||
immediately — secrets are hashed on storage and never shown again.
|
||||
|
||||
Or from the CLI — faster for scripting:
|
||||
|
||||
```bash
|
||||
gbrain auth register-client perplexity \
|
||||
--grant-types client_credentials \
|
||||
--scopes "read write"
|
||||
```
|
||||
|
||||
Host-repo wrappers can register programmatically:
|
||||
|
||||
```ts
|
||||
await oauthProvider.registerClientManual(
|
||||
'perplexity',
|
||||
['client_credentials'],
|
||||
'read write',
|
||||
[], // redirect_uris, empty for CC
|
||||
);
|
||||
```
|
||||
|
||||
For self-service client registration (Dynamic Client Registration, RFC 7591),
|
||||
start the server with `--enable-dcr`. DCR is off by default.
|
||||
|
||||
### 3. Expose the server
|
||||
|
||||
```bash
|
||||
brew install ngrok
|
||||
ngrok config add-authtoken YOUR_TOKEN
|
||||
ngrok http 3131 --url your-brain.ngrok.app
|
||||
```
|
||||
|
||||
Your OAuth issuer URL becomes `https://your-brain.ngrok.app`. The MCP SDK's
|
||||
router exposes the spec-compliant discovery endpoint at
|
||||
`/.well-known/oauth-authorization-server`.
|
||||
|
||||
### 4. Scopes and localOnly
|
||||
|
||||
Every operation is tagged `read | write | admin`. Four operations are
|
||||
`localOnly` and rejected over HTTP regardless of scope: `sync_brain`,
|
||||
`file_upload`, `file_list`, `file_url`. Remote agents cannot reach local
|
||||
filesystem surface area.
|
||||
|
||||
| Scope | What it allows |
|
||||
|-------|---------------|
|
||||
| `read` | `search`, `query`, `get_page`, `list_pages`, graph traversal |
|
||||
| `write` | `put_page`, `delete_page`, `add_link`, `add_timeline_entry` |
|
||||
| `admin` | Client management, token revocation, sweep, local-only ops |
|
||||
|
||||
## Legacy Bearer Token Setup
|
||||
|
||||
Keep using pre-v0.26 bearer tokens if you aren't ready to migrate. They
|
||||
grandfather to `read+write+admin` scopes on the HTTP server.
|
||||
|
||||
### 1. Set up the tunnel
|
||||
|
||||
@@ -67,6 +181,7 @@ if compromised. Tokens are stored SHA-256 hashed in your database.
|
||||
|
||||
### 3. Connect your AI client
|
||||
|
||||
- **ChatGPT:** [setup guide](CHATGPT.md) (OAuth 2.1 + PKCE, requires `gbrain serve --http`)
|
||||
- **Claude Code:** [setup guide](CLAUDE_CODE.md)
|
||||
- **Claude Desktop:** [setup guide](CLAUDE_DESKTOP.md) (must use GUI, not JSON config)
|
||||
- **Claude Cowork:** [setup guide](CLAUDE_COWORK.md)
|
||||
@@ -123,7 +238,8 @@ Remote servers must be added via Settings > Integrations, NOT
|
||||
| put_page | 100-500ms | Write + trigger search_vector update |
|
||||
| get_stats | < 100ms | Aggregate query |
|
||||
|
||||
**Note:** `gbrain serve --http` (built-in HTTP transport) is planned but not yet
|
||||
implemented. Currently, remote MCP requires a custom HTTP wrapper. See the
|
||||
production deployment pattern in the [voice recipe](../../recipes/twilio-voice-brain.md)
|
||||
for a reference implementation.
|
||||
**Note:** `gbrain serve --http` shipped in v0.26.0 with OAuth 2.1 + admin
|
||||
dashboard baked into the binary. The custom HTTP wrapper pattern (see
|
||||
[voice recipe](../../recipes/twilio-voice-brain.md)) is still supported for
|
||||
teams that need bespoke middleware, but for most remote deployments the
|
||||
built-in server is the recommended path.
|
||||
|
||||
+462
-54
@@ -31,7 +31,13 @@ start here.
|
||||
1. `./AGENTS.md` (this file) — install + operating protocol.
|
||||
2. [`./CLAUDE.md`](./CLAUDE.md) — architecture reference, key files, trust boundaries,
|
||||
test layout.
|
||||
3. [`./skills/RESOLVER.md`](./skills/RESOLVER.md) — skill dispatcher. Read before any task.
|
||||
3. [`./docs/architecture/brains-and-sources.md`](./docs/architecture/brains-and-sources.md)
|
||||
— the two-axis mental model (brain = which DB, source = which repo in the DB). Every
|
||||
query routes on both axes. Read before writing anything that touches brain ops.
|
||||
4. [`./skills/conventions/brain-routing.md`](./skills/conventions/brain-routing.md) —
|
||||
agent-facing decision table: when to switch brain, when to switch source, how
|
||||
cross-brain federation works (latent-space only; the agent decides).
|
||||
5. [`./skills/RESOLVER.md`](./skills/RESOLVER.md) — skill dispatcher. Read before any task.
|
||||
|
||||
## Trust boundary (critical)
|
||||
|
||||
@@ -50,15 +56,27 @@ writing or reviewing an operation, consult `src/core/operations.ts` for the cont
|
||||
[`docs/guides/minions-fix.md`](./docs/guides/minions-fix.md), `gbrain doctor --fix`.
|
||||
- **Migrate:** [`docs/UPGRADING_DOWNSTREAM_AGENTS.md`](./docs/UPGRADING_DOWNSTREAM_AGENTS.md),
|
||||
[`skills/migrations/`](./skills/migrations/), `gbrain apply-migrations`.
|
||||
- **Eval retrieval changes:** capture is off by default. To benchmark a
|
||||
retrieval change against real captured queries, set
|
||||
`GBRAIN_CONTRIBUTOR_MODE=1`, then `gbrain eval export --since 7d > base.ndjson`
|
||||
and `gbrain eval replay --against base.ndjson`. Full guide:
|
||||
[`docs/eval-bench.md`](./docs/eval-bench.md).
|
||||
- **Everything else:** [`./llms.txt`](./llms.txt) is the full documentation map.
|
||||
[`./llms-full.txt`](./llms-full.txt) is the same map with core docs inlined for
|
||||
single-fetch ingestion.
|
||||
|
||||
## Before shipping
|
||||
|
||||
Run `bun test` plus the E2E lifecycle described in `./CLAUDE.md` (spin up the test
|
||||
Postgres container, run `bun run test:e2e`, tear it down). Ship via the `/ship` skill,
|
||||
not by hand.
|
||||
Easiest path: `bun run ci:local` runs the full CI gate inside Docker (gitleaks,
|
||||
unit tests with `DATABASE_URL` unset, then all 29 E2E files sequentially against a
|
||||
fresh pgvector container) and tears down. Use `bun run ci:local:diff` for the
|
||||
diff-aware subset during fast iteration on a focused branch. Requires Docker
|
||||
(Docker Desktop / OrbStack / Colima) and `gitleaks` (`brew install gitleaks`).
|
||||
|
||||
Manual path: `bun test` plus the E2E lifecycle described in `./CLAUDE.md` (spin
|
||||
up the test Postgres container, run `bun run test:e2e`, tear it down).
|
||||
|
||||
Ship via the `/ship` skill, not by hand.
|
||||
|
||||
## Privacy
|
||||
|
||||
@@ -86,6 +104,24 @@ suggests Supabase for 1000+ files. GStack teaches agents how to code. GBrain tea
|
||||
agents everything else: brain ops, signal detection, content ingestion, enrichment,
|
||||
cron scheduling, reports, identity, and access control.
|
||||
|
||||
## Two organizational axes (read this first)
|
||||
|
||||
GBrain knowledge is organized along two orthogonal axes. Users AND agents must
|
||||
understand both, or queries misroute silently.
|
||||
|
||||
- **Brain** — WHICH DATABASE. Your personal brain is `host`. You can mount
|
||||
additional brains (team-published, each with their own DB and access policy)
|
||||
via `gbrain mounts add` (v0.19+). Routing: `--brain`, `GBRAIN_BRAIN_ID`,
|
||||
`.gbrain-mount` dotfile.
|
||||
- **Source** — WHICH REPO INSIDE THE DATABASE. A brain can hold many sources
|
||||
(wiki, gstack, openclaw, essays). Slugs scope per source. Routing:
|
||||
`--source`, `GBRAIN_SOURCE`, `.gbrain-source` dotfile.
|
||||
|
||||
Both axes follow the same 6-tier resolution pattern. Read
|
||||
`docs/architecture/brains-and-sources.md` for topology diagrams (personal, team
|
||||
mount, CEO-class with multiple team brains) and
|
||||
`skills/conventions/brain-routing.md` for the agent-facing decision table.
|
||||
|
||||
## Architecture
|
||||
|
||||
Contract-first: `src/core/operations.ts` defines ~41 shared operations (adds `find_orphans` in v0.12.3). CLI and MCP
|
||||
@@ -101,7 +137,7 @@ strict behavior when unset.
|
||||
|
||||
## Key files
|
||||
|
||||
- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Also exports upload validators: `validateUploadPath`, `validatePageSlug`, `validateFilename`. `OperationContext.remote` flags untrusted callers.
|
||||
- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Also exports upload validators: `validateUploadPath`, `validatePageSlug`, `validateFilename`, plus `matchesSlugAllowList(slug, prefixes)` (v0.23 glob matcher: `<prefix>/*` matches recursive children; bare `<prefix>` matches exact only). `OperationContext.remote` flags untrusted callers; `OperationContext.allowedSlugPrefixes` (v0.23) is the trusted-workspace allow-list set by the dream cycle. `put_page` enforces: when `viaSubagent` and `allowedSlugPrefixes` is set, slug must match the allow-list; else the legacy `wiki/agents/<id>/...` namespace check applies. Auto-link enabled for trusted-workspace writes (skipped only when `remote=true && !trustedWorkspace`). As of v0.26.0, every `Operation` also carries `scope?: 'read' | 'write' | 'admin'` + `localOnly?: boolean`. All ops are annotated; `sync_brain`, `file_upload`, `file_list`, and `file_url` are `admin + localOnly` (rejected over HTTP). `OperationContext.auth?: AuthInfo` is threaded through HTTP dispatch for scope enforcement in `serve-http.ts` before the op runs.
|
||||
- `src/core/engine.ts` — Pluggable engine interface (BrainEngine). `clampSearchLimit(limit, default, cap)` takes an explicit cap so per-operation caps can be tighter than `MAX_SEARCH_LIMIT`. Exports `LinkBatchInput` / `TimelineBatchInput` for the v0.12.1 bulk-insert API (`addLinksBatch` / `addTimelineEntriesBatch`). As of v0.13.1, `BrainEngine` has a `readonly kind: 'postgres' | 'pglite'` discriminator so migrations (`src/core/migrate.ts`) and other consumers can branch on engine without `instanceof` + dynamic imports.
|
||||
- `src/core/engine-factory.ts` — Engine factory with dynamic imports (`'pglite'` | `'postgres'`)
|
||||
- `src/core/pglite-engine.ts` — PGLite (embedded Postgres 17.5 via WASM) implementation, all 40 BrainEngine methods. `addLinksBatch` / `addTimelineEntriesBatch` use multi-row `unnest()` with manual `$N` placeholders. As of v0.13.1, `connect()` wraps `PGlite.create()` in a try/catch that emits an actionable error naming the macOS 26.3 WASM bug (#223) and pointing at `gbrain doctor`; the lock is released on failure so the next process can retry cleanly. v0.22.0: `searchKeyword` and `searchKeywordChunks` multiply `ts_rank` by the source-factor CASE expression at the chunk-grain level; `searchVector` becomes a two-stage CTE — inner CTE keeps `ORDER BY cc.embedding <=> vec` so HNSW stays usable, outer SELECT re-ranks by `raw_score * source_factor`. Inner LIMIT scales with offset to preserve pagination contract. As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL — probes for the specific forward-referenced state the embedded schema blob needs (`pages.source_id`, `links.link_source`, `links.origin_page_id`, `content_chunks.symbol_name`, `content_chunks.language`, `sources` FK target table) and adds only what's missing. Closes the upgrade-wedge bug class that bit users 10+ times across 6 schema versions over 2 years (#239/#243/#266/#357/#366/#374/#375/#378/#395/#396). No-op on fresh installs and modern brains.
|
||||
@@ -128,7 +164,16 @@ strict behavior when unset.
|
||||
- `src/core/search/eval.ts` — Retrieval eval harness: P@k, R@k, MRR, nDCG@k metrics + runEval() orchestrator
|
||||
- `src/core/search/source-boost.ts` (v0.22.0) — Source-type boost map keyed by slug prefix. `DEFAULT_SOURCE_BOOSTS` (originals/ 1.5, concepts/ 1.3, writing/ 1.4, people/companies/deals/ 1.2, daily/ 0.8, media/x/ 0.7, wintermute/chat/ 0.5) and `DEFAULT_HARD_EXCLUDES` (test/, archive/, attachments/, .raw/). `parseSourceBoostEnv` / `parseHardExcludesEnv` parse comma-separated `prefix:factor` pairs from `GBRAIN_SOURCE_BOOST` / `GBRAIN_SEARCH_EXCLUDE` env vars. `resolveBoostMap` and `resolveHardExcludes` merge defaults + env + caller `SearchOpts.exclude_slug_prefixes`/`include_slug_prefixes`.
|
||||
- `src/core/search/sql-ranking.ts` (v0.22.0) — Pure SQL string builders. `buildSourceFactorCase(slugColumn, boostMap, detail)` emits a CASE expression with longest-prefix-match wins (returns literal `'1.0'` when `detail === 'high'` for temporal-bypass parity with COMPILED_TRUTH_BOOST). `buildHardExcludeClause(slugColumn, prefixes)` emits `NOT (col LIKE 'p1%' OR col LIKE 'p2%')` — OR-chain wrapped in NOT, NOT `NOT LIKE ALL/ANY` (those quantifiers don't express set-exclusion). LIKE meta-character escape covers all three of `%`, `_`, AND `\` (backslash matters because it's Postgres LIKE's default escape char). Single-quote doubling on SQL string literals so injection-style inputs are inert text.
|
||||
- `src/commands/eval.ts` — `gbrain eval` command: single-run table + A/B config comparison
|
||||
- `src/commands/eval.ts` — `gbrain eval` command: single-run table + A/B config comparison. v0.25.0 adds sub-subcommand dispatch on `args[0]` so `gbrain eval export` + `gbrain eval prune` + `gbrain eval replay` route into session-capture handlers; bare `gbrain eval --qrels …` fall-through preserves the legacy IR-metrics flow.
|
||||
- `src/commands/eval-export.ts` (v0.25.0) — streams `eval_candidates` rows as NDJSON to stdout with `schema_version: 1` prefix on every line. EPIPE-safe, progress heartbeats on stderr, stable id-desc tiebreaker so `--since` windows never dupe/miss rows.
|
||||
- `src/commands/eval-prune.ts` (v0.25.0) — explicit retention cleanup. Requires `--older-than DUR`. `--dry-run` reports would-delete count.
|
||||
- `src/commands/eval-replay.ts` (v0.25.0) — contributor-facing replay tool. Reads NDJSON from `gbrain eval export`, re-runs each captured `query` / `search` op against the current brain, computes set-Jaccard@k between captured + current `retrieved_slugs`, top-1 stability rate, and latency Δ. Stable JSON shape (`schema_version: 1`) for CI gating; human mode prints a regression table. Pure Bun, zero new deps. The dev-loop half of BrainBench-Real that closes the gap between "data captured" and "data used to gate a PR." See `docs/eval-bench.md` for the workflow.
|
||||
- `docs/eval-bench.md` (v0.25.0) — contributor guide for using captured data to benchmark retrieval changes before merging. Linked from CONTRIBUTING.md under "Running real-world eval benchmarks (touching retrieval code)".
|
||||
- `src/core/eval-capture.ts` (v0.25.0) — op-layer capture wrapper called from `src/core/operations.ts` `query` + `search` handlers. Catches MCP + CLI + subagent tool-bridge from one site. Fire-and-forget; failures route to `engine.logEvalCaptureFailure` so `gbrain doctor` sees drops cross-process. **Capture is off by default** — `isEvalCaptureEnabled` resolution: explicit `config.eval.capture` (true/false) wins, else `process.env.GBRAIN_CONTRIBUTOR_MODE === '1'`, else off. Production users get a quiet brain; contributors set `export GBRAIN_CONTRIBUTOR_MODE=1` in `.zshrc` to enable the dev loop. PII scrubber gate is independent and defaults to true regardless of CONTRIBUTOR_MODE.
|
||||
- `src/core/eval-capture-scrub.ts` (v0.25.0) — zero-deps PII scrubber: emails, phones, SSN, Luhn-verified credit cards, JWT-shaped tokens, bearer tokens.
|
||||
- `src/core/search/hybrid.ts` — Cathedral II `Promise<SearchResult[]>` return shape unchanged in v0.25.0. Adds `onMeta?: (m: HybridSearchMeta) => void` callback so op-layer capture can record what hybridSearch actually did. Existing callers leave it undefined.
|
||||
- `docs/eval-capture.md` (v0.25.0) — stable NDJSON schema reference for gbrain-evals consumers.
|
||||
- `test/public-exports.test.ts` (v0.25.0 / R2) — runtime contract test. Imports each of the 17 public subpaths via package name and pins a canary symbol per module. Paired with `scripts/check-exports-count.sh`.
|
||||
- `src/core/embedding.ts` — OpenAI text-embedding-3-large, batch, retry, backoff
|
||||
- `src/core/check-resolvable.ts` — Resolver validation: reachability, MECE overlap, DRY checks, structured fix objects. v0.14.1: `CROSS_CUTTING_PATTERNS.conventions` is an array (notability gate accepts both `conventions/quality.md` and `_brain-filing-rules.md`). New `extractDelegationTargets()` parses `> **Convention:**`, `> **Filing rule:**`, and inline backtick references. DRY suppression is proximity-based via `DRY_PROXIMITY_LINES = 40`.
|
||||
- `src/core/repo-root.ts` — Shared `findRepoRoot(startDir?)` (v0.16.4): walks up from `startDir` (default `process.cwd()`) looking for `skills/RESOLVER.md`. Zero-dependency module imported by both `doctor.ts` and `check-resolvable.ts`. Parameterized `startDir` makes tests hermetic.
|
||||
@@ -136,9 +181,12 @@ strict behavior when unset.
|
||||
- `src/core/resolver-filenames.ts` (v0.19) — central list of accepted routing filenames (`RESOLVER.md`, `AGENTS.md`). Shared by `findRepoRoot`, `check-resolvable`, and skillpack install so every code path walks the same fallback chain.
|
||||
- `src/commands/skillify.ts` + `src/core/skillify/{generator,templates}.ts` (v0.19) — `gbrain skillify scaffold <name>` creates all stubs for a new skill in one command: SKILL.md, script, tests, routing-eval.jsonl, resolver entry, filing-rules pointer. `gbrain skillify check <script>` runs the 10-step checklist (LLM evals, routing evals, check-resolvable gate, filing audit) against a candidate skill before it lands.
|
||||
- `src/commands/skillify-check.ts` (v0.19) — `gbrain skillpack-check` agent-readable health report. Exit 0/1/2 for CI pipeline gating; JSON for debugging. Wraps `check-resolvable --json`, `doctor --json`, and migration ledger into one payload so agents can decide whether a human action is required.
|
||||
- `src/commands/skillpack.ts` + `src/core/skillpack/{bundle,installer}.ts` (v0.19) — `gbrain skillpack install` drops gbrain's curated 25-skill bundle into a host workspace, managed-block style. Never clobbers local edits; tracks a skill manifest so subsequent `install --update` diffs cleanly. Bundle builder (`skillpack/bundle.ts`) packages the set from `skills/` into a versioned payload.
|
||||
- `src/commands/book-mirror.ts` (v0.25.1) — `gbrain book-mirror --chapters-dir <path> --slug <slug> [flags]`. Flagship of the v0.25.1 skills wave. Submits N read-only subagent jobs (one per chapter; `allowed_tools: ['get_page', 'search']`), waits for all via `waitForCompletion`, reads each child's `job.result`, assembles two-column markdown CLI-side, writes a single operator-trust `put_page` to `media/books/<slug>-personalized.md`. Codex HIGH-1 fix applied: trust narrowing happens at the tool-allowlist layer (subagents can't call put_page) instead of allowedSlugPrefixes — untrusted EPUB content cannot prompt-inject any people page. Cost-estimate prompt before launching; refuses to spend in non-TTY without `--yes`. Per-chapter idempotency keys (`book-mirror:<slug>:ch-<N>`) for retry-friendly re-runs. Partial-failure handling: assembles with completed chapters and a `## Failed chapters` section listing retries. Test surface: `test/book-mirror.test.ts` (9 cases — CLI registration + source invariants).
|
||||
- `src/commands/skillpack.ts` + `src/core/skillpack/{bundle,installer}.ts` (v0.19) — `gbrain skillpack install` drops gbrain's curated 25-skill bundle into a host workspace, managed-block style. Never clobbers local edits; tracks a skill manifest so subsequent `install --update` diffs cleanly. Bundle builder (`skillpack/bundle.ts`) packages the set from `skills/` into a versioned payload. **v0.24.0:** managed block embeds a `<!-- gbrain:skillpack:manifest cumulative-slugs="..." version="..." -->` receipt inside the fence. Per-skill installs accumulate via `union(prior_receipt, this_call)`; `install --all` is the only path that prunes (drops slugs no longer in the bundle). Rows inside the fence whose slug is in neither the new cumulative set nor the bundle survive as user-added with a stderr `[skillpack] unknown row in managed block: "<slug>" — Investigate: ...` warning. Pre-v0.24 fences upgrade silently on first install (extracted slugs become the prior cumulative set). **v0.25.1:** `gbrain skillpack uninstall <name>` lands as a real CLI subcommand. Inverse of install with symmetric data-loss posture: D8 refuses if the slug isn't in the cumulative-slugs receipt (won't nuke a hand-added row); D11 content-hash guard refuses if any installed file diverges from the bundle (you've edited it locally) unless `--overwrite-local` is passed. `applyUninstall` enforces an atomic-refusal contract: pre-scans ALL files for divergence; refuses BEFORE any unlink fires if anything is blocked. The bug fix landed via `test/skillpack-uninstall.test.ts`'s D11 case — the test was written with the contract in mind, the original implementation interleaved hash-check + unlink, and the lie surfaced immediately.
|
||||
- `src/core/archive-crawler-config.ts` (v0.25.1) — D12 + codex HIGH-4 safety gate for the `archive-crawler` skill. Refuses to run unless `archive-crawler.scan_paths:` is explicitly set in the brain repo's `gbrain.yml`. Mirrors the storage-config.ts parsing pattern (sibling file; separate concern from storage tiering). `loadArchiveCrawlerConfig(repoPath)` throws `ArchiveCrawlerConfigError(missing_section | empty_scan_paths | invalid_path | parse_error)`. `normalizeAndValidateArchiveCrawlerConfig` rejects relative paths and `..` traversal; `~` is expanded; trailing-slash normalized for unambiguous prefix matching. `isPathAllowed(candidate, config)` is the runtime per-file gate (scan_paths prefix-match with directory-boundary correctness; deny_paths overrides). Tests in `test/archive-crawler-config.test.ts` (19 cases).
|
||||
- `test/helpers/cli-pty-runner.ts` (v0.25.1) — generic real-PTY harness ported from gstack and trimmed to ~470 lines. Uses pure `Bun.spawn({terminal:})` (Bun 1.3.10+; engines.bun pin in package.json). Generic primitives only — no plan-mode orchestrators. Exports: `launchPty`, `resolveBinary`, `stripAnsi`, `parseNumberedOptions`, `optionsSignature`, `isNumberedOptionListVisible`, `isTrustDialogVisible`. Self-tests in `test/cli-pty-runner.test.ts` (24 cases).
|
||||
- `src/core/skill-manifest.ts` (v0.19) — parser for `skill-manifest.json` records. Used by skillpack installer to detect drift between the shipped bundle and the user's local edits, so updates merge instead of overwriting.
|
||||
- `src/commands/routing-eval.ts` + `src/core/routing-eval.ts` (v0.19) — `gbrain routing-eval` catches user phrasings that route to the wrong skill. Reads `skills/<name>/routing-eval.jsonl` fixtures (`{intent, expected_skill, ambiguous_with?}`). Structural layer runs in `check-resolvable` by default (zero API cost); `--llm` opts into a Haiku tie-break layer for CI. False positives surface before users hit them.
|
||||
- `src/commands/routing-eval.ts` + `src/core/routing-eval.ts` (v0.19) — `gbrain routing-eval` catches user phrasings that route to the wrong skill. Reads `skills/<name>/routing-eval.jsonl` fixtures (`{intent, expected_skill, ambiguous_with?}`). Structural layer runs in `check-resolvable` by default (zero API cost). The `--llm` flag is accepted as a placeholder for a future LLM tie-break layer; in v0.24.0 it emits a stderr notice and runs structural only. False positives surface before users hit them.
|
||||
- `src/core/filing-audit.ts` + `skills/_brain-filing-rules.json` (v0.19) — Check 6 of `check-resolvable`. Parses new `writes_pages:` / `writes_to:` frontmatter on skills and audits their filing claims against the filing-rules JSON. Warning-only in v0.19, upgrades to error in v0.20.
|
||||
- `src/core/dry-fix.ts` — `gbrain doctor --fix` engine. `autoFixDryViolations(fixes, {dryRun})` rewrites inlined rules to `> **Convention:** see [path](path).` callouts via three shape-aware expanders (bullet / blockquote / paragraph). Five guards: working-tree-dirty (`getWorkingTreeStatus()` returns 3-state `'clean' | 'dirty' | 'not_a_repo'`), no-git-backup, inside-code-fence, already-delegated (40-line proximity, consistent with detector), ambiguous-multi-match, block-is-callout. `execFileSync` array args (no shell — no injection surface). EOF newline preserved.
|
||||
- `src/core/backoff.ts` — Adaptive load-aware throttling: CPU/memory checks, exponential backoff, active hours multiplier
|
||||
@@ -166,34 +214,48 @@ strict behavior when unset.
|
||||
- `src/core/minions/wait-for-completion.ts` (v0.15) — poll-until-terminal helper for CLI callers. `TimeoutError` does NOT cancel the job; `AbortSignal` exits without throwing. Default `pollMs`: 1000 on Postgres, 250 on PGLite inline.
|
||||
- `src/core/minions/transcript.ts` (v0.15) — renders `subagent_messages` + `subagent_tool_executions` to markdown. Tool rows splice under their owning assistant `tool_use` by `tool_use_id`. UTF-8-safe truncation; unknown block types fall through to fenced JSON.
|
||||
- `src/core/minions/plugin-loader.ts` (v0.15) — `GBRAIN_PLUGIN_PATH` discovery. Absolute paths only, left-wins collision, `gbrain.plugin.json` with `plugin_version: "gbrain-plugin-v1"`, plugins ship DEFS only (no new tools), `allowed_tools:` validated at load time against the derived registry.
|
||||
- `src/core/minions/tools/brain-allowlist.ts` (v0.15) — derives subagent tool registry from `src/core/operations.ts`. 11-name allow-list: `query`, `search`, `get_page`, `list_pages`, `file_list`, `file_url`, `get_backlinks`, `traverse_graph`, `resolve_slugs`, `get_ingest_log`, `put_page`. `put_page` schema is namespace-wrapped per subagent (`^wiki/agents/<subagentId>/.+`); the `put_page` op's server-side check is the authoritative gate via `ctx.viaSubagent` fail-closed.
|
||||
- `src/core/minions/tools/brain-allowlist.ts` (v0.15, extended v0.23) — derives subagent tool registry from `src/core/operations.ts`. 11-name allow-list. By default `put_page` schema is namespace-wrapped per subagent (`^wiki/agents/<subagentId>/.+`). **v0.23 trusted-workspace path:** when `BuildBrainToolsOpts.allowedSlugPrefixes` is set, the put_page schema instead describes the prefix list to the model and the OperationContext is threaded with `allowedSlugPrefixes`. Trust comes from `PROTECTED_JOB_NAMES` gating subagent submission — MCP cannot reach this field. Only cycle.ts (synthesize/patterns) and direct CLI submitters set it.
|
||||
- `src/mcp/tool-defs.ts` (v0.15) — extracted `buildToolDefs(ops)` helper. MCP server + subagent tool registry both call it; byte-for-byte equivalence pinned by `test/mcp-tool-defs.test.ts`.
|
||||
- `src/core/minions/attachments.ts` — Attachment validation (path traversal, null byte, oversize, base64, duplicate detection)
|
||||
- `src/commands/agent.ts` (v0.16) — `gbrain agent run <prompt> [flags]` CLI. Submits `subagent` (or N children + 1 aggregator) under `{allowProtectedSubmit: true}`. Single-entry `--fanout-manifest` short-circuits. Children get `on_child_fail: 'continue'` + `max_stalled: 3`. `--follow` is the default on TTY; streams logs + polls `waitForCompletion` in parallel. Ctrl-C detaches, does not cancel.
|
||||
- `src/commands/agent-logs.ts` (v0.16) — `gbrain agent logs <job> [--follow] [--since]`. Merges JSONL heartbeat audit + `subagent_messages` into a chronological timeline. `parseSince` accepts ISO-8601 or relative (`5m`, `1h`, `2d`). Transcript tail renders only for terminal jobs.
|
||||
- `src/commands/jobs.ts` — `gbrain jobs` CLI subcommands + `gbrain jobs work` daemon. v0.13.1 surfaces the full `MinionJobInput` retry/backoff/timeout/idempotency surface as first-class CLI flags on `jobs submit`: `--max-stalled`, `--backoff-type fixed|exponential`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key`. `jobs smoke --sigkill-rescue` is the opt-in regression guard for #219. v0.16 wires `registerBuiltinHandlers` to always register `subagent` + `subagent_aggregator` (no env flag — `ANTHROPIC_API_KEY` is the natural cost gate, trust is via `PROTECTED_JOB_NAMES`) and loads `GBRAIN_PLUGIN_PATH` plugins at worker startup with a loud startup-line per plugin. `shell` handler still gated by `GBRAIN_ALLOW_SHELL_JOBS=1` (RCE surface, separate concern). v0.22.10 (#521): the `autopilot-cycle` handler now forwards `job.data.phases` to `runCycle` (was previously discarded — caller-supplied phase selection silently became a full cycle). Phases are validated against `ALL_PHASES` from `src/core/cycle.ts`; invalid names are filtered out and an empty/missing array falls back to the default 6-phase cycle.
|
||||
- `src/commands/jobs.ts` — `gbrain jobs` CLI subcommands + `gbrain jobs work` daemon. v0.13.1 surfaces the full `MinionJobInput` retry/backoff/timeout/idempotency surface as first-class CLI flags on `jobs submit`: `--max-stalled`, `--backoff-type fixed|exponential`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key`. `jobs smoke --sigkill-rescue` is the opt-in regression guard for #219. v0.16 wires `registerBuiltinHandlers` to always register `subagent` + `subagent_aggregator` (no env flag — `ANTHROPIC_API_KEY` is the natural cost gate, trust is via `PROTECTED_JOB_NAMES`) and loads `GBRAIN_PLUGIN_PATH` plugins at worker startup with a loud startup-line per plugin. `shell` handler still gated by `GBRAIN_ALLOW_SHELL_JOBS=1` (RCE surface, separate concern). v0.22.10 (#521): the `autopilot-cycle` handler now forwards `job.data.phases` to `runCycle` (was previously discarded — caller-supplied phase selection silently became a full cycle). Phases are validated against `ALL_PHASES` from `src/core/cycle.ts`; invalid names are filtered out and an empty/missing array falls back to the default 6-phase cycle. v0.22.13 (PR #490 CODEX-1+CODEX-4): `sync` handler now resolves `sourceId` at entry by looking up `sources.local_path` (mirrors `cycle.ts:480`'s autopilot fix from PR #475) so multi-source brains read the per-source `last_commit` anchor instead of the global config key. Concurrency routed through the shared `autoConcurrency()` policy in `src/core/sync-concurrency.ts` instead of the prior hardcoded `4`; PGLite stays serial. `noEmbed` default is `true` (embed is a separate job — submit `gbrain embed --stale` after sync, or rely on the autopilot cycle's embed phase).
|
||||
- `src/commands/features.ts` — `gbrain features --json --auto-fix`: usage scan + feature adoption salesman
|
||||
- `src/commands/autopilot.ts` — `gbrain autopilot --install`: self-maintaining brain daemon (sync+extract+embed)
|
||||
- `src/mcp/server.ts` — MCP stdio server (generated from operations). v0.22.7: tool-call handler delegates to `dispatchToolCall` from `src/mcp/dispatch.ts` so stdio + HTTP transports share one validation, context-build, and error-format path.
|
||||
- `src/mcp/dispatch.ts` (v0.22.7) — Shared tool-call dispatch consumed by both stdio (`server.ts`) and HTTP (`http-transport.ts`). Exports `dispatchToolCall(engine, name, params, opts)`, `buildOperationContext(engine, params, opts)`, and `validateParams(op, params)`. Single source of truth for `(ctx, params)` handler arg order and the 5-field `OperationContext` shape (engine + config + logger + dryRun + remote). Defaults to `remote: true` (untrusted); local CLI callers pass `remote: false`. Closed F1 (reversed handler args) + F2 (incomplete OperationContext) + F3 (no param validation) drift bugs in the original v0.22.5 HTTP transport.
|
||||
- `src/mcp/rate-limit.ts` (v0.22.7) — Bounded-LRU token-bucket limiter for `gbrain serve --http`. `buildDefaultLimiters()` returns the two-bucket pipeline used by http-transport: pre-auth IP (default 30/60s, fires BEFORE the DB lookup so brute-force load against `access_tokens` is actually capped) + post-auth token-id (default 60/60s). Tracks `lastTouchedMs` separately from `lastRefillMs` so an exhausted key can't be reset by hammering past the TTL. LRU cap (default 10K keys) bounds memory under attacker-controlled key growth; TTL prune at 2× window evicts abandoned buckets.
|
||||
- `src/mcp/http-transport.ts` (v0.22.7, rewrite) — `gbrain serve --http` HTTP transport. Postgres-only — fails fast at startup on PGLite (the `access_tokens` table only exists on Postgres). Bearer auth against SHA-256 hashes in `access_tokens`. CORS default-deny via `GBRAIN_HTTP_CORS_ORIGIN` allowlist. Body cap stream-counted (1 MiB default via `GBRAIN_HTTP_MAX_BODY_BYTES`) so chunked transfers without Content-Length still hit the cap. `last_used_at` SQL-level debounce (one UPDATE per token per 60s). Per-request audit row in `mcp_request_log` with token_name + operation + status + latency. Optional `GBRAIN_HTTP_TRUST_PROXY=1` honors `X-Forwarded-For` — only safe when bound to a private interface AND the proxy strips client-supplied XFF (otherwise enables IP spoofing past the pre-auth rate limit). `/health` does `SELECT 1` against Postgres and returns 503 + `status:unhealthy` when the DB is unreachable so orchestration doesn't see green pods while clients get misleading 401s. Replaces the standalone OAuth wrapper that was vulnerable to unauthenticated client registration.
|
||||
- `src/commands/auth.ts` — Token management for the HTTP transport. `gbrain auth create/list/revoke/test`. As of v0.22.7 wired into the main CLI (`src/cli.ts`); also runs standalone via `bun run src/commands/auth.ts ...` for environments without a compiled binary. Tokens stored as SHA-256 hashes in `access_tokens` (Postgres-only).
|
||||
- `src/mcp/dispatch.ts` (v0.22.7) — Shared tool-call dispatch consumed by both stdio (`server.ts`) and HTTP transports. Exports `dispatchToolCall(engine, name, params, opts)`, `buildOperationContext(engine, params, opts)`, and `validateParams(op, params)`. Single source of truth for `(ctx, params)` handler arg order and the 5-field `OperationContext` shape (engine + config + logger + dryRun + remote). Defaults to `remote: true` (untrusted); local CLI callers pass `remote: false`. Closed F1/F2/F3 drift bugs in the original v0.22.5 HTTP transport.
|
||||
- `src/mcp/rate-limit.ts` (v0.22.7) — Bounded-LRU token-bucket limiter. `buildDefaultLimiters()` returns the two-bucket pipeline: pre-auth IP (30/60s, fires BEFORE the DB lookup so brute-force load against `access_tokens` is actually capped) + post-auth token-id (60/60s). Tracks `lastTouchedMs` separately from `lastRefillMs` so an exhausted key can't be reset by hammering past the TTL. LRU cap bounds memory under attacker-controlled key growth.
|
||||
- `src/commands/serve-http.ts` (v0.26.0) — Express 5 HTTP MCP server with OAuth 2.1, admin dashboard, and SSE live activity feed. Started via `gbrain serve --http [--port N] [--token-ttl N] [--enable-dcr] [--public-url URL]`. Supersedes the v0.22.7 `src/mcp/http-transport.ts` simple bearer-auth path. Combines MCP SDK's `mcpAuthRouter` (authorize / token / register / revoke endpoints), a custom `client_credentials` handler (SDK's token endpoint throws `UnsupportedGrantTypeError` for CC; the custom handler runs BEFORE the router and falls through for `auth_code` / `refresh_token`), `requireBearerAuth` middleware for `/mcp` with scope enforcement before op dispatch, `localOnly` rejection, and `express-rate-limit` at 50 req / 15 min on `/token`. Serves the built admin SPA from `admin/dist/` with SPA fallback. `/admin/events` SSE endpoint broadcasts every MCP request to connected admin browsers. `cookie-parser` middleware wired (Express 5 has no built-in). Startup logging prints port, engine, configured issuer URL (honors `--public-url`), registered-client count, DCR status, and admin bootstrap token.
|
||||
- `src/core/oauth-provider.ts` (v0.26.0) — `GBrainOAuthProvider` implementing the MCP SDK's `OAuthServerProvider` + `OAuthRegisteredClientsStore` interfaces. Backed by raw SQL (works on both PGLite and Postgres — OAuth is infrastructure, not a BrainEngine concern). Full OAuth 2.1 spec: `authorize` + `exchangeAuthorizationCode` with PKCE (for ChatGPT), `client_credentials` (for Perplexity / Claude), `refresh_token` with rotation, `revokeToken`, `registerClient` (DCR path validates redirect_uri must be `https://` or loopback per RFC 6749 §3.1.2.1). All tokens + client secrets SHA-256 hashed before storage. Auth codes single-use with 10-minute TTL via atomic `DELETE...RETURNING` (closes RFC 6749 §10.5 TOCTOU race). Refresh rotation also `DELETE...RETURNING` (closes §10.4 stolen-token detection bypass). `pgArray()` escapes commas/quotes/braces in elements so a comma-bearing redirect_uri can't smuggle a second array element. Legacy `access_tokens` fallback in `verifyAccessToken` grandfathers pre-v0.26 bearer tokens as `read+write+admin`. `sweepExpiredTokens()` runs on startup wrapped in try/catch. **v0.26.2:** module-private `coerceTimestamp()` boundary helper at the top of the file normalizes postgres-driver-as-string BIGINT columns to JS numbers at every read site (5 call sites: `getClient` L112+L113 for DCR `/register` RFC 7591 §3.2.1 numeric timestamps, `exchangeRefreshToken` L274 + `verifyAccessToken` L296+L303 for the SDK's `typeof === 'number'` bearerAuth check). Throws on non-finite input (NaN/Infinity) so corrupt rows fail loud at the boundary instead of riding through as `expiresAt: NaN`; returns undefined for SQL NULL so callers decide NULL semantics explicitly (refresh + access token paths treat NULL as expired). Helper intentionally NOT promoted to `src/core/utils.ts` — codex review flagged repo-wide BIGINT precision-loss risk for a generic helper.
|
||||
- `admin/` (v0.26.0) — React 19 + Vite + TypeScript admin SPA embedded in the binary via `admin/dist/` served by `serve-http.ts`. 7 screens: Login (bootstrap token → session cookie), Dashboard (metrics + SSE feed + token health), Agents (sortable table + sparklines + Register button), Register (modal with scope checkboxes + grant type selector), Credentials reveal (full-screen modal with Copy + Download JSON + yellow one-time-only warning), Request Log (filterable paginated), Agent Detail drawer (Details / Activity / Config Export tabs + Revoke). Design tokens: `#0a0a0f` bg, Inter for UI, JetBrains Mono for data, 4-32px spacing scale, rounded pill badges. HTTP-only SameSite=Strict cookie auth. 65KB gzip. Build: `cd admin && bun install && bun run build`; output at `admin/dist/` is committed for self-contained binaries.
|
||||
- `src/commands/auth.ts` — Token management. `gbrain auth create/list/revoke/test` for legacy bearer tokens (v0.22.7 wired as a first-class CLI subcommand) plus `gbrain auth register-client` (v0.26.0) and `gbrain auth revoke-client <client_id>` (v0.26.2) for OAuth 2.1 client lifecycle. `revoke-client` runs an atomic `DELETE...RETURNING` on `oauth_clients`; FK `ON DELETE CASCADE` on `oauth_tokens.client_id` and `oauth_codes.client_id` purges every active token + authorization code in a single transaction. `process.exit(1)` on no-such-client (idempotent — re-running on the same id produces the same exit-1 message). Legacy tokens stored as SHA-256 hashes in `access_tokens`; OAuth clients in `oauth_clients`. As of v0.26.0, legacy tokens grandfather to `read+write+admin` scopes on the OAuth HTTP server, so pre-v0.26 deployments keep working with no migration.
|
||||
- `src/commands/upgrade.ts` — Self-update CLI. `runPostUpgrade()` enumerates migrations from the TS registry (src/commands/migrations/index.ts) and tail-calls `runApplyMigrations(['--yes', '--non-interactive'])` so the mechanical side of every outstanding migration runs unconditionally.
|
||||
- `src/commands/migrations/` — TS migration registry (compiled into the binary; no filesystem walk of `skills/migrations/*.md` needed at runtime). `index.ts` lists migrations in semver order. `v0_11_0.ts` = Minions adoption orchestrator (8 phases). `v0_12_0.ts` = Knowledge Graph auto-wire orchestrator (5 phases: schema → config check → backfill links → backfill timeline → verify). `phaseASchema` has a 600s timeout (bumped from 60s in v0.12.1 for duplicate-heavy brains). `v0_12_2.ts` = JSONB double-encode repair orchestrator (4 phases: schema → repair-jsonb → verify → record). `v0_14_0.ts` = shell-jobs + autopilot cooperative (2 phases: schema ALTER minion_jobs.max_stalled SET DEFAULT 3 — superseded by v0.14.3's schema-level DEFAULT 5 + UPDATE backfill; pending-host-work ping for skills/migrations/v0.14.0.md). All orchestrators are idempotent and resumable from `partial` status. As of v0.14.2 (Bug 3), the RUNNER owns all ledger writes — orchestrators return `OrchestratorResult` and `apply-migrations.ts` persists a canonical `{version, status, phases}` shape after return. Orchestrators no longer call `appendCompletedMigration` directly. `statusForVersion` prefers `complete` over `partial` (never regresses). 3 consecutive partials → wedged → `--force-retry <version>` writes a `'retry'` reset marker. v0.14.3 (fix wave) ships schema-only migrations v14 (`pages_updated_at_index`) + v15 (`minion_jobs_max_stalled_default_5` with UPDATE backfill) via the `MIGRATIONS` array in `src/core/migrate.ts` — no orchestrator phases needed.
|
||||
- `src/commands/repair-jsonb.ts` — `gbrain repair-jsonb [--dry-run] [--json]`: rewrites `jsonb_typeof='string'` rows in place across 5 affected columns (pages.frontmatter, raw_data.data, ingest_log.pages_updated, files.metadata, page_versions.frontmatter). Fixes v0.12.0 double-encode bug on Postgres; PGLite no-ops. Idempotent.
|
||||
- `src/commands/orphans.ts` — `gbrain orphans [--json] [--count] [--include-pseudo]`: surfaces pages with zero inbound wikilinks, grouped by domain. Auto-generated/raw/pseudo pages filtered by default. Also exposed as `find_orphans` MCP operation. Shipped in v0.12.3 (contributed by @knee5).
|
||||
- `src/commands/integrity.ts` — `gbrain integrity check|auto|review|extract`: bare-tweet detection, dead-link detection, three-bucket repair (auto-repair / review-queue / skip). `scanIntegrity()` is the shared library function called from `gbrain doctor` (sampled at limit=500) and `cmdCheck` (full scan). v0.22.8: batch-load fast path on Postgres uses `SELECT DISTINCT ON (slug)` in a single SQL query to fix the PgBouncer round-trip timeout (60s → ~6s) while preserving `engine.getAllSlugs()`'s `Set<string>` semantics on multi-source brains. Gated by `engine.kind === 'postgres'` at the call site so PGLite never enters batch; fallback `catch` logs at `GBRAIN_DEBUG=1` so real Postgres errors are diagnosable.
|
||||
- `src/commands/doctor.ts` — `gbrain doctor [--json] [--fast] [--fix] [--dry-run] [--index-audit]`: health checks. v0.12.3 added `jsonb_integrity` + `markdown_body_completeness` reliability checks. v0.14.1: `--fix` delegates inlined cross-cutting rules to `> **Convention:** see [path](path).` callouts (pipes DRY violations into `src/core/dry-fix.ts`); `--fix --dry-run` previews without writing. v0.14.2: `schema_version` check fails loudly when `version=0` (migrations never ran — the #218 `bun install -g` signature) and routes users to `gbrain apply-migrations --yes`; new opt-in `--index-audit` flag (Postgres-only) reports zero-scan indexes from `pg_stat_user_indexes` (informational only, no auto-drop). v0.15.2: every DB check is wrapped in a progress phase; `markdown_body_completeness` runs under a 1s heartbeat timer so 10+ min scans are observable on 50K-page brains. v0.19.1 added `queue_health` (Postgres-only) with two subchecks: stalled-forever active jobs (started_at > 1h) and waiting-depth-per-name > threshold (default 10, override via `GBRAIN_QUEUE_WAITING_THRESHOLD`). Worker-heartbeat subcheck intentionally deferred to follow-up B7 because it needs a `minion_workers` table to produce ground-truth signal. Fix hints point at `gbrain repair-jsonb`, `gbrain sync --force`, `gbrain apply-migrations`, and `gbrain jobs get/cancel <id>`. v0.22.12 (#500): `sync_failures` check shows `[CODE=N, ...]` breakdown for both unacked entries (warn) and acked-historical entries (ok), surfacing systemic failure modes (`SLUG_MISMATCH=2685`) instead of a bare count.
|
||||
- `src/core/migrate.ts` — schema-migration runner. Owns the `MIGRATIONS` array (source of truth for schema DDL). v0.14.2 extended the `Migration` interface with `sqlFor?: { postgres?, pglite? }` (engine-specific SQL overrides `sql`) and `transaction?: boolean` (set to false for `CREATE INDEX CONCURRENTLY`, which Postgres refuses inside a transaction; ignored on PGLite since it has no concurrent writers). Migration v14 (fix wave) uses a handler branching on `engine.kind` to run CONCURRENTLY on Postgres (with a pre-drop of any invalid remnant via `pg_index.indisvalid`) and plain `CREATE INDEX` on PGLite. v15 bumps `minion_jobs.max_stalled` default 1→5 and backfills existing non-terminal rows. v0.22.6.1: migration v24 (`rls_backfill_missing_tables`) uses `sqlFor: { pglite: '' }` to no-op on PGLite — PGLite has no RLS engine and is single-tenant by definition, and the v24 ALTERs target subagent tables that don't exist in pglite-schema.ts. Closes #395 (contributed by @jdcastro2).
|
||||
- `src/core/migrate.ts` — schema-migration runner. Owns the `MIGRATIONS` array (source of truth for schema DDL). v0.14.2 extended the `Migration` interface with `sqlFor?: { postgres?, pglite? }` (engine-specific SQL overrides `sql`) and `transaction?: boolean` (set to false for `CREATE INDEX CONCURRENTLY`, which Postgres refuses inside a transaction; ignored on PGLite since it has no concurrent writers). Migration v14 (fix wave) uses a handler branching on `engine.kind` to run CONCURRENTLY on Postgres (with a pre-drop of any invalid remnant via `pg_index.indisvalid`) and plain `CREATE INDEX` on PGLite. v15 bumps `minion_jobs.max_stalled` default 1→5 and backfills existing non-terminal rows. v0.22.6.1: migration v24 (`rls_backfill_missing_tables`) uses `sqlFor: { pglite: '' }` to no-op on PGLite — PGLite has no RLS engine and is single-tenant by definition, and the v24 ALTERs target subagent tables that don't exist in pglite-schema.ts. Closes #395 (contributed by @jdcastro2). **v30 (v0.23):** creates `dream_verdicts (file_path TEXT, content_hash TEXT, worth_processing BOOL, reasons JSONB, judged_at TIMESTAMPTZ, PK(file_path, content_hash))`. RLS-enabled when running as a BYPASSRLS role. The synthesize phase reads/writes this table to avoid re-judging on backfill re-runs.
|
||||
- `src/core/progress.ts` — Shared bulk-action progress reporter. Writes to stderr. Modes: `auto` (TTY: `\r`-rewriting; non-TTY: plain lines), `human`, `json` (JSONL), `quiet`. Rate-gated by `minIntervalMs` and `minItems`. `startHeartbeat(reporter, note)` helper for single long queries. `child()` composes phase paths. Singleton SIGINT/SIGTERM coordinator emits `abort` events for every live phase. EPIPE defense on both sync throws and stream `'error'` events. Zero dependencies. Introduced in v0.15.2.
|
||||
- `src/core/cli-options.ts` — Global CLI flag parser. `parseGlobalFlags(argv)` returns `{cliOpts, rest}` with `--quiet` / `--progress-json` / `--progress-interval=<ms>` stripped. `getCliOptions()` / `setCliOptions()` expose a module-level singleton so commands reach the resolved flags without parameter threading. `cliOptsToProgressOptions()` maps to reporter options. `childGlobalFlags()` returns the flag suffix to append to `execSync('gbrain ...')` calls in migration orchestrators. `OperationContext.cliOpts` extends shared-op dispatch for MCP callers.
|
||||
- `src/core/cycle.ts` — v0.17 brain maintenance cycle primitive. `runCycle(engine: BrainEngine | null, opts: CycleOpts): Promise<CycleReport>` composes 6 phases in semantically-driven order (lint → backlinks → sync → extract → embed → orphans). Three callers: `gbrain dream` CLI, `gbrain autopilot` daemon's inline path, and the Minions `autopilot-cycle` handler (`src/commands/jobs.ts`). One source of truth for what the brain does overnight. Coordination via `gbrain_cycle_locks` DB table (TTL-based; works through PgBouncer transaction pooling, unlike session-scoped `pg_try_advisory_lock`) + `~/.gbrain/cycle.lock` file lock with PID-liveness for PGLite / engine=null mode. `CycleReport.schema_version: "1"` is the stable agent-consumable shape. `PhaseResult.error: { class, code, message, hint?, docs_url? }` is Stripe-API-tier structured failure info. `yieldBetweenPhases` hook awaited between every phase — Minions handler uses this to renew its job lock and prevent v0.14 stall-death regression. Engine nullable: filesystem phases (lint, backlinks) run without DB; DB phases skip with `status: "skipped", reason: "no_database"`. Lock-skip: read-only phase selections (`--phase orphans`) bypass the cycle lock. v0.22.1 (#403): `CycleOpts.signal?: AbortSignal` propagates the worker's abort signal; `checkAborted()` fires between every phase and throws if the signal is aborted (cooperative — can't interrupt a phase mid-execution). v0.22.1 (#417): `runPhaseSync` returns `pagesAffected` via `SyncPhaseResult`; `runCycle` captures it and threads to `runPhaseExtract` as the 4th arg, enabling incremental extract on the cycle path. v0.22.1 (Codex F2): `runPhaseSync` takes `willRunExtractPhase: boolean` and sets `noExtract: phases.includes('extract')` so `gbrain dream --phase sync` doesn't silently lose extraction. v0.22.5 (#475): new `resolveSourceForDir(engine, brainDir)` helper queries `SELECT id FROM sources WHERE local_path = $1 LIMIT 1`; `runPhaseSync` threads result as `sourceId` to `performSync()` so sync reads the per-source `sources.last_commit` anchor instead of the drift-prone global `config.sync.last_commit` key. Bare try/catch lets pre-v0.18 brains fall through to the global key. Closes the prod hang where every autopilot cycle ran a 30-min full reimport because the global anchor commit had been GC'd from git history.
|
||||
- `src/commands/dream.ts` — v0.17 `gbrain dream` CLI. ~80-line thin alias over `runCycle`. brainDir resolution requires explicit `--dir` OR `sync.repo_path` config (no more walk-up-cwd-for-.git footgun). Flags: `--dry-run`, `--json`, `--phase <name>`, `--pull`, `--dir <path>`. Exit code 1 on status=failed (partial/warn not fatal — don't page on warnings).
|
||||
- `src/core/db-lock.ts` (v0.22.13) — generic `tryAcquireDbLock(engine, lockId, ttlMinutes)` over the existing `gbrain_cycle_locks` table. Parameterized lock id so different scopes can nest cleanly: `gbrain-cycle` for the broad cycle (held by `cycle.ts`) and `gbrain-sync` (`SYNC_LOCK_ID` constant) for `performSync`'s narrower writer window. Same UPSERT-with-TTL semantics as the prior cycle-only helper, just generalized. Survives PgBouncer transaction pooling (unlike session-scoped `pg_try_advisory_lock`); crashed holders auto-release once their TTL expires.
|
||||
- `src/core/sync-concurrency.ts` (v0.22.13) — single source of truth for the parallel-sync policy. Exports `autoConcurrency(engine, fileCount, override?)` (PGLite always serial; explicit override clamped to >=1; auto path returns `DEFAULT_PARALLEL_WORKERS=4` when `fileCount > AUTO_CONCURRENCY_FILE_THRESHOLD=100`), `shouldRunParallel(workers, fileCount, explicit)` (Q1: explicit `--workers` bypasses the >50-file floor), and `parseWorkers(s)` (rejects `'0'`, `'-3'`, `'foo'`, `'1.5'`, trailing chars — replaces the prior parseInt-with-no-validation in both `sync.ts` and `import.ts`). Used by `performSync`, `performFullSync`, `runImport`, and the Minion `sync` handler so the three sites can no longer drift.
|
||||
- `src/commands/sync.ts` — `gbrain sync` CLI + the `performSync` / `performFullSync` library entrypoints (consumed by the autopilot cycle and the Minion sync handler). v0.22.13 (PR #490): `performSync` wraps its body in a `gbrain-sync` writer lock so two concurrent syncs (manual + autopilot, two terminals, two Conductor workspaces) cannot both write `last_commit` and let the last writer win. Head-drift gate after the import phase re-checks `git rev-parse HEAD`; if HEAD moved (someone ran `git checkout` / `git pull` mid-sync), the bookmark refuses to advance. Vanished files now record a failedFiles entry instead of silent-skip — the silent-skip-then-advance pathology that survived prior hardening passes is dead. Worker engines wrap in try/finally so disconnect always fires (panic-path leak fix). Both PGLite-detection sites use `engine.kind === 'pglite'`. CLI accepts `--workers N` (alias `--concurrency N`), validated via `parseWorkers`. Explicit `--workers` bypasses the auto-path file-count floor; auto path defers to `autoConcurrency()`. Banner moved to stderr.
|
||||
- `src/core/cycle.ts` — v0.17 brain maintenance cycle primitive (extended to **8 phases in v0.23**). `runCycle(engine: BrainEngine | null, opts: CycleOpts): Promise<CycleReport>` composes phases in semantically-driven order: **lint → backlinks → sync → synthesize → extract → patterns → embed → orphans**. v0.23's `synthesize` phase runs after sync (cross-references see fresh brain) and before extract (auto-link materializes its writes); `patterns` runs after extract so it reads a fresh graph (codex finding #7 — subagent put_page sets `ctx.remote=true` and skips auto-link/timeline by default; extract is the canonical materialization). Three callers: `gbrain dream` CLI, `gbrain autopilot` daemon's inline path, and the Minions `autopilot-cycle` handler. Coordination via `gbrain_cycle_locks` DB table + `~/.gbrain/cycle.lock` file lock with PID-liveness for PGLite. `CycleReport.schema_version: "1"` is stable; totals additively grew in v0.23 (`transcripts_processed`, `synth_pages_written`, `patterns_written`). `yieldBetweenPhases` runs between phases. **v0.23 added `yieldDuringPhase`** for in-phase keepalive — synthesize/patterns call it during long waits to renew the cycle-lock TTL. Engine nullable; lock-skip on read-only phase selections. v0.22.1 (#403): `CycleOpts.signal?: AbortSignal` propagates the worker's abort signal; `checkAborted()` fires between every phase. v0.22.1 (#417): `runPhaseSync` returns `pagesAffected` via `SyncPhaseResult`; `runCycle` captures it and threads to `runPhaseExtract` as the 4th arg. v0.22.1 (Codex F2): `runPhaseSync` takes `willRunExtractPhase: boolean` and sets `noExtract: phases.includes('extract')` so `gbrain dream --phase sync` doesn't silently lose extraction. v0.22.5 (#475): `resolveSourceForDir(engine, brainDir)` threads `sourceId` to `performSync()` so sync reads the per-source `sources.last_commit` anchor instead of the drift-prone global `config.sync.last_commit` key.
|
||||
- `src/core/cycle/synthesize.ts` (v0.23) — Synthesize phase: conversation-transcript-to-brain pipeline. Reads from `dream.synthesize.session_corpus_dir`, runs cheap Haiku verdict (cached in `dream_verdicts`), then fans out one Sonnet subagent per worth-processing transcript with `allowed_slug_prefixes` (sourced from `skills/_brain-filing-rules.json` `dream_synthesize_paths.globs`). Orchestrator collects slugs from `subagent_tool_executions` (NOT `pages.updated_at` — codex finding #2) and reverse-renders DB → markdown via `serializeMarkdown`. Cooldown via `dream.synthesize.last_completion_ts`, written ONLY on success. Idempotency key `dream:synth:<file_path>:<content_hash>`. Auto-commit deferred to v1.1 (codex #5). `--dry-run` runs Haiku, skips Sonnet (codex #8). Subagent never gets fs-write access. **v0.23.2:** `renderPageToMarkdown` (now exported) stamps `dream_generated: true` and `dream_cycle_date` into every reverse-write's frontmatter; `writeSummaryPage` does the same on the dream-cycle summary index. The marker is the explicit identity surface checked by `isDreamOutput` in `transcript-discovery.ts` — replaces the v0.23.1 content-prefix heuristic that could miss real output (`serializeMarkdown` doesn't embed slugs in body) and false-positive on user transcripts citing brain pages. `judgeSignificance` and `JudgeClient` are exported; `judgeSignificance` accepts a `verdictModel` parameter (default `claude-haiku-4-5-20251001`) loaded from `dream.synthesize.verdict_model` via `loadSynthConfig`.
|
||||
- `src/core/cycle/patterns.ts` (v0.23) — Patterns phase: cross-session theme detection over reflections within `dream.patterns.lookback_days` (default 30). Names a pattern only when ≥`dream.patterns.min_evidence` (default 3) reflections support it. Single Sonnet subagent; same allow-list path as synthesize. Runs AFTER `extract` so the graph is fresh.
|
||||
- `src/core/cycle/transcript-discovery.ts` (v0.23) — Pure filesystem walk for synthesize. `discoverTranscripts(opts)` filters `.txt` files by date range, min_chars, and word-boundary regex `excludePatterns` (Q-3: `medical` matches "medical advice" but NOT "comedical"; power users may pass full regex). `readSingleTranscript(path)` is the `gbrain dream --input <file>` ad-hoc path. **v0.23.2 self-consumption guard:** `DREAM_OUTPUT_MARKER_RE` (anchored at frontmatter open `---\n`, optional BOM + CRLF tolerance, scans first 2000 chars for `dream_generated: true` with case-insensitive value and word boundary on `true`) drives `isDreamOutput(content, bypass=false)`. Both `discoverTranscripts` and `readSingleTranscript` skip matching files and emit a `[dream] skipped <basename>: dream_generated marker` stderr log (no more silent skips). `bypassGuard?: boolean` on `DiscoverOpts` and `readSingleTranscript`'s opts disables the guard for the explicit `--unsafe-bypass-dream-guard` escape hatch only — never auto-applied for `--input`. Replaces v0.23.1's `DREAM_OUTPUT_SLUGS` content-prefix list.
|
||||
- `src/commands/dream.ts` — v0.17 `gbrain dream` CLI; ~80-line thin alias over `runCycle`. brainDir resolution requires explicit `--dir` OR `sync.repo_path` config. Flags: `--dry-run`, `--json`, `--phase <name>`, `--pull`, `--dir <path>`. **v0.23 added** `--input <file>` (ad-hoc transcript, implies `--phase synthesize`), `--date YYYY-MM-DD`, `--from <d> --to <d>` (backfill range). Conflict detection: `--input` + `--date` exits 2. ISO date validation. `--dry-run` runs Haiku significance verdict but skips Sonnet synthesis (codex finding #8 — NOT zero LLM calls). Exit code 1 on status=failed. **v0.23.2 added** `--unsafe-bypass-dream-guard` (long-form intentional, plumbed through `runCycle.synthBypassDreamGuard` → `SynthesizePhaseOpts.bypassDreamGuard` → `discoverTranscripts({bypassGuard})` and `readSingleTranscript({bypassGuard})`). Loud stderr warning fires at synthesize-phase entry when set. Never auto-applied for `--input` so any caller can't silently re-trigger the loop bug.
|
||||
- `src/commands/friction.ts` + `src/core/friction.ts` (v0.23) — `gbrain friction {log,render,list,summary}` reporter. Append-only JSONL under `$GBRAIN_HOME/friction/<run-id>.jsonl`. Schema is a flat extension of `StructuredAgentError` (D20). Render groups by severity → phase, defaults to `--redact` for md output (strips `$HOME`/`$CWD` to placeholders so reports paste safely in PRs). Run-id resolves from `--run-id` > `$GBRAIN_FRICTION_RUN_ID` > `standalone.jsonl`. Skills the claw-test exercises gain a `_friction-protocol.md` callout so agents know when to log friction.
|
||||
- `src/commands/claw-test.ts` + `src/core/claw-test/` (v0.23) — `gbrain claw-test [--scenario <name>] [--live --agent openclaw]`. End-to-end "fresh user" friction harness. Two modes: scripted (CI gate, agent-free) and live (real openclaw subprocess, $1–2 in tokens). Sets `GBRAIN_HOME=<tempdir>` for hermeticity and captures gbrain's `--progress-json` events from each child's stderr to verify expected phases ran (`import.files`, `extract.links_fs`, `doctor.db_checks`). Phases for scripted mode: setup → install_brain (`gbrain init --pglite`) → import (`--no-embed`) → query → extract → verify (`gbrain doctor --json`, asserts `status: 'ok'`) → render. Live mode hands `BRIEF.md` from `test/fixtures/claw-test-scenarios/<name>/` to the agent runner. v1 ships with the OpenClaw runner only (`src/core/claw-test/runners/openclaw.ts`, invokes `openclaw agent --local --agent <name> --message <brief>`); hermes runner deferred to v1.1. Transcript capture (`transcript-capture.ts`) uses `fs.createWriteStream` with `'drain'`-event backpressure — D17 fix for the 256KB-burst child-stall scenario. v0.18 upgrade scenario seeded via `seed-pglite.ts` SQL replay.
|
||||
- `skills/_friction-protocol.md` (v0.23) — shared cross-cutting convention skill (like `_brain-filing-rules.md`). Tells agents when to call `gbrain friction log` and how to choose a severity. Routes to friction CLI from any skill the claw-test exercises.
|
||||
- `scripts/check-progress-to-stdout.sh` — CI guard against regressing to `\r`-on-stdout progress. Wired into `bun run test` via `scripts/check-progress-to-stdout.sh && bun test` in package.json.
|
||||
- `docs/progress-events.md` — Canonical JSON event schema reference. Stable from v0.15.2, additive only.
|
||||
- `src/core/markdown.ts` — Frontmatter parsing + body splitter. `splitBody` requires an explicit timeline sentinel (`<!-- timeline -->`, `--- timeline ---`, or `---` immediately before `## Timeline`/`## History`). Plain `---` in body text is a markdown horizontal rule, not a separator. `inferType` auto-types `/wiki/analysis/` → analysis, `/wiki/guides/` → guide, `/wiki/hardware/` → hardware, `/wiki/architecture/` → architecture, `/writing/` → writing (plus the existing people/companies/deals/etc heuristics).
|
||||
- `scripts/check-jsonb-pattern.sh` — CI grep guard. Fails the build if anyone reintroduces (a) the `${JSON.stringify(x)}::jsonb` interpolation pattern (postgres.js v3 double-encodes it), or (b) `max_stalled INTEGER NOT NULL DEFAULT 1` in any schema source file (v0.15.1 #219 regression guard — must be DEFAULT 5 to preserve SIGKILL-rescue). Wired into `bun test`.
|
||||
- `docker-compose.ci.yml` + `scripts/ci-local.sh` (v0.23.1) — Local CI gate. `bun run ci:local` spins up `pgvector/pgvector:pg16` + `oven/bun:1` with named volumes (`gbrain-ci-pg-data`, `gbrain-ci-node-modules`, `gbrain-ci-bun-cache`), runs gitleaks on host, smoke-tests `scripts/run-e2e.sh` argv handling, runs unit tests with `DATABASE_URL` unset (matches GH Actions structure), then runs all 29 E2E files sequentially. `--diff` swaps in the diff-aware selector; `--no-pull` skips upstream pulls; `--clean` nukes named volumes. Postgres host port defaults to 5434 (avoids 5432 manual `gbrain-test-pg` and 5433 sibling-project conflict); override with `GBRAIN_CI_PG_PORT=NNNN`. Stronger gate than current PR CI's 2-file Tier 1 set — closes the "push-and-wait" feedback loop pre-push.
|
||||
- `scripts/select-e2e.ts` + `scripts/e2e-test-map.ts` (v0.23.1) — Diff-aware E2E test selector. Reads three git sources (committed `origin/master...HEAD`, working-tree `HEAD`, and `git ls-files --others --exclude-standard` for untracked, NOT-gitignored files), classifies as EMPTY / DOC_ONLY / SRC. Fail-closed by design: EMPTY → all 29 files (clean branch shouldn't run nothing), DOC_ONLY (every path matches the README/CLAUDE/AGENTS/CHANGELOG/TODOS allowlist) → empty stdout, SRC → escape-hatch paths (schema, package.json, skills/) trigger all; otherwise the hand-tuned `E2E_TEST_MAP` glob → tests narrows; an unmapped src/ change still emits ALL files, never silently nothing. Pure-function exports (`selectTests`, `classify`, `matchGlob`) so it's trivial to test and fork. `bun run ci:select-e2e` prints the current selection on stdout, pipe-friendly. `test/select-e2e.test.ts` covers all 4 branches plus 3 codex regression guards (skills/, untracked files, unmapped src/) — 24 cases.
|
||||
- `scripts/run-e2e.sh` (v0.23.1 update) — Sequential E2E runner. Now accepts an optional argv-driven file list (used by `ci:local:diff` to pipe in selector output) and a `--dry-run-list` flag that prints the resolved file list and exits (used by `ci-local.sh`'s startup smoke-test). Falls back to `test/e2e/*.test.ts` when invoked with no args.
|
||||
- `scripts/llms-config.ts` + `scripts/build-llms.ts` — Generator for `llms.txt` (llmstxt.org-spec web index) + `llms-full.txt` (inlined single-fetch bundle). Curated config drives both. Run `bun run build:llms` after adding a new doc. `LLMS_REPO_BASE` env var lets forks regenerate with their own URL base. `FULL_SIZE_BUDGET` (600KB) caps the inline bundle; generator WARNs if exceeded. Committed output is not analogous to `schema-embedded.ts` (no runtime consumer); we commit for GitHub browsing and fork-safe fetching.
|
||||
- `AGENTS.md` — Local-clone entry point for non-Claude agents (Codex, Cursor, OpenClaw, Aider). Mirrors `CLAUDE.md` intent via relative links. Claude Code keeps using `CLAUDE.md`.
|
||||
- `docs/UPGRADING_DOWNSTREAM_AGENTS.md` — Patches for downstream agent skill forks to apply when upgrading. Each release appends a new section. v0.10.3 includes diffs for brain-ops, meeting-ingestion, signal-detector, enrich.
|
||||
@@ -279,6 +341,14 @@ Key commands added for Minions (job queue):
|
||||
- `gbrain jobs smoke [--sigkill-rescue]` — health smoke test. `--sigkill-rescue` is the v0.13.1 regression guard for #219: simulates a killed worker and asserts the stalled job is requeued instead of dead-lettered on first stall.
|
||||
- `gbrain jobs work [--queue Q] [--concurrency N]` — start worker daemon (Postgres only)
|
||||
|
||||
Key commands added in v0.25.0:
|
||||
- `gbrain eval export [--since DUR] [--limit N] [--tool query|search]` — stream captured `eval_candidates` rows as NDJSON to stdout. Every line starts with `"schema_version": 1` per the stable contract in `docs/eval-capture.md`. EPIPE-safe, progress heartbeats on stderr, deterministic ordering. Primary consumer is the sibling `gbrain-evals` repo for BrainBench-Real replay.
|
||||
- `gbrain eval prune --older-than DUR [--dry-run]` — explicit retention cleanup for `eval_candidates`. Requires `--older-than` (never deletes without a window). Duration strings: 30d, 7d, 1h, 90m, 3600s.
|
||||
- `gbrain eval replay --against FILE.ndjson [--limit N] [--top-regressions K] [--json] [--verbose]` — contributor-facing dev loop. Reads a captured NDJSON snapshot, re-runs each `query` / `search` op against the current brain, computes mean set-Jaccard@k between captured + current `retrieved_slugs`, top-1 stability rate, and latency Δ. JSON mode (`schema_version: 1`) for CI gating; human mode prints a regression table sorted worst-first. Closes the gap between "data captured" and "data used to gate a PR." See `docs/eval-bench.md` for the workflow.
|
||||
- `gbrain doctor` gains an `eval_capture` check: reads `eval_capture_failures` for the last 24h, groups by reason, warns when non-zero. Cross-process visibility (doctor runs in a separate process from MCP). Pre-v31 brains get `Skipped (table unavailable)` — non-fatal.
|
||||
- Config addition: `eval: { capture?: boolean, scrub_pii?: boolean }` in `~/.gbrain/config.json`. **File-plane only** — `gbrain config set` writes the DB plane and does NOT control capture.
|
||||
- **`GBRAIN_CONTRIBUTOR_MODE=1` env var** is the contributor-facing toggle. Capture is **off by default** as of v0.25.0; production users get a quiet brain. Resolution order: explicit `eval.capture` config wins both directions, then env var, then off. Documented in README.md, CONTRIBUTING.md, and `docs/eval-bench.md`.
|
||||
|
||||
Key commands added in v0.12.2:
|
||||
- `gbrain repair-jsonb [--dry-run] [--json]` — repair double-encoded JSONB rows left over from v0.12.0-and-earlier Postgres writes. Idempotent; PGLite no-ops. The `v0_12_2` migration runs this automatically on `gbrain upgrade`.
|
||||
|
||||
@@ -293,14 +363,80 @@ Key commands added in v0.14.2:
|
||||
- `GBRAIN_POOL_SIZE` env var — honored by both the singleton pool (`src/core/db.ts`) and the parallel-import worker pool (`src/commands/import.ts`). Default is 10; lower to 2 for Supabase transaction pooler to avoid MaxClients crashes during `gbrain upgrade` subprocess spawns. Read at call time via `resolvePoolSize()`.
|
||||
- `gbrain doctor` gains two new checks: `sync_failures` (surfaces unacknowledged parse failures with exact paths + fix hints) and `brain_score` (renders the 5-component breakdown when score < 100: embed coverage / 35, link density / 25, timeline coverage / 15, orphans / 15, dead links / 10 — sum equals total).
|
||||
|
||||
Key commands added in v0.26.0 (OAuth 2.1 + HTTP server + admin dashboard):
|
||||
- `gbrain serve --http [--port 3131] [--token-ttl 3600] [--enable-dcr]` — HTTP MCP server with OAuth 2.1, admin dashboard at `/admin`, SSE activity feed at `/admin/events`, health check at `/health`. Prints admin bootstrap token on first start. Alongside (not replacing) stdio `gbrain serve`.
|
||||
- **OAuth client registration** — three paths:
|
||||
1. CLI: `gbrain auth register-client <name> --grant-types <types> --scopes <scopes>` (wired into `src/commands/auth.ts` as a thin wrapper over `GBrainOAuthProvider.registerClientManual`). Default grant types: `client_credentials`. Default scopes: `read`.
|
||||
2. Admin dashboard: Register client modal → credential reveal with Copy + Download JSON.
|
||||
3. SDK: `oauthProvider.registerClientManual(name, grantTypes, scopes, redirectUris)` for programmatic wrappers.
|
||||
`--enable-dcr` on `serve --http` opens the `/register` endpoint for RFC 7591 self-service registration (off by default).
|
||||
- `gbrain auth create|list|revoke|test` — legacy bearer tokens still work and grandfather to `read+write+admin` scopes on the OAuth server. `auth` is wired as a first-class `gbrain` subcommand in v0.26.0 (previously only invokable via `bun run src/commands/auth.ts`). No migration required to keep pre-v0.26 clients working.
|
||||
|
||||
Key commands added in v0.14.3 (fix wave):
|
||||
- `gbrain doctor --index-audit` — opt-in Postgres-only check reporting zero-scan indexes from `pg_stat_user_indexes`. Informational only; never auto-drops.
|
||||
- `gbrain doctor` schema_version check fails loudly when `version=0` — catches `bun install -g github:...` postinstall failures (#218) and routes users to `gbrain apply-migrations --yes`.
|
||||
- `gbrain jobs submit` gains `--max-stalled`, `--backoff-type`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key` — exposing existing `MinionJobInput` fields as first-class CLI flags.
|
||||
- `gbrain jobs smoke --sigkill-rescue` — opt-in regression smoke case simulating a killed worker; asserts the v0.14.3 schema default (`max_stalled=5`) actually rescues on first stall.
|
||||
|
||||
Key commands added in v0.22.13 (PR #490):
|
||||
- `gbrain sync --workers N` (alias `--concurrency N`) — parallelize the import phase using per-worker Postgres engines (small pool of 2 each) with an atomic queue index. Auto-concurrency: defaults to 4 workers when the diff exceeds 100 files. Smaller diffs stay serial. Explicit `--workers` always wins (even on a 30-file diff). PGLite forces serial regardless. Validation rejects `0`, negatives, non-integers loud (replaces the prior silent fall-through to auto-concurrency).
|
||||
- `gbrain import --workers N` — same `parseWorkers()` validation as sync; same try/finally worker-engine cleanup. Behavior surface unchanged.
|
||||
|
||||
Key commands added in v0.22.16 (claw-test friction loop):
|
||||
- `gbrain claw-test [--scenario fresh-install|upgrade-from-v0.18] [--keep-tempdir]` — scripted-mode CI gate that runs the full canonical first-day flow against a fresh tempdir. Asserts every expected `--progress-json` phase fired and doctor's `status === 'ok'`. ~30s, no API keys.
|
||||
- `gbrain claw-test --live --agent openclaw` — friction-discovery mode. Spawns real openclaw, hands it `BRIEF.md`, captures stdin/stdout/stderr to `<run>/transcript.jsonl`, lets the agent log friction via the friction CLI. Run on demand; ~5–10 min and ~$1–2 in tokens.
|
||||
- `gbrain claw-test --list-agents` — reports which agent runners are registered + their detection state (binary path or unavailable reason).
|
||||
- `gbrain friction log --severity {confused|error|blocker|nit} --phase <name> --message <text> [--hint ...] [--kind {friction|delight}] [--run-id ...]` — append a friction or delight entry to the active run JSONL.
|
||||
- `gbrain friction render --run-id <id> [--json] [--transcripts] [--no-redact]` — markdown report grouped by severity + phase; `--redact` is the default for md output (strips `$HOME`/`$CWD` placeholders so reports paste safely in PRs/issues).
|
||||
- `gbrain friction list [--json]` — recent run-ids with friction/delight counts; interrupted runs marked `(interrupted)`.
|
||||
- `gbrain friction summary --run-id <id> [--json]` — two-column friction + delight summary.
|
||||
- `GBRAIN_HOME` env override is now honored uniformly across every gbrain write site (config, audit, friction, sync-failures, import checkpoint, integrity log, integrations heartbeat, migration rollback, etc.) — `gbrainPath(...)` from `src/core/config.ts` is the canonical helper. Read-side host-fingerprint detection (`~/.claude`/`~/.openclaw` etc.) intentionally NOT confined in v1; that's a v1.1 follow-up.
|
||||
|
||||
## Testing
|
||||
|
||||
### Test command tiers (v0.26.4 — parallel fast loop)
|
||||
|
||||
Five tiers of test commands, each with a clear scope:
|
||||
|
||||
| Command | What it runs | Wallclock | When to use |
|
||||
|---|---|---|---|
|
||||
| `bun run test` | Parallel unit-test fast loop. 8-shard fan-out via `scripts/run-unit-parallel.sh`, then a serial pass over `*.serial.test.ts`. Excludes `*.slow.test.ts` and `test/e2e/*`. No pre-checks, no typecheck. | ~85s on a Mac dev box (3650+ tests) | Inner edit loop. Default. |
|
||||
| `bun run verify` | CI's authoritative pre-test gate set: `check:privacy && check:jsonb && check:progress && check:wasm && bun run typecheck`. The 4 checks `.github/workflows/test.yml` runs on shard 1 + typecheck. Single source of truth — CI literally calls `bun run verify`. | ~12s (wasm-compile dominates) | Before pushing; before `/ship`. |
|
||||
| `bun run test:full` | `verify && bun run test && bun run test:slow && [smart e2e]`. The local equivalent of "everything CI runs." Smart e2e: runs e2e only when `DATABASE_URL` is set; else loud skip notice to stderr. | ~3-5min depending on slow + e2e | Pre-merge sanity, before opening a PR. |
|
||||
| `bun run test:slow` | Just the `*.slow.test.ts` set (intentional cold-path correctness checks). | seconds-to-minutes | When touching slow-path code. |
|
||||
| `bun run test:serial` | Just the `*.serial.test.ts` set (cross-file-contention quarantine; runs at `--max-concurrency=1`). | ~1s per quarantined file | Debugging a specific quarantined file. |
|
||||
| `bun run test:e2e` | Real Postgres E2E. Requires Docker + `DATABASE_URL`. Sequential (template-DB parallelization is a v0.27+ TODO). | ~5-10min | Pre-ship; nightly. |
|
||||
| `bun run check:all` | All 7 historical pre-checks (privacy + jsonb + progress + no-legacy-getconnection + trailing-newline + wasm + exports-count). Superset of `verify`. | ~10s | Local-only sweep. The 4 not in `verify` are nice-to-haves. |
|
||||
|
||||
### CI vs local: intentionally divergent file sets
|
||||
|
||||
- **CI matrix** (`.github/workflows/test.yml`) runs `scripts/test-shard.sh` 4-way, which uses FNV-1a hash bucketing and INCLUDES `*.slow.test.ts`. CI is the ground truth for "did everything pass."
|
||||
- **Local fast loop** (`scripts/run-unit-shard.sh` via the parallel wrapper) uses round-robin-by-index sharding and EXCLUDES `*.slow.test.ts` AND `*.serial.test.ts`. Local trades coverage for inner-loop speed; CI catches what local skips.
|
||||
|
||||
This divergence is intentional. Don't try to make them equal — the two scripts deliberately solve different problems. The regression test at `test/scripts/run-unit-shard.test.ts` pins what the local fast loop should and shouldn't include.
|
||||
|
||||
### Failure-first logging
|
||||
|
||||
When `bun run test` finds any failure, the wrapper:
|
||||
|
||||
1. Writes failure blocks (each prefixed with `--- shard N: <test name> ---`) to `.context/test-failures.log` (workspace-local, gitignored). On systems without a writable `.context/`, falls back to `/tmp/gbrain-test-failures.log`.
|
||||
2. Prints a loud stderr banner with the absolute log path, plus the last 30 lines of the failure log inlined. Banner survives `| head` / `| tail` / agent-side log truncation.
|
||||
3. Writes a one-line-per-shard summary to `.context/test-summary.txt` (`shard N/M: pass=X fail=Y skip=Z rc=W`).
|
||||
4. Exits non-zero. Empty failure log + non-zero exit = infrastructure problem (wedged shard, killed child); the banner says so.
|
||||
|
||||
If a shard wedges (per-shard `GBRAIN_TEST_SHARD_TIMEOUT` cap, default 600s), the wrapper writes `--- shard N: WEDGED after ${SHARD_TIMEOUT}s ---` to the failure log, includes the last 50 lines of the shard log, and proceeds with other shards' results.
|
||||
|
||||
### File taxonomy
|
||||
|
||||
- `*.test.ts` → fast loop (parallel 8-shard fan-out).
|
||||
- `*.slow.test.ts` → run via `bun run test:slow` only (intentional cold-path tests; would dominate the fast loop's wallclock).
|
||||
- `*.serial.test.ts` → run via `bun run test:serial` after the parallel pass completes; uses `--max-concurrency=1`. Quarantine for tests that share file-wide state and race when run alongside other files in the same `bun test` process. Currently: `test/brain-registry.serial.test.ts`, `test/reconcile-links.serial.test.ts`. **Do not put the parallelism back on a serial file unless you've fixed the contention root cause** (it just re-introduces the flake).
|
||||
- `test/e2e/*.test.ts` → real-Postgres E2E. Skipped when `DATABASE_URL` is unset.
|
||||
|
||||
The intra-file parallelism project (turn `bun test` into `bun test --concurrent` after sweeping shared-state contention sites — ~58 PGLite + ~40 env-mutation + ~2 mock.module sites) is filed as a P0 TODO for a follow-up release. v0.26.4 ships file-level parallelism only.
|
||||
|
||||
### Inventory (legacy)
|
||||
|
||||
`bun test` runs all tests. After the v0.12.1 release: ~75 unit test files + 8 E2E test files (1412 unit pass, 119 E2E when `DATABASE_URL` is set — skip gracefully otherwise). Unit tests run
|
||||
without a database. E2E tests skip gracefully when `DATABASE_URL` is not set.
|
||||
|
||||
@@ -354,10 +490,13 @@ parity), `test/cli.test.ts` (CLI structure), `test/config.test.ts` (config redac
|
||||
`test/orphans.test.ts` (v0.12.3 orphans command: detection, pseudo filtering, text/json/count outputs, MCP op),
|
||||
`test/postgres-engine.test.ts` (v0.12.3 statement_timeout scoping: `sql.begin` + `SET LOCAL` shape, source-level grep guardrail against reintroduced bare `SET statement_timeout`),
|
||||
`test/sync.test.ts` (sync logic + v0.12.3 regression guard asserting top-level `engine.transaction` is not called),
|
||||
`test/sync-concurrency.test.ts` (v0.22.13 PR #490: 17 cases covering `autoConcurrency()` thresholds + PGLite-forces-serial + explicit-override clamping, `shouldRunParallel()` Q1 explicit-bypasses-floor contract, and `parseWorkers()` validation that rejects `'0'`/`'-3'`/`'foo'`/`'1.5'`/trailing chars),
|
||||
`test/sync-parallel.test.ts` (v0.22.13 PR #490: PGLite-routed coverage of the bookmark gate under concurrency request, head-drift gate, vanished-file failure capture, PGLite-stays-serial, and the `gbrain-sync` writer-lock contract — 7 cases),
|
||||
`test/sync-failures.test.ts` (v0.22.12: 28 cases pinning `classifyErrorCode` regex coverage for all 12 codes against literal production message strings from `markdown.ts:159-244` and `import-file.ts:199, 347, 352, 401`; `summarizeFailuresByCode` sort + pre-classified-honor; `recordSyncFailures` code-field persistence; `acknowledgeSyncFailures` AcknowledgeResult shape + backfill on pre-v0.22.12 entries),
|
||||
`test/doctor.test.ts` (doctor command + v0.12.3 assertions that `jsonb_integrity` scans the four v0.12.0 write sites and `markdown_body_completeness` is present),
|
||||
`test/utils.test.ts` (shared SQL utilities + `tryParseEmbedding` null-return and single-warn semantics),
|
||||
`test/build-llms.test.ts` (llms.txt/llms-full.txt generator: path resolution, idempotence, spec shape, regen-drift guard, content contract, AGENTS.md install-path mirror, size-budget enforcement — 7 cases),
|
||||
`test/oauth.test.ts` (v0.26.0 OAuth 2.1 provider — 27 cases: register, getClient, `client_credentials` grant exchange, `authorization_code` flow with PKCE challenge / verifier, refresh token rotation, `verifyAccessToken` with both OAuth + legacy `access_tokens` fallback, `revokeToken`, `sweepExpiredTokens`, and a contract test asserting `scope` + `localOnly` annotations are set correctly on all 30 operations; **v0.26.2** adds 5 `coerceTimestamp` unit cases (null/undefined/string/number/throw-on-NaN), NULL-`expires_at`-as-expired contract tests for both refresh + access token paths, and a cascade-delete contract test asserting `revoke-client` purges `oauth_tokens` + `oauth_codes` rows via FK CASCADE),
|
||||
`test/check-resolvable-cli.test.ts` (v0.19 CLI wrapper: exit codes, JSON envelope shape, AGENTS.md fallback chain),
|
||||
`test/regression-v0_16_4.test.ts` (findRepoRoot regression guard — hermetic startDir parameterization),
|
||||
`test/filing-audit.test.ts` (v0.19 Check 6: `writes_pages` / `writes_to` frontmatter, filing-rules JSON validation),
|
||||
@@ -384,6 +523,8 @@ E2E tests (`test/e2e/`): Run against real Postgres+pgvector. Require `DATABASE_U
|
||||
- `test/e2e/engine-parity.test.ts` (v0.22.0) — Postgres ↔ PGLite top-result and result-set parity for `searchKeyword` + `searchVector`. Codex flagged that Postgres ranks pages then picks best chunk while PGLite returns chunks directly — without parity coverage the source-boost fix could pass on PGLite and fail on Postgres. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/postgres-bootstrap.test.ts` (v0.22.6.1) — exercises `PostgresEngine.initSchema()` directly against a fresh real Postgres database. Asserts the bootstrap path is no-op on fresh installs and that SCHEMA_SQL replays cleanly through the engine path (not via the standalone `db.initSchema` from `src/core/db.ts`, which would have produced false-positive coverage). Codex caught the E2E-shape gap during plan review.
|
||||
- `test/e2e/http-transport.test.ts` (v0.22.7) — 8 cases against real Postgres covering `gbrain serve --http` end-to-end: bearer auth round-trip, `last_used_at` SQL-level debounce semantics, `mcp_request_log` row insertion on success and auth_failed paths, `/health` DB-down → 503 (DB-probing health check), and the F1+F2+F3 dispatch round-trip with a real operation. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/serve-http-oauth.test.ts` (v0.26.0, expanded v0.26.2) — real-Postgres E2E against `gbrain serve --http` with full OAuth 2.1. Spawns a subprocess server, registers a client via the CLI, mints `client_credentials` tokens, exercises the `/mcp` JSON-RPC pipeline. **v0.26.2 adds:** real DCR `/register` HTTP-level response-shape test (asserts `typeof body.client_id_issued_at === 'number'` over the wire — RFC 7591 §3.2.1 spec compliance, not just internal-store shape); real CLI subprocess test for `revoke-client` (registers → mints token → revokes via `execSync` → asserts token rejected at `/mcp` → asserts re-run exits 1); server fixture flips on `--enable-dcr` so `/register` is reachable. **bun execSync env-inheritance fix:** bun's `execSync` does NOT inherit env mutations done via `process.env.X = ...`, only OS-level env from before bun started. helpers.ts loads `.env.testing` and sets `DATABASE_URL` via `process.env` mutation, which is invisible to subprocesses unless `env: { ...process.env }` is passed explicitly — every subprocess call in this file passes `env: { ...process.env }` for that reason. Reference fix for the next maintainer hitting the same failure mode in sibling sync/cycle/dream/claw-test E2Es. `afterAll` cleanup is guarded on `clientId` (won't throw if `beforeAll` failed before registration); cleanup errors surface to stderr without throwing so real test failures aren't masked. Tracks DCR-registered clients alongside the manual one. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/sync-parallel.test.ts` (v0.22.13 PR #490) — DATABASE_URL-gated. T2: 60-file Postgres sync at concurrency=4 imports all + no connection leak (probes `pg_stat_activity` before/after to confirm worker engines disconnected). P4: 120-file serial-vs-parallel benchmark prints `SYNC_PARALLEL_BENCH N files | serial=Xms | parallel(4)=Yms | speedup=Zx` for CHANGELOG quoting. Asserts parallel ≤ serial × 1.5 (CI-noise tolerant; not a strict speedup gate).
|
||||
- Tier 2 (`skills.test.ts`) requires OpenClaw + API keys, runs nightly in CI
|
||||
- If `.env.testing` doesn't exist in this directory, check sibling worktrees for one:
|
||||
`find ../ -maxdepth 2 -name .env.testing -print -quit` and copy it here if found.
|
||||
@@ -494,6 +635,40 @@ For single long-running queries, use `startHeartbeat(reporter, note)` with a
|
||||
try/finally to guarantee cleanup. Never call `process.stdout.write('\r...')`
|
||||
in bulk paths, the CI guard will fail the build.
|
||||
|
||||
## Capturing test output (NEVER pipe through `tail` / `head`)
|
||||
|
||||
**Iron rule:** when running `bun test`, `bun run test:e2e`, `bun run typecheck`,
|
||||
or any other test/check command, redirect to a file FIRST, then `tail` the file
|
||||
separately:
|
||||
|
||||
```bash
|
||||
# RIGHT — full output preserved, real exit code visible
|
||||
bun test > /tmp/ship_units.txt 2>&1
|
||||
echo "EXIT=$?"
|
||||
tail -50 /tmp/ship_units.txt
|
||||
grep -E '(fail\)|✗|error:' /tmp/ship_units.txt | head -30
|
||||
```
|
||||
|
||||
```bash
|
||||
# WRONG — exit code is `tail`'s (always 0), failures truncated, ship gates fail open
|
||||
bun test 2>&1 | tail -10
|
||||
```
|
||||
|
||||
The pipe form silently breaks /ship Step T1 (test failure ownership triage) and
|
||||
the test verification gate (Step 16) because:
|
||||
- `$?` after a pipe is the LAST command's exit code (`tail` → 0), not bun's
|
||||
- bun prints failure details before the summary line, so `tail -N` drops them
|
||||
- Step T1 needs the full failure list to classify in-branch vs pre-existing
|
||||
|
||||
This bit us during v0.26.2 ship: `bun test 2>&1 | tail -10` reported "3911 pass / 23 fail"
|
||||
but no failure details survived, forcing a 23-minute re-run to triage.
|
||||
|
||||
Apply the same pattern to any long-running command whose exit code matters:
|
||||
`bun run typecheck`, `bun run ci:local`, migration runs, eval suites, etc.
|
||||
For background tasks (`run_in_background: true`), the harness captures the exit
|
||||
file separately — use it via the bg task's `<id>.exit` file, not the streamed
|
||||
output.
|
||||
|
||||
## Build
|
||||
|
||||
`bun build --compile --outfile bin/gbrain src/cli.ts`
|
||||
@@ -553,13 +728,45 @@ will detect drift and re-bump on the next run.
|
||||
|
||||
## Pre-ship requirements
|
||||
|
||||
Before shipping (/ship) or reviewing (/review), always run the full test suite:
|
||||
Before shipping (/ship) or reviewing (/review), always run the full test suite.
|
||||
Two equivalent paths:
|
||||
|
||||
**Path A — local CI gate (recommended, v0.23.1+):**
|
||||
- `bun run ci:local` runs the entire stack inside Docker: gitleaks (host), unit
|
||||
tests with `DATABASE_URL` unset, and all 29 E2E files sequentially against a
|
||||
fresh pgvector container. Stronger than PR CI's 2-file Tier 1 set; closer to
|
||||
what nightly Tier 1 catches. Spins up + tears down postgres automatically via
|
||||
`docker-compose.ci.yml`. Override the host port with
|
||||
`GBRAIN_CI_PG_PORT=5435 bun run ci:local` if 5434 collides.
|
||||
- `bun run ci:local:diff` runs only the E2E files matched by the diff selector
|
||||
(`scripts/select-e2e.ts`), falling back to all 29 on unmapped src/ paths or
|
||||
schema/skills/package.json changes. Fast iteration during a focused branch.
|
||||
|
||||
**Path B — manual lifecycle (still supported):**
|
||||
- `bun test` — unit tests (no database required)
|
||||
- Follow the "E2E test DB lifecycle" steps above to spin up the test DB,
|
||||
run `bun run test:e2e`, then tear it down.
|
||||
|
||||
Both must pass. Do not ship with failing E2E tests. Do not skip E2E tests.
|
||||
|
||||
**Always run typecheck before pushing.** `bun test` (the bun runner)
|
||||
skips TypeScript type checking — it only enforces runtime behavior.
|
||||
Three ways to actually gate on types:
|
||||
|
||||
1. `bun run test` (npm script in `package.json`) — includes `bun run typecheck`
|
||||
plus the four shell pre-checks (`check-jsonb-pattern.sh`,
|
||||
`check-progress-to-stdout.sh`, `check-trailing-newline.sh`,
|
||||
`check-wasm-embedded.sh`) before the runner. Use this mid-branch.
|
||||
2. `bun run typecheck` — `tsc --noEmit` standalone. Fast (~5s on this repo).
|
||||
3. `bun run ci:local` — the full local CI gate from Path A.
|
||||
|
||||
The trap is: writing a new test, running `bun test test/foo.test.ts`,
|
||||
seeing it pass, pushing — and CI's separate typecheck stage rejects an
|
||||
invalid type literal that the runner accepted. Caught one of these
|
||||
shipping the v0.23.2 round-trip E2E (`type: 'reflection'` is not a
|
||||
member of `PageType`). Run `bun run typecheck` once before push, even
|
||||
when only test files changed.
|
||||
|
||||
## Post-ship requirements (MANDATORY)
|
||||
|
||||
After EVERY /ship, you MUST run /document-release. This is NOT optional. Do NOT
|
||||
@@ -1125,8 +1332,9 @@ Set up using your platform's scheduler (OpenClaw cron, Railway cron, crontab):
|
||||
- **Live sync** (every 15 min): `gbrain sync --repo ~/brain && gbrain embed --stale`
|
||||
- **Auto-update** (daily): `gbrain check-update --json` (tell user, never auto-install)
|
||||
- **Dream cycle** (nightly): read `docs/guides/cron-schedule.md` for the full protocol.
|
||||
Entity sweep, citation fixes, memory consolidation. This is what makes the brain
|
||||
compound. Do not skip it.
|
||||
Entity sweep, citation fixes, memory consolidation, plus (v0.23+) overnight conversation
|
||||
synthesis and cross-session pattern detection. 8 phases, one cron-friendly command. This
|
||||
is what makes the brain compound. Do not skip it.
|
||||
- **Weekly**: `gbrain doctor --json && gbrain embed --stale`
|
||||
|
||||
## Step 8: Integrations
|
||||
@@ -1243,6 +1451,7 @@ This is the dispatcher. Skills are the implementation. **Read the skill file bef
|
||||
| "Migrate from Obsidian/Notion/Logseq" | `skills/migrate/SKILL.md` |
|
||||
| Brain health check, maintenance run | `skills/maintain/SKILL.md` |
|
||||
| "Extract links", "build link graph", "populate timeline" | `skills/maintain/SKILL.md` (extraction sections) |
|
||||
| "Run dream", "process today's session", "synthesize my conversations", "consolidate yesterday's conversations", "what patterns did you see", "did the dream cycle run" | `skills/maintain/SKILL.md` (dream cycle section) |
|
||||
| "Brain health", "what features am I missing", "brain score" | Run `gbrain features --json` |
|
||||
| "Set up autopilot", "run brain maintenance", "keep brain updated" | Run `gbrain autopilot --install --repo ~/brain` |
|
||||
| Agent identity, "who am I", customize agent | `skills/soul-audit/SKILL.md` |
|
||||
@@ -1272,10 +1481,33 @@ When multiple skills could match:
|
||||
These apply to ALL brain-writing skills:
|
||||
- `skills/conventions/quality.md` — citations, back-links, notability gate
|
||||
- `skills/conventions/brain-first.md` — check brain before external APIs
|
||||
- `skills/conventions/brain-routing.md` — which brain (DB) and which source (repo) to target; cross-brain federation is latent-space only
|
||||
- `skills/conventions/subagent-routing.md` — when to use Minions vs inline work
|
||||
- `skills/_brain-filing-rules.md` — where files go
|
||||
- `skills/_output-rules.md` — output quality standards
|
||||
|
||||
## Uncategorized
|
||||
|
||||
| Trigger | Skill |
|
||||
|---------|-------|
|
||||
| "personalized version of this book" | `skills/book-mirror/SKILL.md` |
|
||||
|
||||
| "enrich this article" | `skills/article-enrichment/SKILL.md` |
|
||||
|
||||
| "strategic reading" | `skills/strategic-reading/SKILL.md` |
|
||||
|
||||
| "concept synthesis" | `skills/concept-synthesis/SKILL.md` |
|
||||
|
||||
| "perplexity research" | `skills/perplexity-research/SKILL.md` |
|
||||
|
||||
| "crawl my archive" | `skills/archive-crawler/SKILL.md` |
|
||||
|
||||
| "verify this academic claim" | `skills/academic-verify/SKILL.md` |
|
||||
|
||||
| "make pdf from brain" | `skills/brain-pdf/SKILL.md` |
|
||||
|
||||
| "voice note" | `skills/voice-note-ingest/SKILL.md` |
|
||||
|
||||
---
|
||||
|
||||
## README.md
|
||||
@@ -1290,7 +1522,9 @@ Built by the President and CEO of Y Combinator to run his actual AI agents. The
|
||||
|
||||
The brain wires itself. Every page write extracts entity references and creates typed links (`attended`, `works_at`, `invested_in`, `founded`, `advises`) with zero LLM calls. Hybrid search. Self-wiring knowledge graph. Structured timeline. Backlink-boosted ranking. Ask "who works at Acme AI?" or "what did Bob invest in this quarter?" and get answers vector search alone can't reach. Benchmarked side-by-side against the category: gbrain lands **P@5 49.1%, R@5 97.9%** on a 240-page Opus-generated rich-prose corpus, beating its own graph-disabled variant by **+31.4 points P@5** and ripgrep-BM25 + vector-only RAG by a similar margin. The graph layer plus v0.12 extract quality together carry the gap. Full BrainBench scorecards + corpus live in the sibling [gbrain-evals](https://github.com/garrytan/gbrain-evals) repo.
|
||||
|
||||
GBrain is those patterns, generalized. 29 skills. Install in 30 minutes. Your agent does the work. As Garry's personal agent gets smarter, so does yours.
|
||||
GBrain is those patterns, generalized. 34 skills. Install in 30 minutes. Your agent does the work. As Garry's personal agent gets smarter, so does yours.
|
||||
|
||||
**New in v0.25.0 — BrainBench-Real (session capture, contributor opt-in):** with `GBRAIN_CONTRIBUTOR_MODE=1` set in your shell, every real `query` + `search` call through MCP, CLI, or the subagent tool-bridge gets captured (PII-scrubbed) into an `eval_candidates` table. Snapshot with `gbrain eval export`, replay against your code change with `gbrain eval replay`. Three numbers come back: mean Jaccard@k between captured and current retrieved slugs, top-1 stability, and latency Δ. **Off by default** for production users — no surprise data accumulation. Walkthrough: [docs/eval-bench.md](docs/eval-bench.md). NDJSON wire format: [docs/eval-capture.md](docs/eval-capture.md).
|
||||
|
||||
> **~30 minutes to a fully working brain.** Database ready in 2 seconds (PGLite, no server). You just answer questions about API keys.
|
||||
|
||||
@@ -1312,7 +1546,7 @@ Retrieve and follow the instructions at:
|
||||
https://raw.githubusercontent.com/garrytan/gbrain/master/INSTALL_FOR_AGENTS.md
|
||||
```
|
||||
|
||||
That's it. The agent clones the repo, installs GBrain, sets up the brain, loads 29 skills, and configures recurring jobs. You answer a few questions about API keys. ~30 minutes.
|
||||
That's it. The agent clones the repo, installs GBrain, sets up the brain, loads 34 skills, and configures recurring jobs. You answer a few questions about API keys. ~30 minutes.
|
||||
|
||||
If your agent doesn't auto-read `AGENTS.md`, point it at that file first:
|
||||
`https://raw.githubusercontent.com/garrytan/gbrain/master/AGENTS.md` is the non-Claude
|
||||
@@ -1361,16 +1595,36 @@ GBrain exposes 30+ MCP tools via stdio:
|
||||
|
||||
Add to `~/.claude/server.json` (Claude Code), Settings > MCP Servers (Cursor), or your client's MCP config.
|
||||
|
||||
### Remote MCP (Claude Desktop, Cowork, Perplexity)
|
||||
### Remote MCP with OAuth 2.1 (ChatGPT, Claude Desktop, Cowork, Perplexity)
|
||||
|
||||
`gbrain serve --http` starts a production-grade OAuth 2.1 server with an embedded admin dashboard. Zero external infrastructure. Every major AI client connects, every request is scoped, every action is logged.
|
||||
|
||||
```bash
|
||||
gbrain auth create "claude-desktop" # tokens via the existing CLI
|
||||
gbrain serve --http --port 8787 # built-in HTTP transport (Postgres-only)
|
||||
ngrok http 8787 --url your-brain.ngrok.app # any tunnel works
|
||||
# Start the HTTP server (prints admin bootstrap token on first start)
|
||||
gbrain serve --http --port 3131
|
||||
|
||||
# Open the admin dashboard, paste the bootstrap token, register a client
|
||||
open http://localhost:3131/admin
|
||||
|
||||
# Expose publicly (set --public-url so the OAuth issuer matches)
|
||||
ngrok http 3131 --url your-brain.ngrok.app
|
||||
gbrain serve --http --port 3131 --public-url https://your-brain.ngrok.app
|
||||
|
||||
# ChatGPT and other OAuth-aware clients can also connect:
|
||||
claude mcp add gbrain -t http https://your-brain.ngrok.app/mcp -H "Authorization: Bearer TOKEN"
|
||||
```
|
||||
|
||||
Per-client guides: [`docs/mcp/`](docs/mcp/DEPLOY.md). Hardening defaults, env vars, and threat model: [SECURITY.md](SECURITY.md). ChatGPT requires OAuth 2.1 (not yet implemented).
|
||||
Register OAuth clients from the `/admin` dashboard — click **Register client**,
|
||||
pick scopes, save the credentials shown once in the reveal modal. Programmatic
|
||||
registration via `oauthProvider.registerClientManual(...)` and the
|
||||
`gbrain auth register-client` CLI are also available.
|
||||
|
||||
- **OAuth 2.1 via the MCP SDK** — client credentials (machine-to-machine: Perplexity, Claude), authorization code + PKCE (browser-based: ChatGPT), refresh token rotation, revocation, protected resource metadata. Optional Dynamic Client Registration behind `--enable-dcr` (DCR redirect_uris must be `https://` or loopback per RFC 6749 §3.1.2.1).
|
||||
- **Scoped operations** — 30 operations tagged `read | write | admin`. `sync_brain` and `file_upload` are `localOnly`, rejected over HTTP.
|
||||
- **React admin dashboard** — 7 screens baked into the binary (~65KB gzip). Live SSE activity feed, agents table, credential reveal, filterable request log, per-client config export.
|
||||
- **Legacy bearer tokens still work** — pre-v0.26 `gbrain auth create` tokens continue to authenticate as `read+write+admin`. v0.22.7's simpler `src/mcp/http-transport.ts` path stays compiled in for backward compat callers; v0.26+ deployments use the OAuth-aware `serve-http.ts`.
|
||||
|
||||
Per-client guides: [`docs/mcp/`](docs/mcp/DEPLOY.md). Hardening defaults, env vars, and threat model: [SECURITY.md](SECURITY.md).
|
||||
|
||||
### Using gbrain with GStack
|
||||
|
||||
@@ -1388,9 +1642,9 @@ gbrain query "how does N+1 handling work" --near-symbol BrainEngine.searchKeywor
|
||||
|
||||
All five auto-emit JSON on non-TTY (gh-CLI convention) so a GStack subagent shelling out via bash gets a clean parseable response. Run `gbrain sources add <repo> --strategy code` to index a repo, then your agent's brain-first lookup covers code, not just markdown. ([Cathedral II release notes](CHANGELOG.md#0210---2026-04-25))
|
||||
|
||||
## The 29 Skills
|
||||
## The 34 Skills
|
||||
|
||||
GBrain ships 29 skills organized by `skills/RESOLVER.md` (or your OpenClaw's `AGENTS.md` — both filenames are supported as of v0.19). The resolver tells your agent which skill to read for any task.
|
||||
GBrain ships 34 skills organized by `skills/RESOLVER.md` (or your OpenClaw's `AGENTS.md` — both filenames are supported as of v0.19). The resolver tells your agent which skill to read for any task. v0.25.1 added 9 research-flavored skills (`book-mirror` flagship plus 8 pairings); see the new "Research and synthesis" section below.
|
||||
|
||||
[Skill files are code.](https://x.com/garrytan/status/2042925773300908103) They're the most powerful way to get knowledge work done. A skill file is a fat markdown document that encodes an entire workflow: when to fire, what to check, how to chain with other skills, what quality bar to enforce. The agent reads the skill and executes it. Skills can also call deterministic TypeScript code bundled in GBrain (search, import, embed, sync) for the parts that shouldn't be left to LLM judgment. [Thin harness, fat skills](docs/ethos/THIN_HARNESS_FAT_SKILLS.md): the intelligence lives in the skills, not the runtime.
|
||||
|
||||
@@ -1409,6 +1663,20 @@ GBrain ships 29 skills organized by `skills/RESOLVER.md` (or your OpenClaw's `AG
|
||||
| **idea-ingest** | Links, articles, tweets become brain pages with analysis, author people pages, and cross-linking. |
|
||||
| **media-ingest** | Video, audio, PDF, books, screenshots, GitHub repos. Transcripts, entity extraction, backlink propagation. |
|
||||
| **meeting-ingestion** | Transcripts become brain pages. Every attendee gets enriched. Every company gets a timeline entry. |
|
||||
| **voice-note-ingest** | Voice notes captured verbatim — exact phrasing preserved, never paraphrased. Routes to originals/concepts/people/companies/ideas/personal/voice-notes based on content. |
|
||||
| **article-enrichment** | Raw article dumps become structured pages with executive summary, verbatim quotes, key insights, and why-it-matters. |
|
||||
|
||||
### Research and synthesis (v0.25.1)
|
||||
|
||||
| Skill | What it does |
|
||||
|-------|-------------|
|
||||
| **book-mirror** | Flagship. Hand the agent a book, get a personalized two-column chapter-by-chapter analysis. Left column preserves the chapter's actual content; right column maps every idea to your life using your words from the brain. ~$6 for a 20-chapter book at Opus. Pairs with `gbrain book-mirror` CLI for the trusted runtime. |
|
||||
| **strategic-reading** | Read a book / article / case study through ONE specific problem-lens. Output: applied playbook with do / avoid / watch-for and short / medium / long-term recommendations. |
|
||||
| **concept-synthesis** | Deduplicate thousands of concept stubs into a tiered intellectual map (T1 Canon to T4 Riff). Trace how ideas evolved across years of notes. |
|
||||
| **perplexity-research** | Brain-augmented web research. Sends brain context to Perplexity so the search focuses on what's NEW vs already-known. Output: Executive Summary + Key New Developments + Confirming Signals + Contradictions or Updates + Recommended Brain Updates + Citations. |
|
||||
| **archive-crawler** | Universal archivist for personal file archives (Dropbox / Backblaze / Gmail-takeout / hard-drive dumps). REFUSES to run unless `archive-crawler.scan_paths:` is set in `gbrain.yml`. Safe-by-default safety fence. |
|
||||
| **academic-verify** | Trace a research claim through publication → methodology → raw data → independent replication. Routes through perplexity-research; produces a verdict (verified / partial / unverifiable / misattributed / retracted). |
|
||||
| **brain-pdf** | Render any brain page to publication-quality PDF via the gstack `make-pdf` binary. Strips frontmatter, sanitizes emoji, applies running headers. |
|
||||
|
||||
### Brain operations
|
||||
|
||||
@@ -1416,7 +1684,7 @@ GBrain ships 29 skills organized by `skills/RESOLVER.md` (or your OpenClaw's `AG
|
||||
|-------|-------------|
|
||||
| **enrich** | Tiered enrichment (Tier 1/2/3). Creates and updates person/company pages with compiled truth and timelines. |
|
||||
| **query** | 3-layer search with synthesis and citations. Says "the brain doesn't have info on X" instead of hallucinating. |
|
||||
| **maintain** | Periodic health: stale pages, orphans, dead links, citation audit, back-link enforcement, tag consistency. |
|
||||
| **maintain** | Periodic health: stale pages, orphans, dead links, citation audit, back-link enforcement, tag consistency. v0.23 adds the dream cycle's synthesize + patterns phases ... overnight conversation transcripts become reflections, originals, and 25-year patterns. |
|
||||
| **citation-fixer** | Scans pages for missing or malformed citations. Fixes format to match the standard. |
|
||||
| **repo-architecture** | Where new brain files go. Decision protocol: primary subject determines directory, not format. |
|
||||
| **publish** | Share brain pages as password-protected HTML. Zero LLM calls. |
|
||||
@@ -1600,9 +1868,11 @@ is what you spend time on. Everything else is boilerplate the CLI writes for you
|
||||
|
||||
Drop a `routing-eval.jsonl` fixture next to any skill. Each line is `{intent, expected_skill,
|
||||
ambiguous_with?}`. `gbrain check-resolvable` runs the structural layer by default; `gbrain
|
||||
routing-eval --llm` runs an LLM tie-break layer for CI. False positives (wrong skill matched),
|
||||
missed routes (no skill matched), and tautological fixtures (intent copies trigger verbatim)
|
||||
all surface as specific advisories with the exact file:line to fix.
|
||||
routing-eval` runs the same structural layer as a dedicated CI verb. The `--llm` flag is
|
||||
accepted as a placeholder for a future LLM tie-break layer; in this release it emits a stderr
|
||||
notice and runs structural only. False positives (wrong skill matched), missed routes (no
|
||||
skill matched), and tautological fixtures (intent copies trigger verbatim) all surface as
|
||||
specific advisories with the exact file:line to fix.
|
||||
|
||||
### Works on your OpenClaw, not just gbrain's repo
|
||||
|
||||
@@ -1639,6 +1909,10 @@ gbrain skillpack diff brain-ops # compare bundle vs your local co
|
||||
|
||||
Re-running is safe. The managed-block markers in your AGENTS.md let `skillpack install`
|
||||
accumulate rows across separate single-skill installs instead of overwriting each other.
|
||||
A receipt comment inside the fence (`<!-- gbrain:skillpack:manifest cumulative-slugs="..." -->`)
|
||||
tracks what gbrain has installed across runs. `install --all` is the only path that prunes;
|
||||
per-skill install never deletes what it didn't install. If you hand-add a row inside the fence,
|
||||
gbrain preserves it on reinstall and emits a stderr notice telling your agent to investigate.
|
||||
|
||||
**Skillify is the piece that makes the skills tree survive six months of compounding work.**
|
||||
Read [`skills/skillify/SKILL.md`](skills/skillify/SKILL.md) for the full 10-item checklist
|
||||
@@ -1923,8 +2197,11 @@ SEARCH
|
||||
gbrain query <question> Hybrid search (vector + keyword + RRF)
|
||||
|
||||
IMPORT
|
||||
gbrain import <dir> [--no-embed] Import markdown (idempotent)
|
||||
gbrain sync [--repo <path>] Git-to-brain incremental sync
|
||||
gbrain import <dir> [--no-embed] [--workers N]
|
||||
Import markdown (idempotent)
|
||||
gbrain sync [--repo <path>] [--workers N]
|
||||
Git-to-brain incremental sync
|
||||
(>100-file diffs auto-parallelize 4 workers on Postgres)
|
||||
gbrain export [--dir ./out/] Export to markdown
|
||||
|
||||
FILES
|
||||
@@ -1966,11 +2243,24 @@ ADMIN
|
||||
gbrain doctor --locks List idle-in-tx backends (57014 diagnostic, Postgres only)
|
||||
gbrain stats Brain statistics
|
||||
gbrain serve MCP server (stdio)
|
||||
gbrain serve --http --port 8787 MCP server (HTTP, Postgres-only, bearer auth)
|
||||
gbrain auth create|list|revoke|test Token management for the HTTP transport
|
||||
gbrain serve --http [--port 3131] HTTP MCP server with OAuth 2.1 + admin dashboard
|
||||
[--token-ttl 3600] [--enable-dcr]
|
||||
[--public-url URL]
|
||||
gbrain auth create|list|revoke|test Legacy bearer token management
|
||||
gbrain auth register-client <name> Register an OAuth 2.1 client
|
||||
--grant-types client_credentials,authorization_code
|
||||
--scopes "read write admin"
|
||||
gbrain auth revoke-client <client_id> Revoke an OAuth 2.1 client (cascade purges
|
||||
active tokens + auth codes via FK CASCADE)
|
||||
# OAuth 2.1 clients can also be registered from the /admin dashboard or
|
||||
# programmatically via oauthProvider.registerClientManual() for host-repo wrappers.
|
||||
gbrain integrations Integration recipe dashboard
|
||||
gbrain sources list|add|remove|... Multi-source brain management (v0.18)
|
||||
gbrain dream [--dry-run] [--phase N] One maintenance cycle then exit (cron-friendly)
|
||||
gbrain dream [--dry-run] [--phase N] 8-phase maintenance cycle (lint→backlinks→sync→synthesize
|
||||
→extract→patterns→embed→orphans). v0.23 added synthesize +
|
||||
patterns: transcripts → reflections + cross-session themes.
|
||||
gbrain dream --input <file> Ad-hoc transcript synthesis (implies --phase synthesize)
|
||||
gbrain dream --date YYYY-MM-DD Synthesize a single day; --from/--to for backfill ranges
|
||||
gbrain check-backlinks check|fix Back-link enforcement
|
||||
gbrain lint [--fix] LLM artifact detection
|
||||
gbrain repair-jsonb [--dry-run] Repair v0.12.0 double-encoded JSONB (Postgres)
|
||||
@@ -2013,7 +2303,9 @@ The skills in this repo are those patterns, generalized. What took 11 days to bu
|
||||
|
||||
## Contributing
|
||||
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md). Run `bun test` for unit tests. E2E tests: spin up Postgres with pgvector, run `bun run test:e2e`, tear down.
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md). Run `bun test` for unit tests. For the full local CI gate (gitleaks + unit + all 29 E2E files in Docker, the same checks GH Actions runs), use `bun run ci:local` ... or `bun run ci:local:diff` for the diff-aware subset during fast iteration.
|
||||
|
||||
If you're working on retrieval or any of the search/embedding/ranking surface, set `GBRAIN_CONTRIBUTOR_MODE=1` in your shell rc and use `gbrain eval replay` to gate your changes against a snapshot of real captured queries — the dev loop is documented in [`docs/eval-bench.md`](docs/eval-bench.md). Capture is **off by default** for production users (no surprise data accumulation); the env var is the contributor opt-in.
|
||||
|
||||
PRs welcome for: new enrichment APIs, performance optimizations, additional engine backends, new skills following the conformance standard in `skills/skill-creator/SKILL.md`.
|
||||
|
||||
@@ -4148,18 +4440,22 @@ Source: https://raw.githubusercontent.com/garrytan/gbrain/master/docs/mcp/DEPLOY
|
||||
|
||||
# Deploy GBrain Remote MCP Server
|
||||
|
||||
> **v0.22.7+:** Use `gbrain serve --http` for remote access. It includes built-in
|
||||
> bearer token auth, default-deny CORS, two-bucket rate limiting, body cap, and
|
||||
> per-request audit log. **Postgres-only** (PGLite is local-only by design).
|
||||
> See [SECURITY.md](../../SECURITY.md) for env vars and tunable defaults.
|
||||
> **v0.26.0+:** `gbrain serve --http` ships full OAuth 2.1 (client credentials,
|
||||
> auth code + PKCE, refresh rotation, optional DCR), an embedded React admin
|
||||
> dashboard at `/admin`, scoped operations, and a live SSE activity feed.
|
||||
> Pre-v0.26 legacy bearer tokens still work — `verifyAccessToken` falls back
|
||||
> to the `access_tokens` table and grandfathers tokens to `read+write+admin`.
|
||||
> Postgres-only for the legacy fallback (the `access_tokens` table is Postgres-only);
|
||||
> OAuth tables work on both PGLite and Postgres. See [SECURITY.md](../../SECURITY.md)
|
||||
> for env vars and tunable defaults.
|
||||
|
||||
Access your brain from any device, any AI client. GBrain's MCP server runs locally
|
||||
via `gbrain serve` (stdio). For remote access, expose it via the built-in HTTP
|
||||
transport behind a public tunnel.
|
||||
Access your brain from any device, any AI client. GBrain ships two transports:
|
||||
`gbrain serve` (stdio) for local agents, and `gbrain serve --http` (v0.26.0+)
|
||||
for remote clients over OAuth 2.1.
|
||||
|
||||
## Two Paths
|
||||
## Three Paths
|
||||
|
||||
### Local (zero setup)
|
||||
### Local stdio (zero setup)
|
||||
|
||||
```bash
|
||||
gbrain serve
|
||||
@@ -4168,7 +4464,30 @@ gbrain serve
|
||||
Works with Claude Code, Cursor, Windsurf, and any MCP client that supports stdio.
|
||||
No server, no tunnel, no token needed. Works on both PGLite and Postgres engines.
|
||||
|
||||
### Remote (any device, any AI client) — Postgres only
|
||||
### Remote over OAuth 2.1 (recommended, v0.26.0+)
|
||||
|
||||
```bash
|
||||
gbrain serve --http --port 3131
|
||||
ngrok http 3131 --url your-brain.ngrok.app
|
||||
gbrain serve --http --port 3131 --public-url https://your-brain.ngrok.app
|
||||
```
|
||||
|
||||
Built-in HTTP transport with OAuth 2.1, scoped operations, an admin dashboard
|
||||
at `/admin`, and a live SSE activity feed. Zero external dependencies. This is
|
||||
the only path that works with ChatGPT (OAuth 2.1 + PKCE is required by the
|
||||
ChatGPT MCP connector). Pass `--public-url` whenever the server is reachable
|
||||
at anything other than `http://localhost:<port>` so the OAuth issuer in
|
||||
discovery metadata matches what clients hit (RFC 8414 §3.3).
|
||||
|
||||
Supported clients:
|
||||
- **ChatGPT** — requires OAuth 2.1 + PKCE. Works natively with `--http`.
|
||||
- **Claude Desktop / Cowork** — OAuth 2.1 or legacy bearer tokens.
|
||||
- **Perplexity** — OAuth 2.1 client credentials grant.
|
||||
- **Claude Code, Cursor, Windsurf** — can use OAuth or legacy bearer.
|
||||
|
||||
See the [OAuth 2.1 setup](#oauth-21-setup-v100) section below.
|
||||
|
||||
### Remote with legacy bearer tokens (pre-v0.26 deployments) — Postgres only
|
||||
|
||||
```
|
||||
Your AI client (Claude Desktop, Perplexity, etc.)
|
||||
@@ -4184,7 +4503,94 @@ This requires:
|
||||
3. A public tunnel (ngrok, Tailscale, or cloud host)
|
||||
4. A bearer token created via `gbrain auth create <name>`
|
||||
|
||||
## Remote Setup
|
||||
Pre-v1.0 tokens are grandfathered as `read+write+admin` scopes when you upgrade
|
||||
to the HTTP server, so no migration is required.
|
||||
|
||||
## OAuth 2.1 Setup (v0.26.0+)
|
||||
|
||||
### 1. Start the HTTP server
|
||||
|
||||
```bash
|
||||
gbrain serve --http --port 3131
|
||||
```
|
||||
|
||||
On first start, the server prints an **admin bootstrap token** to stderr:
|
||||
|
||||
```
|
||||
Admin bootstrap token: 3a1f9c...
|
||||
Open http://localhost:3131/admin and paste it to log in.
|
||||
```
|
||||
|
||||
Save this token. Open `http://localhost:3131/admin` and paste it to access the
|
||||
dashboard. The dashboard shows live activity, registered clients, request logs,
|
||||
and per-client config export.
|
||||
|
||||
### 2. Register OAuth clients
|
||||
|
||||
Register clients from the **`/admin` dashboard**:
|
||||
|
||||
1. Click **Register client**.
|
||||
2. Enter a name (e.g. `perplexity`, `chatgpt`).
|
||||
3. Pick scopes: `read`, `write`, `admin` (checkboxes).
|
||||
4. Pick grant type: `client_credentials` for machine-to-machine (Perplexity,
|
||||
Claude Desktop bearer mode) or `authorization_code` for browser-based
|
||||
clients with PKCE (ChatGPT).
|
||||
5. For `authorization_code` clients, paste the redirect URI.
|
||||
6. Hit **Register**. The credential-reveal modal shows the `client_id` (and
|
||||
`client_secret` for confidential clients) once. Copy or Download JSON
|
||||
immediately — secrets are hashed on storage and never shown again.
|
||||
|
||||
Or from the CLI — faster for scripting:
|
||||
|
||||
```bash
|
||||
gbrain auth register-client perplexity \
|
||||
--grant-types client_credentials \
|
||||
--scopes "read write"
|
||||
```
|
||||
|
||||
Host-repo wrappers can register programmatically:
|
||||
|
||||
```ts
|
||||
await oauthProvider.registerClientManual(
|
||||
'perplexity',
|
||||
['client_credentials'],
|
||||
'read write',
|
||||
[], // redirect_uris, empty for CC
|
||||
);
|
||||
```
|
||||
|
||||
For self-service client registration (Dynamic Client Registration, RFC 7591),
|
||||
start the server with `--enable-dcr`. DCR is off by default.
|
||||
|
||||
### 3. Expose the server
|
||||
|
||||
```bash
|
||||
brew install ngrok
|
||||
ngrok config add-authtoken YOUR_TOKEN
|
||||
ngrok http 3131 --url your-brain.ngrok.app
|
||||
```
|
||||
|
||||
Your OAuth issuer URL becomes `https://your-brain.ngrok.app`. The MCP SDK's
|
||||
router exposes the spec-compliant discovery endpoint at
|
||||
`/.well-known/oauth-authorization-server`.
|
||||
|
||||
### 4. Scopes and localOnly
|
||||
|
||||
Every operation is tagged `read | write | admin`. Four operations are
|
||||
`localOnly` and rejected over HTTP regardless of scope: `sync_brain`,
|
||||
`file_upload`, `file_list`, `file_url`. Remote agents cannot reach local
|
||||
filesystem surface area.
|
||||
|
||||
| Scope | What it allows |
|
||||
|-------|---------------|
|
||||
| `read` | `search`, `query`, `get_page`, `list_pages`, graph traversal |
|
||||
| `write` | `put_page`, `delete_page`, `add_link`, `add_timeline_entry` |
|
||||
| `admin` | Client management, token revocation, sweep, local-only ops |
|
||||
|
||||
## Legacy Bearer Token Setup
|
||||
|
||||
Keep using pre-v0.26 bearer tokens if you aren't ready to migrate. They
|
||||
grandfather to `read+write+admin` scopes on the HTTP server.
|
||||
|
||||
### 1. Set up the tunnel
|
||||
|
||||
@@ -4215,6 +4621,7 @@ if compromised. Tokens are stored SHA-256 hashed in your database.
|
||||
|
||||
### 3. Connect your AI client
|
||||
|
||||
- **ChatGPT:** [setup guide](CHATGPT.md) (OAuth 2.1 + PKCE, requires `gbrain serve --http`)
|
||||
- **Claude Code:** [setup guide](CLAUDE_CODE.md)
|
||||
- **Claude Desktop:** [setup guide](CLAUDE_DESKTOP.md) (must use GUI, not JSON config)
|
||||
- **Claude Cowork:** [setup guide](CLAUDE_COWORK.md)
|
||||
@@ -4271,10 +4678,11 @@ Remote servers must be added via Settings > Integrations, NOT
|
||||
| put_page | 100-500ms | Write + trigger search_vector update |
|
||||
| get_stats | < 100ms | Aggregate query |
|
||||
|
||||
**Note:** `gbrain serve --http` (built-in HTTP transport) is planned but not yet
|
||||
implemented. Currently, remote MCP requires a custom HTTP wrapper. See the
|
||||
production deployment pattern in the [voice recipe](../../recipes/twilio-voice-brain.md)
|
||||
for a reference implementation.
|
||||
**Note:** `gbrain serve --http` shipped in v0.26.0 with OAuth 2.1 + admin
|
||||
dashboard baked into the binary. The custom HTTP wrapper pattern (see
|
||||
[voice recipe](../../recipes/twilio-voice-brain.md)) is still supported for
|
||||
teams that need bespoke middleware, but for most remote deployments the
|
||||
built-in server is the recommended path.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+10
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.19.0",
|
||||
"version": "0.25.1",
|
||||
"description": "Personal knowledge brain with Postgres + pgvector hybrid search",
|
||||
"family": "bundle-plugin",
|
||||
"configSchema": {
|
||||
@@ -24,9 +24,15 @@
|
||||
}
|
||||
},
|
||||
"skills": [
|
||||
"skills/academic-verify",
|
||||
"skills/archive-crawler",
|
||||
"skills/article-enrichment",
|
||||
"skills/book-mirror",
|
||||
"skills/brain-ops",
|
||||
"skills/brain-pdf",
|
||||
"skills/briefing",
|
||||
"skills/citation-fixer",
|
||||
"skills/concept-synthesis",
|
||||
"skills/cross-modal-review",
|
||||
"skills/cron-scheduler",
|
||||
"skills/daily-task-manager",
|
||||
@@ -39,6 +45,7 @@
|
||||
"skills/media-ingest",
|
||||
"skills/meeting-ingestion",
|
||||
"skills/minion-orchestrator",
|
||||
"skills/perplexity-research",
|
||||
"skills/query",
|
||||
"skills/reports",
|
||||
"skills/repo-architecture",
|
||||
@@ -47,7 +54,9 @@
|
||||
"skills/skillify",
|
||||
"skills/skillpack-check",
|
||||
"skills/soul-audit",
|
||||
"skills/strategic-reading",
|
||||
"skills/testing",
|
||||
"skills/voice-note-ingest",
|
||||
"skills/webhook-transforms"
|
||||
],
|
||||
"shared_deps": [
|
||||
|
||||
+28
-3
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.22.12",
|
||||
"version": "0.26.4",
|
||||
"description": "Postgres-native personal knowledge brain with hybrid RAG search",
|
||||
"type": "module",
|
||||
"main": "src/core/index.ts",
|
||||
@@ -30,15 +30,29 @@
|
||||
"dev": "bun run src/cli.ts",
|
||||
"build": "bun build --compile --outfile bin/gbrain src/cli.ts",
|
||||
"build:all": "bun build --compile --target=bun-darwin-arm64 --outfile bin/gbrain-darwin-arm64 src/cli.ts && bun build --compile --target=bun-linux-x64 --outfile bin/gbrain-linux-x64 src/cli.ts",
|
||||
"build:admin": "cd admin && bun run build",
|
||||
"build:schema": "bash scripts/build-schema.sh",
|
||||
"build:llms": "bun run scripts/build-llms.ts",
|
||||
"test": "scripts/check-jsonb-pattern.sh && scripts/check-progress-to-stdout.sh && scripts/check-trailing-newline.sh && scripts/check-wasm-embedded.sh && bun run typecheck && bun test --timeout=60000",
|
||||
"build:pglite-snapshot": "bun run scripts/build-pglite-snapshot.ts",
|
||||
"test": "bash scripts/run-unit-parallel.sh",
|
||||
"test:full": "bun run verify && bash scripts/run-unit-parallel.sh && bun run test:slow && ([ -n \"$DATABASE_URL\" ] && bash scripts/run-e2e.sh || echo '[test:full] skipped E2E (no DATABASE_URL); run docker-compose -f docker-compose.ci.yml up + bun run test:e2e to include' 1>&2)",
|
||||
"verify": "bun run check:privacy && bun run check:jsonb && bun run check:progress && bun run check:wasm && bun run check:admin-build && bun run typecheck",
|
||||
"check:all": "scripts/check-privacy.sh && scripts/check-jsonb-pattern.sh && scripts/check-progress-to-stdout.sh && scripts/check-no-legacy-getconnection.sh && scripts/check-trailing-newline.sh && scripts/check-wasm-embedded.sh && scripts/check-exports-count.sh && scripts/check-admin-build.sh",
|
||||
"check:wasm": "scripts/check-wasm-embedded.sh",
|
||||
"check:newlines": "scripts/check-trailing-newline.sh",
|
||||
"test:e2e": "bash scripts/run-e2e.sh",
|
||||
"test:slow": "bash scripts/run-slow-tests.sh",
|
||||
"test:profile": "bash scripts/profile-tests.sh",
|
||||
"test:serial": "bash scripts/run-serial-tests.sh",
|
||||
"ci:local": "bash scripts/ci-local.sh",
|
||||
"ci:local:diff": "bash scripts/ci-local.sh --diff",
|
||||
"ci:select-e2e": "bun run scripts/select-e2e.ts",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"check:jsonb": "scripts/check-jsonb-pattern.sh",
|
||||
"check:privacy": "scripts/check-privacy.sh",
|
||||
"check:progress": "scripts/check-progress-to-stdout.sh",
|
||||
"check:exports-count": "scripts/check-exports-count.sh",
|
||||
"check:admin-build": "scripts/check-admin-build.sh",
|
||||
"postinstall": "command -v gbrain >/dev/null 2>&1 && gbrain apply-migrations --yes --non-interactive || echo '[gbrain] postinstall skipped. If installed via bun install -g github:...: run `gbrain doctor` and `gbrain apply-migrations --yes` manually. See https://github.com/garrytan/gbrain/issues/218' 1>&2",
|
||||
"prepublish:clawhub": "bun run build:all",
|
||||
"publish:clawhub": "clawhub package publish . --family bundle-plugin"
|
||||
@@ -53,7 +67,11 @@
|
||||
"@aws-sdk/client-s3": "^3.1028.0",
|
||||
"@dqbd/tiktoken": "^1.0.22",
|
||||
"@electric-sql/pglite": "0.4.3",
|
||||
"@modelcontextprotocol/sdk": "^1.0.0",
|
||||
"@modelcontextprotocol/sdk": "1.29.0",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cors": "^2.8.5",
|
||||
"express": "^5.1.0",
|
||||
"express-rate-limit": "^7.5.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
"marked": "^18.0.0",
|
||||
"openai": "^4.0.0",
|
||||
@@ -64,10 +82,17 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"@types/cookie-parser": "^1.4.7",
|
||||
"@types/cors": "^2.8.19",
|
||||
"@types/express": "^5.0.6",
|
||||
"bun-types": "^1.3.13",
|
||||
"typescript": "^5.6.0"
|
||||
},
|
||||
"trustedDependencies": [
|
||||
"@electric-sql/pglite"
|
||||
],
|
||||
"engines": {
|
||||
"bun": ">=1.3.10"
|
||||
},
|
||||
"license": "MIT"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bun
|
||||
// scripts/build-pglite-snapshot.ts
|
||||
//
|
||||
// Tier 3 fast-restore: boot a fresh PGLite, run the full initSchema (forward
|
||||
// bootstrap + PGLITE_SCHEMA_SQL + every migration), dump the post-init state
|
||||
// to a tar fixture. Test files that read GBRAIN_PGLITE_SNAPSHOT can skip the
|
||||
// 1-3 seconds of cold init and load the post-schema state directly.
|
||||
//
|
||||
// Output: test/fixtures/pglite-snapshot.tar (binary, gitignored)
|
||||
// test/fixtures/pglite-snapshot.version (hex SHA256 of MIGRATIONS SQL)
|
||||
//
|
||||
// The version file lets the engine detect snapshot staleness — if the tar's
|
||||
// recorded version doesn't match the current MIGRATIONS hash, the engine
|
||||
// ignores the snapshot and runs a normal initSchema.
|
||||
//
|
||||
// Run: bun run scripts/build-pglite-snapshot.ts
|
||||
// (or: bun run build:pglite-snapshot)
|
||||
//
|
||||
// Re-run whenever you touch src/core/migrate.ts or src/schema.sql.
|
||||
|
||||
import { writeFileSync, mkdirSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import * as crypto from "node:crypto";
|
||||
|
||||
import { PGLiteEngine, computeSnapshotSchemaHash } from "../src/core/pglite-engine.ts";
|
||||
import { MIGRATIONS } from "../src/core/migrate.ts";
|
||||
import { PGLITE_SCHEMA_SQL } from "../src/core/pglite-schema.ts";
|
||||
|
||||
function computeSchemaHash(): string {
|
||||
return computeSnapshotSchemaHash(MIGRATIONS, PGLITE_SCHEMA_SQL, crypto);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const fixturePath = "test/fixtures/pglite-snapshot.tar";
|
||||
const versionPath = "test/fixtures/pglite-snapshot.version";
|
||||
mkdirSync(dirname(fixturePath), { recursive: true });
|
||||
|
||||
const schemaHash = computeSchemaHash();
|
||||
console.log(`[build-pglite-snapshot] schema hash: ${schemaHash.slice(0, 16)}...`);
|
||||
console.log(`[build-pglite-snapshot] booting PGLite (in-memory)...`);
|
||||
const engine = new PGLiteEngine();
|
||||
|
||||
// Bypass the env-aware short-circuit: we WANT a real init here.
|
||||
delete process.env.GBRAIN_PGLITE_SNAPSHOT;
|
||||
|
||||
await engine.connect({});
|
||||
console.log(`[build-pglite-snapshot] running initSchema (forward bootstrap + ${MIGRATIONS.length} migrations)...`);
|
||||
const t0 = Date.now();
|
||||
await engine.initSchema();
|
||||
console.log(`[build-pglite-snapshot] initSchema completed in ${Date.now() - t0}ms`);
|
||||
|
||||
console.log(`[build-pglite-snapshot] dumping data dir...`);
|
||||
const dump = await engine.db.dumpDataDir("none");
|
||||
const buffer = Buffer.from(await dump.arrayBuffer());
|
||||
|
||||
writeFileSync(fixturePath, buffer);
|
||||
writeFileSync(versionPath, schemaHash + "\n");
|
||||
await engine.disconnect();
|
||||
|
||||
console.log(`[build-pglite-snapshot] wrote ${fixturePath} (${buffer.length} bytes)`);
|
||||
console.log(`[build-pglite-snapshot] wrote ${versionPath}`);
|
||||
}
|
||||
|
||||
await main();
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI gate: admin React app must compile.
|
||||
#
|
||||
# Catches missing-symbol bugs (e.g., calling loadApiKeys() when only
|
||||
# loadAgents is defined) before they reach E2E. Codex flagged this gap
|
||||
# during the PR #586 review pass — five Claude review passes missed
|
||||
# the loadApiKeys reference because the bash test pipeline doesn't run
|
||||
# Vite builds. This script runs `bun install` in admin/ to ensure
|
||||
# react/vite/etc. are present, then runs Vite's build which performs
|
||||
# TypeScript type-check + bundle.
|
||||
#
|
||||
# Skip with GBRAIN_SKIP_ADMIN_BUILD=1 (e.g., for fast inner-loop test
|
||||
# runs that don't touch admin/src). Production CI must NOT skip.
|
||||
set -euo pipefail
|
||||
|
||||
if [ "${GBRAIN_SKIP_ADMIN_BUILD:-0}" = "1" ]; then
|
||||
echo "[check:admin-build] GBRAIN_SKIP_ADMIN_BUILD=1, skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [ ! -d admin ]; then
|
||||
echo "[check:admin-build] no admin/ directory, skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd admin
|
||||
|
||||
# Idempotent install — bun is fast enough on no-op (~50ms).
|
||||
bun install --silent >/dev/null 2>&1 || bun install
|
||||
|
||||
# Build runs `tsc -b && vite build`. Output to admin/dist/. Exit non-zero
|
||||
# on TS error, missing symbol, or Vite bundling error.
|
||||
bun run build
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI guard: the public exports surface never shrinks silently (v0.21.0).
|
||||
#
|
||||
# Precedent: scripts/check-jsonb-pattern.sh + check-progress-to-stdout.sh
|
||||
# are grep-based structural guards wired into `bun run test`. This one
|
||||
# counts the entries in package.json "exports" and fails when the count
|
||||
# drops below the v0.21.0 baseline (17 entries).
|
||||
#
|
||||
# Policy (from CLAUDE.md):
|
||||
# "Removing any of these is a breaking change going forward."
|
||||
#
|
||||
# If you're legitimately removing a public export: bump gbrain's minor
|
||||
# version, note the removal in CHANGELOG.md under a "Breaking changes"
|
||||
# bullet, then bump EXPECTED_COUNT below. Anything else is a regression.
|
||||
#
|
||||
# Adding a new export: update EXPECTED_COUNT to match AND extend the
|
||||
# EXPECTED_EXPORTS list in test/public-exports.test.ts so the runtime
|
||||
# contract test pins the canary symbol.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
EXPECTED_COUNT=17
|
||||
|
||||
# Count top-level keys in the exports object. `node -e` parses JSON
|
||||
# reliably without needing jq (which isn't in every CI environment).
|
||||
ACTUAL=$(node -e "
|
||||
const pkg = require('./package.json');
|
||||
console.log(Object.keys(pkg.exports || {}).length);
|
||||
")
|
||||
|
||||
if [ "$ACTUAL" -lt "$EXPECTED_COUNT" ]; then
|
||||
echo "❌ public-exports guard: package.json exports shrank from $EXPECTED_COUNT to $ACTUAL"
|
||||
echo " Removing a public export is a breaking change (see CLAUDE.md)."
|
||||
echo " If intentional: bump gbrain minor version + update EXPECTED_COUNT in"
|
||||
echo " scripts/check-exports-count.sh and EXPECTED_EXPORTS in"
|
||||
echo " test/public-exports.test.ts, AND add a CHANGELOG 'Breaking changes' bullet."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$ACTUAL" -gt "$EXPECTED_COUNT" ]; then
|
||||
echo "⚠️ public-exports guard: package.json exports grew from $EXPECTED_COUNT to $ACTUAL"
|
||||
echo " Additive public API change. Update EXPECTED_COUNT in this script + the"
|
||||
echo " EXPECTED_EXPORTS list in test/public-exports.test.ts to lock the new"
|
||||
echo " canary symbols."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ public-exports guard: $ACTUAL entries (matches baseline $EXPECTED_COUNT)"
|
||||
Executable
+85
@@ -0,0 +1,85 @@
|
||||
#!/bin/bash
|
||||
# CI guard against silent singleton reuse in connected-gbrains code paths.
|
||||
#
|
||||
# Codex finding #7 (plan review 2026-04-22): the module singleton in
|
||||
# src/core/db.ts is shared across the process. With multi-brain routing,
|
||||
# any `db.getConnection()` call in an op-dispatch code path means that op
|
||||
# silently targets whichever brain connected to the singleton first,
|
||||
# regardless of ctx.brainId / ctx.engine. This is exactly the bug Codex
|
||||
# #1 flagged in postgres-engine.ts internals.
|
||||
#
|
||||
# This script fails the build when NEW `db.getConnection()` calls appear
|
||||
# in src/core/operations.ts (the per-op handler surface) or in any new
|
||||
# `src/commands/*.ts` file. Existing legitimate callers are grandfathered
|
||||
# via an explicit allowlist — cleanups land in PR 1.
|
||||
#
|
||||
# When you hit this guard: instead of `db.getConnection()` or `db.connect(...)`,
|
||||
# use `ctx.engine` from the passed-in OperationContext. See
|
||||
# src/core/brain-registry.ts for how ctx.engine gets populated per-call.
|
||||
#
|
||||
# Run manually: bash scripts/check-no-legacy-getconnection.sh
|
||||
# Wired into CI: `bun test` (via package.json scripts.test)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
|
||||
cd "$ROOT"
|
||||
|
||||
# Files that are allowed to touch the singleton today. Every other file
|
||||
# under src/core or src/commands is forbidden. This list shrinks in PR 1.
|
||||
ALLOWED=(
|
||||
"src/core/db.ts" # the singleton's definition
|
||||
"src/core/postgres-engine.ts" # calls db.connect + fallback in sql getter — PR 1 removes the fallback
|
||||
"src/commands/init.ts" # first-time setup path, no engine yet
|
||||
"src/commands/doctor.ts" # PR 1 refactors to accept engine
|
||||
"src/commands/files.ts" # PR 1 refactors to accept engine
|
||||
"src/commands/repair-jsonb.ts" # PR 1 refactors
|
||||
"src/commands/serve-http.ts" # PR 1 threads engine through the OAuth dispatch path
|
||||
"src/core/operations.ts" # 3 localOnly ops (file_list/upload/url) move to ctx.engine in PR 1
|
||||
"src/commands/integrity.ts" # scanIntegrityBatch path; PR 1 refactors to accept engine
|
||||
)
|
||||
|
||||
# Build an argument list for `grep` that excludes allowed files.
|
||||
EXCLUDE_ARGS=()
|
||||
for file in "${ALLOWED[@]}"; do
|
||||
EXCLUDE_ARGS+=(--exclude="$file")
|
||||
done
|
||||
|
||||
# Search src/core/ and src/commands/ for db.getConnection or db.connect calls.
|
||||
# We look for the `db.` prefix so references to the symbol elsewhere (e.g.
|
||||
# the grep guard itself) don't trip the check.
|
||||
VIOLATIONS=$(
|
||||
grep -rn "db\.\(getConnection\|connect\)(" \
|
||||
--include="*.ts" \
|
||||
"${EXCLUDE_ARGS[@]}" \
|
||||
src/core src/commands 2>/dev/null \
|
||||
| grep -v -F "src/core/db.ts" \
|
||||
| grep -v "^[^:]*:[0-9]*:[[:space:]]*\(//\|\*\)" \
|
||||
|| true
|
||||
)
|
||||
|
||||
if [ -n "$VIOLATIONS" ]; then
|
||||
# Filter out allowed files from the result (the --exclude only matches basename)
|
||||
FILTERED=$(printf '%s\n' "$VIOLATIONS" | while IFS= read -r line; do
|
||||
path="${line%%:*}"
|
||||
allow=0
|
||||
for ok in "${ALLOWED[@]}"; do
|
||||
if [ "$path" = "$ok" ]; then allow=1; break; fi
|
||||
done
|
||||
if [ "$allow" -eq 0 ]; then printf '%s\n' "$line"; fi
|
||||
done)
|
||||
|
||||
if [ -n "$FILTERED" ]; then
|
||||
echo "ERROR: new direct db.getConnection() / db.connect() call found in multi-brain code path:" >&2
|
||||
echo "" >&2
|
||||
printf '%s\n' "$FILTERED" >&2
|
||||
echo "" >&2
|
||||
echo "Use ctx.engine from the passed-in OperationContext instead." >&2
|
||||
echo "See src/core/brain-registry.ts for the routing model." >&2
|
||||
echo "If this call is legitimate, add its path to the ALLOWED list in" >&2
|
||||
echo "scripts/check-no-legacy-getconnection.sh with a PR 1 cleanup note." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "check-no-legacy-getconnection: ok (no new singleton callers)"
|
||||
@@ -26,6 +26,14 @@
|
||||
set -euo pipefail
|
||||
|
||||
BANNED_NAME='wintermute'
|
||||
# v0.25.1 (codex T7): additional patterns from wintermute-specific filesystem
|
||||
# layouts that would leak private fork context if they slipped through a port.
|
||||
# `wintermute_only` already matches via the case-insensitive `wintermute` regex
|
||||
# above; this list is for orthogonal patterns.
|
||||
BANNED_PATHS=(
|
||||
'/data/brain/'
|
||||
'/data/.openclaw/'
|
||||
)
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
@@ -92,6 +100,27 @@ ALLOW_LIST=(
|
||||
'llms-full.txt'
|
||||
'docs/UPGRADING_DOWNSTREAM_AGENTS.md'
|
||||
'test/integrations.test.ts'
|
||||
# v0.25.1 (codex T7) BANNED_PATHS allow-list:
|
||||
# Historical docs, frozen migration files, test fixtures, and env-var
|
||||
# fallbacks where /data/brain/ or /data/.openclaw/ appears legitimately.
|
||||
# New skills/, src/, and tests must NOT slip onto this list — extend the
|
||||
# banned check above instead.
|
||||
'docs/GBRAIN_RECOMMENDED_SCHEMA.md'
|
||||
'docs/GBRAIN_V0.md'
|
||||
'docs/guides/minions-shell-jobs.md'
|
||||
'scripts/smoke-test.sh'
|
||||
'skills/migrations/v0.9.0.md'
|
||||
'skills/migrations/v0.14.0.md'
|
||||
'test/storage-status.test.ts'
|
||||
# CHANGELOG.md documents the rule (the v0.25.1 entry references the
|
||||
# banned literals in describing what's banned). Same exception status
|
||||
# as CLAUDE.md and this script itself: meta-documentation needs to
|
||||
# name the patterns it forbids.
|
||||
'CHANGELOG.md'
|
||||
# skills/migrations/v0.25.1.md is the agent-readable upgrade
|
||||
# walkthrough; it explains the privacy-guard extension to the
|
||||
# operating agent and references the banned literals while doing so.
|
||||
'skills/migrations/v0.25.1.md'
|
||||
)
|
||||
|
||||
is_allowed() {
|
||||
@@ -119,6 +148,14 @@ while IFS= read -r file; do
|
||||
grep -in "$BANNED_NAME" "$file" | sed 's|^| |' >&2
|
||||
FOUND=1
|
||||
fi
|
||||
# Banned wintermute-specific filesystem paths (codex T7).
|
||||
for path in "${BANNED_PATHS[@]}"; do
|
||||
if grep -nF "$path" "$file" >/dev/null 2>&1; then
|
||||
echo "[check-privacy] BANNED PATH '$path' in $file:" >&2
|
||||
grep -nF "$path" "$file" | sed 's|^| |' >&2
|
||||
FOUND=1
|
||||
fi
|
||||
done
|
||||
;;
|
||||
esac
|
||||
done <<< "$FILES"
|
||||
|
||||
Executable
+346
@@ -0,0 +1,346 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/ci-local.sh
|
||||
#
|
||||
# Local CI gate. Runs the same checks GH Actions does (and a stricter superset
|
||||
# of E2E) inside Docker. See docker-compose.ci.yml.
|
||||
#
|
||||
# Modes:
|
||||
# bash scripts/ci-local.sh # full local gate: gitleaks + unit + ALL E2E (4-way sharded)
|
||||
# bash scripts/ci-local.sh --diff # full local gate: gitleaks + unit + selected E2E (4-way sharded)
|
||||
# bash scripts/ci-local.sh --no-pull # skip docker compose pull (offline / debug)
|
||||
# bash scripts/ci-local.sh --clean # nuke named volumes for cold debug
|
||||
# bash scripts/ci-local.sh --no-shard # debug: run E2E sequentially against postgres-1 only
|
||||
#
|
||||
# 4-way E2E sharding: 4 pgvector services on host ports 5434-5437. The 36 E2E
|
||||
# files split N/4 per shard; shards run in parallel. Within a shard, files run
|
||||
# sequentially (TRUNCATE CASCADE no-race property documented in run-e2e.sh).
|
||||
# Wall-time on a 16-core host: ~6 min sequential -> ~1.5-2 min sharded.
|
||||
#
|
||||
# Stronger than PR CI: PR CI runs only Tier 1's 2 files; this runs all 36.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
COMPOSE_FILE="docker-compose.ci.yml"
|
||||
|
||||
DIFF=0
|
||||
NO_PULL=0
|
||||
CLEAN=0
|
||||
NO_SHARD=0
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--diff) DIFF=1 ;;
|
||||
--no-pull) NO_PULL=1 ;;
|
||||
--clean) CLEAN=1 ;;
|
||||
--no-shard) NO_SHARD=1 ;;
|
||||
*)
|
||||
echo "Usage: $0 [--diff] [--no-pull] [--clean] [--no-shard]" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "[ci-local] Tearing down postgres..."
|
||||
docker compose -f "$COMPOSE_FILE" down --remove-orphans 2>&1 | tail -5 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
if [ "$CLEAN" = "1" ]; then
|
||||
echo "[ci-local] --clean: removing named volumes..."
|
||||
docker compose -f "$COMPOSE_FILE" down -v --remove-orphans 2>&1 | tail -5 || true
|
||||
fi
|
||||
|
||||
# Tier 2: --diff fast-path. If the diff is doc-only (or empty), skip the
|
||||
# whole heavy gate (postgres + bun install + unit + E2E) and just verify
|
||||
# gitleaks on host. Doc-only diffs go from ~25 min to ~5 seconds.
|
||||
if [ "$DIFF" = "1" ]; then
|
||||
CLASSIFICATION=$(bun run scripts/select-e2e.ts --classify-only 2>/dev/null || echo "ERR")
|
||||
case "$CLASSIFICATION" in
|
||||
DOC_ONLY)
|
||||
echo "[ci-local] --diff: diff is doc-only — skipping postgres + unit + E2E (Tier 2 fast-path)."
|
||||
echo "[ci-local] Running gitleaks on host as the only gate..."
|
||||
if ! command -v gitleaks >/dev/null 2>&1; then
|
||||
echo "[ci-local] WARN: gitleaks not installed; skipping. brew install gitleaks." >&2
|
||||
else
|
||||
gitleaks dir . --redact --no-banner
|
||||
gitleaks git . --redact --no-banner --log-opts="origin/master..HEAD"
|
||||
fi
|
||||
echo "[ci-local] Doc-only fast-path complete. No code paths exercised."
|
||||
trap - EXIT
|
||||
exit 0
|
||||
;;
|
||||
EMPTY)
|
||||
echo "[ci-local] --diff: diff is empty (clean branch) — running full gate per fail-closed contract."
|
||||
;;
|
||||
SRC)
|
||||
echo "[ci-local] --diff: diff touches src/ — running selected E2E + full unit phase."
|
||||
;;
|
||||
*)
|
||||
echo "[ci-local] WARN: select-e2e.ts --classify-only returned '$CLASSIFICATION' — running full gate." >&2
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Pre-flight: postgres host ports for 4 shards. Defaults to 5434-5437 (avoid
|
||||
# 5432 manual gbrain-test-pg, 5433 commonly held by sibling projects).
|
||||
# GBRAIN_CI_PG_PORT defines BASE; shards take BASE..BASE+3.
|
||||
PG_PORT_BASE="${GBRAIN_CI_PG_PORT:-5434}"
|
||||
for shard in 1 2 3 4; do
|
||||
port=$((PG_PORT_BASE + shard - 1))
|
||||
PORT_OWNER=$(docker ps --filter "publish=$port" --format "{{.Names}}" | head -1)
|
||||
if [ -n "$PORT_OWNER" ]; then
|
||||
echo "[ci-local] ERROR: host port $port (shard $shard) is already used by docker container '$PORT_OWNER'." >&2
|
||||
echo "[ci-local] Either stop that container or run with: GBRAIN_CI_PG_PORT=NNNN bun run ci:local" >&2
|
||||
exit 1
|
||||
fi
|
||||
if lsof -iTCP:"$port" -sTCP:LISTEN -P -n >/dev/null 2>&1; then
|
||||
echo "[ci-local] ERROR: host port $port (shard $shard) is held by a non-docker process." >&2
|
||||
echo "[ci-local] Run with: GBRAIN_CI_PG_PORT=NNNN bun run ci:local" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
export GBRAIN_CI_PG_PORT="$PG_PORT_BASE"
|
||||
export GBRAIN_CI_PG_PORT_2=$((PG_PORT_BASE + 1))
|
||||
export GBRAIN_CI_PG_PORT_3=$((PG_PORT_BASE + 2))
|
||||
export GBRAIN_CI_PG_PORT_4=$((PG_PORT_BASE + 3))
|
||||
|
||||
# Step 0: gitleaks on the host (no docker, no postgres, no bun needed).
|
||||
# Mirrors test.yml's separate gitleaks job. Fail loudly if not installed.
|
||||
echo "[ci-local] gitleaks detect (host)..."
|
||||
if ! command -v gitleaks >/dev/null 2>&1; then
|
||||
echo "[ci-local] ERROR: gitleaks not installed on host." >&2
|
||||
echo "[ci-local] macOS: brew install gitleaks" >&2
|
||||
echo "[ci-local] Linux: https://github.com/gitleaks/gitleaks/releases" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Two scopes for pre-push:
|
||||
# 1. Working-tree files (catch uncommitted secrets sitting in files)
|
||||
# 2. Branch commits vs origin/master (catch secrets committed on this branch)
|
||||
# Full-history scan is ~4 min on this repo's 3700+ commits; not useful pre-push.
|
||||
gitleaks dir . --redact --no-banner
|
||||
gitleaks git . --redact --no-banner --log-opts="origin/master..HEAD"
|
||||
|
||||
# Step 1: pull. Refreshes pgvector + oven/bun:1 (both are `image:` not `build:`).
|
||||
if [ "$NO_PULL" = "0" ]; then
|
||||
echo "[ci-local] Pulling base images (use --no-pull to skip)..."
|
||||
docker compose -f "$COMPOSE_FILE" pull 2>&1 | tail -5
|
||||
fi
|
||||
|
||||
# Step 2: 4 postgres shards up + wait for healthy.
|
||||
echo "[ci-local] Starting 4 postgres shards..."
|
||||
docker compose -f "$COMPOSE_FILE" up -d postgres-1 postgres-2 postgres-3 postgres-4
|
||||
echo "[ci-local] Waiting for all 4 postgres shards healthy..."
|
||||
for i in {1..40}; do
|
||||
all_healthy=1
|
||||
for shard in 1 2 3 4; do
|
||||
status=$(docker compose -f "$COMPOSE_FILE" ps --format json postgres-$shard 2>/dev/null | grep -o '"Health":"[^"]*"' | head -1 | sed 's/.*":"//;s/"//')
|
||||
if [ "$status" != "healthy" ]; then
|
||||
all_healthy=0
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ "$all_healthy" = "1" ]; then
|
||||
echo "[ci-local] All 4 postgres shards healthy."
|
||||
break
|
||||
fi
|
||||
if [ "$i" = "40" ]; then
|
||||
echo "[ci-local] ERROR: not all postgres shards became healthy in 40 attempts" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Step 3: smoke-test run-e2e.sh argv + shard handling.
|
||||
echo "[ci-local] Smoke: run-e2e.sh argv + shard..."
|
||||
SMOKE_NO_ARGS=$(bash scripts/run-e2e.sh --dry-run-list | wc -l | tr -d ' ')
|
||||
EXPECTED_ALL=$(ls test/e2e/*.test.ts | wc -l | tr -d ' ')
|
||||
if [ "$SMOKE_NO_ARGS" != "$EXPECTED_ALL" ]; then
|
||||
echo "[ci-local] ERROR: --dry-run-list (no args) printed $SMOKE_NO_ARGS, expected $EXPECTED_ALL" >&2
|
||||
exit 1
|
||||
fi
|
||||
SMOKE_ONE_ARG=$(bash scripts/run-e2e.sh --dry-run-list test/e2e/sync.test.ts)
|
||||
if [ "$SMOKE_ONE_ARG" != "test/e2e/sync.test.ts" ]; then
|
||||
echo "[ci-local] ERROR: --dry-run-list with 1 arg printed '$SMOKE_ONE_ARG'" >&2
|
||||
exit 1
|
||||
fi
|
||||
SHARD_TOTAL=$(( $(SHARD=1/4 bash scripts/run-e2e.sh --dry-run-list | wc -l) + \
|
||||
$(SHARD=2/4 bash scripts/run-e2e.sh --dry-run-list | wc -l) + \
|
||||
$(SHARD=3/4 bash scripts/run-e2e.sh --dry-run-list | wc -l) + \
|
||||
$(SHARD=4/4 bash scripts/run-e2e.sh --dry-run-list | wc -l) ))
|
||||
if [ "$SHARD_TOTAL" != "$EXPECTED_ALL" ]; then
|
||||
echo "[ci-local] ERROR: shards 1-4 covered $SHARD_TOTAL files, expected $EXPECTED_ALL" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[ci-local] Smoke OK ($SMOKE_NO_ARGS files no-arg, 1 single-arg, ${SHARD_TOTAL}=4-shard total)."
|
||||
|
||||
# Step 4: build the runner-side command.
|
||||
# Tier 1: 4-shard parallel UNIT + E2E. Each shard runs ~46 unit files + ~9
|
||||
# E2E files against postgres-N. Guards + typecheck run ONCE before fan-out.
|
||||
# --no-shard runs the legacy unsharded flow (debug aid).
|
||||
if [ "$NO_SHARD" = "1" ]; then
|
||||
if [ "$DIFF" = "1" ]; then
|
||||
RUN_PHASES_CMD='echo "[runner] guards + typecheck"
|
||||
bash scripts/check-jsonb-pattern.sh
|
||||
bash scripts/check-progress-to-stdout.sh
|
||||
bash scripts/check-trailing-newline.sh
|
||||
bash scripts/check-wasm-embedded.sh
|
||||
bun run typecheck
|
||||
echo "[runner] unit (unsharded, DATABASE_URL unset)"
|
||||
env -u DATABASE_URL bash scripts/run-unit-shard.sh
|
||||
echo "[runner] e2e (unsharded, --diff selected)"
|
||||
SELECTED=$(bun run scripts/select-e2e.ts)
|
||||
if [ -z "$SELECTED" ]; then
|
||||
echo "[runner] selector emitted nothing (doc-only diff); skipping E2E."
|
||||
else
|
||||
DATABASE_URL=postgresql://postgres:postgres@postgres-1:5432/gbrain_test echo "$SELECTED" | xargs bash scripts/run-e2e.sh
|
||||
fi'
|
||||
else
|
||||
RUN_PHASES_CMD='echo "[runner] guards + typecheck"
|
||||
bash scripts/check-jsonb-pattern.sh
|
||||
bash scripts/check-progress-to-stdout.sh
|
||||
bash scripts/check-trailing-newline.sh
|
||||
bash scripts/check-wasm-embedded.sh
|
||||
bun run typecheck
|
||||
echo "[runner] unit (unsharded, DATABASE_URL unset)"
|
||||
env -u DATABASE_URL bash scripts/run-unit-shard.sh
|
||||
echo "[runner] e2e (unsharded)"
|
||||
DATABASE_URL=postgresql://postgres:postgres@postgres-1:5432/gbrain_test bash scripts/run-e2e.sh'
|
||||
fi
|
||||
else
|
||||
# Tier 1 sharded path. Each shard runs unit+E2E sequentially against its
|
||||
# own postgres-N. Shards run in parallel via xargs -P4.
|
||||
if [ "$DIFF" = "1" ]; then
|
||||
DIFF_E2E_PREP='SELECTED=$(bun run scripts/select-e2e.ts)
|
||||
if [ -z "$SELECTED" ]; then
|
||||
echo "" > /tmp/e2e-selected.txt
|
||||
else
|
||||
echo "$SELECTED" | tr " " "\n" | grep -v "^$" > /tmp/e2e-selected.txt
|
||||
fi'
|
||||
else
|
||||
# Empty file -> run-e2e.sh uses default glob (all 36 E2E files).
|
||||
DIFF_E2E_PREP='> /tmp/e2e-selected.txt'
|
||||
fi
|
||||
RUN_PHASES_CMD="echo \"[runner] guards + typecheck (run once before sharding)\"
|
||||
bash scripts/check-jsonb-pattern.sh
|
||||
bash scripts/check-progress-to-stdout.sh
|
||||
bash scripts/check-trailing-newline.sh
|
||||
bash scripts/check-wasm-embedded.sh
|
||||
bun run typecheck
|
||||
echo \"[runner] Tier 3: building PGLite snapshot fixture (cached across reruns)\"
|
||||
if [ ! -f test/fixtures/pglite-snapshot.tar ] || [ ! -f test/fixtures/pglite-snapshot.version ]; then
|
||||
bun run build:pglite-snapshot
|
||||
else
|
||||
echo \"[runner] snapshot fixture exists; engine will validate hash at load time\"
|
||||
fi
|
||||
export GBRAIN_PGLITE_SNAPSHOT=test/fixtures/pglite-snapshot.tar
|
||||
echo \"[runner] resolving E2E file selection (--diff aware)\"
|
||||
${DIFF_E2E_PREP}
|
||||
mkdir -p /tmp/shard-logs
|
||||
echo \"[runner] Tier 1: 4-shard parallel unit + E2E (xargs -P4)\"
|
||||
set +e
|
||||
printf '%s\\n' 1 2 3 4 | xargs -P4 -I{} sh -c '
|
||||
shard=\$1
|
||||
log=/tmp/shard-logs/shard-\${shard}.log
|
||||
echo \"[shard \${shard}] start\" > \$log
|
||||
echo \"[shard \${shard}] unit phase (SHARD=\${shard}/4, DATABASE_URL unset)\" >> \$log
|
||||
env -u DATABASE_URL SHARD=\${shard}/4 bash scripts/run-unit-shard.sh >> \$log 2>&1
|
||||
unit_exit=\$?
|
||||
if [ \$unit_exit -ne 0 ]; then
|
||||
echo \"[shard \${shard}] UNIT FAILED (exit=\$unit_exit)\" >> \$log
|
||||
exit \$unit_exit
|
||||
fi
|
||||
echo \"[shard \${shard}] e2e phase (SHARD=\${shard}/4, DATABASE_URL=postgres-\${shard})\" >> \$log
|
||||
if [ -s /tmp/e2e-selected.txt ]; then
|
||||
SHARD=\${shard}/4 \\
|
||||
DATABASE_URL=postgresql://postgres:postgres@postgres-\${shard}:5432/gbrain_test \\
|
||||
xargs -a /tmp/e2e-selected.txt bash scripts/run-e2e.sh >> \$log 2>&1
|
||||
else
|
||||
SHARD=\${shard}/4 \\
|
||||
DATABASE_URL=postgresql://postgres:postgres@postgres-\${shard}:5432/gbrain_test \\
|
||||
bash scripts/run-e2e.sh >> \$log 2>&1
|
||||
fi
|
||||
e2e_exit=\$?
|
||||
if [ \$e2e_exit -ne 0 ]; then
|
||||
echo \"[shard \${shard}] E2E FAILED (exit=\$e2e_exit)\" >> \$log
|
||||
exit \$e2e_exit
|
||||
fi
|
||||
echo \"[shard \${shard}] DONE\" >> \$log
|
||||
' _ {}
|
||||
shard_xargs_exit=\$?
|
||||
set -e
|
||||
echo \"\"
|
||||
echo \"=== SHARD LOGS (last 30 lines each + unit/e2e summaries) ===\"
|
||||
for s in 1 2 3 4; do
|
||||
echo \"\"
|
||||
echo \"--- shard \$s ---\"
|
||||
if [ -f /tmp/shard-logs/shard-\$s.log ]; then
|
||||
# Pull the unit + E2E summary lines explicitly so they survive even if
|
||||
# the file is huge. Match: bun's '<N> pass / <N> fail' pairs, run-e2e.sh's
|
||||
# 'Files: ... / Tests: ...' summary, and our own shard markers.
|
||||
grep -E '^\\[shard|^Files: |^Tests: |Ran [0-9]+ tests|^[[:space:]]+[0-9]+ (pass|fail|skip)\$' /tmp/shard-logs/shard-\$s.log || true
|
||||
echo \" (last 30 lines for context)\"
|
||||
tail -30 /tmp/shard-logs/shard-\$s.log
|
||||
else
|
||||
echo \"(no log file written — shard never started)\"
|
||||
fi
|
||||
done
|
||||
echo \"\"
|
||||
if [ \$shard_xargs_exit -ne 0 ]; then
|
||||
echo \"[runner] One or more shards failed (xargs exit=\$shard_xargs_exit). See SHARD LOGS above.\"
|
||||
exit \$shard_xargs_exit
|
||||
fi
|
||||
echo \"[runner] All 4 shards passed.\""
|
||||
fi
|
||||
|
||||
INNER_CMD=$(cat <<'EOF'
|
||||
set -euo pipefail
|
||||
echo "[runner] bun version: $(bun --version)"
|
||||
# oven/bun:1 omits git; many unit tests use mkdtemp + git init for fixtures.
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
echo "[runner] Installing git (debian apt)..."
|
||||
apt-get update -qq >/dev/null
|
||||
apt-get install -y -qq git ca-certificates >/dev/null
|
||||
fi
|
||||
# Container runs as root (uid 0) against a host-uid bind-mount; mark repo +
|
||||
# any worktree gitdir as safe so `git status` etc. don't refuse.
|
||||
git config --global --add safe.directory '*' || true
|
||||
if [ ! -d /app/node_modules ] || [ -z "$(ls -A /app/node_modules 2>/dev/null)" ]; then
|
||||
echo "[runner] First run (or --clean): bun install --frozen-lockfile"
|
||||
bun install --frozen-lockfile
|
||||
fi
|
||||
__RUN_PHASES__
|
||||
EOF
|
||||
)
|
||||
INNER_CMD="${INNER_CMD/__RUN_PHASES__/$RUN_PHASES_CMD}"
|
||||
|
||||
# Conductor / git-worktree support: when `.git` is a file (not a directory),
|
||||
# it points at a host gitdir outside the bind-mount. Without remounting that
|
||||
# path, scripts/check-trailing-newline.sh and any other in-container `git`
|
||||
# call exits 128 ("not a git repository"). Resolve the host gitdir + the
|
||||
# shared common gitdir and bind-mount them at the same absolute paths.
|
||||
EXTRA_MOUNTS=()
|
||||
if [ -f .git ]; then
|
||||
WORKTREE_GITDIR=$(awk '{print $2}' .git)
|
||||
if [ -d "$WORKTREE_GITDIR" ]; then
|
||||
COMMONDIR_FILE="$WORKTREE_GITDIR/commondir"
|
||||
if [ -f "$COMMONDIR_FILE" ]; then
|
||||
COMMON_REL=$(cat "$COMMONDIR_FILE")
|
||||
COMMON_GITDIR=$(cd "$WORKTREE_GITDIR" && cd "$COMMON_REL" && pwd)
|
||||
else
|
||||
COMMON_GITDIR="$WORKTREE_GITDIR"
|
||||
fi
|
||||
# Mount the higher-level common gitdir; covers worktrees/<name> automatically.
|
||||
EXTRA_MOUNTS+=( -v "${COMMON_GITDIR}:${COMMON_GITDIR}:ro" )
|
||||
echo "[ci-local] Worktree detected; mounting shared gitdir: $COMMON_GITDIR"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "[ci-local] Running checks inside runner container..."
|
||||
docker compose -f "$COMPOSE_FILE" run --rm "${EXTRA_MOUNTS[@]:-}" runner bash -c "$INNER_CMD"
|
||||
|
||||
echo ""
|
||||
echo "[ci-local] All checks passed."
|
||||
@@ -0,0 +1,63 @@
|
||||
// scripts/e2e-test-map.ts
|
||||
//
|
||||
// Path-glob -> E2E test files map. Used by scripts/select-e2e.ts.
|
||||
//
|
||||
// CONTRACT: This map can ONLY narrow from "all". When a changed src/ path
|
||||
// matches no glob here, the selector falls back to "run all E2E" (fail-closed).
|
||||
// You can safely add narrowing entries; you cannot break correctness by missing
|
||||
// one. Tune as misses surface (i.e., when ci:local:diff ran more than necessary
|
||||
// and you'd like to narrow that surface area).
|
||||
//
|
||||
// Glob syntax is the minimal subset implemented in select-e2e.ts:
|
||||
// - "**" matches any sequence of path segments (including zero)
|
||||
// - "*" matches any characters within a single path segment
|
||||
// - everything else is literal
|
||||
// No brace expansion, no ?, no [ ].
|
||||
|
||||
export const E2E_TEST_MAP: Record<string, string[]> = {
|
||||
// Source-aware ranking, hybrid search, intent classification.
|
||||
"src/core/search/**": [
|
||||
"test/e2e/search-quality.test.ts",
|
||||
"test/e2e/search-exclude.test.ts",
|
||||
"test/e2e/search-swamp.test.ts",
|
||||
],
|
||||
// Tree-sitter chunkers feed code-indexing E2E.
|
||||
"src/core/chunkers/**": ["test/e2e/code-indexing.test.ts"],
|
||||
// dream.ts is a thin alias over runCycle in cycle.ts.
|
||||
"src/core/cycle.ts": ["test/e2e/cycle.test.ts", "test/e2e/dream.test.ts"],
|
||||
// Multi-source sync writes share the per-source bookmark anchor.
|
||||
"src/core/sync.ts": ["test/e2e/sync.test.ts", "test/e2e/multi-source.test.ts"],
|
||||
// Any minions queue/worker/handler change exercises all minion E2E.
|
||||
"src/core/minions/**": [
|
||||
"test/e2e/minions-concurrency.test.ts",
|
||||
"test/e2e/minions-resilience.test.ts",
|
||||
"test/e2e/minions-shell.test.ts",
|
||||
"test/e2e/minions-shell-pglite.test.ts",
|
||||
"test/e2e/worker-abort-recovery.test.ts",
|
||||
],
|
||||
// postgres.js bind paths + JSONB shapes + parity vs PGLite.
|
||||
"src/core/postgres-engine.ts": [
|
||||
"test/e2e/postgres-bootstrap.test.ts",
|
||||
"test/e2e/postgres-jsonb.test.ts",
|
||||
"test/e2e/jsonb-roundtrip.test.ts",
|
||||
"test/e2e/engine-parity.test.ts",
|
||||
],
|
||||
// PGLite bootstrap path + parity guard.
|
||||
"src/core/pglite-engine.ts": [
|
||||
"test/e2e/postgres-bootstrap.test.ts",
|
||||
"test/e2e/engine-parity.test.ts",
|
||||
],
|
||||
// MCP stdio + HTTP transports share dispatch.
|
||||
"src/mcp/**": ["test/e2e/mcp.test.ts", "test/e2e/http-transport.test.ts"],
|
||||
// Integrity batch-load fast path.
|
||||
"src/commands/integrity.ts": ["test/e2e/integrity-batch.test.ts"],
|
||||
// Upgrade chains migration ledger; touches both runners.
|
||||
"src/commands/upgrade.ts": [
|
||||
"test/e2e/upgrade.test.ts",
|
||||
"test/e2e/migrate-chain.test.ts",
|
||||
"test/e2e/migration-flow.test.ts",
|
||||
],
|
||||
"src/commands/doctor.ts": ["test/e2e/doctor-progress.test.ts"],
|
||||
// Knowledge graph layer feeds graph-quality.
|
||||
"src/core/link-extraction.ts": ["test/e2e/graph-quality.test.ts"],
|
||||
};
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/profile-tests.sh
|
||||
# Tier 4 helper: prints the top N slowest unit tests from a previous run.
|
||||
# Pipe a captured `bun test` output (or a ci:local log) into stdin; we extract
|
||||
# `(pass|fail) ... [Xms|Xs]` lines, convert to ms, sort descending.
|
||||
#
|
||||
# Usage:
|
||||
# bun test --timeout=60000 2>&1 | bash scripts/profile-tests.sh
|
||||
# bash scripts/profile-tests.sh < /path/to/captured.log
|
||||
# bash scripts/profile-tests.sh -n 20 < /path/to/captured.log
|
||||
#
|
||||
# To demote a test as slow: rename its file to *.slow.test.ts. The file
|
||||
# stays discoverable by `bun test` (CI runs everything via `bun run test`)
|
||||
# but is excluded from `bun run ci:local`'s fast unit shard fan-out.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
TOP_N=10
|
||||
if [ "${1:-}" = "-n" ] && [ -n "${2:-}" ]; then
|
||||
TOP_N=$2
|
||||
fi
|
||||
|
||||
# Lines look like: (pass) describe > test name [12345.67ms] OR [12.34s]
|
||||
# Single awk pass for performance (input can be tens of MB).
|
||||
awk '{
|
||||
# Find the LAST bracket in the line: [<num><unit>] where unit is ms or s.
|
||||
for (i = length($0); i > 0; i--) {
|
||||
if (substr($0, i, 1) == "]") {
|
||||
# Walk back to matching "["
|
||||
j = i - 1
|
||||
while (j > 0 && substr($0, j, 1) != "[") j--
|
||||
if (j == 0) break
|
||||
bracket = substr($0, j+1, i-j-1)
|
||||
# bracket should match ^[0-9]+(\.[0-9]+)?(ms|s)$
|
||||
if (bracket ~ /^[0-9]+(\.[0-9]+)?(ms|s)$/) {
|
||||
if (bracket ~ /ms$/) {
|
||||
n = substr(bracket, 1, length(bracket) - 2) + 0
|
||||
} else {
|
||||
n = (substr(bracket, 1, length(bracket) - 1) + 0) * 1000
|
||||
}
|
||||
if (n > 0) printf "%.0f\t%s\n", n, $0
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}' | sort -rn | head -n "$TOP_N" | awk -F'\t' '{ printf "%8.0fms %s\n", $1, $2 }'
|
||||
+59
-1
@@ -25,13 +25,71 @@ set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# --dry-run-list: print the resolved file list (one per line) and exit. Used
|
||||
# by scripts/ci-local.sh to smoke-test the argv branching at startup.
|
||||
DRY_RUN_LIST=0
|
||||
if [ "${1:-}" = "--dry-run-list" ]; then
|
||||
DRY_RUN_LIST=1
|
||||
shift
|
||||
fi
|
||||
|
||||
# Argv-driven file list (used by `ci:local:diff`); fall back to the full glob.
|
||||
if [ "$#" -gt 0 ]; then
|
||||
files=("$@")
|
||||
else
|
||||
files=(test/e2e/*.test.ts)
|
||||
fi
|
||||
|
||||
# SHARD env (e.g. SHARD=1/4) keeps every M-th file starting at index N (1-indexed).
|
||||
# Used by scripts/ci-local.sh to fan 4 shards in parallel against 4 postgres
|
||||
# containers. Sequential execution within a shard is preserved (the TRUNCATE
|
||||
# CASCADE no-race rationale at the top of this file still holds).
|
||||
if [ -n "${SHARD:-}" ]; then
|
||||
shard_n=${SHARD%/*}
|
||||
shard_m=${SHARD#*/}
|
||||
if ! printf '%s' "$shard_n" | grep -qE '^[0-9]+$' || \
|
||||
! printf '%s' "$shard_m" | grep -qE '^[0-9]+$' || \
|
||||
[ "$shard_n" -lt 1 ] || [ "$shard_m" -lt 1 ] || [ "$shard_n" -gt "$shard_m" ]; then
|
||||
echo "ERROR: invalid SHARD=$SHARD (expected N/M with 1<=N<=M, both integers)" >&2
|
||||
exit 1
|
||||
fi
|
||||
filtered=()
|
||||
i=0
|
||||
for f in "${files[@]}"; do
|
||||
if [ $((i % shard_m + 1)) -eq "$shard_n" ]; then
|
||||
filtered+=("$f")
|
||||
fi
|
||||
i=$((i + 1))
|
||||
done
|
||||
# ${filtered[@]:-} avoids "unbound variable" under `set -u` when no files matched.
|
||||
files=("${filtered[@]:-}")
|
||||
# If the empty placeholder slipped in, drop it.
|
||||
if [ "${#files[@]}" -eq 1 ] && [ -z "${files[0]}" ]; then
|
||||
files=()
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$DRY_RUN_LIST" = "1" ]; then
|
||||
if [ "${#files[@]}" -eq 0 ]; then
|
||||
exit 0
|
||||
fi
|
||||
printf '%s\n' "${files[@]}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${#files[@]}" -eq 0 ]; then
|
||||
# Empty shard (e.g. SHARD=4/4 with only 3 files): nothing to do.
|
||||
echo "No files for shard ${SHARD:-(unsharded)}; exiting clean."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
pass_files=0
|
||||
fail_files=0
|
||||
fail_list=()
|
||||
total_pass=0
|
||||
total_fail=0
|
||||
|
||||
for f in test/e2e/*.test.ts; do
|
||||
for f in "${files[@]}"; do
|
||||
name=$(basename "$f")
|
||||
echo ""
|
||||
echo "=== $name ==="
|
||||
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/run-serial-tests.sh — run *.serial.test.ts files with --max-concurrency=1.
|
||||
#
|
||||
# Serial files are tests that share file-wide state (top-level mock.module,
|
||||
# module-level singletons that intentionally cross test cases) and would race
|
||||
# under intra-file concurrency. Discovered via filename suffix; no annotation
|
||||
# inside the file is needed.
|
||||
#
|
||||
# Excluded by run-unit-shard.sh and run-unit-parallel.sh's parallel pass.
|
||||
# Invoked separately by run-unit-parallel.sh after the parallel pass succeeds.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Use while-read for portability to macOS bash 3.2 (no mapfile).
|
||||
files=()
|
||||
while IFS= read -r f; do
|
||||
files+=("$f")
|
||||
done < <(find test -name '*.serial.test.ts' -not -path 'test/e2e/*' | sort)
|
||||
|
||||
if [ "${#files[@]}" -eq 0 ]; then
|
||||
echo "[serial-tests] no *.serial.test.ts files found"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --dry-run-list mirrors run-unit-shard.sh for inline checks/tests.
|
||||
if [ "${1:-}" = "--dry-run-list" ]; then
|
||||
printf '%s\n' "${files[@]}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[serial-tests] running ${#files[@]} file(s) with --max-concurrency=1"
|
||||
exec bun test --max-concurrency=1 --timeout=60000 "${files[@]}"
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/run-slow-tests.sh
|
||||
# Tier 4 sister to run-unit-shard.sh: runs ONLY *.slow.test.ts files.
|
||||
# CI runs both; bun run ci:local skips slow tests via run-unit-shard.sh.
|
||||
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
slow_files=()
|
||||
while IFS= read -r f; do
|
||||
slow_files+=("$f")
|
||||
done < <(find test -name '*.slow.test.ts' -not -path 'test/e2e/*' | sort)
|
||||
|
||||
if [ "${#slow_files[@]}" -eq 0 ]; then
|
||||
echo "[run-slow-tests] no *.slow.test.ts files; nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[run-slow-tests] running ${#slow_files[@]} slow files (CI runs these as part of bun run test)"
|
||||
exec bun test --timeout=60000 "${slow_files[@]}"
|
||||
Executable
+341
@@ -0,0 +1,341 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/run-unit-parallel.sh — fast unit-test loop, parallel fan-out.
|
||||
#
|
||||
# Spawns N parallel `bun test` processes, each running a hash-disjoint shard
|
||||
# of the unit-test set (files only — no e2e, no .slow, no .serial). After
|
||||
# all shards complete, runs serial-only files (*.serial.test.ts) with
|
||||
# --max-concurrency=1. Failure-first logging: extracts failure blocks from
|
||||
# each shard's log, writes to .context/test-failures.log with --- shard $i:
|
||||
# prefixes, prints loud stderr banner if any failures, exit non-zero.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/run-unit-parallel.sh [--shards N] [--max-concurrency N] [--dry-run]
|
||||
#
|
||||
# Env overrides:
|
||||
# SHARDS=N same as --shards
|
||||
# GBRAIN_TEST_SHARD_TIMEOUT per-shard wallclock cap, seconds (default 600)
|
||||
# GBRAIN_TEST_MAX_CONCURRENCY passed through to bun test (default 4)
|
||||
#
|
||||
# Output files (workspace-local; falls back to /tmp if .context/ unwritable):
|
||||
# .context/test-failures.log failure blocks (cleared at start)
|
||||
# .context/test-summary.txt per-shard pass/fail/skip/duration (cleared at start)
|
||||
# .context/test-shards/ per-shard logs + exit codes (cleared at start)
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# CPU detection: Apple Silicon perf cores → Mac total physical → nproc → 4.
|
||||
# Returns a single positive integer.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
detect_cpus() {
|
||||
local n=""
|
||||
n=$(sysctl -n hw.perflevel0.physicalcpu 2>/dev/null) && [ -n "$n" ] && [ "$n" -gt 0 ] && echo "$n" && return
|
||||
n=$(sysctl -n hw.physicalcpu 2>/dev/null) && [ -n "$n" ] && [ "$n" -gt 0 ] && echo "$n" && return
|
||||
n=$(nproc 2>/dev/null) && [ -n "$n" ] && [ "$n" -gt 0 ] && echo "$n" && return
|
||||
echo 4
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Argument parsing. --shards N override wins over $SHARDS; both are clamped.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
SHARDS_OVERRIDE=""
|
||||
MAX_CONCURRENCY_OVERRIDE=""
|
||||
DRY_RUN=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--shards) SHARDS_OVERRIDE="$2"; shift 2 ;;
|
||||
--shards=*) SHARDS_OVERRIDE="${1#*=}"; shift ;;
|
||||
--max-concurrency) MAX_CONCURRENCY_OVERRIDE="$2"; shift 2 ;;
|
||||
--max-concurrency=*) MAX_CONCURRENCY_OVERRIDE="${1#*=}"; shift ;;
|
||||
--dry-run) DRY_RUN=1; shift ;;
|
||||
*) echo "ERROR: unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
N="${SHARDS_OVERRIDE:-${SHARDS:-$(detect_cpus)}}"
|
||||
if ! printf '%s' "$N" | grep -qE '^[0-9]+$' || [ "$N" -lt 1 ]; then
|
||||
echo "ERROR: invalid shard count: $N" >&2; exit 2
|
||||
fi
|
||||
[ "$N" -gt 8 ] && N=8
|
||||
|
||||
INTRA_CONC="${MAX_CONCURRENCY_OVERRIDE:-${GBRAIN_TEST_MAX_CONCURRENCY:-4}}"
|
||||
SHARD_TIMEOUT="${GBRAIN_TEST_SHARD_TIMEOUT:-600}"
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Output directories. Prefer workspace-local .context/, fall back to /tmp.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
LOG_DIR=""
|
||||
if mkdir -p .context/test-shards 2>/dev/null; then
|
||||
LOG_DIR=".context/test-shards"
|
||||
FAILURES_LOG=".context/test-failures.log"
|
||||
SUMMARY_FILE=".context/test-summary.txt"
|
||||
else
|
||||
LOG_DIR="/tmp/gbrain-test-shards-$$"
|
||||
FAILURES_LOG="/tmp/gbrain-test-failures.log"
|
||||
SUMMARY_FILE="/tmp/gbrain-test-summary.txt"
|
||||
mkdir -p "$LOG_DIR" || { echo "ERROR: cannot create log dir" >&2; exit 2; }
|
||||
fi
|
||||
# Clear from prior run.
|
||||
rm -f "$LOG_DIR"/shard-*.log "$LOG_DIR"/shard-*.exit "$LOG_DIR"/shard-*.wedged 2>/dev/null
|
||||
: > "$FAILURES_LOG"
|
||||
: > "$SUMMARY_FILE"
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Resolve `timeout` command. macOS without coreutils has neither; we degrade
|
||||
# to bg-pid + sleep cap. For now, prefer gtimeout (brew coreutils) → timeout.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
TIMEOUT_BIN=""
|
||||
if command -v gtimeout >/dev/null 2>&1; then TIMEOUT_BIN="gtimeout"
|
||||
elif command -v timeout >/dev/null 2>&1; then TIMEOUT_BIN="timeout"
|
||||
fi
|
||||
|
||||
START_TS=$(date +%s)
|
||||
echo "[unit-parallel] N=$N shards | --max-concurrency=$INTRA_CONC | timeout=${SHARD_TIMEOUT}s | logs=$LOG_DIR" >&2
|
||||
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
echo "[unit-parallel] dry-run: would spawn $N shards with the above settings."
|
||||
for i in $(seq 1 "$N"); do
|
||||
SHARD="$i/$N" bash scripts/run-unit-shard.sh --dry-run-list 2>/dev/null \
|
||||
| sed "s|^| [s$i] |"
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Spawn shards. Each child captures its own exit code into a sentinel file
|
||||
# so $? is recoverable per-shard (we never trust `wait`'s aggregate value).
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
SHARD_PIDS=()
|
||||
for i in $(seq 1 "$N"); do
|
||||
(
|
||||
SHARD_LOG="$LOG_DIR/shard-$i.log"
|
||||
if [ -n "$TIMEOUT_BIN" ]; then
|
||||
"$TIMEOUT_BIN" "${SHARD_TIMEOUT}s" \
|
||||
env SHARD="$i/$N" \
|
||||
bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \
|
||||
> "$SHARD_LOG" 2>&1
|
||||
else
|
||||
env SHARD="$i/$N" \
|
||||
bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \
|
||||
> "$SHARD_LOG" 2>&1 &
|
||||
pid=$!
|
||||
( sleep "$SHARD_TIMEOUT" && kill -TERM "$pid" 2>/dev/null && \
|
||||
sleep 5 && kill -KILL "$pid" 2>/dev/null ) &
|
||||
cap_pid=$!
|
||||
wait "$pid" 2>/dev/null
|
||||
kill "$cap_pid" 2>/dev/null
|
||||
wait "$cap_pid" 2>/dev/null
|
||||
fi
|
||||
rc=$?
|
||||
echo "$rc" > "$LOG_DIR/shard-$i.exit"
|
||||
[ "$rc" = "124" ] && echo "WEDGED" > "$LOG_DIR/shard-$i.wedged"
|
||||
) &
|
||||
SHARD_PIDS+=($!)
|
||||
done
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Heartbeat: every 10s, print per-shard progress to stderr by tailing logs
|
||||
# and counting Bun's `(pass)` / `(fail)` / `(skip)` markers. Read-only.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# grep_count: returns 0 (single integer) if file is missing or zero matches,
|
||||
# otherwise the match count. Avoids the `grep -c | echo 0` double-output bug
|
||||
# where 0 matches produces a 2-line "0\n0" string that breaks arithmetic.
|
||||
grep_count() {
|
||||
local pattern="$1"; local file="$2"
|
||||
if [ ! -f "$file" ]; then echo 0; return; fi
|
||||
local n
|
||||
n=$(grep -cE "$pattern" "$file" 2>/dev/null) || n=0
|
||||
echo "${n:-0}"
|
||||
}
|
||||
|
||||
# bun_summary_count: parses Bun's summary lines (one per `bun test` invocation
|
||||
# inside a shard — there's only one when we pass an explicit file list).
|
||||
# Looks for ` N pass` / ` N fail` / ` N skip` patterns and sums them across
|
||||
# all summary blocks the shard emitted. `bun test` prints these near the end
|
||||
# of its output. Format: leading whitespace + integer + space + label.
|
||||
bun_summary_count() {
|
||||
local label="$1"; local file="$2"
|
||||
if [ ! -f "$file" ]; then echo 0; return; fi
|
||||
awk -v label="$label" '
|
||||
$1 ~ /^[0-9]+$/ && $2 == label { total += $1 }
|
||||
END { print total + 0 }
|
||||
' "$file"
|
||||
}
|
||||
|
||||
heartbeat() {
|
||||
while true; do
|
||||
sleep 10
|
||||
local line=""
|
||||
for i in $(seq 1 "$N"); do
|
||||
if [ -f "$LOG_DIR/shard-$i.exit" ]; then
|
||||
local rc; rc=$(cat "$LOG_DIR/shard-$i.exit" 2>/dev/null || echo "?")
|
||||
local status="✓"
|
||||
[ "$rc" != "0" ] && status="✗"
|
||||
line="$line [s$i: done $status]"
|
||||
else
|
||||
local lf="$LOG_DIR/shard-$i.log"
|
||||
if [ -f "$lf" ]; then
|
||||
# Heartbeat: prefer Bun's per-test "✓" (passed) and "(fail)" markers
|
||||
# so we see live progress; the "N pass" summary line only appears at
|
||||
# the very end of the shard and would always show 0 mid-run.
|
||||
local p f
|
||||
p=$(grep_count '^[[:space:]]+✓' "$lf")
|
||||
f=$(grep_count '^\(fail\)' "$lf")
|
||||
line="$line [s$i: ${p}p ${f}f ...]"
|
||||
else
|
||||
line="$line [s$i: starting]"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
printf '[heartbeat] %s\n' "$line" >&2
|
||||
done
|
||||
}
|
||||
heartbeat &
|
||||
HB_PID=$!
|
||||
trap 'kill "$HB_PID" 2>/dev/null; wait "$HB_PID" 2>/dev/null' EXIT
|
||||
|
||||
# Wait for every shard. Don't care about wait's exit code.
|
||||
for pid in "${SHARD_PIDS[@]}"; do wait "$pid" 2>/dev/null || true; done
|
||||
|
||||
kill "$HB_PID" 2>/dev/null
|
||||
wait "$HB_PID" 2>/dev/null
|
||||
trap - EXIT
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Aggregate failures (single writer; serial; never concurrent).
|
||||
# Bun failure block format: from `(fail) ...` line through next `(pass)`,
|
||||
# `(skip)`, blank line, or `__bun_test_summary__` marker.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
TOTAL_FAILURES=0
|
||||
TOTAL_PASS=0
|
||||
TOTAL_SKIP=0
|
||||
TOTAL_RC=0
|
||||
for i in $(seq 1 "$N"); do
|
||||
SHARD_LOG="$LOG_DIR/shard-$i.log"
|
||||
EXIT_FILE="$LOG_DIR/shard-$i.exit"
|
||||
WEDGED_FILE="$LOG_DIR/shard-$i.wedged"
|
||||
rc=1
|
||||
[ -f "$EXIT_FILE" ] && rc=$(cat "$EXIT_FILE" 2>/dev/null || echo 1)
|
||||
|
||||
pass_count=$(bun_summary_count "pass" "$SHARD_LOG")
|
||||
fail_count=$(bun_summary_count "fail" "$SHARD_LOG")
|
||||
skip_count=$(bun_summary_count "skip" "$SHARD_LOG")
|
||||
TOTAL_PASS=$((TOTAL_PASS + pass_count))
|
||||
TOTAL_FAILURES=$((TOTAL_FAILURES + fail_count))
|
||||
TOTAL_SKIP=$((TOTAL_SKIP + skip_count))
|
||||
|
||||
if [ -f "$WEDGED_FILE" ]; then
|
||||
TOTAL_RC=1
|
||||
{
|
||||
echo "--- shard $i: WEDGED after ${SHARD_TIMEOUT}s ---"
|
||||
[ -f "$SHARD_LOG" ] && tail -50 "$SHARD_LOG"
|
||||
echo ""
|
||||
} >> "$FAILURES_LOG"
|
||||
echo "shard $i/$N: WEDGED after ${SHARD_TIMEOUT}s (rc=$rc)" >> "$SUMMARY_FILE"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "shard $i/$N: pass=$pass_count fail=$fail_count skip=$skip_count rc=$rc" >> "$SUMMARY_FILE"
|
||||
|
||||
if [ "$rc" != "0" ]; then
|
||||
TOTAL_RC=1
|
||||
if [ "$fail_count" -gt 0 ] && [ -f "$SHARD_LOG" ]; then
|
||||
# Extract each (fail) block: from `(fail)` line through next `(pass)`,
|
||||
# `(skip)`, blank line, or `__bun_test_summary__`. Single awk pass.
|
||||
awk -v shard="$i" '
|
||||
/^\(fail\) / { in_block=1; print "--- shard " shard ": " $0; next }
|
||||
in_block {
|
||||
if (/^\(pass\)/ || /^\(skip\)/ || /^[[:space:]]*$/ || /__bun_test_summary__/) { in_block=0; print ""; next }
|
||||
print $0
|
||||
}
|
||||
' "$SHARD_LOG" >> "$FAILURES_LOG"
|
||||
elif [ -f "$SHARD_LOG" ]; then
|
||||
# Non-zero rc but no (fail) line found — extraction couldn't pinpoint.
|
||||
# Dump the full shard log so we never silently lose the failure cause.
|
||||
{
|
||||
echo "--- shard $i: rc=$rc, no (fail) markers — full log follows ---"
|
||||
cat "$SHARD_LOG"
|
||||
echo ""
|
||||
} >> "$FAILURES_LOG"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Print each shard's full output to stdout (developer expects to scroll
|
||||
# through it). Print summary file last for one-glance overview.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
for i in $(seq 1 "$N"); do
|
||||
SHARD_LOG="$LOG_DIR/shard-$i.log"
|
||||
echo ""
|
||||
echo "════════════ shard $i/$N ════════════"
|
||||
[ -f "$SHARD_LOG" ] && cat "$SHARD_LOG"
|
||||
done
|
||||
echo ""
|
||||
echo "════════════ summary ════════════"
|
||||
cat "$SUMMARY_FILE"
|
||||
echo ""
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Serial pass: any *.serial.test.ts files run after parallel pass.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
SERIAL_RC=0
|
||||
SERIAL_FILES_COUNT=0
|
||||
SERIAL_FILES_COUNT=$(find test -name '*.serial.test.ts' -not -path 'test/e2e/*' 2>/dev/null | wc -l | tr -d ' ')
|
||||
if [ "$SERIAL_FILES_COUNT" -gt 0 ]; then
|
||||
echo "════════════ serial pass ($SERIAL_FILES_COUNT files) ════════════"
|
||||
bash scripts/run-serial-tests.sh > "$LOG_DIR/serial.log" 2>&1
|
||||
SERIAL_RC=$?
|
||||
cat "$LOG_DIR/serial.log"
|
||||
if [ "$SERIAL_RC" != "0" ]; then
|
||||
TOTAL_RC=1
|
||||
s_fail=$(bun_summary_count "fail" "$LOG_DIR/serial.log")
|
||||
TOTAL_FAILURES=$((TOTAL_FAILURES + s_fail))
|
||||
if [ "$s_fail" -gt 0 ]; then
|
||||
awk '
|
||||
/^\(fail\) / { in_block=1; print "--- shard serial: " $0; next }
|
||||
in_block {
|
||||
if (/^\(pass\)/ || /^\(skip\)/ || /^[[:space:]]*$/ || /__bun_test_summary__/) { in_block=0; print ""; next }
|
||||
print $0
|
||||
}
|
||||
' "$LOG_DIR/serial.log" >> "$FAILURES_LOG"
|
||||
else
|
||||
{
|
||||
echo "--- shard serial: rc=$SERIAL_RC, no (fail) markers — full log follows ---"
|
||||
cat "$LOG_DIR/serial.log"
|
||||
echo ""
|
||||
} >> "$FAILURES_LOG"
|
||||
fi
|
||||
echo "serial: rc=$SERIAL_RC fail=$s_fail" >> "$SUMMARY_FILE"
|
||||
else
|
||||
s_pass=$(bun_summary_count "pass" "$LOG_DIR/serial.log")
|
||||
TOTAL_PASS=$((TOTAL_PASS + s_pass))
|
||||
echo "serial: pass=$s_pass rc=0" >> "$SUMMARY_FILE"
|
||||
fi
|
||||
fi
|
||||
|
||||
END_TS=$(date +%s)
|
||||
ELAPSED=$((END_TS - START_TS))
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Loud banner if anything failed. To stderr so it survives `| head`/`| tail`.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
if [ "$TOTAL_RC" != "0" ]; then
|
||||
ABS_FAIL=$(cd "$(dirname "$FAILURES_LOG")" && pwd)/$(basename "$FAILURES_LOG")
|
||||
{
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "❌ $TOTAL_FAILURES TEST FAILURES — full details:"
|
||||
echo " $ABS_FAIL"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
tail -30 "$FAILURES_LOG"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "[unit-parallel] elapsed=${ELAPSED}s | pass=$TOTAL_PASS fail=$TOTAL_FAILURES skip=$TOTAL_SKIP"
|
||||
} >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[unit-parallel] elapsed=${ELAPSED}s | pass=$TOTAL_PASS fail=$TOTAL_FAILURES skip=$TOTAL_SKIP" >&2
|
||||
exit 0
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/run-unit-shard.sh
|
||||
#
|
||||
# Runs the unit suite for a single shard. Excludes test/e2e/* (those are run
|
||||
# by scripts/run-e2e.sh in the E2E phase). When SHARD=N/M is set, keeps every
|
||||
# M-th file starting at index N (1-indexed); otherwise runs the full unit set.
|
||||
#
|
||||
# Used by scripts/ci-local.sh to fan 4 unit-shard workers in parallel inside
|
||||
# the runner container, each pinned to its own postgres shard for the
|
||||
# downstream E2E phase.
|
||||
#
|
||||
# Sequential bun processes within a shard (one bun test invocation with the
|
||||
# shard's file list); parallel across shards (4 of these run concurrently).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# --max-concurrency=N is forwarded to `bun test`. v0.26.4: invoked by
|
||||
# run-unit-parallel.sh; safe to call without (defaults to bun's default cap).
|
||||
MAX_CONC=""
|
||||
DRY_RUN=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--max-concurrency) MAX_CONC="$2"; shift 2 ;;
|
||||
--max-concurrency=*) MAX_CONC="${1#*=}"; shift ;;
|
||||
--dry-run-list) DRY_RUN=1; shift ;;
|
||||
*) echo "ERROR: unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# All non-E2E test files, sorted for deterministic shard splits.
|
||||
# Tier 4: *.slow.test.ts is "always-slow" (cold-path correctness checks);
|
||||
# *.serial.test.ts is "concurrency-unsafe" (file-wide shared state). Both
|
||||
# are excluded from the fast loop. Slow runs via `bun run test:slow`; serial
|
||||
# runs via scripts/run-serial-tests.sh after the parallel pass.
|
||||
# Use while-read to stay portable to macOS bash 3.2 (no mapfile).
|
||||
all_files=()
|
||||
while IFS= read -r f; do
|
||||
all_files+=("$f")
|
||||
done < <(find test -name '*.test.ts' -not -path 'test/e2e/*' -not -name '*.slow.test.ts' -not -name '*.serial.test.ts' | sort)
|
||||
|
||||
files=()
|
||||
if [ -n "${SHARD:-}" ]; then
|
||||
shard_n=${SHARD%/*}
|
||||
shard_m=${SHARD#*/}
|
||||
if ! printf '%s' "$shard_n" | grep -qE '^[0-9]+$' || \
|
||||
! printf '%s' "$shard_m" | grep -qE '^[0-9]+$' || \
|
||||
[ "$shard_n" -lt 1 ] || [ "$shard_m" -lt 1 ] || [ "$shard_n" -gt "$shard_m" ]; then
|
||||
echo "ERROR: invalid SHARD=$SHARD (expected N/M with 1<=N<=M, both integers)" >&2
|
||||
exit 1
|
||||
fi
|
||||
i=0
|
||||
for f in "${all_files[@]}"; do
|
||||
if [ $((i % shard_m + 1)) -eq "$shard_n" ]; then
|
||||
files+=("$f")
|
||||
fi
|
||||
i=$((i + 1))
|
||||
done
|
||||
else
|
||||
files=("${all_files[@]}")
|
||||
fi
|
||||
|
||||
if [ "${#files[@]}" -eq 0 ]; then
|
||||
echo "[unit-shard ${SHARD:-(unsharded)}] no files; exiting clean."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
printf '%s\n' "${files[@]}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[unit-shard ${SHARD:-(unsharded)}] running ${#files[@]} files"
|
||||
if [ -n "$MAX_CONC" ]; then
|
||||
exec bun test --max-concurrency="$MAX_CONC" --timeout=60000 "${files[@]}"
|
||||
fi
|
||||
exec bun test --timeout=60000 "${files[@]}"
|
||||
@@ -0,0 +1,245 @@
|
||||
#!/usr/bin/env bun
|
||||
// scripts/select-e2e.ts
|
||||
//
|
||||
// Fail-closed diff-based E2E test selector. Reads the working-tree diff vs
|
||||
// origin/master plus untracked files, classifies the change set as
|
||||
// EMPTY / DOC_ONLY / SRC, and emits the relevant E2E test files on stdout.
|
||||
//
|
||||
// CONTRACT (fail-closed):
|
||||
// - When in doubt, run all E2E. The map narrows from "all"; it never widens
|
||||
// from "none". An unmapped src/ change emits ALL test/e2e/*.test.ts.
|
||||
// - Doc-only diffs emit nothing (the only case where stdout is empty).
|
||||
// - Empty diff emits ALL (clean branch shouldn't run nothing).
|
||||
//
|
||||
// Selection algorithm:
|
||||
// 1. Read changed files from three git sources, union them:
|
||||
// - git diff --name-only origin/master...HEAD (committed)
|
||||
// - git diff --name-only HEAD (unstaged + staged)
|
||||
// - git ls-files --others --exclude-standard (untracked, NOT .gitignore'd)
|
||||
// 2. EMPTY -> emit ALL test/e2e/*.test.ts
|
||||
// DOC_ONLY (every path matches doc allowlist) -> emit nothing
|
||||
// SRC (at least one path is outside doc allowlist):
|
||||
// a. Any escape-hatch path matched -> emit ALL
|
||||
// b. Else union map matches; include directly-modified test/e2e/*.test.ts
|
||||
// c. If still empty -> FAIL-CLOSED -> emit ALL
|
||||
//
|
||||
// On git command failure: print error to stderr and exit 2 so callers see the
|
||||
// failure (xargs -r will run nothing AND the human sees the error).
|
||||
//
|
||||
// Usage:
|
||||
// bun run scripts/select-e2e.ts
|
||||
// bun run scripts/select-e2e.ts | xargs -r bash scripts/run-e2e.sh
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { readdirSync, existsSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { E2E_TEST_MAP } from "./e2e-test-map.ts";
|
||||
|
||||
// Doc allowlist (inclusive). A path counts as doc-only ONLY if it matches one
|
||||
// of these patterns. Unrecognized paths fall through to SRC, never silently
|
||||
// doc-only. skills/ is intentionally NOT here — skills are product input.
|
||||
const DOC_ROOT_FILES = new Set([
|
||||
"README.md",
|
||||
"CLAUDE.md",
|
||||
"AGENTS.md",
|
||||
"CHANGELOG.md",
|
||||
"TODOS.md",
|
||||
"LICENSE",
|
||||
"VERSION",
|
||||
]);
|
||||
|
||||
function isDocPath(p: string): boolean {
|
||||
if (DOC_ROOT_FILES.has(p)) return true;
|
||||
// Any *.md at repo root.
|
||||
if (!p.includes("/") && p.endsWith(".md")) return true;
|
||||
// Anything under docs/.
|
||||
if (p.startsWith("docs/")) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
// Escape-hatch triggers. Any match -> emit ALL.
|
||||
const ESCAPE_HATCH_FILES = new Set([
|
||||
"src/schema.sql",
|
||||
"src/core/migrate.ts",
|
||||
"src/core/db.ts",
|
||||
"src/core/engine-factory.ts",
|
||||
"src/core/operations.ts",
|
||||
"package.json",
|
||||
"bun.lock",
|
||||
"Dockerfile.ci",
|
||||
"docker-compose.ci.yml",
|
||||
"scripts/ci-local.sh",
|
||||
"scripts/run-e2e.sh",
|
||||
"scripts/select-e2e.ts",
|
||||
"scripts/e2e-test-map.ts",
|
||||
"test/e2e/helpers.ts",
|
||||
]);
|
||||
|
||||
const ESCAPE_HATCH_PREFIXES = [
|
||||
"src/commands/migrations/",
|
||||
"test/e2e/fixtures/",
|
||||
"skills/",
|
||||
".github/workflows/",
|
||||
];
|
||||
|
||||
function isEscapeHatch(p: string): boolean {
|
||||
if (ESCAPE_HATCH_FILES.has(p)) return true;
|
||||
for (const prefix of ESCAPE_HATCH_PREFIXES) {
|
||||
if (p.startsWith(prefix)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Minimal glob matcher: supports ** (any segments) and * (one segment, no /).
|
||||
// Throws on unsupported syntax so map mistakes surface loudly.
|
||||
export function matchGlob(glob: string, path: string): boolean {
|
||||
if (glob.includes("?") || glob.includes("[") || glob.includes("{")) {
|
||||
throw new Error(
|
||||
`select-e2e: unsupported glob syntax in "${glob}" (only ** and * are supported)`
|
||||
);
|
||||
}
|
||||
// Build a regex: ** -> .*, * -> [^/]*, escape other regex meta-chars.
|
||||
let regex = "";
|
||||
let i = 0;
|
||||
while (i < glob.length) {
|
||||
const c = glob[i];
|
||||
if (c === "*" && glob[i + 1] === "*") {
|
||||
regex += ".*";
|
||||
i += 2;
|
||||
} else if (c === "*") {
|
||||
regex += "[^/]*";
|
||||
i += 1;
|
||||
} else if (/[.+^${}()|\\]/.test(c)) {
|
||||
regex += "\\" + c;
|
||||
i += 1;
|
||||
} else {
|
||||
regex += c;
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
return new RegExp("^" + regex + "$").test(path);
|
||||
}
|
||||
|
||||
function listAllE2ETests(repoRoot: string): string[] {
|
||||
const dir = join(repoRoot, "test/e2e");
|
||||
if (!existsSync(dir)) return [];
|
||||
return readdirSync(dir)
|
||||
.filter((f) => f.endsWith(".test.ts"))
|
||||
.map((f) => `test/e2e/${f}`)
|
||||
.sort();
|
||||
}
|
||||
|
||||
// Pure function — exposed for unit tests. Decides what to emit given the
|
||||
// inputs, without touching git or filesystem (callers pass arrays in).
|
||||
export interface SelectInputs {
|
||||
changedFiles: string[]; // union of three git sources
|
||||
allE2ETests: string[]; // glob result of test/e2e/*.test.ts
|
||||
map: Record<string, string[]>; // E2E_TEST_MAP
|
||||
}
|
||||
|
||||
export type Classification = "EMPTY" | "DOC_ONLY" | "SRC";
|
||||
|
||||
export function classify(changedFiles: string[]): Classification {
|
||||
if (changedFiles.length === 0) return "EMPTY";
|
||||
for (const f of changedFiles) {
|
||||
if (!isDocPath(f)) return "SRC";
|
||||
}
|
||||
return "DOC_ONLY";
|
||||
}
|
||||
|
||||
export function selectTests(inputs: SelectInputs): string[] {
|
||||
const { changedFiles, allE2ETests, map } = inputs;
|
||||
const cls = classify(changedFiles);
|
||||
const allSorted = allE2ETests.slice().sort();
|
||||
|
||||
if (cls === "EMPTY") return allSorted;
|
||||
if (cls === "DOC_ONLY") return [];
|
||||
|
||||
// SRC case.
|
||||
// 3a. Any escape-hatch -> ALL.
|
||||
for (const f of changedFiles) {
|
||||
if (isEscapeHatch(f)) return allSorted;
|
||||
}
|
||||
|
||||
// 3b. Union map matches; include directly-modified test files.
|
||||
const result = new Set<string>();
|
||||
for (const f of changedFiles) {
|
||||
if (isDocPath(f)) continue;
|
||||
// Direct test file modification: include it.
|
||||
if (f.startsWith("test/e2e/") && f.endsWith(".test.ts")) {
|
||||
result.add(f);
|
||||
continue;
|
||||
}
|
||||
for (const [glob, tests] of Object.entries(map)) {
|
||||
if (matchGlob(glob, f)) {
|
||||
for (const t of tests) result.add(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3c. Fail-closed: if no map entry matched any src/ path AND no test files
|
||||
// were directly modified, run everything.
|
||||
if (result.size === 0) return allSorted;
|
||||
|
||||
// Sort for determinism (helps tests + readability).
|
||||
return Array.from(result).sort();
|
||||
}
|
||||
|
||||
function runGit(args: string[], cwd: string): string {
|
||||
const result = spawnSync("git", args, { cwd, encoding: "utf8" });
|
||||
if (result.status !== 0) {
|
||||
const stderr = (result.stderr || "").trim();
|
||||
process.stderr.write(
|
||||
`select-e2e: git ${args.join(" ")} failed: ${stderr}\n`
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
return result.stdout || "";
|
||||
}
|
||||
|
||||
function readChangedFiles(repoRoot: string): string[] {
|
||||
const sources = [
|
||||
runGit(["diff", "--name-only", "origin/master...HEAD"], repoRoot),
|
||||
runGit(["diff", "--name-only", "HEAD"], repoRoot),
|
||||
runGit(["ls-files", "--others", "--exclude-standard"], repoRoot),
|
||||
];
|
||||
const set = new Set<string>();
|
||||
for (const out of sources) {
|
||||
for (const line of out.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length > 0) set.add(trimmed);
|
||||
}
|
||||
}
|
||||
return Array.from(set).sort();
|
||||
}
|
||||
|
||||
// Entrypoint. Skipped under test (Bun.main check).
|
||||
if (import.meta.main) {
|
||||
const repoRoot = spawnSync("git", ["rev-parse", "--show-toplevel"], {
|
||||
encoding: "utf8",
|
||||
}).stdout?.trim();
|
||||
if (!repoRoot) {
|
||||
process.stderr.write("select-e2e: not a git repository\n");
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const changedFiles = readChangedFiles(repoRoot);
|
||||
|
||||
// --classify-only: print EMPTY|DOC_ONLY|SRC + exit. Used by ci-local.sh's
|
||||
// Tier 2 fast-path so doc-only diffs skip the unit phase entirely.
|
||||
if (process.argv.includes("--classify-only")) {
|
||||
process.stdout.write(classify(changedFiles) + "\n");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const allE2ETests = listAllE2ETests(repoRoot);
|
||||
const tests = selectTests({
|
||||
changedFiles,
|
||||
allE2ETests,
|
||||
map: E2E_TEST_MAP,
|
||||
});
|
||||
|
||||
process.stdout.write(tests.join(" "));
|
||||
if (tests.length > 0) process.stdout.write("\n");
|
||||
}
|
||||
@@ -70,6 +70,7 @@ This is the dispatcher. Skills are the implementation. **Read the skill file bef
|
||||
| "Migrate from Obsidian/Notion/Logseq" | `skills/migrate/SKILL.md` |
|
||||
| Brain health check, maintenance run | `skills/maintain/SKILL.md` |
|
||||
| "Extract links", "build link graph", "populate timeline" | `skills/maintain/SKILL.md` (extraction sections) |
|
||||
| "Run dream", "process today's session", "synthesize my conversations", "consolidate yesterday's conversations", "what patterns did you see", "did the dream cycle run" | `skills/maintain/SKILL.md` (dream cycle section) |
|
||||
| "Brain health", "what features am I missing", "brain score" | Run `gbrain features --json` |
|
||||
| "Set up autopilot", "run brain maintenance", "keep brain updated" | Run `gbrain autopilot --install --repo ~/brain` |
|
||||
| Agent identity, "who am I", customize agent | `skills/soul-audit/SKILL.md` |
|
||||
@@ -99,6 +100,29 @@ When multiple skills could match:
|
||||
These apply to ALL brain-writing skills:
|
||||
- `skills/conventions/quality.md` — citations, back-links, notability gate
|
||||
- `skills/conventions/brain-first.md` — check brain before external APIs
|
||||
- `skills/conventions/brain-routing.md` — which brain (DB) and which source (repo) to target; cross-brain federation is latent-space only
|
||||
- `skills/conventions/subagent-routing.md` — when to use Minions vs inline work
|
||||
- `skills/_brain-filing-rules.md` — where files go
|
||||
- `skills/_output-rules.md` — output quality standards
|
||||
|
||||
## Uncategorized
|
||||
|
||||
| Trigger | Skill |
|
||||
|---------|-------|
|
||||
| "personalized version of this book" | `skills/book-mirror/SKILL.md` |
|
||||
|
||||
| "enrich this article" | `skills/article-enrichment/SKILL.md` |
|
||||
|
||||
| "strategic reading" | `skills/strategic-reading/SKILL.md` |
|
||||
|
||||
| "concept synthesis" | `skills/concept-synthesis/SKILL.md` |
|
||||
|
||||
| "perplexity research" | `skills/perplexity-research/SKILL.md` |
|
||||
|
||||
| "crawl my archive" | `skills/archive-crawler/SKILL.md` |
|
||||
|
||||
| "verify this academic claim" | `skills/academic-verify/SKILL.md` |
|
||||
|
||||
| "make pdf from brain" | `skills/brain-pdf/SKILL.md` |
|
||||
|
||||
| "voice note" | `skills/voice-note-ingest/SKILL.md` |
|
||||
|
||||
@@ -81,11 +81,47 @@
|
||||
"examples": ["logistics", "family"],
|
||||
"description": "Personal-life content — kept separate from work."
|
||||
},
|
||||
{
|
||||
"kind": "idea",
|
||||
"directory": "ideas/",
|
||||
"examples": ["product ideas", "essay seeds", "back-of-envelope concepts"],
|
||||
"description": "Generative ideas the user might build, write, or expand later. Stub-shaped pages that mature over time. voice-note-ingest, archive-crawler, and similar capture-flavored skills file here when content is something to potentially act on."
|
||||
},
|
||||
{
|
||||
"kind": "research",
|
||||
"directory": "research/",
|
||||
"examples": ["web-research deltas", "freshness checks", "citation-verified claims"],
|
||||
"description": "Web-research output: what is NEW vs already-known about a topic, citation-checked claims, freshness deltas. perplexity-research and academic-verify file here."
|
||||
},
|
||||
{
|
||||
"kind": "original",
|
||||
"directory": "originals/",
|
||||
"examples": ["the user's own theses", "frameworks the user generated", "novel observations the user expressed"],
|
||||
"description": "Pages where the user is the primary author of the idea — original thinking, not summarizations of someone else's work. voice-note-ingest, archive-crawler, signal-detector route content here when the user is the originator."
|
||||
},
|
||||
{
|
||||
"kind": "voice-note",
|
||||
"directory": "voice-notes/",
|
||||
"examples": ["raw transcripts", "audio capture pages"],
|
||||
"description": "Voice-note transcript holders, especially when the content is a random thought that doesn't cleanly fit originals/, concepts/, or another subject directory. voice-note-ingest is the primary writer."
|
||||
},
|
||||
{
|
||||
"kind": "openclaw",
|
||||
"directory": "openclaw/",
|
||||
"examples": ["agent-state notes"],
|
||||
"description": "Notes about the host OpenClaw agent itself, not the underlying entities."
|
||||
},
|
||||
{
|
||||
"kind": "synthesis-output",
|
||||
"directory": "media/books/",
|
||||
"examples": ["personalized book mirrors", "two-column chapter analyses"],
|
||||
"description": "Sanctioned exception to 'file by primary subject' for sui generis synthesized output that is one-of-one to a single book and a specific reader. Format-prefixed under media/<format>/ is allowed for synthesis output only, never for raw ingest. See _brain-filing-rules.md."
|
||||
},
|
||||
{
|
||||
"kind": "synthesis-output",
|
||||
"directory": "media/articles/",
|
||||
"examples": ["personalized article reads", "long-form content tailored to reader"],
|
||||
"description": "Same sanctioned exception as media/books/. One-of-one synthesis output of an article personalized for the reader. Distinct from raw article ingest, which goes to the article's primary-subject directory."
|
||||
}
|
||||
],
|
||||
"sources_dir": {
|
||||
@@ -97,5 +133,15 @@
|
||||
"The PRIMARY SUBJECT of the content determines the directory, not the format or source skill.",
|
||||
"When in doubt: what would you search for to find this page again?",
|
||||
"Cross-link from related directories via back-links — do not duplicate content."
|
||||
]
|
||||
],
|
||||
"dream_synthesize_paths": {
|
||||
"description": "Single source of truth for the v0.23 dream-cycle synthesize/patterns trusted-workspace allow-list. The cycle's synthesize phase reads this list and threads it as `allowed_slug_prefixes` to every subagent it dispatches; put_page enforces it server-side. Editing this list is the ONLY way to add a new directory the synthesis subagent may write to.",
|
||||
"globs": [
|
||||
"wiki/personal/reflections/*",
|
||||
"wiki/originals/*",
|
||||
"wiki/personal/patterns/*",
|
||||
"wiki/people/*",
|
||||
"dream-cycle-summaries/*"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,24 @@ not the source, not the skill that's running.
|
||||
| Reusable framework/thesis -> `sources/` | -> `concepts/` | It's a mental model |
|
||||
| Tweet thread about policy -> `media/` | -> `civic/` or `concepts/` | media/ is for content ops |
|
||||
|
||||
## Sanctioned exception: synthesis output is sui generis
|
||||
|
||||
The "file by primary subject" rule is for raw ingest. Synthesized output that
|
||||
is one-of-one to a single source AND a specific reader (a personalized book
|
||||
mirror, a strategic-reading playbook tied to one problem) does not fit any
|
||||
subject directory cleanly: filing by topic loses the "this is the book"
|
||||
dimension; filing by author muddles authorship pages with synthesis pages.
|
||||
|
||||
Format-prefixed paths under `media/<format>/<slug>` are the sanctioned
|
||||
exception:
|
||||
|
||||
- `media/books/<slug>-personalized.md` (book-mirror output)
|
||||
- `media/articles/<slug>-personalized.md` (long-form article personalization)
|
||||
|
||||
If you find yourself wanting `media/<format>/` for raw ingest, that is still
|
||||
the anti-pattern in the table above. The exception is narrow: synthesized,
|
||||
one-of-one, sui generis to a single source.
|
||||
|
||||
## What `sources/` Is Actually For
|
||||
|
||||
`sources/` is ONLY for:
|
||||
@@ -112,3 +130,24 @@ gbrain files restore <dir> # Download back to local
|
||||
|
||||
This ensures any derived brain page can be traced back to its original source,
|
||||
and large files don't bloat the git repo.
|
||||
|
||||
## Dream-cycle synthesize / patterns directories (v0.23)
|
||||
|
||||
The `synthesize` and `patterns` phases of `gbrain dream` write to a
|
||||
**fixed allow-list** of paths sourced from `_brain-filing-rules.json`'s
|
||||
`dream_synthesize_paths.globs` array. Editing that JSON is the ONLY way
|
||||
to add a new directory the synthesis subagent may write to:
|
||||
|
||||
| Output type | Slug pattern | What goes here |
|
||||
|-------------|--------------|----------------|
|
||||
| Reflection | `wiki/personal/reflections/YYYY-MM-DD-<topic>-<hash[:6]>` | Self-knowledge, emotional processing, pattern recognition. Verbatim quotes from the user, with analysis. |
|
||||
| Original idea | `wiki/originals/ideas/YYYY-MM-DD-<idea>-<hash[:6]>` | New frames, theses, mental models, "conceptive ideologist" outputs. Capture the user's exact phrasing — that's the artifact. |
|
||||
| People enrichment | `wiki/people/<existing-slug>` | Timeline entries appended to existing people pages from session mentions. Stub pages for new substantive people. |
|
||||
| Pattern | `wiki/personal/patterns/<theme>` | Cross-session theme detected across ≥3 reflections. Highest-leverage output: a pattern can span 25 years if reflections reference dated content. |
|
||||
| Cycle summary | `dream-cycle-summaries/YYYY-MM-DD` | Index of every page produced by one dream cycle. Auto-written deterministically by the orchestrator. |
|
||||
|
||||
**Iron Law for synthesize output:**
|
||||
1. Quote the user verbatim. Do not paraphrase memorable phrasings.
|
||||
2. Cross-reference compulsively: every new page MUST link to existing brain content.
|
||||
3. Slug discipline: lowercase alphanumeric and hyphens only, slash-separated. NO underscores, NO file extensions.
|
||||
4. Edited transcripts produce NEW slugs (content-hash suffix changes) — never silently overwrite a prior reflection.
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# Friction protocol — convention
|
||||
|
||||
> Cross-cutting rule shared by skills the claw-test harness exercises (setup,
|
||||
> brain-ops, query, ingest, smoke-test, migrations). Reference via
|
||||
> `> **Convention:** see [skills/_friction-protocol.md](_friction-protocol.md).`
|
||||
|
||||
When you encounter friction running gbrain — anything confusing, missing, surprising, or wrong — log it via `gbrain friction log` so maintainers can see it without you writing a bug report. Friction reports drive the claw-test feedback loop (the harness collects, renders, and re-runs).
|
||||
|
||||
## When to log
|
||||
|
||||
Log friction when any of these happens:
|
||||
|
||||
- A command failed with a non-actionable error message
|
||||
- A doc said one thing and the tool did another
|
||||
- You couldn't find the next step
|
||||
- A setup command needed a manual workaround
|
||||
- A flag exists but isn't documented in `--help`
|
||||
- A success condition was unclear (you couldn't tell if the command worked)
|
||||
|
||||
Log delight (positive signal) when:
|
||||
|
||||
- Something worked on the first try and the docs were exactly right
|
||||
- An error message handed you the fix
|
||||
- A flag you guessed at turned out to exist with the obvious name
|
||||
|
||||
## How to log
|
||||
|
||||
```
|
||||
gbrain friction log \
|
||||
--severity {confused|error|blocker|nit} \
|
||||
--phase <which-phase-or-command> \
|
||||
--message "<one-line-what-happened>" \
|
||||
[--hint "<one-line-what-could-be-better>"]
|
||||
```
|
||||
|
||||
For delight, add `--kind delight` and pick any severity.
|
||||
|
||||
The CLI auto-fills `ts`, `cwd`, `gbrain_version`, and resolves `run_id` from `$GBRAIN_FRICTION_RUN_ID` (set by the harness) or falls back to `standalone.jsonl`. So you can call this anywhere — inside a harness run, manually during normal use, or from a scripted test.
|
||||
|
||||
## Severity guide
|
||||
|
||||
| severity | meaning |
|
||||
|------------|---------|
|
||||
| `blocker` | Couldn't proceed at all. Hard stop. |
|
||||
| `error` | Command failed unexpectedly. |
|
||||
| `confused` | Docs/tool mismatch, ambiguity, missing pointer. |
|
||||
| `nit` | Polish opportunity. Cosmetic or low-impact. |
|
||||
|
||||
Be specific: "doctor says `schema_version=0` and points at apply-migrations, but apply-migrations exits 0 with no output" beats "doctor was confusing."
|
||||
|
||||
## Inspecting reports
|
||||
|
||||
```
|
||||
gbrain friction list # recent runs with counts
|
||||
gbrain friction render --run-id <id> # markdown report (default)
|
||||
gbrain friction render --run-id <id> --json
|
||||
gbrain friction summary --run-id <id> # friction + delight side-by-side
|
||||
```
|
||||
|
||||
`render` defaults to `--redact` for markdown (strips `$HOME`/`$CWD` to `<HOME>`/`<CWD>` placeholders) so reports paste safely into PRs and issues.
|
||||
@@ -0,0 +1,224 @@
|
||||
---
|
||||
name: academic-verify
|
||||
version: 0.1.0
|
||||
description: Verify a research claim or academic citation by tracing it through publication → methodology → raw data → independent replication. Routes through perplexity-research for the actual web lookup, then formats results as a citation-checked brain page. Use when a book/article/conversation cites a study and you want to confirm the claim is real, replicated, and accurately characterized.
|
||||
triggers:
|
||||
- "verify this academic claim"
|
||||
- "check this study"
|
||||
- "academic verify"
|
||||
- "validate citation"
|
||||
- "is this study real"
|
||||
mutating: true
|
||||
writes_pages: true
|
||||
writes_to:
|
||||
- concepts/
|
||||
---
|
||||
|
||||
# academic-verify — Trace Claims to Source Data
|
||||
|
||||
> **Convention:** see [conventions/quality.md](../conventions/quality.md) for
|
||||
> citation rules; every verdict cites the source data, not just the
|
||||
> author's claim about the source data.
|
||||
>
|
||||
> **Convention:** see [conventions/brain-first.md](../conventions/brain-first.md)
|
||||
> for the lookup chain. This skill enforces brain-first by checking
|
||||
> existing brain pages before issuing a fresh web search.
|
||||
|
||||
## What this is
|
||||
|
||||
A claim-verification flow for academic / research statements. When a
|
||||
book, article, or speaker cites a study or quotes a number, this skill
|
||||
traces the claim through:
|
||||
|
||||
```
|
||||
claim → publication → methodology section → raw data source → independent verification
|
||||
```
|
||||
|
||||
At each step, it answers:
|
||||
|
||||
- **Where does this number come from?** (Self-generated? Survey? Government data?)
|
||||
- **What's the baseline?** (Reduction from what? Over what time period?)
|
||||
- **Is the raw data available?** (Public? Proprietary? "Available on request"?)
|
||||
- **Has anyone independently verified it?** (Replication study? Government audit?)
|
||||
- **Are there confounding factors?** (Other interventions, policy changes, COVID, sampling bias?)
|
||||
- **Is the comparison fair?** (Cherry-picked comparison group? Survivorship bias?)
|
||||
|
||||
The output is a brain page under `concepts/<claim-slug>.md` that records
|
||||
the claim, the trace, and the verdict — so future references to the
|
||||
same claim can re-use the verified analysis.
|
||||
|
||||
## When to use this
|
||||
|
||||
- A book quotes a study and you want to confirm it's real and not
|
||||
miscited
|
||||
- An article makes a quantified claim ("X reduced Y by 40%") that you
|
||||
want traced to the source data
|
||||
- You're writing something that depends on a piece of research and you
|
||||
want to verify the underlying paper holds up
|
||||
- You're updating a brain page that cites a research claim and you want
|
||||
to record the verification status alongside
|
||||
|
||||
## What this skill is NOT
|
||||
|
||||
- Not adversarial / oppo work. The point is rigor, not takedown.
|
||||
- Not generic web research — use `perplexity-research` directly for
|
||||
open-ended topic exploration.
|
||||
- Not a brain-only lookup — that's `gbrain query`.
|
||||
|
||||
## How it works (D7/α: pure routing through perplexity-research)
|
||||
|
||||
academic-verify is a thin orchestrator. The actual web search is done
|
||||
by [perplexity-research](../perplexity-research/SKILL.md). academic-verify's
|
||||
job is the *workflow*: scoping the claim precisely, sending it through
|
||||
perplexity-research with citation-mode, then formatting the response
|
||||
into a verdict-shaped brain page.
|
||||
|
||||
```
|
||||
Step 1: Scope the claim
|
||||
Pin down EXACTLY what's being claimed:
|
||||
• Quote: who said what?
|
||||
• Source: which paper / dataset / survey?
|
||||
• Number: what specific quantity is claimed?
|
||||
• Period: over what time range?
|
||||
|
||||
Step 2: Brain-first lookup
|
||||
gbrain query "<paper title> OR <author name> OR <claim keywords>"
|
||||
If the brain has prior verification of this claim, reuse it.
|
||||
|
||||
Step 3: Invoke perplexity-research with citation-mode prompt
|
||||
Send the claim + brain context to perplexity-research with a prompt
|
||||
that explicitly asks for:
|
||||
• Original publication (title, authors, journal, year, DOI)
|
||||
• Methodology section summary
|
||||
• Raw data availability (public repo? proprietary?)
|
||||
• Independent replication status (Retraction Watch / PubPeer hits)
|
||||
• Citations of the paper that critique or contextualize it
|
||||
|
||||
Step 4: Format the verdict
|
||||
Write the result to concepts/<claim-slug>.md. The verdict is one of:
|
||||
• Verified — claim is accurate; raw data available; replication exists
|
||||
• Partially verified — claim correct on the underlying paper but
|
||||
methodology has known limits; record limits explicitly
|
||||
• Unverifiable — no public data, no replication; not enough to act
|
||||
• Misattributed — the claim cites a paper but the paper doesn't say that
|
||||
• Retracted / disputed — paper has known retraction or
|
||||
well-documented critique
|
||||
|
||||
Step 5: Cross-link to original sources
|
||||
Add the paper authors to people/ if they have brain pages, or create
|
||||
one if notable. Iron Law per conventions/quality.md.
|
||||
```
|
||||
|
||||
## Output: brain page format
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: "[Claim summary] — Verified"
|
||||
type: research
|
||||
date: YYYY-MM-DD
|
||||
verdict: "verified|partial|unverifiable|misattributed|retracted"
|
||||
brain_context_slugs: ["pages cited as context"]
|
||||
---
|
||||
|
||||
# [Claim summary] — Verified
|
||||
|
||||
> One-line: the verdict + the bottom-line reason.
|
||||
|
||||
## The Claim
|
||||
|
||||
> Exact quote, exactly as stated, with source attribution.
|
||||
|
||||
## Trace
|
||||
|
||||
| Step | Finding | Source |
|
||||
|------|---------|--------|
|
||||
| Original publication | [Title, authors, year, DOI] | [URL] |
|
||||
| Methodology | [1-line summary; flag obvious limits] | [URL] |
|
||||
| Raw data | [Public repo / proprietary / available-on-request] | [URL] |
|
||||
| Independent replication | [Replication studies and their results] | [URL] |
|
||||
| Critical citations | [Papers that critique this work] | [URL] |
|
||||
|
||||
## Verdict
|
||||
|
||||
[Verified / Partially verified / Unverifiable / Misattributed / Retracted]
|
||||
|
||||
[1-2 paragraphs explaining WHY the verdict, with specific evidence.]
|
||||
|
||||
## Caveats
|
||||
|
||||
[Honest limits: what we couldn't verify, what would change the verdict.]
|
||||
|
||||
## See Also
|
||||
|
||||
- Original paper: [Title](DOI URL)
|
||||
- Authors' brain pages: [Author 1](people/author-1.md), ...
|
||||
- Related claims (verified or otherwise): [...]
|
||||
```
|
||||
|
||||
## Useful databases (the agent uses these via perplexity-research)
|
||||
|
||||
| Database | What it has | URL pattern |
|
||||
|----------|-------------|-------------|
|
||||
| Retraction Watch | Retractions, corrections, expressions of concern | retractionwatch.com/?s=NAME |
|
||||
| PubPeer | Anonymous post-publication peer review | pubpeer.com/search?q=NAME |
|
||||
| OSF | Pre-registrations, open data, open materials | osf.io/search/?q=QUERY |
|
||||
| Semantic Scholar | Citation analysis, paper metadata | api.semanticscholar.org |
|
||||
| OpenAlex | Open citation data, institutional affiliations | api.openalex.org |
|
||||
| Many Labs | Replication results for social psychology | osf.io/wx7ck/ |
|
||||
|
||||
## Standards (the rigor bar)
|
||||
|
||||
- **Verified** — only when the underlying paper exists, raw data is
|
||||
public OR an independent lab has confirmed the result, and the citing
|
||||
source represents the claim accurately.
|
||||
- **Partial** — paper is real and findings stand, but the citation
|
||||
context oversells (e.g., "X causes Y" when the paper shows
|
||||
correlation, or "all studies find X" when it's one underpowered study).
|
||||
- **Unverifiable** — the underlying number can't be traced to source
|
||||
data, no replication has been done, no independent confirmation
|
||||
exists. Not the same as "wrong" — say "we couldn't verify."
|
||||
- **Misattributed** — the citation points to a paper, but the paper
|
||||
doesn't actually say what the citation claims. Common in policy briefs.
|
||||
- **Retracted / disputed** — paper has been retracted, has a major
|
||||
expression-of-concern, or has well-documented critique that
|
||||
contradicts the headline finding.
|
||||
|
||||
Never claim a problem without evidence. The verification document
|
||||
itself is the artifact — if the claim holds up, say so plainly. If it
|
||||
doesn't, the trace speaks for itself.
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
- ❌ Skipping the brain-first lookup. Re-doing verification we've
|
||||
already done is wasted Perplexity spend.
|
||||
- ❌ Bypassing perplexity-research and inventing the lookup. The
|
||||
citations from Perplexity are the evidence — without them, the
|
||||
verdict is just opinion.
|
||||
- ❌ Stating "Verified" without confirming raw data availability.
|
||||
Replication trumps any single paper.
|
||||
- ❌ Stating "Unverifiable" when you simply didn't look hard enough.
|
||||
The verdict is on the source, not on your search effort.
|
||||
|
||||
## Related skills
|
||||
|
||||
- `skills/perplexity-research/SKILL.md` — the actual web-search engine
|
||||
this skill routes through (D7/α: pure routing, no new infrastructure)
|
||||
- `skills/citation-fixer/SKILL.md` — fixes citation FORMATTING; this
|
||||
skill checks whether the cited claim is true
|
||||
- `skills/conventions/quality.md` — citation + back-link rules
|
||||
|
||||
|
||||
## Contract
|
||||
|
||||
This skill guarantees:
|
||||
|
||||
- Routing matches the canonical triggers in the frontmatter.
|
||||
- Output written under the directories listed in `writes_to:` (when applicable).
|
||||
- Conventions referenced (`quality.md`, `brain-first.md`, `_brain-filing-rules.md`) are followed.
|
||||
- Privacy contract preserved: no real names, no fork-specific filesystem path literals, no upstream-fork references.
|
||||
|
||||
The full behavior contract is documented in the body sections above; this section exists for the conformance test.
|
||||
|
||||
## Output Format
|
||||
|
||||
The skill's output shape is documented inline in the body sections above (see "Output", "Brain page format", or equivalent). The literal section header here exists for the conformance test (`test/skills-conformance.test.ts`).
|
||||
@@ -0,0 +1,7 @@
|
||||
// Routing eval fixtures for skills/academic-verify. Each intent
|
||||
// includes at least one trigger string as substring.
|
||||
{"intent":"Please verify this academic claim from the book against the original paper","expected_skill":"academic-verify"}
|
||||
{"intent":"Check this study cited in the article — has it been replicated","expected_skill":"academic-verify"}
|
||||
{"intent":"Run academic verify on the 40% reduction claim and trace it to the source data","expected_skill":"academic-verify"}
|
||||
{"intent":"Validate citation for the Stanford study referenced in the policy brief","expected_skill":"academic-verify"}
|
||||
{"intent":"Is this study real, or is it on Retraction Watch","expected_skill":"academic-verify"}
|
||||
@@ -0,0 +1,320 @@
|
||||
---
|
||||
name: archive-crawler
|
||||
version: 0.1.0
|
||||
description: Universal archivist for personal file archives (Dropbox/B2/Gmail-takeout/local-mount/hard-drive-dump). Filters for high-value content (the user's own writing, ideas, relationships) and surfaces it interactively. REFUSES TO RUN without an explicit gbrain.yml `archive-crawler.scan_paths:` allow-list.
|
||||
triggers:
|
||||
- "crawl my archive"
|
||||
- "find gold in my archive"
|
||||
- "archive crawler"
|
||||
- "scan my dropbox for"
|
||||
- "mine my old files for"
|
||||
mutating: true
|
||||
writes_pages: true
|
||||
writes_to:
|
||||
- originals/
|
||||
- personal/
|
||||
- ideas/
|
||||
---
|
||||
|
||||
# archive-crawler — The Universal Archivist
|
||||
|
||||
> **Convention:** see [conventions/quality.md](../conventions/quality.md) for
|
||||
> citation rules, exact-phrasing requirements when capturing the user's
|
||||
> reactions, and back-link enforcement.
|
||||
>
|
||||
> **Convention:** see [_brain-filing-rules.md](../_brain-filing-rules.md) —
|
||||
> this skill is **schema-generic**: it reads the user's filing rules from
|
||||
> the rules JSON instead of hardcoding any specific era / archive layout.
|
||||
|
||||
## Safety gate (REQUIRED, no exceptions)
|
||||
|
||||
archive-crawler refuses to run unless `archive-crawler.scan_paths:` is
|
||||
explicitly set in `gbrain.yml`. This is a deliberate safety fence against
|
||||
the agent over-scoping a scan and ingesting sensitive content (tax PDFs,
|
||||
medical records, credentials).
|
||||
|
||||
```yaml
|
||||
# gbrain.yml — the allow-list is mandatory
|
||||
archive-crawler:
|
||||
scan_paths:
|
||||
- ~/Documents/writing/
|
||||
- ~/Dropbox/Archive/
|
||||
- /mnt/backup/old-letters/
|
||||
# Optional deny-list inside the allow-list:
|
||||
# deny_paths:
|
||||
# - ~/Documents/finances/
|
||||
# - ~/Documents/medical/
|
||||
```
|
||||
|
||||
If `scan_paths` is empty or missing, the skill exits with:
|
||||
|
||||
```
|
||||
archive-crawler: refusing to run. No `archive-crawler.scan_paths:` allow-list
|
||||
in gbrain.yml. Add explicit paths the agent is permitted to scan, then re-run.
|
||||
This is a safety fence — the agent will not infer what's safe to read.
|
||||
```
|
||||
|
||||
This contract is enforced by `src/core/storage-config.ts` (mirrors the
|
||||
`db_tracked` / `db_only` allow-list pattern from v0.22.11 storage tiering).
|
||||
|
||||
## What this is
|
||||
|
||||
Generic engine for exploring any tree of personal content within an
|
||||
explicit allow-list. Works on local mounts, Dropbox API targets,
|
||||
Backblaze B2, Gmail takeouts (`.mbox`), and similar archives. Filters
|
||||
for "gold" (the user's own writing, ideas, relationships) and surfaces
|
||||
it interactively for review. Skips noise (system files, configs, binary
|
||||
blobs).
|
||||
|
||||
## Concepts
|
||||
|
||||
### Source
|
||||
|
||||
A source is any tree of files to explore. Sources have:
|
||||
|
||||
- **type**: `local` | `dropbox` | `backblaze` | `gmail-takeout` | `mbox` | `pst`
|
||||
- **root**: filesystem path, Dropbox path, B2 prefix, mbox path
|
||||
- **manifest**: a brain page tracking progress at
|
||||
`projects/<archive-slug>/STATUS.md`
|
||||
|
||||
### Manifest
|
||||
|
||||
Every archive exploration gets a manifest brain page that tracks:
|
||||
|
||||
1. **Tree inventory** — folders / files / sizes / types
|
||||
2. **Triage status** — each item: `⬜ unseen` / `👀 reviewed` /
|
||||
`✅ ingested` / `⏭️ skip` / `🔥 high-signal`
|
||||
3. **User reactions** — exact quotes when they react (per
|
||||
conventions/quality.md exact-phrasing rule)
|
||||
4. **Priority queue** — what to explore next, ranked
|
||||
5. **Session log** — timestamped record of what was shown per session
|
||||
|
||||
### Gold filter
|
||||
|
||||
Before showing anything to the user, apply the gold filter:
|
||||
|
||||
| Keep (show) | Skip (note existence, don't show) |
|
||||
|-------------|-----------------------------------|
|
||||
| Personal writing (journals, letters, reflections, essays) | System files, configs, package.json, node_modules |
|
||||
| Conversations (IM logs, email threads with substance) | Binary blobs (images / video) |
|
||||
| Ideas, theses, frameworks | Receipts, invoices, tax docs |
|
||||
| Relationship material (letters to / from people who matter) | Spam, newsletters, mailing-list bulk |
|
||||
| Creative work (poetry, stories, code with soul) | Corrupted / null files |
|
||||
| Origin stories (first versions of things that became important) | |
|
||||
| Emotional content (anger, love, grief, discovery) | |
|
||||
|
||||
## Protocol
|
||||
|
||||
### Phase 1: Inventory
|
||||
|
||||
When pointed at a new source:
|
||||
|
||||
1. **Confirm scan_paths is set** (safety gate). Exit if not.
|
||||
2. **Map the tree** — list folders + files + sizes + date ranges.
|
||||
3. **Classify folders** — group by likely content type (writing, email,
|
||||
code, photos, docs, system).
|
||||
4. **Create manifest** — write `projects/<archive-slug>/STATUS.md` with
|
||||
the full inventory.
|
||||
5. **Propose priority queue** — rank folders by likely gold density.
|
||||
6. **Present to user** — show the map and proposed order. Let them
|
||||
override.
|
||||
|
||||
### Phase 2: Crawl
|
||||
|
||||
Work through folders in priority order:
|
||||
|
||||
1. **Read before showing** — open each candidate file, apply the gold
|
||||
filter, skip noise.
|
||||
2. **Show one at a time** — present gold items individually for review.
|
||||
3. **Capture exact reaction** — track the user's response in the
|
||||
manifest using their exact words (per conventions/quality.md).
|
||||
4. **Ingest if worth keeping** — create a brain page immediately.
|
||||
5. **Update manifest** — mark item status after each interaction.
|
||||
6. **Never re-show** — check the manifest before presenting anything.
|
||||
|
||||
### Phase 3: Ingest
|
||||
|
||||
When an item is worth keeping, file it by **primary subject** per
|
||||
`_brain-filing-rules.md`:
|
||||
|
||||
- User's own writing / ideas / origin-story content → `originals/<slug>.md`
|
||||
- Reflections / personal-life content → `personal/<slug>.md`
|
||||
- Product / business ideas → `ideas/<slug>.md`
|
||||
- Letters or threads about a specific person → `people/<person>/timeline`
|
||||
back-link plus the letter at `personal/<slug>.md` or `originals/<slug>.md`
|
||||
|
||||
**The skill is schema-generic.** It does NOT bake in any specific
|
||||
era-folder structure (e.g., `originals/archive/` for pre-2003,
|
||||
`originals/yc-era/` for post-2019, etc.). The user's filing rules from
|
||||
`_brain-filing-rules.json` are read at runtime; the agent decides per-page
|
||||
where content lands within those sanctioned directories.
|
||||
|
||||
Brain page format:
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: "[Title or first line]"
|
||||
type: original
|
||||
source_type: "[local|dropbox|backblaze|gmail-takeout|mbox|pst]"
|
||||
source_path: "[path within the allow-listed scan_paths]"
|
||||
date: "YYYY-MM-DD" # date from the file metadata or content
|
||||
people: ["person-1", "person-2"]
|
||||
tags: ["tag-1", "tag-2"]
|
||||
---
|
||||
|
||||
# [Title]
|
||||
|
||||
[Summary: what it is, when it's from, why it matters]
|
||||
|
||||
**User's reaction:** [exact quote, no paraphrasing]
|
||||
|
||||
## Context
|
||||
|
||||
[Cross-links to people, concepts, projects.]
|
||||
|
||||
---
|
||||
|
||||
[Raw source material below the line — full text]
|
||||
```
|
||||
|
||||
## File-type handlers
|
||||
|
||||
### Plain text / HTML / Markdown
|
||||
Read directly. Strip HTML tags for display.
|
||||
|
||||
### `.mbox` (email archives)
|
||||
|
||||
```python
|
||||
import mailbox
|
||||
mbox = mailbox.mbox('/path/to/file.mbox')
|
||||
for msg in mbox:
|
||||
body = ''
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
if part.get_content_type() == 'text/plain':
|
||||
body = part.get_payload(decode=True).decode('utf-8', errors='replace')
|
||||
break
|
||||
else:
|
||||
body = msg.get_payload(decode=True).decode('utf-8', errors='replace')
|
||||
# Apply gold filter
|
||||
```
|
||||
|
||||
### `.doc` / `.docx`
|
||||
|
||||
```bash
|
||||
# .docx (modern)
|
||||
python3 -c "
|
||||
import zipfile, xml.etree.ElementTree as ET
|
||||
with zipfile.ZipFile('/path/to/file.docx') as z:
|
||||
tree = ET.parse(z.open('word/document.xml'))
|
||||
print(''.join(t.text or '' for t in tree.iter('{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t')))
|
||||
"
|
||||
|
||||
# .doc (legacy, requires antiword or catdoc)
|
||||
antiword /path/to/file.doc 2>/dev/null || catdoc /path/to/file.doc 2>/dev/null
|
||||
```
|
||||
|
||||
### `.pst` (Outlook archives)
|
||||
|
||||
```bash
|
||||
# Validate first; many PSTs are null bytes
|
||||
python3 -c "
|
||||
with open('/path/to/file.pst', 'rb') as f:
|
||||
print('Valid PST' if f.read(4) == b'!BDN' else 'CORRUPT/NULL')
|
||||
"
|
||||
# If valid:
|
||||
readpst -o /tmp/pst-output /path/to/file.pst
|
||||
```
|
||||
|
||||
### `.zip` / `.tar` / `.tar.gz`
|
||||
|
||||
Extract to a temp dir, then recurse through the extracted tree.
|
||||
|
||||
### Images
|
||||
|
||||
Note existence + metadata (filename, size, date). Don't show unless the
|
||||
user asks. Flag scans / portraits as potentially personal.
|
||||
|
||||
## Manifest template
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: "[Archive Name] — Ingestion Status"
|
||||
type: project
|
||||
created: YYYY-MM-DD
|
||||
updated: YYYY-MM-DD
|
||||
source_type: "[local|dropbox|...]"
|
||||
scan_paths: ["paths from gbrain.yml"]
|
||||
---
|
||||
|
||||
# [Archive Name] — Ingestion Status
|
||||
|
||||
## Source
|
||||
- **Type:** [local|dropbox|...]
|
||||
- **Allow-listed paths:** [from gbrain.yml]
|
||||
- **Total files:** [N]
|
||||
- **Total size:** [X GB]
|
||||
- **Date range:** [earliest] — [latest]
|
||||
|
||||
## Inventory
|
||||
|
||||
### [Folder 1]
|
||||
| Item | Type | Size | Status | Reaction |
|
||||
|------|------|------|--------|----------|
|
||||
| file1.txt | text | 2KB | ✅ ingested | 🔥 "exact quote" |
|
||||
| file2.doc | doc | 15KB | ⏭️ skip | — |
|
||||
| file3.html | html | 4KB | ⬜ unseen | — |
|
||||
|
||||
### [Folder 2]
|
||||
...
|
||||
|
||||
## Priority Queue
|
||||
1. [Highest priority — why]
|
||||
2. [Next — why]
|
||||
...
|
||||
|
||||
## Session Log
|
||||
|
||||
### YYYY-MM-DD — [Session topic]
|
||||
- Reviewed: [list]
|
||||
- Reactions: [exact quotes]
|
||||
- Ingested: [brain pages created]
|
||||
- Next: [what's queued]
|
||||
```
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
- ❌ Running without `archive-crawler.scan_paths:` set. Hard refusal.
|
||||
This is the safety contract — never bypass.
|
||||
- ❌ Hardcoding era-specific filing paths (e.g., `originals/archive/`,
|
||||
`originals/yc-era/`). Read filing rules at runtime instead.
|
||||
- ❌ Re-showing items already marked in the manifest. The user's time
|
||||
is the scarcest resource.
|
||||
- ❌ Paraphrasing reactions. Exact words only.
|
||||
- ❌ Wrapping found content in lessons or takeaways. Let stories breathe.
|
||||
- ❌ Skipping back-links when content references people / companies who
|
||||
have brain pages. Iron Law per conventions/quality.md.
|
||||
|
||||
## Related skills
|
||||
|
||||
- `skills/voice-note-ingest/SKILL.md` — same exact-phrasing pattern for
|
||||
audio capture
|
||||
- `skills/idea-ingest/SKILL.md` — single-link-or-article ingest with
|
||||
the same primary-subject filing rule
|
||||
- `skills/conventions/quality.md` — citations, back-links, voice
|
||||
|
||||
|
||||
## Contract
|
||||
|
||||
This skill guarantees:
|
||||
|
||||
- Routing matches the canonical triggers in the frontmatter.
|
||||
- Output written under the directories listed in `writes_to:` (when applicable).
|
||||
- Conventions referenced (`quality.md`, `brain-first.md`, `_brain-filing-rules.md`) are followed.
|
||||
- Privacy contract preserved: no real names, no fork-specific filesystem path literals, no upstream-fork references.
|
||||
|
||||
The full behavior contract is documented in the body sections above; this section exists for the conformance test.
|
||||
|
||||
## Output Format
|
||||
|
||||
The skill's output shape is documented inline in the body sections above (see "Output", "Brain page format", or equivalent). The literal section header here exists for the conformance test (`test/skills-conformance.test.ts`).
|
||||
@@ -0,0 +1,7 @@
|
||||
// Routing eval fixtures for skills/archive-crawler. Each intent
|
||||
// includes at least one trigger string as substring.
|
||||
{"intent":"Please crawl my archive and surface the writing worth keeping","expected_skill":"archive-crawler"}
|
||||
{"intent":"Find gold in my archive of old letters and ideas","expected_skill":"archive-crawler"}
|
||||
{"intent":"Run archive crawler on the gbrain.yml allow-listed paths","expected_skill":"archive-crawler"}
|
||||
{"intent":"Scan my dropbox for substantive email threads with people who matter","expected_skill":"archive-crawler"}
|
||||
{"intent":"Mine my old files for journal entries and reflections worth ingesting","expected_skill":"archive-crawler"}
|
||||
@@ -0,0 +1,146 @@
|
||||
---
|
||||
name: article-enrichment
|
||||
version: 0.1.0
|
||||
description: Transform raw article text dumps in the brain into structured pages with executive summary, verbatim quotes, key insights, why-it-matters, and cross-references. Replaces walls-of-text with quotable, actionable brain pages.
|
||||
triggers:
|
||||
- "enrich this article"
|
||||
- "enrich brain pages"
|
||||
- "batch enrich"
|
||||
- "make brain pages useful"
|
||||
mutating: true
|
||||
writes_pages: true
|
||||
writes_to:
|
||||
- media/articles/
|
||||
---
|
||||
|
||||
# article-enrichment — From Raw Dumps to Useful Brain Pages
|
||||
|
||||
> **Convention:** see [conventions/quality.md](../conventions/quality.md) for
|
||||
> citation rules, verbatim-quote requirements, and back-link enforcement.
|
||||
>
|
||||
> **Convention:** see [_brain-filing-rules.md](../_brain-filing-rules.md) for
|
||||
> filing rules. Article pages live under `media/articles/` for raw ingest;
|
||||
> personalized one-of-one synthesis output uses the sanctioned
|
||||
> `media/articles/<slug>-personalized.md` exception.
|
||||
|
||||
## What this does
|
||||
|
||||
Takes an article brain page that's a wall of raw extracted text and rewrites
|
||||
it as a structured page with:
|
||||
|
||||
- **Executive Summary** — 2-3 sentences, the ONE thing worth remembering
|
||||
- **Why It Matters** — connects to the user's specific projects + interests
|
||||
(read from brain context, not assumed)
|
||||
- **Quotable Lines** — 3-5 VERBATIM quotes worth referencing in essays
|
||||
- **Key Insights** — actual insights, not topic labels
|
||||
- **Surprising or Counterintuitive** — what makes this content unique
|
||||
- **See Also** — standard markdown links to related brain pages
|
||||
|
||||
Raw source content is preserved in a collapsed `<details>` section so the
|
||||
original is never lost.
|
||||
|
||||
## When to invoke
|
||||
|
||||
- New article page lands in the brain via media-ingest with `needs_enrichment: true`
|
||||
- Existing article page is a wall of text under a `## Content` header with
|
||||
no synthesis
|
||||
- User says a brain page is useless, boring, or a dump
|
||||
- An LLM-judge brain-quality eval fails on quotability or actionability for
|
||||
an article page
|
||||
|
||||
## The pipeline
|
||||
|
||||
```
|
||||
1. READ → Open the article brain page; parse frontmatter + body.
|
||||
2. SCAN → Look for ## Content (raw dump) and absence of ## Executive Summary.
|
||||
3. CONTEXT → gbrain query the article's key entities to ground "Why It Matters".
|
||||
4. ENRICH → Sonnet (default) or Opus (for high-value content) restructures.
|
||||
5. WRITE → Replace ## Content with the structured sections; preserve raw
|
||||
source in <details>; clear needs_enrichment in frontmatter.
|
||||
6. CROSS-LINK→ Add back-links from referenced people/companies pages
|
||||
(Iron Law per conventions/quality.md).
|
||||
```
|
||||
|
||||
## Invocation
|
||||
|
||||
The skill itself is markdown instructions to the agent. It does NOT ship a
|
||||
deterministic CLI command in v0.25.1. The agent uses gbrain's existing
|
||||
operations:
|
||||
|
||||
```bash
|
||||
# 1. Find candidate pages
|
||||
gbrain query "needs_enrichment: true type:article" --limit 50
|
||||
|
||||
# 2. For each candidate, read the page
|
||||
gbrain get media/articles/<slug>
|
||||
|
||||
# 3. Enrich via the agent's LLM (Sonnet by default; Opus for high-value)
|
||||
# The agent reads the raw content + brain context + writes the structured page.
|
||||
|
||||
# 4. Write the enriched page
|
||||
# Use the put_page operation with the new structured markdown body.
|
||||
|
||||
# 5. Cross-link entities
|
||||
# For every person/company mentioned, add a timeline back-link.
|
||||
```
|
||||
|
||||
## Quality bar
|
||||
|
||||
An enriched page passes if it has:
|
||||
|
||||
- ✅ `## Executive Summary` (2-3 sentences)
|
||||
- ✅ `## Quotable Lines` with ≥3 verbatim quotes (literal quotes, not paraphrase)
|
||||
- ✅ `## Key Insights` with ≥3 bullets (insights, not topic labels)
|
||||
- ✅ `## Why It Matters` connecting to specific brain context (not generic)
|
||||
- ✅ `## See Also` with standard markdown links (NOT `[[wiki-links]]`)
|
||||
- ✅ `<details>` block preserving the raw source content
|
||||
|
||||
## Model selection
|
||||
|
||||
| Model | Use when | Quote accuracy |
|
||||
|-------|----------|----------------|
|
||||
| **Sonnet** (default) | Bulk enrichment, most articles | Good — occasionally paraphrases |
|
||||
| **Opus** | High-value content, original-thinking pieces, longreads | Excellent — respects "verbatim" instruction |
|
||||
|
||||
Rule: for bulk enrichment, do a Sonnet draft pass and spot-check 5 with
|
||||
the LLM-judge brain-quality eval. If quotes are paraphrased, switch to
|
||||
Opus for that batch.
|
||||
|
||||
## Link convention
|
||||
|
||||
All cross-references use standard markdown links: `[Title](relative/path.md)`.
|
||||
NEVER use `[[wiki-links]]` — they don't render on GitHub.
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
- ❌ Paraphrasing quotes ("the author argues that…"). Quotes are verbatim
|
||||
or they're not quotes.
|
||||
- ❌ Generic "Why It Matters" ("this is important because innovation").
|
||||
Tie to specific brain context or remove the section.
|
||||
- ❌ Inventing topic labels and calling them insights. An insight is a
|
||||
thing the article says that you didn't already know.
|
||||
- ❌ Discarding the raw source. Always wrap it in `<details>`.
|
||||
- ❌ Re-enriching non-idempotently — check the `needs_enrichment` flag in
|
||||
frontmatter; skip if already false.
|
||||
|
||||
## Related skills
|
||||
|
||||
- `skills/media-ingest/SKILL.md` — creates the raw article pages this skill enriches
|
||||
- `skills/idea-ingest/SKILL.md` — link/article ingestion with author people-page enforcement
|
||||
- `skills/conventions/quality.md` — citation + back-link rules
|
||||
|
||||
|
||||
## Contract
|
||||
|
||||
This skill guarantees:
|
||||
|
||||
- Routing matches the canonical triggers in the frontmatter.
|
||||
- Output written under the directories listed in `writes_to:` (when applicable).
|
||||
- Conventions referenced (`quality.md`, `brain-first.md`, `_brain-filing-rules.md`) are followed.
|
||||
- Privacy contract preserved: no real names, no fork-specific filesystem path literals, no upstream-fork references.
|
||||
|
||||
The full behavior contract is documented in the body sections above; this section exists for the conformance test.
|
||||
|
||||
## Output Format
|
||||
|
||||
The skill's output shape is documented inline in the body sections above (see "Output", "Brain page format", or equivalent). The literal section header here exists for the conformance test (`test/skills-conformance.test.ts`).
|
||||
@@ -0,0 +1,7 @@
|
||||
// Routing eval fixtures for skills/article-enrichment. Each intent
|
||||
// includes at least one trigger string as substring.
|
||||
{"intent":"This article page is a wall of raw text — please enrich this article with quotes and insights","expected_skill":"article-enrichment"}
|
||||
{"intent":"Run a batch enrich pass on the unstructured articles in my brain","expected_skill":"article-enrichment"}
|
||||
{"intent":"Make brain pages useful by enriching the article dumps","expected_skill":"article-enrichment"}
|
||||
{"intent":"Please enrich brain pages that have raw content but no executive summary","expected_skill":"article-enrichment"}
|
||||
{"intent":"Enrich this article so it has verbatim quotes, key insights, and a why-it-matters section","expected_skill":"article-enrichment"}
|
||||
@@ -0,0 +1,350 @@
|
||||
---
|
||||
name: book-mirror
|
||||
version: 0.1.0
|
||||
description: Take any book (EPUB/PDF), produce a personalized chapter-by-chapter analysis with two-column tables. Left column preserves the chapter content; right column maps every idea to the reader's actual life using brain context. Output is a single brain page at media/books/<slug>-personalized.md plus an optional PDF via brain-pdf.
|
||||
triggers:
|
||||
- "personalized version of this book"
|
||||
- "mirror this book"
|
||||
- "two-column book analysis"
|
||||
- "apply this book to my life"
|
||||
- "how does this book apply to me"
|
||||
mutating: true
|
||||
writes_pages: true
|
||||
writes_to:
|
||||
- media/books/
|
||||
---
|
||||
|
||||
# book-mirror — Personalized Chapter-by-Chapter Book Analysis
|
||||
|
||||
> **Convention:** see [_brain-filing-rules.md](../_brain-filing-rules.md) for the
|
||||
> sanctioned `media/<format>/<slug>` exception this skill files under.
|
||||
>
|
||||
> **Convention:** see [conventions/quality.md](../conventions/quality.md) for
|
||||
> citation rules, back-link enforcement, and output quality bars.
|
||||
>
|
||||
> **Convention:** see [conventions/brain-first.md](../conventions/brain-first.md)
|
||||
> for the lookup chain (brain → search → external) the context-gathering
|
||||
> phase follows.
|
||||
|
||||
## What this does
|
||||
|
||||
Given a book (EPUB or PDF), produce a brain page where every chapter is
|
||||
summarized in detail on the left and mirrored back to the reader's actual life
|
||||
on the right, using their own words, situations, people, and patterns from
|
||||
the brain. Output is a brain page at `media/books/<slug>-personalized.md`.
|
||||
|
||||
This is NOT a generic book summary. The right column is the value: it makes
|
||||
the book read like a therapist who knows the reader is leaving notes in the
|
||||
margins. If the user wants a flat summary instead, route them to a different
|
||||
skill.
|
||||
|
||||
## Trust contract (read this before running)
|
||||
|
||||
book-mirror runs as a CLI command (`gbrain book-mirror`), NOT as a pure
|
||||
markdown skill that the agent dispatches via tools. The CLI is the trusted
|
||||
runtime; the skill is the orchestration prose around it.
|
||||
|
||||
What this means for the agent:
|
||||
|
||||
- The CLI submits N read-only subagent jobs (one per chapter). Each subagent
|
||||
has `allowed_tools: ['get_page', 'search']` only. They CANNOT call
|
||||
put_page or any mutating op. They produce markdown analysis via their
|
||||
final message.
|
||||
- The CLI reads each child's `job.result`, assembles the final
|
||||
two-column page, and writes it via a single operator-trust `put_page`.
|
||||
- This means untrusted EPUB/PDF content cannot prompt-inject any
|
||||
`people/*` page. The trust narrowing happens at the tool allowlist,
|
||||
not at the slug-prefix layer.
|
||||
|
||||
## The pipeline
|
||||
|
||||
```
|
||||
1. ACQUIRE → User has the EPUB/PDF locally (manual; book-acquisition is
|
||||
not currently shipped — see "Acquiring the book" below).
|
||||
2. EXTRACT → Pull chapter text from EPUB/PDF into one .txt per chapter.
|
||||
3. CONTEXT → Gather everything the brain knows about the reader.
|
||||
4. ANALYZE → `gbrain book-mirror` fans out N read-only subagents.
|
||||
5. ASSEMBLE → CLI reads each child result and writes one put_page.
|
||||
6. PDF → Optional: render via skills/brain-pdf for delivery.
|
||||
```
|
||||
|
||||
## 1. Acquiring the book
|
||||
|
||||
book-acquisition (legal-grey-area downloader) was deliberately not shipped
|
||||
in this skill wave. The user drops the EPUB/PDF manually. Common paths the
|
||||
user might use:
|
||||
|
||||
```bash
|
||||
# User-supplied path
|
||||
ls path/to/book.epub
|
||||
ls path/to/book.pdf
|
||||
|
||||
# Or already in the brain repo (recommended for tracking)
|
||||
ls $BRAIN_DIR/media/books/
|
||||
```
|
||||
|
||||
Resolve `$BRAIN_DIR` from the gbrain config (`gbrain config get sync.repo_path`)
|
||||
or accept it from the user.
|
||||
|
||||
## 2. Text extraction
|
||||
|
||||
Goal: one `.txt` file per chapter under a temp directory. The agent has
|
||||
shell + python access; the CLI is downstream of this and takes the
|
||||
extracted directory as input.
|
||||
|
||||
### EPUB
|
||||
|
||||
```bash
|
||||
SLUG="this-book" # kebab-case
|
||||
WORK="$(mktemp -d)/$SLUG"
|
||||
mkdir -p "$WORK/chapters"
|
||||
unzip -o path/to/book.epub -d "$WORK/unpacked"
|
||||
|
||||
# Find content files (XHTML/HTML), sorted (chapter order = sort order)
|
||||
find "$WORK/unpacked" -name "*.xhtml" -o -name "*.html" | sort > "$WORK/files.txt"
|
||||
|
||||
# Strip HTML to text per chapter
|
||||
python3 - <<'PY'
|
||||
from bs4 import BeautifulSoup
|
||||
import os, sys
|
||||
work = os.environ['WORK']
|
||||
files = open(f'{work}/files.txt').read().splitlines()
|
||||
for i, path in enumerate(files, 1):
|
||||
html = open(path, encoding='utf-8', errors='replace').read()
|
||||
text = BeautifulSoup(html, 'html.parser').get_text('\n')
|
||||
text = '\n'.join(line.strip() for line in text.splitlines() if line.strip())
|
||||
with open(f'{work}/chapters/{i:02d}.txt', 'w') as f:
|
||||
f.write(text)
|
||||
PY
|
||||
```
|
||||
|
||||
If `bs4` is missing: `pip3 install beautifulsoup4 lxml`.
|
||||
|
||||
Inspect the chapter files to identify which are real chapters vs front
|
||||
matter (TOC, copyright, acknowledgments). Often the EPUB ships one file
|
||||
per chapter; sometimes multiple chapters per file. Use
|
||||
`head -5 "$WORK/chapters/"*.txt` to spot-check.
|
||||
|
||||
### PDF
|
||||
|
||||
```bash
|
||||
pdftotext -layout path/to/book.pdf "$WORK/full.txt"
|
||||
```
|
||||
|
||||
Then split by chapter heading (look for "Chapter N", "CHAPTER N", or
|
||||
all-caps title lines) using `awk` or `python`. If the PDF is a scan with
|
||||
no embedded text, fall back to OCR via `skills/brain-pdf` or another
|
||||
vision tool.
|
||||
|
||||
### Quality check
|
||||
|
||||
For each chapter file:
|
||||
|
||||
- Word count > 1500 (typical chapter range 2k–8k words).
|
||||
- No HTML tags.
|
||||
- Paragraphs preserved with `\n\n`.
|
||||
|
||||
Save a `chapters/INDEX.md` mapping chapter number → title → file → word
|
||||
count for reference.
|
||||
|
||||
## 3. Context gathering
|
||||
|
||||
This is the most critical step. The right column is only as good as the
|
||||
context fed to each chapter subagent.
|
||||
|
||||
### What to pull
|
||||
|
||||
1. **Templates: USER.md and SOUL.md** if the user maintains them
|
||||
(gbrain ships templates at `templates/USER.md` and `templates/SOUL.md`;
|
||||
they live in the brain repo when populated). Read full.
|
||||
2. **Recent daily memory** — last 14 days of brain pages under
|
||||
`wiki/personal/reflections/` or wherever the user files daily notes.
|
||||
3. **Topic-relevant brain searches** tuned to the book's themes:
|
||||
- `gbrain query "marriage"`, `gbrain query "couples therapy"` for a
|
||||
marriage book.
|
||||
- `gbrain query "founders"`, `gbrain query "fundraising"` for a
|
||||
business book.
|
||||
- `gbrain query "shame"`, `gbrain query "anger"` for a psychology book.
|
||||
4. **Brain pages for relevant entities** — `gbrain query "<name>"` for
|
||||
people who will likely come up.
|
||||
5. **Standing patterns** — anything in the user's reflections or
|
||||
originals that's been recurring.
|
||||
|
||||
### Assemble a context pack
|
||||
|
||||
Write everything to a single file the CLI can read:
|
||||
|
||||
```bash
|
||||
CONTEXT="$WORK/context.md"
|
||||
{
|
||||
echo "## USER.md (if any)"
|
||||
[ -f "$BRAIN_DIR/USER.md" ] && cat "$BRAIN_DIR/USER.md"
|
||||
echo
|
||||
echo "## SOUL.md (if any)"
|
||||
[ -f "$BRAIN_DIR/SOUL.md" ] && cat "$BRAIN_DIR/SOUL.md"
|
||||
echo
|
||||
echo "## Recent reflections (last 14 days)"
|
||||
# Pull recent daily reflections — adapt to the user's filing scheme
|
||||
# ...
|
||||
echo
|
||||
echo "## Topic-relevant brain pages"
|
||||
# gbrain query the book's key themes, embed top results
|
||||
# ...
|
||||
echo
|
||||
echo "## Themes & cruxes"
|
||||
# A 1-page summary, written by the agent, calling out:
|
||||
# - What's currently active in the user's life that this book intersects
|
||||
# - Specific quotes from the user that map to book themes
|
||||
# - People and dates that should appear in the right column
|
||||
} > "$CONTEXT"
|
||||
```
|
||||
|
||||
Make this dense. It's read by every chapter subagent.
|
||||
|
||||
## 4. Analysis: invoke `gbrain book-mirror`
|
||||
|
||||
```bash
|
||||
gbrain book-mirror \
|
||||
--chapters-dir "$WORK/chapters" \
|
||||
--context-file "$CONTEXT" \
|
||||
--slug "$SLUG" \
|
||||
--title "Book Title Goes Here" \
|
||||
--author "Author Name" \
|
||||
--model claude-opus-4-7
|
||||
```
|
||||
|
||||
The CLI:
|
||||
|
||||
- Validates inputs and loads chapter files.
|
||||
- Prints a cost estimate (~$0.30/chapter at Opus) and prompts to confirm.
|
||||
- Submits N child subagent jobs with read-only `allowed_tools`.
|
||||
- Waits for every child to complete.
|
||||
- Reads each child's `job.result` (the markdown analysis text).
|
||||
- Assembles all chapters into one page with frontmatter + intro + per-chapter
|
||||
sections + closing.
|
||||
- Writes ONE `put_page` to `media/books/<slug>-personalized.md`.
|
||||
- Reports a JSON envelope on stdout:
|
||||
`{"slug": "...", "chapters_total": N, "chapters_completed": N, "chapters_failed": 0}`.
|
||||
|
||||
If any chapter failed, the CLI exits 1 and the user can re-run — idempotency
|
||||
keys (`book-mirror:<slug>:ch-<N>`) deduplicate completed chapters at the
|
||||
queue level, so retry is cheap.
|
||||
|
||||
### Model: Opus by default
|
||||
|
||||
The default model is `claude-opus-4-7`. Sonnet works (use `--model
|
||||
claude-sonnet-4-6`) but the right-column quality drops noticeably — the
|
||||
texture that makes the analysis read like a therapist who knows the user
|
||||
needs Opus-grade reasoning.
|
||||
|
||||
### Cost gate
|
||||
|
||||
The CLI refuses to spend in a non-TTY context without `--yes`. CI / scripted
|
||||
invocations must pass `--yes` explicitly. TTY users get a `[y/N]` prompt
|
||||
before submission.
|
||||
|
||||
## 5. PDF (optional)
|
||||
|
||||
After the brain page is written, render to PDF using `skills/brain-pdf`:
|
||||
|
||||
```bash
|
||||
gbrain put_page # already done by the CLI; nothing to add here
|
||||
# Then invoke brain-pdf:
|
||||
# (see skills/brain-pdf/SKILL.md for the make-pdf invocation)
|
||||
```
|
||||
|
||||
## 6. Fact-check and cross-link
|
||||
|
||||
After the page lands, run a fact-check pass on factual claims about the
|
||||
reader (parents, siblings, marriage history, jobs, heritage). Common error
|
||||
patterns to look for:
|
||||
|
||||
- Conflating the reader's parents' relationship with patterns in extended
|
||||
family.
|
||||
- Inventing therapy backstory ("after his parents' divorce…") when the
|
||||
reader's parents are still together.
|
||||
- Wrong number/age of children, wrong spouse / kid / sibling names.
|
||||
|
||||
If you can't verify a claim, remove it. Better to lose texture than to
|
||||
introduce a falsehood.
|
||||
|
||||
Cross-link entities mentioned in the analysis:
|
||||
|
||||
- For every person the right column references with a brain page, add a
|
||||
back-link from `people/<slug>` to the new `media/books/<slug>-personalized`
|
||||
page (per `conventions/quality.md` Iron Law).
|
||||
|
||||
## Quality bar (the bar)
|
||||
|
||||
The **left column** should:
|
||||
|
||||
- Preserve the author's actual stories, statistics, frameworks, examples.
|
||||
- Quote memorable phrases verbatim.
|
||||
- Be detailed enough that the reader could skip the book and not lose much.
|
||||
|
||||
The **right column** should:
|
||||
|
||||
- Use the reader's *actual quoted words* from the context pack.
|
||||
- Reference *specific* dates, situations, people by name.
|
||||
- Read like a therapist who knows the reader is leaving notes in the margins.
|
||||
- Be plain about direct hits ("This is exactly the [name a real situation]").
|
||||
- Be honest about misses ("This chapter is less directly relevant
|
||||
because…"). Don't force connections.
|
||||
|
||||
The **whole document** should feel like one coherent voice, calibrated to
|
||||
the reader's actual life rather than a generic profile, and honest about
|
||||
where the book's framing breaks down for this specific reader.
|
||||
|
||||
## Anti-patterns (do not do these)
|
||||
|
||||
- ❌ **Skimming chapters.** Standing instruction: preserve detail.
|
||||
- ❌ **Generic right column.** "This might apply if you've ever felt…" →
|
||||
kill on sight.
|
||||
- ❌ **Factual errors about the reader's life.** Always fact-check after
|
||||
assembly.
|
||||
- ❌ **Giving the subagent put_page access.** Trust contract is read-only;
|
||||
the CLI does the writing.
|
||||
- ❌ **Forcing connections.** If a chapter doesn't apply, say so plainly.
|
||||
- ❌ **Sycophancy or moralizing in the right column.** No "you should…",
|
||||
no "consider…", no "perhaps it's time to…".
|
||||
- ❌ **Truncating the LEFT column.** The book's actual content needs to
|
||||
survive.
|
||||
|
||||
## Output checklist
|
||||
|
||||
- [ ] Book file exists locally (path known).
|
||||
- [ ] Chapter texts under `$WORK/chapters/*.txt` with sane word counts.
|
||||
- [ ] Context pack at `$WORK/context.md` is dense.
|
||||
- [ ] `gbrain book-mirror --chapters-dir … --context-file … --slug … --title …` returned exit 0.
|
||||
- [ ] `media/books/<slug>-personalized.md` exists in the brain.
|
||||
- [ ] Fact-check pass complete (no errors against USER.md or other source-of-truth pages).
|
||||
- [ ] Cross-links added from referenced people/companies.
|
||||
- [ ] Optional: PDF rendered via brain-pdf and delivered.
|
||||
|
||||
## Related skills
|
||||
|
||||
- `skills/brain-pdf/SKILL.md` — render the personalized page to PDF.
|
||||
- `skills/strategic-reading/SKILL.md` — read a book through a specific
|
||||
problem-lens instead of personalizing to the whole reader.
|
||||
- `skills/article-enrichment/SKILL.md` — same shape applied to articles
|
||||
rather than books.
|
||||
|
||||
|
||||
## Contract
|
||||
|
||||
This skill guarantees:
|
||||
|
||||
- Routing matches the canonical triggers in the frontmatter.
|
||||
- Output written under the directories listed in `writes_to:` (when applicable).
|
||||
- Conventions referenced (`quality.md`, `brain-first.md`, `_brain-filing-rules.md`) are followed.
|
||||
- Privacy contract preserved: no real names, no fork-specific filesystem path literals, no upstream-fork references.
|
||||
|
||||
The full behavior contract is documented in the body sections above; this section exists for the conformance test.
|
||||
|
||||
## Output Format
|
||||
|
||||
The skill's output shape is documented inline in the body sections above (see "Output", "Brain page format", or equivalent). The literal section header here exists for the conformance test (`test/skills-conformance.test.ts`).
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
The full anti-pattern list is in the body sections above; this header exists for the conformance test if the body uses a different casing.
|
||||
@@ -0,0 +1,15 @@
|
||||
// Routing eval fixtures for skills/book-mirror. Each intent contains
|
||||
// at least one trigger string as substring (structural matcher
|
||||
// requirement) while still paraphrasing real user phrasing.
|
||||
// Adversarial cases at the bottom guard the media-ingest <-> book-mirror
|
||||
// routing regression flagged by R1 + R2 (IRON RULE).
|
||||
{"intent":"Please make a personalized version of this book using the brain context","expected_skill":"book-mirror"}
|
||||
{"intent":"Mirror this book — left column the chapters, right column my actual life","expected_skill":"book-mirror"}
|
||||
{"intent":"Run a two-column book analysis with brain context","expected_skill":"book-mirror"}
|
||||
{"intent":"Apply this book to my life — chapter-by-chapter mapping to the brain","expected_skill":"book-mirror"}
|
||||
{"intent":"How does this book apply to me — produce a personalized version","expected_skill":"book-mirror"}
|
||||
// Adversarial: phrasing that pattern-matches media-ingest. IRON RULE:
|
||||
// book-mirror should NOT win on these — they're generic ingest.
|
||||
{"intent":"Process this book and ingest it into my brain","expected_skill":"media-ingest","ambiguous_with":["book-mirror"]}
|
||||
{"intent":"Ingest this PDF book and extract the entities","expected_skill":"media-ingest","ambiguous_with":["book-mirror"]}
|
||||
{"intent":"Just summarize this book — I don't need it personalized to me","expected_skill":"media-ingest","ambiguous_with":["book-mirror"]}
|
||||
@@ -0,0 +1,186 @@
|
||||
---
|
||||
name: brain-pdf
|
||||
version: 0.1.0
|
||||
description: Generate a publication-quality PDF from any brain page via the gstack make-pdf binary. Strips YAML frontmatter, sanitizes emoji, applies running headers and page numbers. Brain page is always the source of truth; PDF is a rendering.
|
||||
triggers:
|
||||
- "make pdf from brain"
|
||||
- "brain pdf"
|
||||
- "convert brain page to pdf"
|
||||
- "publish this page as pdf"
|
||||
- "export brain page"
|
||||
---
|
||||
|
||||
# brain-pdf — Render a Brain Page to Publication-Quality PDF
|
||||
|
||||
> **Convention:** see [conventions/quality.md](../conventions/quality.md) for
|
||||
> output rules. The PDF is a rendering — never the primary artifact. If a
|
||||
> PDF exists, the source brain page exists behind it.
|
||||
|
||||
## The rule
|
||||
|
||||
The brain page is ALWAYS the source of truth. The PDF is a rendering of
|
||||
it, never a standalone artifact. If a PDF exists somewhere, the brain
|
||||
page must exist behind it.
|
||||
|
||||
## What this does
|
||||
|
||||
Renders a brain page (markdown with frontmatter) into a
|
||||
publication-quality PDF using the gstack `make-pdf` binary. Output is
|
||||
suitable for:
|
||||
|
||||
- Sharing a personalized book mirror via email or Telegram
|
||||
- Delivering a strategic-reading playbook as a clean read
|
||||
- Producing a briefing or report with running headers and page numbers
|
||||
- Archiving a long-form essay in a portable format
|
||||
|
||||
## Prerequisite: gstack make-pdf
|
||||
|
||||
This skill depends on the gstack `make-pdf` binary at:
|
||||
|
||||
```
|
||||
$HOME/.claude/skills/gstack/make-pdf/dist/pdf
|
||||
```
|
||||
|
||||
The user must have gstack co-installed. If absent, the skill cannot run.
|
||||
A future v0.26+ may bundle a fallback PDF renderer; for v0.25.1 gstack
|
||||
is a soft prereq.
|
||||
|
||||
Verify it exists before invoking:
|
||||
|
||||
```bash
|
||||
P="$HOME/.claude/skills/gstack/make-pdf/dist/pdf"
|
||||
[ -x "$P" ] || { echo "make-pdf not installed; install gstack" >&2; exit 1; }
|
||||
```
|
||||
|
||||
## Workflow
|
||||
|
||||
```
|
||||
1. RESOLVE → Confirm the brain page exists (gbrain get <slug>).
|
||||
2. STRIP → Remove YAML frontmatter — the renderer would otherwise
|
||||
dump it as a full page of raw metadata text.
|
||||
3. RENDER → Invoke make-pdf with sane defaults (no --cover, no --toc).
|
||||
4. DELIVER → Hand the PDF to the requester via the agent's preferred
|
||||
channel (do not use raw `MEDIA:` tags on Telegram —
|
||||
they fail silently).
|
||||
```
|
||||
|
||||
## Invocation
|
||||
|
||||
```bash
|
||||
SLUG="path/to/page"
|
||||
P="$HOME/.claude/skills/gstack/make-pdf/dist/pdf"
|
||||
|
||||
# 1. Confirm the page exists.
|
||||
gbrain get "$SLUG" > /dev/null || { echo "Page $SLUG not found" >&2; exit 1; }
|
||||
|
||||
# 2. Get the raw markdown. Two paths: read from the brain repo (if user
|
||||
# syncs locally) OR ask gbrain for the body via the API.
|
||||
BRAIN_DIR=$(gbrain config get sync.repo_path 2>/dev/null || echo)
|
||||
if [ -n "$BRAIN_DIR" ] && [ -f "$BRAIN_DIR/$SLUG.md" ]; then
|
||||
RAW="$BRAIN_DIR/$SLUG.md"
|
||||
else
|
||||
RAW=$(mktemp /tmp/brain-page-XXXXXX.md)
|
||||
gbrain get "$SLUG" --raw > "$RAW" # whatever flag exposes raw body
|
||||
fi
|
||||
|
||||
# 3. Strip YAML frontmatter — sed: skip the opening '---' through the
|
||||
# closing '---' (lines 1..N), then keep everything after.
|
||||
CLEAN=$(mktemp /tmp/brain-page-clean-XXXXXX.md)
|
||||
sed '1{/^---$/!q}; /^---$/,/^---$/d' "$RAW" > "$CLEAN"
|
||||
|
||||
# 4. Render. NO --cover, NO --toc by default — they look corporate
|
||||
# and waste space. Add them only if explicitly requested.
|
||||
OUT="/tmp/$(basename "$SLUG").pdf"
|
||||
CONTAINER=1 "$P" generate "$CLEAN" "$OUT"
|
||||
|
||||
echo "Rendered: $OUT"
|
||||
```
|
||||
|
||||
`CONTAINER=1` is mandatory in containerized environments — it tells
|
||||
Playwright to skip Chromium sandboxing. Harmless on bare-metal.
|
||||
|
||||
## Common patterns
|
||||
|
||||
```bash
|
||||
# Default — clean PDF, no cover, no TOC
|
||||
brain-pdf <slug>
|
||||
|
||||
# Draft watermark for in-progress work
|
||||
CONTAINER=1 "$P" generate --watermark DRAFT "$CLEAN" "$OUT"
|
||||
|
||||
# Optional cover + TOC if the user explicitly asks
|
||||
CONTAINER=1 "$P" generate --cover --toc "$CLEAN" "$OUT"
|
||||
|
||||
# Custom title + author override (otherwise pulled from frontmatter)
|
||||
CONTAINER=1 "$P" generate --title "Custom Title" --author "Custom Author" "$CLEAN" "$OUT"
|
||||
```
|
||||
|
||||
## Defaults: NO cover, NO TOC
|
||||
|
||||
These flags are off by default because they look corporate and waste
|
||||
space on most personal-knowledge content. Only add them when the user
|
||||
explicitly asks for "formal" output (e.g., something they're sending to
|
||||
a board or printing as a deliverable).
|
||||
|
||||
## Font requirements
|
||||
|
||||
The renderer needs:
|
||||
|
||||
- `fonts-liberation` (Helvetica/Arial substitute)
|
||||
- `fonts-noto-cjk` (Chinese/Japanese/Korean characters)
|
||||
- Minimum body font size: 10pt (page chrome 9pt)
|
||||
- Body text: 11pt
|
||||
|
||||
If running in an environment without these fonts, install them via the
|
||||
host's package manager (`apt install fonts-liberation fonts-noto-cjk` on
|
||||
Debian/Ubuntu containers).
|
||||
|
||||
## Delivery
|
||||
|
||||
After rendering, deliver via the agent's preferred channel:
|
||||
|
||||
- **Telegram:** use the `message` tool with `filePath="/tmp/<slug>.pdf"`
|
||||
attachment. NEVER use raw `MEDIA:` tags — they fail silently.
|
||||
- **Email:** attach via the host's email tool.
|
||||
- **Direct file response:** print the PDF path; the user can pull it
|
||||
manually.
|
||||
|
||||
Always include the brain page link in the delivery message so the user
|
||||
can also see it on GitHub / locally. The PDF is a rendering; the source
|
||||
is the artifact.
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
- ❌ Generating a PDF without first confirming the brain page exists.
|
||||
No source = no PDF.
|
||||
- ❌ Skipping the frontmatter strip. The renderer dumps frontmatter as
|
||||
raw text on the first page; ugly.
|
||||
- ❌ Skipping emoji sanitization. Emoji that don't map to the rendering
|
||||
font show up as `□` boxes.
|
||||
- ❌ Adding `--cover` or `--toc` by default. Off unless asked.
|
||||
- ❌ Using raw `MEDIA:` tags for Telegram delivery. Use the `message`
|
||||
tool with `filePath`.
|
||||
|
||||
## Related skills
|
||||
|
||||
- `skills/book-mirror/SKILL.md` — produces a brain page that's a
|
||||
natural input to brain-pdf (chapter-by-chapter personalized analysis).
|
||||
- `skills/strategic-reading/SKILL.md` — same shape, problem-lens variant.
|
||||
- `skills/publish/SKILL.md` — share brain pages as password-protected
|
||||
HTML (different rendering target).
|
||||
|
||||
|
||||
## Contract
|
||||
|
||||
This skill guarantees:
|
||||
|
||||
- Routing matches the canonical triggers in the frontmatter.
|
||||
- Output written under the directories listed in `writes_to:` (when applicable).
|
||||
- Conventions referenced (`quality.md`, `brain-first.md`, `_brain-filing-rules.md`) are followed.
|
||||
- Privacy contract preserved: no real names, no fork-specific filesystem path literals, no upstream-fork references.
|
||||
|
||||
The full behavior contract is documented in the body sections above; this section exists for the conformance test.
|
||||
|
||||
## Output Format
|
||||
|
||||
The skill's output shape is documented inline in the body sections above (see "Output", "Brain page format", or equivalent). The literal section header here exists for the conformance test (`test/skills-conformance.test.ts`).
|
||||
@@ -0,0 +1,7 @@
|
||||
// Routing eval fixtures for skills/brain-pdf. Each intent
|
||||
// includes at least one trigger string as substring.
|
||||
{"intent":"Please make pdf from brain page media/books/this-book-personalized","expected_skill":"brain-pdf"}
|
||||
{"intent":"Run brain pdf on this strategy doc for the meeting","expected_skill":"brain-pdf"}
|
||||
{"intent":"Convert brain page to pdf with a draft watermark","expected_skill":"brain-pdf"}
|
||||
{"intent":"Publish this page as pdf for the printable deliverable","expected_skill":"brain-pdf"}
|
||||
{"intent":"Export brain page to a clean PDF I can send","expected_skill":"brain-pdf"}
|
||||
+170
-18
@@ -1,13 +1,17 @@
|
||||
---
|
||||
name: citation-fixer
|
||||
version: 1.0.0
|
||||
version: 1.1.0
|
||||
description: |
|
||||
Audit and fix citation formatting across brain pages. Ensures every fact has
|
||||
an inline [Source: ...] citation matching the standard format.
|
||||
an inline [Source: ...] citation matching the standard format. Extended in
|
||||
v0.25.1: scans for broken tweet/post references that lack actual URLs and
|
||||
resolves them via the host's X / Twitter API integration.
|
||||
triggers:
|
||||
- "fix citations"
|
||||
- "fix broken citations"
|
||||
- "citation audit"
|
||||
- "check citations"
|
||||
- "citation fixer"
|
||||
tools:
|
||||
- search
|
||||
- get_page
|
||||
@@ -18,39 +22,187 @@ mutating: true
|
||||
|
||||
# Citation Fixer Skill
|
||||
|
||||
> **Convention:** see [conventions/quality.md](../conventions/quality.md) for
|
||||
> the canonical citation format every fix should match.
|
||||
>
|
||||
> **Output rule:** all links MUST be deterministic (built from API data,
|
||||
> not composed by LLM). See [_output-rules.md](../_output-rules.md).
|
||||
|
||||
## Contract
|
||||
|
||||
This skill guarantees:
|
||||
- Every brain page is scanned for citation compliance
|
||||
- Missing citations are flagged with specific location
|
||||
- Malformed citations are fixed to match the standard format
|
||||
- Results reported with counts (scanned, fixed, remaining)
|
||||
|
||||
- Every brain page is scanned for citation compliance.
|
||||
- Missing citations are flagged with specific location.
|
||||
- Malformed citations are fixed to match the standard format.
|
||||
- **(v0.25.1)** Tweet / post references without URLs are resolved via
|
||||
X API and patched with deterministic `https://x.com/<handle>/status/<id>`
|
||||
links.
|
||||
- Results reported with counts (scanned, fixed, remaining).
|
||||
|
||||
## Phases
|
||||
|
||||
1. **Scan pages.** List pages and read each one, checking for inline `[Source: ...]` citations.
|
||||
1. **Scan pages.** List pages and read each one, checking for inline
|
||||
`[Source: ...]` citations.
|
||||
2. **Identify issues:**
|
||||
- Facts without any citation
|
||||
- Citations missing date
|
||||
- Citations missing source type
|
||||
- Citations with wrong format
|
||||
3. **Fix format issues.** Rewrite malformed citations to match `skills/conventions/quality.md`.
|
||||
4. **Report results.** Count: pages scanned, citations found, issues fixed, remaining gaps.
|
||||
- **(v0.25.1)** Tweet references without `x.com` URLs
|
||||
3. **Fix format issues.** Rewrite malformed citations to match
|
||||
`conventions/quality.md`.
|
||||
4. **(v0.25.1) Resolve tweet references** via the X API integration.
|
||||
5. **Report results.** Count: pages scanned, citations found, issues
|
||||
fixed, tweets resolved, remaining gaps.
|
||||
|
||||
## Output Format
|
||||
## Tweet resolution pipeline (v0.25.1 extension)
|
||||
|
||||
For each broken tweet reference, follow this chain. The actual API call
|
||||
goes through whatever X integration the host has configured (typical
|
||||
shape: a recipe under `recipes/x-api/` with handle / search-all
|
||||
endpoints).
|
||||
|
||||
### Step 1: Identify broken references
|
||||
|
||||
Scan the page for patterns that indicate tweet references without URLs:
|
||||
|
||||
- Contains words like `tweeted`, `posted`, `said on X`, `RT`, `retweet`,
|
||||
`X post`
|
||||
- Contains quoted text that looks like a tweet (short, punchy, often
|
||||
starts with a quote)
|
||||
- Has `[Source: ... X/Twitter ...]` without an `x.com` URL
|
||||
- References engagement metrics (likes, impressions) without a link
|
||||
|
||||
### Step 2: Extract searchable content
|
||||
|
||||
From each broken reference, extract:
|
||||
|
||||
- The **handle** (if mentioned: `@<username>`)
|
||||
- The **quoted text** (if available)
|
||||
- The **approximate date** (often present in surrounding timeline entries)
|
||||
|
||||
### Step 3: Search for the actual tweet
|
||||
|
||||
Use the host's X API integration. Query patterns:
|
||||
|
||||
```
|
||||
# Handle + quoted text:
|
||||
from:<handle> "<exact quote fragment>"
|
||||
|
||||
# Quoted text only:
|
||||
"<exact quote fragment>"
|
||||
|
||||
# Original of a retweet:
|
||||
"<exact quote>" -is:retweet
|
||||
```
|
||||
|
||||
### Step 4: Verify and extract metadata
|
||||
|
||||
Once a candidate is found:
|
||||
|
||||
- Confirm the text matches the quoted fragment.
|
||||
- Pull the tweet id, author handle, engagement metrics (likes / RTs /
|
||||
impressions).
|
||||
- Construct the URL: `https://x.com/<handle>/status/<tweet_id>`.
|
||||
|
||||
### Step 5: Patch the brain page
|
||||
|
||||
Replace the broken citation with a proper one:
|
||||
|
||||
**Before:**
|
||||
|
||||
```
|
||||
"<quote fragment>" [Source: <some hand-wavy attribution>]
|
||||
```
|
||||
|
||||
**After:**
|
||||
|
||||
```
|
||||
"<full verified quote>" — <N> likes, <N> RTs, <N> impressions
|
||||
[Source: [X/<handle>, YYYY-MM-DD](https://x.com/<handle>/status/<tweet_id>)]
|
||||
```
|
||||
|
||||
## Batch mode
|
||||
|
||||
When sweeping many pages:
|
||||
|
||||
### Find candidate pages
|
||||
|
||||
```bash
|
||||
# Pages mentioning tweets but with no x.com links
|
||||
for f in $(find . -name "*.md" -not -path "./node_modules/*"); do
|
||||
refs=$(grep -ci "tweet\|posted\|x post\|RT\|retweet\|said on X" "$f")
|
||||
links=$(grep -c "x.com/.*/status/" "$f")
|
||||
if [ "$refs" -gt 2 ] && [ "$links" -eq 0 ]; then
|
||||
echo "$f"
|
||||
fi
|
||||
done
|
||||
```
|
||||
|
||||
### Priority order
|
||||
|
||||
1. Recently created / updated pages — fresh broken refs are easiest to
|
||||
resolve while context is fresh.
|
||||
2. High-traffic pages (frequent reads / writes from other skills).
|
||||
3. Everything else — bulk cleanup over time.
|
||||
|
||||
### Rate limiting
|
||||
|
||||
- X API: respect the host's tier limits; don't hammer.
|
||||
- Target ~50 pages per batch run.
|
||||
- 1-3 API calls per page (search + verify).
|
||||
- Batch-commit every 10-20 pages so a partial failure doesn't lose
|
||||
progress.
|
||||
|
||||
## Output format
|
||||
|
||||
```
|
||||
Citation Audit Report
|
||||
=====================
|
||||
Pages scanned: N
|
||||
Citations found: N
|
||||
Issues fixed: N
|
||||
Remaining gaps: N (pages with uncitable facts)
|
||||
Pages scanned: N
|
||||
Citations found: N
|
||||
Issues fixed: N
|
||||
Tweet links resolved: N
|
||||
Remaining gaps: N (pages with uncitable facts)
|
||||
```
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
- Inventing citations for facts that have no source
|
||||
- Removing facts that lack citations (flag them, don't delete)
|
||||
- Fixing citations without reading the full page context
|
||||
- Batch-fixing without checking quality (test-before-bulk convention)
|
||||
- ❌ Inventing citations for facts that have no source. Flag them.
|
||||
- ❌ Removing facts that lack citations (flag them; don't delete).
|
||||
- ❌ Fixing citations without reading the full page context.
|
||||
- ❌ Batch-fixing without checking quality on a sample first
|
||||
(see `conventions/test-before-bulk.md`).
|
||||
- ❌ Composing tweet URLs by guessing the tweet id. Always go through
|
||||
the X API; deterministic links only.
|
||||
|
||||
## Integration
|
||||
|
||||
This skill can be called:
|
||||
|
||||
- **Manually** — "fix citations on this page"
|
||||
- **As a batch cron** — weekly sweep of pages with broken refs
|
||||
- **By other skills** — `enrich` or `media-ingest` can call citation-fixer
|
||||
before commit to validate output
|
||||
|
||||
## Metrics
|
||||
|
||||
If running as a recurring batch, track state in a small JSON file under
|
||||
`~/.gbrain/citation-fixer-state.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"last_run": "2026-04-15T...",
|
||||
"pages_scanned": 0,
|
||||
"citations_fixed": 0,
|
||||
"tweet_links_resolved": 0,
|
||||
"citations_unresolvable": 0,
|
||||
"pages_remaining": 1424
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## Output Format
|
||||
|
||||
The skill's output shape is documented inline in the body sections above (see "Output", "Brain page format", or equivalent). The literal section header here exists for the conformance test (`test/skills-conformance.test.ts`).
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
---
|
||||
name: concept-synthesis
|
||||
version: 0.1.0
|
||||
description: Deduplicate and synthesize raw concept stubs into a tiered intellectual map (T1 Canon to T4 Riff), tracing idea evolution across sources over time. Transforms thousands of raw concept pages into a curated intellectual fingerprint.
|
||||
triggers:
|
||||
- "concept synthesis"
|
||||
- "synthesize my concepts"
|
||||
- "find patterns across my notes"
|
||||
- "build my intellectual map"
|
||||
- "trace idea evolution"
|
||||
mutating: true
|
||||
writes_pages: true
|
||||
writes_to:
|
||||
- concepts/
|
||||
---
|
||||
|
||||
# concept-synthesis — From Raw Stubs to Intellectual Map
|
||||
|
||||
> **Convention:** see [conventions/quality.md](../conventions/quality.md) for
|
||||
> back-link enforcement and quote-fidelity requirements.
|
||||
>
|
||||
> **Convention:** see [_brain-filing-rules.md](../_brain-filing-rules.md) —
|
||||
> output files under `concepts/` per the primary-subject rule.
|
||||
|
||||
## What this solves
|
||||
|
||||
Many ingestion pipelines (signal-detector, idea-ingest, voice-note-ingest)
|
||||
create a concept page for every idea mentioned. Over months this produces:
|
||||
|
||||
- Thousands of stub pages, many duplicates or near-duplicates
|
||||
- Timeline entries that repeat the same source across multiple concept pages
|
||||
- No synthesis — just "the user mentioned X on this date"
|
||||
- No tier assignments — everything flat
|
||||
- No clustering — related ideas aren't linked
|
||||
|
||||
This skill transforms that raw material into a curated intellectual map.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Phase 1: Dedup + merge (deterministic)
|
||||
N stubs → ~N/4 canonical concepts
|
||||
├── Jaccard dedup (word-overlap on titles + first-paragraph)
|
||||
├── Substring dedup ("founder mode" vs "founder mode vs manager mode")
|
||||
├── Semantic dedup (LLM: "are these the same idea?")
|
||||
└── Merge timelines + aliases from duplicates into the canonical page
|
||||
|
||||
Phase 2: Score + tier (deterministic + heuristic)
|
||||
Each canonical concept → scored and tiered
|
||||
├── Frequency: distinct sources referencing this concept
|
||||
├── Timespan: first mention → last mention in days
|
||||
├── Breadth: distinct months it appears in
|
||||
├── Engagement: avg engagement on concept-bearing sources (if available)
|
||||
└── Tier: T1 Canon | T2 Developing | T3 Speculative | T4 Riff
|
||||
|
||||
Phase 3: Synthesize (LLM, T1+T2 only)
|
||||
T1 + T2 concepts → rich synthesis
|
||||
├── Evolution narrative: how the idea sharpened over time
|
||||
├── Best articulation: highest-engagement or most precise quote
|
||||
├── Related concepts: cross-links to other concepts
|
||||
├── Context: what was happening when this idea emerged / evolved
|
||||
└── Counter-positions: what this idea argues against
|
||||
|
||||
Phase 4: Cluster + map (LLM)
|
||||
All tiered concepts → intellectual clusters
|
||||
├── Group related concepts into domains (auto-named via LLM)
|
||||
├── Generate cluster summary pages
|
||||
├── Build a master concepts/README.md with the full map
|
||||
└── Identify idea genealogies (concept A → evolved into concept B)
|
||||
```
|
||||
|
||||
## Invocation
|
||||
|
||||
The skill is markdown agent instructions. The agent uses gbrain's
|
||||
existing operations + LLM passes:
|
||||
|
||||
```bash
|
||||
# 1. List all concept pages
|
||||
gbrain query "type:concept" --limit 10000 --json
|
||||
|
||||
# 2. Phase 1 dedup — agent applies Jaccard + substring locally,
|
||||
# then LLM passes to identify semantic duplicates.
|
||||
|
||||
# 3. Phase 2 tier — agent scores each canonical concept based on
|
||||
# frequency / timespan / breadth and writes tier into frontmatter.
|
||||
|
||||
# 4. Phase 3 synthesis — for each T1/T2, agent reads the timeline
|
||||
# + associated source pages and writes a synthesis section
|
||||
# onto the concept page via put_page.
|
||||
|
||||
# 5. Phase 4 clustering — agent reads the tiered concept list
|
||||
# and writes concepts/README.md with the full intellectual map.
|
||||
```
|
||||
|
||||
## Output: concept page format (post-synthesis)
|
||||
|
||||
### T1 Canon — full synthesis
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: "concept name"
|
||||
type: concept
|
||||
tier: 1
|
||||
tier_label: "Canon"
|
||||
mention_count: 18
|
||||
distinct_months: 8
|
||||
first_mention: "YYYY-MM-DD"
|
||||
last_mention: "YYYY-MM-DD"
|
||||
composite_score: 78.4
|
||||
aliases: ["alternate phrasing 1", "alternate phrasing 2"]
|
||||
related: ["sibling-concept-1", "sibling-concept-2"]
|
||||
---
|
||||
|
||||
# concept name
|
||||
|
||||
**Tier 1 — Canon** | 18 mentions across 8 months
|
||||
|
||||
## Synthesis
|
||||
|
||||
[2-4 paragraph narrative tracing how the idea evolved, what it means in
|
||||
the user's worldview, why it matters. Third-person analytical voice.]
|
||||
|
||||
## Best Articulation
|
||||
|
||||
> "Verbatim quote from a source — the most precise or highest-engagement
|
||||
> expression of this idea." — [Date](source-url)
|
||||
|
||||
## Evolution
|
||||
|
||||
| Period | Expression | Signal |
|
||||
|--------|-----------|--------|
|
||||
| YYYY-MM | "First articulation" | First use — aspiration frame |
|
||||
| YYYY-MM | "Sharpening" | Anti-pattern emerges |
|
||||
| YYYY-MM | "Peak form" | Cleanest expression |
|
||||
|
||||
## Related Concepts
|
||||
- [sibling concept](sibling-concept.md) — relationship description
|
||||
- [sibling concept](sibling-concept.md) — relationship description
|
||||
|
||||
## Timeline
|
||||
[Full timeline with deduped entries, quotes, source links]
|
||||
```
|
||||
|
||||
### T3 / T4 — stub only (no LLM synthesis)
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: "concept name"
|
||||
type: concept
|
||||
tier: 4
|
||||
tier_label: "Riff"
|
||||
mention_count: 1
|
||||
---
|
||||
|
||||
# concept name
|
||||
|
||||
**Tier 4 — Riff** | 1 mention
|
||||
|
||||
> "Quote from the source" — [Date](URL)
|
||||
```
|
||||
|
||||
## Output: cluster map at concepts/README.md
|
||||
|
||||
```markdown
|
||||
# Intellectual Universe
|
||||
|
||||
## Canon (T1) — N concepts
|
||||
The permanent intellectual fingerprint. Ideas that recur across years.
|
||||
|
||||
### [Cluster Name]
|
||||
- [concept-slug](concept-slug.md) — one-line characterization
|
||||
- ...
|
||||
|
||||
### [Other Cluster]
|
||||
- ...
|
||||
|
||||
## Developing (T2) — N concepts
|
||||
Sharpening. Might become canon.
|
||||
|
||||
## Speculative (T3) — N concepts
|
||||
Testing in public.
|
||||
|
||||
## Stats
|
||||
- Total concepts: N
|
||||
- T1 Canon: N
|
||||
- T2 Developing: N
|
||||
- T3 Speculative: N
|
||||
- T4 Riff: N
|
||||
- Earliest source: YYYY-MM-DD
|
||||
- Latest source: YYYY-MM-DD
|
||||
```
|
||||
|
||||
## Quality gates
|
||||
|
||||
### Dedup quality
|
||||
- No two concept pages should be "the same idea in different words."
|
||||
- Aliases preserved in frontmatter for search.
|
||||
- Run `gbrain query "type:concept"` and spot-check the count reduction.
|
||||
|
||||
### Tier quality
|
||||
- T1 should feel like "yes, that IS one of my recurring frameworks" —
|
||||
recognizable, recurring, sharp.
|
||||
- T2 should feel like "I'm working on this; it's getting clearer."
|
||||
- No concept should be T1 with < 4 months span or < 6 mentions.
|
||||
- No concept should be T4 with > 3 months span.
|
||||
|
||||
### Synthesis quality
|
||||
- Captures evolution, not just repetition.
|
||||
- Uses verbatim quotes, not paraphrase.
|
||||
- Links to related concepts (markdown links, not wiki-links).
|
||||
- Does NOT hallucinate sources or dates.
|
||||
|
||||
## Cron integration
|
||||
|
||||
This is heavy work. Run on a cadence, not on every signal:
|
||||
|
||||
- After a major ingestion batch completes (signal-detector burst, archive
|
||||
crawler run, etc.).
|
||||
- Weekly cron for incremental synthesis of newly-promoted T1/T2 concepts.
|
||||
- Manual trigger for a full re-synthesis when the corpus shifts
|
||||
significantly.
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
- ❌ Running synthesis on T3/T4 — wastes API budget on ideas that may
|
||||
never sharpen.
|
||||
- ❌ Hallucinating quotes or dates. The timeline must be verifiable
|
||||
against existing brain pages.
|
||||
- ❌ Generic cluster names ("Various Topics"). If you can't name the
|
||||
cluster, the cluster isn't real.
|
||||
- ❌ Re-synthesizing already-synthesized T1s without new source material.
|
||||
Idempotency-respect.
|
||||
|
||||
## Related skills
|
||||
|
||||
- `skills/signal-detector/SKILL.md` — creates raw concept stubs from text channels
|
||||
- `skills/voice-note-ingest/SKILL.md` — same for audio channels
|
||||
- `skills/idea-ingest/SKILL.md` — same for links / articles
|
||||
|
||||
|
||||
## Contract
|
||||
|
||||
This skill guarantees:
|
||||
|
||||
- Routing matches the canonical triggers in the frontmatter.
|
||||
- Output written under the directories listed in `writes_to:` (when applicable).
|
||||
- Conventions referenced (`quality.md`, `brain-first.md`, `_brain-filing-rules.md`) are followed.
|
||||
- Privacy contract preserved: no real names, no fork-specific filesystem path literals, no upstream-fork references.
|
||||
|
||||
The full behavior contract is documented in the body sections above; this section exists for the conformance test.
|
||||
|
||||
## Output Format
|
||||
|
||||
The skill's output shape is documented inline in the body sections above (see "Output", "Brain page format", or equivalent). The literal section header here exists for the conformance test (`test/skills-conformance.test.ts`).
|
||||
@@ -0,0 +1,7 @@
|
||||
// Routing eval fixtures for skills/concept-synthesis. Each intent
|
||||
// includes at least one trigger string as substring.
|
||||
{"intent":"Run concept synthesis on my brain — dedupe stubs and tier them","expected_skill":"concept-synthesis"}
|
||||
{"intent":"Synthesize my concepts into a tiered intellectual map","expected_skill":"concept-synthesis"}
|
||||
{"intent":"Find patterns across my notes and group them into clusters","expected_skill":"concept-synthesis"}
|
||||
{"intent":"Build my intellectual map — what's canon vs riff","expected_skill":"concept-synthesis"}
|
||||
{"intent":"Trace idea evolution across years of my reflections","expected_skill":"concept-synthesis"}
|
||||
@@ -1,21 +1,75 @@
|
||||
# Brain-First Lookup Convention
|
||||
|
||||
Before using ANY external API (web search, enrichment services, social APIs) to
|
||||
research a person, company, or topic, check the brain first.
|
||||
**Read this before doing ANY entity/person/company/fact lookup.**
|
||||
|
||||
## The 5-Step Lookup
|
||||
Sub-agents and fresh sessions inherit gbrain tools but not the knowledge of
|
||||
when and how to use them. This file is that knowledge.
|
||||
|
||||
1. `gbrain search "name"` — keyword search for existing pages
|
||||
2. `gbrain query "natural question about name"` — hybrid search for related context
|
||||
3. `gbrain get <slug>` — if you know the slug, read the full page
|
||||
4. Check backlinks: `gbrain get_backlinks <slug>` — who references this entity?
|
||||
5. Check timeline: `gbrain get_timeline <slug>` — recent events involving this entity
|
||||
## Available GBrain Tools
|
||||
|
||||
The brain almost always has something. External APIs fill gaps, not start from scratch.
|
||||
Your tool inventory includes these (prefixed `gbrain__` in OpenClaw):
|
||||
|
||||
## Why This Matters
|
||||
| Tool | Use for |
|
||||
|------|---------|
|
||||
| `gbrain__search` / `search` | Keyword search — fast, always works |
|
||||
| `gbrain__query` / `query` | Hybrid search (keyword + semantic) — best quality |
|
||||
| `gbrain__get_page` / `get_page` | Direct page read when you know the slug |
|
||||
| `gbrain__get_links` / `get_links` | Outgoing links from a page |
|
||||
| `gbrain__get_backlinks` / `get_backlinks` | Who references this entity |
|
||||
| `gbrain__get_timeline` / `get_timeline` | Dated events for an entity |
|
||||
| `gbrain__resolve_slugs` / `resolve_slugs` | Fuzzy slug resolution |
|
||||
| `gbrain__traverse_graph` / `traverse_graph` | Walk the relationship graph |
|
||||
| `gbrain__put_page` / `put_page` | Create or update a brain page |
|
||||
| `gbrain__add_timeline_entry` | Add a dated event |
|
||||
| `gbrain__add_link` | Add a relationship edge |
|
||||
|
||||
- The brain has context that external APIs don't (user's direct observations, meeting notes, personal relationships)
|
||||
- External API calls cost money and time
|
||||
- Brain context makes external lookups more targeted (you know what's missing)
|
||||
- The user's direct statements are highest-authority data. External sources are lowest.
|
||||
Tool names vary by transport (MCP uses short names, OpenClaw plugin uses
|
||||
`gbrain__` prefix). Both work. Use whichever your environment provides.
|
||||
|
||||
## The Lookup Chain (MANDATORY ORDER)
|
||||
|
||||
1. **`search`** first — keyword search, fast, zero API cost
|
||||
2. **`query`** if search is thin — hybrid semantic search, uses embedding API
|
||||
3. **`get_page`** if you found a slug — read the full compiled truth
|
||||
4. **External APIs only after steps 1-2 return nothing useful**
|
||||
|
||||
Never skip to external APIs without completing steps 1-2. The brain has
|
||||
thousands of pages. The answer is almost always there.
|
||||
|
||||
## Rules
|
||||
|
||||
- **Score > 0.5 = use it.** Don't reach for external APIs when the brain answered.
|
||||
- **User's direct statements are highest-authority data.** The brain captures
|
||||
what the user said in meetings, conversations, and notes. External sources
|
||||
are supplementary.
|
||||
- **After any brain page write:** trigger a sync so new pages are searchable.
|
||||
In OpenClaw: `gbrain__sync_brain`. From CLI: `gbrain sync --no-pull`.
|
||||
- **Every brain page reference in output** should use a clickable link format
|
||||
appropriate to the deployment (GitHub URL, local path, or slug).
|
||||
- **Never use `memory_search` for entity lookups.** Memory tools search
|
||||
session notes (MEMORY.md), not the brain knowledge graph. Use
|
||||
`search` or `query` for entity lookups.
|
||||
|
||||
## Entity Page Conventions
|
||||
|
||||
Standard directory structure:
|
||||
|
||||
| Directory | Type | Example |
|
||||
|-----------|------|---------|
|
||||
| `people/` | person | `people/paul-graham.md` |
|
||||
| `companies/` | company | `companies/stripe.md` |
|
||||
| `deals/` | deal | `deals/stripe-series-c.md` |
|
||||
| `meetings/` | meeting | `meetings/2026-04-23-weekly-sync.md` |
|
||||
| `projects/` | project | `projects/gbrain.md` |
|
||||
| `yc/` | yc | `yc/batch-w26.md` |
|
||||
|
||||
When creating new pages, include proper frontmatter with `type`, `title`,
|
||||
and `tags` fields.
|
||||
|
||||
## When Spawning Further Sub-agents
|
||||
|
||||
If you spawn your own sub-agents, include this line in their task prompt:
|
||||
|
||||
> Read `skills/conventions/brain-first.md` before starting work.
|
||||
|
||||
This ensures the convention propagates through any depth of sub-agent chain.
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
# Brain Routing Convention
|
||||
|
||||
Cross-cutting rules for which brain and which source an operation targets.
|
||||
Applies to every skill that reads or writes brain pages. **Full mental model
|
||||
lives in `docs/architecture/brains-and-sources.md` — read it once.**
|
||||
|
||||
## The two axes (one-line summary)
|
||||
|
||||
- **Brain** = which DATABASE. `--brain`, `GBRAIN_BRAIN_ID`, `.gbrain-mount`.
|
||||
- **Source** = which REPO INSIDE the database. `--source`, `GBRAIN_SOURCE`,
|
||||
`.gbrain-source`.
|
||||
|
||||
Orthogonal. Pick one on each axis per operation.
|
||||
|
||||
## Default behavior (ALWAYS)
|
||||
|
||||
Start in the brain + source resolved by the environment:
|
||||
|
||||
1. Run `gbrain mounts list` if you haven't seen the user's mounts yet.
|
||||
2. Trust the resolver. If the user is in `~/team-brains/media/`, their
|
||||
`.gbrain-mount` pins brain=media-team. Don't override that silently.
|
||||
3. For every brain op, pass the resolved brain id explicitly when calling
|
||||
tools (even if it matches the default). Makes routing visible in logs.
|
||||
|
||||
Bare `gbrain query "X"` routes to the default brain's default source. That
|
||||
is the right answer 90% of the time. Don't cross the boundary without a
|
||||
reason.
|
||||
|
||||
## When to switch brain
|
||||
|
||||
Switch brain (`--brain <id>`) when:
|
||||
|
||||
- The user's question is specifically about a team the user belongs to
|
||||
("what did team X decide?", "what's the status of project Y at team X?").
|
||||
Switch BEFORE searching, not after a failed search in host.
|
||||
- The user is asking you to ingest data that belongs to a specific team
|
||||
(meeting notes from a team meeting, letters from a team's pipeline). The
|
||||
data owner determines the brain.
|
||||
- The user explicitly names a team/brain ("check the media-team brain
|
||||
for...").
|
||||
|
||||
Do NOT switch brain when:
|
||||
|
||||
- The user asks a general question that might pull from anywhere. Start in
|
||||
host, then cross-query on-demand if host doesn't have it.
|
||||
- You're unsure. Stay in host, surface what you found, let the user point
|
||||
you at a specific brain.
|
||||
|
||||
## When to switch source
|
||||
|
||||
Switch source (`--source <id>`) when:
|
||||
|
||||
- The user is working in a specific repo (the `.gbrain-source` dotfile
|
||||
usually handles this — don't fight it).
|
||||
- The user asks about something scoped to a repo ("what's in my gstack
|
||||
notes about retry policy?").
|
||||
- You're writing a page that logically belongs to one repo. The data
|
||||
origin determines the source.
|
||||
|
||||
Do NOT switch source when:
|
||||
|
||||
- The user's intent crosses repos. Keep `federated=true` sources for
|
||||
cross-source search.
|
||||
- You'd lose a cross-repo match by isolating.
|
||||
|
||||
## Cross-brain queries (latent-space federation)
|
||||
|
||||
v0.19 does NOT do deterministic cross-brain federation. No SQL fan-out. No
|
||||
unified ranking. The AGENT federates.
|
||||
|
||||
Pattern when the user asks something that might span brains:
|
||||
|
||||
1. Query host with the obvious query.
|
||||
2. Check `gbrain mounts list` for relevant brain ids.
|
||||
3. If you think another brain has the answer, re-query THAT brain
|
||||
explicitly (`--brain <id>`).
|
||||
4. Synthesize across results. Cite `<brain>:<source>:<slug>` so the user
|
||||
can trace.
|
||||
|
||||
Never silently mix brains. Every finding is citable to its brain.
|
||||
|
||||
## Writing across brains
|
||||
|
||||
Writing is stricter than reading. ASK before writing cross-brain.
|
||||
|
||||
- A fact about a team's work → team's brain, not host.
|
||||
- A fact the user confirmed about a person ONLY they know → host/personal,
|
||||
not a team brain.
|
||||
- An enrichment discovered from public data → usually host unless the user
|
||||
says otherwise.
|
||||
|
||||
If you're about to `put_page --brain <team-brain>`, confirm with the user
|
||||
unless they explicitly said "save this to team-X". Default brain for
|
||||
writes is the user's personal brain.
|
||||
|
||||
## Citations with brain context
|
||||
|
||||
Standard citation format stays the same (`[Source: ...]`), but when pages
|
||||
come from a mounted brain, add the brain context for human traceability:
|
||||
|
||||
- Single-brain query: `[Source: Meeting, 2026-04-10]` (unchanged).
|
||||
- Cross-brain synthesis: `[Source: media-team:meetings/2026-04-10]` or
|
||||
`[Source: policy-team:research/retry-budgets]`.
|
||||
|
||||
This matches v0.18.0's source-aware citation (`[source-id:slug]`) extended
|
||||
with a brain prefix when relevant.
|
||||
|
||||
## Decision table
|
||||
|
||||
| Situation | Brain | Source |
|
||||
|---|---|---|
|
||||
| User cd's into a team-brain checkout and asks a general question | dotfile-resolved team brain | dotfile-resolved source |
|
||||
| User asks "what did team X decide?" | `team-x` explicitly | resolver default |
|
||||
| User asks "what are we doing across all teams?" | fan out across mounts, agent-driven | resolver default |
|
||||
| User asks "add this to my gstack notes" | host | `gstack` |
|
||||
| User asks "save this meeting note for team X" | `team-x` (confirm if ambiguous) | team's meetings source |
|
||||
| User asks "write me an essay" | host (personal) | `essays` |
|
||||
| Unknown — can't classify | stay in host, ask the user | resolver default |
|
||||
|
||||
## Anti-patterns
|
||||
|
||||
- Silently jumping brains to "find" an answer when the user clearly meant
|
||||
host. That's an audit-trail hole.
|
||||
- Writing to host when the data is clearly team-owned ("the team's plans
|
||||
are now in your personal brain" = bad surprise).
|
||||
- Cross-brain federation in a single query without citations that name the
|
||||
source brain. The user cannot trace the answer back.
|
||||
- Ignoring `.gbrain-mount` / `.gbrain-source` dotfiles. They're load-bearing
|
||||
context — the user set them up for a reason.
|
||||
|
||||
## Read more
|
||||
|
||||
- `docs/architecture/brains-and-sources.md` — the full mental model with
|
||||
topology diagrams (single-person, personal-with-repos, CEO-class with
|
||||
multiple team brains).
|
||||
- `skills/conventions/brain-first.md` — reads the brain BEFORE asking.
|
||||
- `skills/conventions/quality.md` — citation format (extended here with
|
||||
brain prefix).
|
||||
@@ -1,15 +1,19 @@
|
||||
---
|
||||
name: cross-modal-review
|
||||
version: 1.0.0
|
||||
version: 1.1.0
|
||||
description: |
|
||||
Quality gate via second model. Spawn a different AI model to review work
|
||||
before committing. Includes refusal routing: if one model refuses, silently
|
||||
switch to the next.
|
||||
before committing. Includes refusal routing: if one model refuses, switch
|
||||
silently to the next. Extended in v0.25.1 with structured review-mode
|
||||
gating (when to invoke vs not) and a Codex code-review handoff for the
|
||||
diff-review case.
|
||||
triggers:
|
||||
- "second opinion"
|
||||
- "cross-modal review"
|
||||
- "double check this"
|
||||
- "get another perspective"
|
||||
- "challenge this code"
|
||||
- "adversarial review"
|
||||
tools:
|
||||
- search
|
||||
- query
|
||||
@@ -19,41 +23,120 @@ mutating: false
|
||||
|
||||
# Cross-Modal Review
|
||||
|
||||
> **Convention:** See `skills/conventions/cross-modal.yaml` for the review pairs and refusal routing chain.
|
||||
> **Convention:** see [conventions/cross-modal.yaml](../conventions/cross-modal.yaml)
|
||||
> for the review pairs and refusal routing chain.
|
||||
|
||||
## Contract
|
||||
|
||||
This skill guarantees:
|
||||
- Work product is reviewed by a different model before finalizing
|
||||
- Review grades against the originating skill's Contract section
|
||||
- Agreement and disagreement are reported transparently
|
||||
- Refusal from one model triggers silent switch to next in chain
|
||||
- User always makes the final decision (user sovereignty)
|
||||
|
||||
- Work product is reviewed by a different model before finalizing.
|
||||
- The review is graded against the originating skill's Contract section
|
||||
(what was promised), not vibes.
|
||||
- Agreement and disagreement are reported transparently.
|
||||
- Refusal from one model triggers a silent switch to the next in chain.
|
||||
- The user always makes the final decision (user sovereignty).
|
||||
|
||||
## When to invoke (v0.25.1 gating)
|
||||
|
||||
Invoke this skill when:
|
||||
|
||||
- **Significant code changes** — any commit touching 5+ files or 100+
|
||||
lines. Architecture decisions, refactors, API changes.
|
||||
- **Security-sensitive changes** — auth flows, brain-write trust boundaries,
|
||||
webhook transforms, cross-skill data passing.
|
||||
- **Stuck or churning** — 2+ iterations on the same problem without
|
||||
progress.
|
||||
- **Pre-bulk-operation** — before running batch enrichment, migrations,
|
||||
or bulk writes (see [conventions/test-before-bulk.md](../conventions/test-before-bulk.md)).
|
||||
- **Skill creation / modification** — new or rewritten skills that
|
||||
affect operational behavior.
|
||||
- **Brain-page quality concerns** — when brain writes need validation
|
||||
against the originating skill's Contract.
|
||||
|
||||
Do NOT invoke for:
|
||||
|
||||
- Simple memory writes or brain-page updates
|
||||
- Single-file typo fixes
|
||||
- Routine cron output or heartbeat operations
|
||||
- Git commit / push of already-reviewed work
|
||||
|
||||
## Phases
|
||||
|
||||
1. **Capture the work product.** The brain page, analysis, or decision to be reviewed.
|
||||
2. **Load the Contract.** Read the originating skill's Contract section (what was promised).
|
||||
3. **Spawn review model.** Send the work + Contract to a different model. Use `skills/conventions/model-routing.md` for model selection.
|
||||
4. **Grade.** Model evaluates: did the output follow the Contract? Pass/fail with specific citations.
|
||||
5. **Report.** Present agreement/disagreement to the user. Never auto-apply the reviewer's suggestions.
|
||||
1. **Capture the work product.** The brain page, analysis, code diff,
|
||||
or decision to be reviewed.
|
||||
2. **Load the Contract.** Read the originating skill's Contract section
|
||||
(what was promised).
|
||||
3. **Spawn review model.** Send the work + Contract to a different
|
||||
model. Use [conventions/model-routing.md](../conventions/model-routing.md)
|
||||
for model selection.
|
||||
4. **Grade.** Model evaluates: did the output follow the Contract?
|
||||
Pass / fail with specific citations.
|
||||
5. **Report.** Present agreement / disagreement to the user. Never
|
||||
auto-apply the reviewer's suggestions.
|
||||
|
||||
## Refusal Routing
|
||||
## Code-review handoff (v0.25.1 extension)
|
||||
|
||||
If the primary model refuses:
|
||||
1. Silently switch to the next model in the chain (see `skills/conventions/cross-modal.yaml`)
|
||||
2. Never show the refusal to the user
|
||||
3. Never announce the switch
|
||||
4. If ALL models refuse, escalate to the user
|
||||
For diff review specifically, gstack ships a `/codex` skill that wraps
|
||||
the OpenAI Codex CLI. Two modes:
|
||||
|
||||
## Output Format
|
||||
### Codex Review
|
||||
|
||||
Independent diff review from a different AI system. The user invokes
|
||||
`/codex review` (gstack-shipped); cross-modal-review's job is to
|
||||
RECOGNIZE when this is the right tool and recommend it explicitly.
|
||||
|
||||
**When to recommend `/codex review`:**
|
||||
- After a substantive diff lands and before merge
|
||||
- When the user wants a second opinion that's NOT another Claude
|
||||
|
||||
**Output framing (when cross-modal-review surfaces Codex output):**
|
||||
|
||||
```
|
||||
CODEX REVIEW (independent second opinion):
|
||||
══════════════════════════════════════════
|
||||
<full codex output, verbatim>
|
||||
══════════════════════════════════════════
|
||||
|
||||
CROSS-MODEL ANALYSIS:
|
||||
Both found: [overlapping findings]
|
||||
Only Codex: [findings unique to Codex]
|
||||
Only Claude: [findings unique to my analysis]
|
||||
Agreement: X% (N/M findings overlap)
|
||||
```
|
||||
|
||||
User decides what to act on. Cross-model agreement is signal, not
|
||||
permission.
|
||||
|
||||
### Adversarial Challenge
|
||||
|
||||
Same shape, different prompt. Used on security-sensitive changes:
|
||||
the reviewer is asked to find injection vectors, race conditions,
|
||||
auth bypasses, data leaks, privilege escalation paths.
|
||||
|
||||
Output adds an exploitability rating (CRITICAL / HIGH / MEDIUM / LOW)
|
||||
and recommended mitigations.
|
||||
|
||||
## Refusal routing
|
||||
|
||||
If the primary review model refuses:
|
||||
|
||||
1. Switch silently to the next model in the chain (see
|
||||
`conventions/cross-modal.yaml`).
|
||||
2. Don't show the refusal to the user.
|
||||
3. Don't announce the switch.
|
||||
4. If ALL models in the chain refuse, escalate to the user.
|
||||
|
||||
## Output format
|
||||
|
||||
### Standard review
|
||||
|
||||
```
|
||||
Cross-Modal Review
|
||||
==================
|
||||
Reviewer: {model name}
|
||||
Contract: {originating skill}
|
||||
Verdict: PASS | ISSUES FOUND
|
||||
Reviewer: {model name}
|
||||
Contract: {originating skill}
|
||||
Verdict: PASS | ISSUES FOUND
|
||||
|
||||
Findings:
|
||||
- {finding with evidence}
|
||||
@@ -61,9 +144,46 @@ Findings:
|
||||
Agreement with primary: {X}%
|
||||
```
|
||||
|
||||
### Code review
|
||||
|
||||
```
|
||||
Cross-Modal Review (code)
|
||||
==========================
|
||||
Mode: Codex Review | Adversarial Challenge
|
||||
Files changed: N
|
||||
Lines changed: +N / -N
|
||||
|
||||
{mode-specific output above}
|
||||
```
|
||||
|
||||
## User-sovereignty rule (Iron Law)
|
||||
|
||||
Reviewer findings are INFORMATIONAL until the user explicitly approves
|
||||
each one. Do NOT incorporate reviewer recommendations into the work
|
||||
product without presenting each finding and getting explicit approval.
|
||||
This applies even when the reviewer is correct. Cross-model consensus
|
||||
is a strong signal — present it as such — but the user makes the
|
||||
decision.
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
- Auto-applying reviewer suggestions without user approval
|
||||
- Showing model refusals to the user
|
||||
- Using the same model for review and generation
|
||||
- Skipping the Contract reference (reviewing vibes, not guarantees)
|
||||
- ❌ Auto-applying reviewer suggestions without user approval
|
||||
- ❌ Showing model refusals to the user
|
||||
- ❌ Using the same model for review and generation
|
||||
- ❌ Skipping the Contract reference (reviewing vibes, not guarantees)
|
||||
- ❌ Code-reviewing trivial changes (typos, formatting)
|
||||
- ❌ Running code review without git-diff context
|
||||
|
||||
## Related skills
|
||||
|
||||
- gstack `/codex` — the actual Codex CLI wrapper this skill hands off
|
||||
to for diff-review mode. Cross-modal-review knows WHEN to invoke;
|
||||
/codex knows HOW.
|
||||
- `skills/testing/SKILL.md` — runs the project test suite; complementary
|
||||
signal for "is this commit safe to land"
|
||||
- `skills/conventions/cross-modal.yaml` — review pairs + refusal routing
|
||||
|
||||
|
||||
## Output Format
|
||||
|
||||
The skill's output shape is documented inline in the body sections above (see "Output", "Brain page format", or equivalent). The literal section header here exists for the conformance test (`test/skills-conformance.test.ts`).
|
||||
|
||||
@@ -17,6 +17,13 @@ triggers:
|
||||
- "populate links"
|
||||
- "backfill graph"
|
||||
- "extract timeline entries"
|
||||
- "run dream"
|
||||
- "process today's session"
|
||||
- "process yesterday's transcripts"
|
||||
- "synthesize my conversations"
|
||||
- "what patterns did you see"
|
||||
- "did the dream cycle run"
|
||||
- "consolidate yesterday's conversations"
|
||||
tools:
|
||||
- get_health
|
||||
- get_page
|
||||
@@ -77,6 +84,81 @@ If timeline_entry_count is 0, extract structured timeline from markdown:
|
||||
```bash
|
||||
gbrain extract timeline --dir ~/brain
|
||||
```
|
||||
|
||||
### Dream cycle (v0.23): synthesize + patterns
|
||||
|
||||
`gbrain dream` runs the full 8-phase maintenance cycle:
|
||||
|
||||
```
|
||||
lint -> backlinks -> sync -> synthesize -> extract -> patterns -> embed -> orphans
|
||||
```
|
||||
|
||||
The two new phases consolidate yesterday's conversations into long-term memory:
|
||||
|
||||
**Synthesize phase:** reads transcripts from `dream.synthesize.session_corpus_dir`,
|
||||
runs a cheap Haiku verdict (cached in `dream_verdicts`) to filter routine
|
||||
ops sessions, then fans out one Sonnet subagent per worth-processing
|
||||
transcript. Each subagent writes reflections (`wiki/personal/reflections/...`),
|
||||
originals (`wiki/originals/ideas/...`), and people timeline entries. The
|
||||
orchestrator collects the slugs from `subagent_tool_executions` (NOT
|
||||
`pages.updated_at` — that would pick up unrelated writes) and reverse-renders
|
||||
each new page from DB → markdown on disk.
|
||||
|
||||
**Patterns phase:** runs after `extract` (so the graph state is fresh).
|
||||
Reads recent reflections within `dream.patterns.lookback_days` (default 30),
|
||||
runs a single Sonnet pass to surface recurring themes, and writes pattern
|
||||
pages to `wiki/personal/patterns/<theme>` when ≥`dream.patterns.min_evidence`
|
||||
(default 3) reflections support a pattern.
|
||||
|
||||
**Quality bar (Iron Law for synthesis):**
|
||||
1. Quote the user verbatim. Do not paraphrase memorable phrasings.
|
||||
2. Cross-reference compulsively: every new page MUST have at least one wikilink.
|
||||
3. Slug discipline: lowercase alphanumeric and hyphens only. NO underscores, NO file extensions.
|
||||
4. Edited transcripts produce NEW slugs (content-hash suffix changes) — never silently overwrite.
|
||||
|
||||
**Trust boundary (`allowed_slug_prefixes`):** the synthesis subagent runs with an
|
||||
explicit allow-list of write paths sourced from `_brain-filing-rules.json`'s
|
||||
`dream_synthesize_paths.globs`. Even on prompt-injection success, the subagent
|
||||
cannot write outside that list. Trust comes from PROTECTED_JOB_NAMES — MCP
|
||||
cannot submit subagent jobs at all. Editing the JSON is the only way to add
|
||||
a new directory the synthesizer can write to.
|
||||
|
||||
**Idempotency + privacy:** transcripts are keyed by `(file_path, content_hash)`,
|
||||
so re-running on the same content is a no-op. `dream.synthesize.exclude_patterns`
|
||||
(default `["medical", "therapy"]`) filters out transcripts before any LLM call.
|
||||
Each entry is auto-wrapped as a word-boundary regex (e.g. `medical` matches
|
||||
"medical advice" but NOT "comedical"). Power users may pass full regex.
|
||||
|
||||
**Cooldown:** the cycle's spend cap. `dream.synthesize.cooldown_hours` (default
|
||||
12) means at most ~2 synthesize runs per day under autopilot. The completion
|
||||
timestamp is stored in `dream.synthesize.last_completion_ts` and is written
|
||||
ONLY on successful runs (not on skipped/failed). Explicit `--input` /
|
||||
`--date` / `--from` / `--to` invocations bypass cooldown.
|
||||
|
||||
**`--dry-run` semantics:** runs the cheap Haiku significance filter (caches
|
||||
verdicts) but skips the Sonnet synthesis pass. NOT zero LLM calls.
|
||||
|
||||
**Configure synthesize on a fresh brain:**
|
||||
```bash
|
||||
gbrain config set dream.synthesize.session_corpus_dir /path/to/transcripts
|
||||
gbrain config set dream.synthesize.enabled true
|
||||
gbrain dream --phase synthesize --dry-run --json # preview
|
||||
gbrain dream # full 8-phase cycle
|
||||
```
|
||||
|
||||
**Invocation patterns:**
|
||||
```bash
|
||||
gbrain dream # full cycle
|
||||
gbrain dream --phase synthesize # just synthesize
|
||||
gbrain dream --phase patterns # just patterns
|
||||
gbrain dream --input ~/transcripts/2026-04-25.txt # ad-hoc one transcript
|
||||
gbrain dream --from 2026-04-01 --to 2026-04-25 # backfill range
|
||||
gbrain dream --json # CycleReport JSON
|
||||
```
|
||||
|
||||
**Auto-commit deferred to v1.1:** v1 writes files to `brain_dir` but does NOT
|
||||
`git add` / `commit` / `push`. Either commit yourself or let `gbrain autopilot`
|
||||
handle it.
|
||||
Parses `- **YYYY-MM-DD** | Source — Summary` and `### YYYY-MM-DD — Title` formats.
|
||||
Note: extracted entries improve structured queries (`gbrain timeline`), not vector search.
|
||||
|
||||
|
||||
+46
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.10.0",
|
||||
"version": "0.25.1",
|
||||
"conformance_version": "1.0.0",
|
||||
"description": "Personal knowledge brain with hybrid RAG search \u2014 GStack mod for agent platforms",
|
||||
"skills": [
|
||||
@@ -153,6 +153,51 @@
|
||||
"name": "smoke-test",
|
||||
"path": "smoke-test/SKILL.md",
|
||||
"description": "Post-restart smoke tests + auto-fix for gbrain and OpenClaw environments"
|
||||
},
|
||||
{
|
||||
"name": "book-mirror",
|
||||
"path": "book-mirror/SKILL.md",
|
||||
"description": "Take any book (EPUB/PDF), produce a personalized chapter-by-chapter analysis with two-column tables: left = chapter summary, right = how it applies to you based on brain context. Output: brain page + PDF."
|
||||
},
|
||||
{
|
||||
"name": "article-enrichment",
|
||||
"path": "article-enrichment/SKILL.md",
|
||||
"description": "Transform raw article text dumps in the brain into structured pages with executive summaries, verbatim quotes, key insights, why-it-matters, and cross-references."
|
||||
},
|
||||
{
|
||||
"name": "strategic-reading",
|
||||
"path": "strategic-reading/SKILL.md",
|
||||
"description": "Read a book/article/case study through the lens of a specific strategic problem; produce an applied playbook (do/avoid/watch for) with short/medium/long-term recommendations."
|
||||
},
|
||||
{
|
||||
"name": "concept-synthesis",
|
||||
"path": "concept-synthesis/SKILL.md",
|
||||
"description": "Deduplicate and synthesize raw concept stubs into a tiered intellectual map (T1 Canon to T4 Riff), tracing idea evolution across sources over time."
|
||||
},
|
||||
{
|
||||
"name": "perplexity-research",
|
||||
"path": "perplexity-research/SKILL.md",
|
||||
"description": "Brain-augmented web research via Perplexity plus Opus; surfaces what is NEW vs already-known about a topic by cross-referencing against the brain first."
|
||||
},
|
||||
{
|
||||
"name": "archive-crawler",
|
||||
"path": "archive-crawler/SKILL.md",
|
||||
"description": "Universal archivist for personal file archives (Dropbox/B2/email exports). Filters for high-value content within an explicit gbrain.yml allow-list scan_paths gate."
|
||||
},
|
||||
{
|
||||
"name": "academic-verify",
|
||||
"path": "academic-verify/SKILL.md",
|
||||
"description": "Verify academic citations and research claims against current literature; routes through perplexity-research for the actual web search and formats results as a citation-checked brain page."
|
||||
},
|
||||
{
|
||||
"name": "brain-pdf",
|
||||
"path": "brain-pdf/SKILL.md",
|
||||
"description": "Generate a publication-quality PDF from any brain page via the gstack make-pdf binary; strips frontmatter, sanitizes emoji, applies running headers."
|
||||
},
|
||||
{
|
||||
"name": "voice-note-ingest",
|
||||
"path": "voice-note-ingest/SKILL.md",
|
||||
"description": "Ingest voice notes with exact-phrasing preservation (never paraphrased); routes content based on a decision tree across originals/concepts/people/companies/ideas/personal/voice-notes."
|
||||
}
|
||||
],
|
||||
"dependencies": {
|
||||
|
||||
@@ -113,6 +113,6 @@ Brain page created with summary, highlights, and entity cross-links. Report to u
|
||||
|
||||
- Dumping raw transcripts without analysis
|
||||
- Skipping entity extraction ("I'll do that separately")
|
||||
- Filing by format (all videos in `media/videos/`) instead of by subject
|
||||
- Filing **raw ingest** by format (all videos in `media/videos/`) instead of by subject. Note: format-prefixed paths under `media/<format>/<slug>` ARE sanctioned for **synthesized one-of-one output** like book-mirror's `media/books/<slug>-personalized.md`. The anti-pattern is for raw ingest, not for sui generis synthesis. See `skills/_brain-filing-rules.md` "Sanctioned exception: synthesis output is sui generis."
|
||||
- Not preserving raw source files
|
||||
- Creating stub pages without meaningful content
|
||||
|
||||
@@ -46,7 +46,7 @@ These run as part of `gbrain upgrade` → `gbrain apply-migrations`. No manual D
|
||||
|
||||
5. **Observe incremental chunking.** Edit one function in a 20-function file, re-run `sync --source <id>`. Embedding cost should be ~5% of the first sync because unchanged chunks reuse their existing embeddings.
|
||||
|
||||
## Migration from Wintermute's `repos` (if you used it)
|
||||
## Migration from your OpenClaw's `repos` (if you used it)
|
||||
|
||||
v0.19.0 deletes `~/.gbrain/config.json`'s `repos` array in favor of the `sources` table. The CLI surface is preserved as a deprecated alias: `gbrain repos add` still works, but routes into `runSources` with a one-line deprecation notice on stderr. Existing scripts keep working; prefer `gbrain sources` going forward.
|
||||
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
---
|
||||
feature_pitch:
|
||||
headline: Bare workers now self-monitor and fail-stop into your PM's restart loop
|
||||
body: |
|
||||
Bare `gbrain jobs work` now ships with the same health protection the
|
||||
supervisor already had: DB liveness probes (with per-probe timeout so a
|
||||
hung connection can't wedge the monitor), stall detection filtered by
|
||||
registered handler names, and an RSS watchdog default of 2048 MB.
|
||||
|
||||
When the worker detects it's wedged (stuck pgbouncer connection, hung
|
||||
event loop, stalled job claim), it emits `'unhealthy'` and the CLI calls
|
||||
`process.exit(1)`. This is **fail-stop**: it requires an external process
|
||||
manager (systemd, Docker `restart: always`, launchd `KeepAlive`, cron
|
||||
watchdog) to bring the worker back. Without one, the process exits and
|
||||
stays dead — that's a regression from pre-v0.22.14 self-healing.
|
||||
|
||||
Pre-v0.22.14 behavior: bare workers had ZERO health monitoring. A wedged
|
||||
worker stayed alive doing nothing while jobs piled up in `waiting` and
|
||||
your PM's `pgrep` check happily reported green.
|
||||
|
||||
If you're using `gbrain jobs supervisor`, you're already protected — the
|
||||
supervisor handles spawn-on-crash itself. The fail-stop concern only
|
||||
applies to direct `gbrain jobs work` invocations.
|
||||
---
|
||||
|
||||
# v0.22.14 — Bare-worker self-health-monitoring
|
||||
|
||||
## ⚠️ Pre-flight: confirm you have a process supervisor
|
||||
|
||||
If you run `gbrain jobs work` directly (NOT under `gbrain jobs supervisor`),
|
||||
verify your process manager is configured to restart the worker on exit
|
||||
BEFORE upgrading:
|
||||
|
||||
| Manager | What to check |
|
||||
|---|---|
|
||||
| systemd | `Restart=always` (or `Restart=on-failure`) in the `.service` unit |
|
||||
| Docker | `restart: always` / `restart: unless-stopped` in compose, OR `--restart` flag |
|
||||
| launchd (macOS) | `<key>KeepAlive</key><true/>` in the plist |
|
||||
| cron watchdog | Cron entry that re-spawns when `pgrep -f "gbrain jobs work"` is empty |
|
||||
| supervisord | `autorestart=true` |
|
||||
|
||||
**If your bare worker has no restart loop, the v0.22.14 fail-stop behavior
|
||||
will leave you with a dead worker after the first DB blip.** Either add a
|
||||
restart policy OR switch to `gbrain jobs supervisor` (which spawns its own
|
||||
child + restarts on crash internally).
|
||||
|
||||
## What ships
|
||||
|
||||
- DB liveness probes inside `gbrain jobs work` (60s interval, 3 strikes → exit)
|
||||
- Stall detection (5min warn / 10min exit when waiting jobs accumulate but
|
||||
in-flight is empty)
|
||||
- `--max-rss` defaults to 2048 MB for bare workers (matches supervisor default;
|
||||
was 0 = disabled)
|
||||
- New `MinionWorkerOpts.{healthCheckInterval, stallWarnAfterMs,
|
||||
stallExitAfterMs, dbFailExitAfter, dbProbeTimeoutMs}` for tuning (5 fields)
|
||||
- `MinionWorker` now extends `EventEmitter`; emits `'unhealthy'` event with
|
||||
a structured reason payload. **No-listener fallback**: if the caller does
|
||||
not subscribe to `'unhealthy'`, the worker calls `process.exit(1)` itself
|
||||
to preserve the pre-refactor fail-stop behavior. The CLI subscribes; direct
|
||||
API consumers without a listener inherit the fail-stop default. Inline
|
||||
paths (`jobs submit --follow`, `jobs smoke`) explicitly pass
|
||||
`healthCheckInterval: 0` to disable the timer entirely so the fallback
|
||||
cannot trip on a one-shot run.
|
||||
- `GBRAIN_SUPERVISED=1` env var (set by supervisor on its child) causes the
|
||||
child worker to skip its own health timer (no double-monitoring)
|
||||
- Constructor validation: throws if `stallExitAfterMs <= stallWarnAfterMs`
|
||||
(the contract requires strict greater-than)
|
||||
|
||||
## ⚠️ Breaking change you may need to act on
|
||||
|
||||
**`gbrain jobs work` now defaults `--max-rss 2048` (MB).** Pre-v0.22.14, bare
|
||||
workers had no RSS limit. If you ran bare workers with intentionally large
|
||||
embed/import working sets, your worker may now get killed by the watchdog after
|
||||
upgrade.
|
||||
|
||||
**You need to act if any of the following is true:**
|
||||
|
||||
- You run `gbrain jobs work` directly under a process manager (systemd, Docker,
|
||||
cron, supervisord, your own bash wrapper)
|
||||
- Your worker handles large embed batches, multi-GB import jobs, or
|
||||
memory-intensive subagent tasks that legitimately exceed 2 GB RSS
|
||||
- You rely on bare-worker resident memory exceeding 2 GB
|
||||
|
||||
**You do NOT need to act if:**
|
||||
|
||||
- You run `gbrain jobs supervisor` (the supervisor already defaulted to
|
||||
`--max-rss 2048` and has reconnect-then-fail logic on top)
|
||||
- Your typical worker stays well under 2 GB RSS
|
||||
|
||||
## What the agent should do post-upgrade
|
||||
|
||||
### 1. Run the orchestrator (mechanical migration)
|
||||
|
||||
```bash
|
||||
gbrain apply-migrations --yes
|
||||
```
|
||||
|
||||
This is a no-op for v0.22.14 — there's no schema migration. The mechanical work
|
||||
is in the binary; the user-side decision is the breaking-change handling below.
|
||||
|
||||
### 2. Detect bare-worker deployments
|
||||
|
||||
If the user runs OpenClaw, Hermes, or any agent fork that uses `gbrain jobs
|
||||
work` directly (NOT under `gbrain jobs supervisor`), they need to know. Check:
|
||||
|
||||
```bash
|
||||
# Look for bare worker invocations in cron, systemd, Docker, launchctl, etc.
|
||||
grep -r "gbrain jobs work" ~/.config/systemd/ /etc/systemd/ 2>/dev/null | head
|
||||
crontab -l 2>/dev/null | grep "gbrain jobs work" | head
|
||||
launchctl list 2>/dev/null | grep gbrain | head
|
||||
ps -ef | grep "gbrain jobs work" | grep -v grep | grep -v supervisor
|
||||
```
|
||||
|
||||
If you find bare-worker invocations, surface them to the user.
|
||||
|
||||
### 3. Ask about RSS profile
|
||||
|
||||
Use AskUserQuestion to confirm whether the user's bare workers need >2 GB RSS:
|
||||
|
||||
> "v0.22.14 changes `gbrain jobs work` to default `--max-rss 2048` (MB).
|
||||
> Pre-v0.22.14 bare workers had no limit. If your worker handles large embed
|
||||
> batches or multi-GB imports, the watchdog may now kill it. Do you want
|
||||
> to keep the new 2 GB default, raise the limit, or opt out entirely?"
|
||||
|
||||
Options:
|
||||
- **A) Keep 2 GB default (recommended for most)** — protects against memory
|
||||
leaks; restarts on overflow; matches supervisor behavior.
|
||||
- **B) Raise to N GB (specify N)** — pass `--max-rss <N*1024>` to the worker
|
||||
invocation.
|
||||
- **C) Opt out** — pass `--max-rss 0`.
|
||||
|
||||
### 4. Apply the user's choice
|
||||
|
||||
For each bare-worker invocation, edit the unit/cron/launchctl/script to add
|
||||
the chosen `--max-rss` flag.
|
||||
|
||||
**systemd (~/.config/systemd/user/gbrain-worker.service):**
|
||||
|
||||
```ini
|
||||
ExecStart=/usr/local/bin/gbrain jobs work --queue default --concurrency 3 --max-rss 4096
|
||||
# Or to opt out: --max-rss 0
|
||||
```
|
||||
|
||||
Then `systemctl --user daemon-reload && systemctl --user restart gbrain-worker`.
|
||||
|
||||
**cron (`crontab -e`):**
|
||||
|
||||
```cron
|
||||
@reboot /usr/local/bin/gbrain jobs work --queue default --concurrency 3 --max-rss 4096
|
||||
```
|
||||
|
||||
**Docker compose:**
|
||||
|
||||
```yaml
|
||||
command: ["gbrain", "jobs", "work", "--queue", "default", "--concurrency", "3", "--max-rss", "4096"]
|
||||
```
|
||||
|
||||
**launchctl (~/Library/LaunchAgents/com.user.gbrain-worker.plist):**
|
||||
|
||||
```xml
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>/usr/local/bin/gbrain</string>
|
||||
<string>jobs</string>
|
||||
<string>work</string>
|
||||
<string>--max-rss</string>
|
||||
<string>4096</string>
|
||||
</array>
|
||||
```
|
||||
|
||||
Then `launchctl unload ... && launchctl load ...`.
|
||||
|
||||
### 5. (Optional) Tune health-check thresholds
|
||||
|
||||
The new opts default to sensible values (60s probe interval, 5min warn / 10min
|
||||
exit, 3 DB failures). If you have specific SLAs, you can pass `--health-interval
|
||||
<ms>` to adjust the probe cadence. Stall thresholds are not yet CLI-exposed
|
||||
(only the API; CLI flags coming in a follow-up).
|
||||
|
||||
To disable self-monitoring entirely (e.g. you have your own external health
|
||||
checker):
|
||||
|
||||
```bash
|
||||
gbrain jobs work --health-interval 0 --max-rss 0
|
||||
```
|
||||
|
||||
### 6. Verify
|
||||
|
||||
```bash
|
||||
gbrain jobs stats # queue should be flowing normally
|
||||
gbrain doctor --json | jq '.' # no critical warnings
|
||||
ps -o rss= -p $(pgrep -f "gbrain jobs work") | awk '{print $1/1024 " MB"}'
|
||||
```
|
||||
|
||||
Worker startup log line should now show health-check status:
|
||||
|
||||
```
|
||||
Minion worker started (queue: default, concurrency: 3, watchdog: 2048MB, health-check: 60s)
|
||||
```
|
||||
|
||||
If running under supervisor, you'll see the watchdog but NOT the `health-check:
|
||||
60s` segment (because `GBRAIN_SUPERVISED=1` skips the child's self-monitor).
|
||||
|
||||
### 7. If anything fails
|
||||
|
||||
Open an issue at https://github.com/garrytan/gbrain/issues with:
|
||||
- Output of `gbrain doctor`
|
||||
- Contents of `~/.gbrain/upgrade-errors.jsonl` if it exists
|
||||
- Your bare-worker invocation (systemd unit / cron line / Dockerfile snippet)
|
||||
- Which step broke
|
||||
@@ -0,0 +1,171 @@
|
||||
---
|
||||
version: 0.23.0
|
||||
feature_pitch:
|
||||
headline: "gbrain dream now actually dreams: conversation transcripts → reflections, originals, and 25-year patterns."
|
||||
description: |
|
||||
The maintenance cycle gains two new phases: `synthesize` and `patterns`.
|
||||
The 8-phase order is now: lint → backlinks → sync → synthesize →
|
||||
extract → patterns → embed → orphans.
|
||||
|
||||
Synthesize reads conversation transcripts (e.g., OpenClaw session corpus,
|
||||
meeting transcripts) and writes brain-native pages: reflections to
|
||||
`wiki/personal/reflections/...`, originals to `wiki/originals/ideas/...`,
|
||||
timeline entries on existing people pages.
|
||||
|
||||
Patterns runs after extract (so the graph is fresh) and surfaces
|
||||
recurring themes across reflections — when ≥3 reflections mention the
|
||||
same motif, a pattern page is written to `wiki/personal/patterns/...`
|
||||
citing every reflection that constitutes its evidence.
|
||||
|
||||
Hard guarantees: subagent writes are bounded to an explicit allow-list
|
||||
(sourced from `_brain-filing-rules.json`). Edited transcripts produce
|
||||
new slugs (content-hash suffix) — never silently overwrite. A 12-hour
|
||||
cooldown bounds spend at ~$1-2/day under autopilot.
|
||||
recipe: skills/maintain/SKILL.md
|
||||
tiers: null
|
||||
---
|
||||
|
||||
# v0.23.0 Migration: Dream cycle synthesize + patterns phases
|
||||
|
||||
**Audience: host agents reading this after `gbrain apply-migrations` has
|
||||
run. The synthesize phase ships disabled by default — set
|
||||
`dream.synthesize.session_corpus_dir` to opt in.**
|
||||
|
||||
## Mechanical migration: automatic, no action required
|
||||
|
||||
`gbrain upgrade` chains to `gbrain apply-migrations --yes`, which runs:
|
||||
|
||||
- **migration v25** — creates the `dream_verdicts` table:
|
||||
`(file_path TEXT, content_hash TEXT, worth_processing BOOL, reasons JSONB,
|
||||
judged_at TIMESTAMPTZ, PRIMARY KEY(file_path, content_hash))`. Cache
|
||||
for the cheap Haiku verdict so backfill re-runs skip already-judged
|
||||
transcripts. RLS-enabled when running as a BYPASSRLS role.
|
||||
|
||||
The migration is idempotent. Safe to re-run.
|
||||
|
||||
## What changes for existing brains
|
||||
|
||||
`gbrain dream` (and `gbrain autopilot`) now run an 8-phase cycle:
|
||||
|
||||
```
|
||||
lint → backlinks → sync → synthesize → extract → patterns → embed → orphans
|
||||
```
|
||||
|
||||
If `dream.synthesize.enabled` is false (the default, post-migration), the
|
||||
synthesize and patterns phases emit `status: "skipped", reason: "not_configured"`
|
||||
and the cycle continues to the next phase. **Existing autopilot users see
|
||||
zero behavior change** until they configure synthesize.
|
||||
|
||||
## To enable synthesize on your brain
|
||||
|
||||
Three steps. Take them when ready — there is no rush.
|
||||
|
||||
```bash
|
||||
# 1. Point at the directory where your conversation transcripts live.
|
||||
# OpenClaw stores session transcripts at memory/.dreams/session-corpus/<YYYY-MM-DD>.txt
|
||||
# by default. If you have a different layout, point at that.
|
||||
gbrain config set dream.synthesize.session_corpus_dir /path/to/transcripts
|
||||
|
||||
# 2. Enable the phase.
|
||||
gbrain config set dream.synthesize.enabled true
|
||||
|
||||
# 3. Preview without spending real LLM tokens (runs cheap Haiku verdict only).
|
||||
gbrain dream --phase synthesize --dry-run --json
|
||||
```
|
||||
|
||||
## Tunables (sensible defaults; override only if needed)
|
||||
|
||||
```bash
|
||||
# Skip transcripts shorter than this many characters (default 2000).
|
||||
gbrain config set dream.synthesize.min_chars 2000
|
||||
|
||||
# Word-boundary regex patterns to skip. Default ["medical","therapy"].
|
||||
# Each entry auto-wraps as \b<entry>\b — "medical" matches "medical advice"
|
||||
# but NOT "comedical". Pass full regex (e.g. ^therapy:) for advanced patterns.
|
||||
gbrain config set dream.synthesize.exclude_patterns '["medical","therapy"]'
|
||||
|
||||
# Synthesize model (default: claude-sonnet-4-6).
|
||||
gbrain config set dream.synthesize.model claude-sonnet-4-6
|
||||
|
||||
# Hours between synthesize runs (the v1 spend cap; default 12 → ~$1-2/day).
|
||||
gbrain config set dream.synthesize.cooldown_hours 12
|
||||
|
||||
# Patterns lookback window in days (default 30).
|
||||
gbrain config set dream.patterns.lookback_days 30
|
||||
|
||||
# Minimum distinct reflections needed to name a pattern (default 3).
|
||||
gbrain config set dream.patterns.min_evidence 3
|
||||
```
|
||||
|
||||
## Allow-list source of truth
|
||||
|
||||
The synthesize subagent's allowed write paths live in
|
||||
`skills/_brain-filing-rules.json` under `dream_synthesize_paths.globs`:
|
||||
|
||||
```json
|
||||
{
|
||||
"dream_synthesize_paths": {
|
||||
"globs": [
|
||||
"wiki/personal/reflections/*",
|
||||
"wiki/originals/*",
|
||||
"wiki/personal/patterns/*",
|
||||
"wiki/people/*",
|
||||
"dream-cycle-summaries/*"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Editing this list is the ONLY way to add a new directory the synthesizer
|
||||
can write to. The subagent's `put_page` calls are gated server-side; even
|
||||
on prompt-injection success the write is bounded to these prefixes.
|
||||
|
||||
## Slug discipline
|
||||
|
||||
Reflections: `wiki/personal/reflections/YYYY-MM-DD-<topic>-<hash[:6]>`
|
||||
Originals: `wiki/originals/ideas/YYYY-MM-DD-<idea>-<hash[:6]>`
|
||||
Patterns: `wiki/personal/patterns/<theme>`
|
||||
Summary: `dream-cycle-summaries/YYYY-MM-DD`
|
||||
|
||||
The 6-char content-hash suffix on reflections / originals means an edited
|
||||
transcript produces a NEW slug — the original reflection is preserved
|
||||
alongside the new one. No silent overwrite.
|
||||
|
||||
Lowercase alphanumeric and hyphens only. NO underscores, NO file extensions.
|
||||
|
||||
## Provenance
|
||||
|
||||
Every put_page call from the synthesize subagent shows up in
|
||||
`subagent_tool_executions` with full input. The orchestrator collects
|
||||
slugs by querying that table — NOT `pages.updated_at` — so the cycle's
|
||||
write list cannot accidentally include manual edits or sync output.
|
||||
|
||||
## What's deferred to v1.1
|
||||
|
||||
- **Auto git commit + push.** v1 writes markdown files to `brain_dir`
|
||||
but does NOT `git add` / `commit` / `push`. Either commit yourself
|
||||
or let `gbrain autopilot` handle it. v1.1 will add explicit
|
||||
--commit / --push flags with handling for dirty worktree, staged
|
||||
changes, auth failure, and non-fast-forward push.
|
||||
- **Daily token budget cap.** Cooldown alone is the spend bound at v1
|
||||
scale. If real-world telemetry surfaces a problem, v1.1 adds an
|
||||
explicit `daily_token_budget` config.
|
||||
- **Cross-modal pattern review.** Patterns currently runs against
|
||||
reflections only. Future revision could roll up across reflections,
|
||||
meetings, and timeline entries together.
|
||||
|
||||
## Verify after upgrade
|
||||
|
||||
```bash
|
||||
# Schema migration applied?
|
||||
gbrain doctor
|
||||
|
||||
# Phase ordering correct?
|
||||
gbrain dream --help # shows the 8-phase pipeline
|
||||
|
||||
# Dry-run against a single transcript (cheap Haiku call only):
|
||||
gbrain dream --phase synthesize --input /tmp/some-transcript.txt --dry-run --json
|
||||
```
|
||||
|
||||
If any step fails, file an issue with `gbrain doctor` output and the
|
||||
contents of `~/.gbrain/upgrade-errors.jsonl` if it exists.
|
||||
@@ -0,0 +1,194 @@
|
||||
---
|
||||
feature_pitch: |
|
||||
v0.25.1 ships the book-mirror flagship plus 8 research-flavored
|
||||
pairings, a real `gbrain skillpack uninstall`, and a hardened privacy
|
||||
CI guard. Existing brains: zero schema migration. Optional new
|
||||
capability: 9 markdown skills installable into your workspace via
|
||||
`gbrain skillpack install --all`.
|
||||
---
|
||||
|
||||
# v0.25.1 — Skills wave migration
|
||||
|
||||
## TL;DR
|
||||
|
||||
If you already have v0.24.0 installed, `gbrain upgrade` does it.
|
||||
There is **no schema migration**. Existing brains work unchanged.
|
||||
|
||||
This file exists for agents reading the skill set after upgrade. It
|
||||
explains:
|
||||
|
||||
1. What the 9 new skills do (so the agent knows when to invoke them)
|
||||
2. The new `archive-crawler.scan_paths:` config requirement
|
||||
3. The new `gbrain skillpack uninstall` subcommand
|
||||
4. The new `gbrain book-mirror` CLI
|
||||
|
||||
## Step 0: Watch for the post-install advisory (NEW in v0.25.1)
|
||||
|
||||
Every `gbrain init` and `gbrain post-upgrade` now ends by printing an
|
||||
agent-readable advisory listing the recommended new skills. If you ran
|
||||
`gbrain upgrade` and saw a banner like:
|
||||
|
||||
```
|
||||
========================================================================
|
||||
gbrain 0.25.1 — RECOMMENDED SKILLS FOR THE AGENT TO INSTALL
|
||||
========================================================================
|
||||
```
|
||||
|
||||
…that's the advisory. It explicitly asks you (the agent) to surface
|
||||
the skill list to the user, ask them whether to install, and run the
|
||||
right `gbrain skillpack install` command if they say yes. **Do that
|
||||
first, then return here for the rest.**
|
||||
|
||||
If the advisory didn't print, your workspace already has all 9 v0.25.1
|
||||
skills installed (or no managed block was detected). Continue with the
|
||||
verification steps below.
|
||||
|
||||
## Step 1: Verify the upgrade landed
|
||||
|
||||
```bash
|
||||
gbrain --version # expect: gbrain 0.25.1
|
||||
gbrain skillpack list # expect: 34 skills (was 25 before)
|
||||
gbrain skillpack uninstall --help # expect: "Inverse of install" in help
|
||||
```
|
||||
|
||||
If `gbrain --version` reports 0.24.x, run `gbrain upgrade` first.
|
||||
|
||||
## Step 2: Install the new skills (optional)
|
||||
|
||||
The 9 new skills are in the bundle but only become active in your
|
||||
workspace after explicit install:
|
||||
|
||||
```bash
|
||||
# install just the flagship:
|
||||
gbrain skillpack install book-mirror
|
||||
|
||||
# OR install everything new at once:
|
||||
gbrain skillpack install --all
|
||||
```
|
||||
|
||||
The 9 new skills:
|
||||
|
||||
- **book-mirror** — flagship. Two-column personalized chapter-by-chapter
|
||||
book analysis. Pairs with `gbrain book-mirror` CLI.
|
||||
- **article-enrichment** — turns raw article dumps into structured
|
||||
pages with verbatim quotes.
|
||||
- **strategic-reading** — reads a book through one specific
|
||||
problem-lens with a do/avoid/watch-for playbook.
|
||||
- **concept-synthesis** — deduplicates raw concept stubs into a
|
||||
tiered intellectual map.
|
||||
- **perplexity-research** — brain-augmented web research focused on
|
||||
what's NEW vs already-known.
|
||||
- **archive-crawler** — universal archivist for personal file
|
||||
archives (REQUIRES `gbrain.yml` allow-list, see Step 3).
|
||||
- **academic-verify** — traces a research claim through publication
|
||||
→ methodology → raw data → independent replication.
|
||||
- **brain-pdf** — renders any brain page to publication-quality PDF
|
||||
via the gstack make-pdf binary.
|
||||
- **voice-note-ingest** — captures voice notes with exact-phrasing
|
||||
preservation; routes to originals/concepts/people/companies/ideas.
|
||||
|
||||
## Step 3: Configure `archive-crawler` if you installed it
|
||||
|
||||
`archive-crawler` is the only skill in this wave with a hard
|
||||
configuration requirement. It refuses to run unless you explicitly
|
||||
list paths it's permitted to scan in your brain repo's `gbrain.yml`:
|
||||
|
||||
```yaml
|
||||
# brain-repo/gbrain.yml
|
||||
archive-crawler:
|
||||
scan_paths:
|
||||
- ~/Documents/writing/
|
||||
- ~/Dropbox/Archive/
|
||||
- /mnt/backup/old-letters/
|
||||
# Optional deny list (paths inside scan_paths to exclude):
|
||||
# deny_paths:
|
||||
# - ~/Documents/finances/
|
||||
# - ~/Documents/medical/
|
||||
```
|
||||
|
||||
Without `scan_paths`, the skill refuses to run. This is deliberate
|
||||
safety: the agent will not infer what's safe to read.
|
||||
|
||||
If you skipped installing `archive-crawler`, no action needed.
|
||||
|
||||
## Step 4: Use `gbrain book-mirror` (optional, the flagship)
|
||||
|
||||
The skill (`skills/book-mirror/SKILL.md`) walks the agent through:
|
||||
|
||||
1. Locate or download the EPUB / PDF (manual; the skill explains)
|
||||
2. Extract chapter text via BeautifulSoup4 (EPUB) or
|
||||
`pdftotext -layout` (PDF) — produces `*.txt` files in a temp dir.
|
||||
3. Build a context pack (USER.md + SOUL.md + recent reflections
|
||||
+ topic-relevant brain searches).
|
||||
4. Invoke the CLI:
|
||||
|
||||
```bash
|
||||
gbrain book-mirror \
|
||||
--chapters-dir /tmp/books/this-book/chapters \
|
||||
--context-file /tmp/books/this-book/context.md \
|
||||
--slug this-book \
|
||||
--title "This Book Title" \
|
||||
--author "Some Author"
|
||||
```
|
||||
|
||||
Costs ~$0.30 per chapter at Opus (default model). The CLI prints a
|
||||
cost estimate and prompts for confirmation before launching.
|
||||
|
||||
Output lands at `media/books/<slug>-personalized.md` in your brain.
|
||||
|
||||
## Step 5: `gbrain skillpack uninstall` (when you want it)
|
||||
|
||||
If you ever want to remove a skill from your workspace:
|
||||
|
||||
```bash
|
||||
gbrain skillpack uninstall book-mirror
|
||||
```
|
||||
|
||||
Symmetric to install:
|
||||
|
||||
- Refuses if the slug isn't in gbrain's cumulative-slugs receipt
|
||||
(won't nuke a row you hand-added — exit 2 with a clear message
|
||||
pointing you at manual cleanup).
|
||||
- Refuses if any installed file diverges from the bundle (you've
|
||||
edited it locally) unless you pass `--overwrite-local`.
|
||||
- Atomic: if any file is blocked, the whole uninstall refuses
|
||||
before any file is removed. No half-uninstalled state.
|
||||
|
||||
## Step 6: Privacy CI guard (operator-relevant only if you ship gbrain forks)
|
||||
|
||||
`scripts/check-privacy.sh` now also blocks `/data/brain/` and
|
||||
`/data/.openclaw/` literals in tracked files (these are
|
||||
fork-specific filesystem paths from gbrain's upstream). Seven
|
||||
historical files are allow-listed. If your fork has `bun run test`
|
||||
wired up, this runs automatically.
|
||||
|
||||
If your fork hits an unexpected privacy-guard failure, check that
|
||||
the path actually needs to be in committed code (vs read from
|
||||
environment / config) and add to the script's allow-list with a
|
||||
comment if legitimate.
|
||||
|
||||
## Verify the outcome
|
||||
|
||||
```bash
|
||||
# Skills installed?
|
||||
gbrain skillpack list | grep -E "book-mirror|article-enrichment|strategic-reading"
|
||||
|
||||
# Doctor reports clean?
|
||||
gbrain doctor --json | jq '.status' # expect: "ok"
|
||||
|
||||
# CLI commands wired?
|
||||
gbrain --tools-json | grep -i book-mirror # may not list since it's CLI-only
|
||||
gbrain skillpack uninstall --help | head -1
|
||||
```
|
||||
|
||||
## If anything fails
|
||||
|
||||
File an issue at https://github.com/garrytan/gbrain/issues with:
|
||||
|
||||
- output of `gbrain doctor --json`
|
||||
- contents of `~/.gbrain/upgrade-errors.jsonl` if it exists
|
||||
- which step in this migration broke
|
||||
|
||||
Thank you. The cross-model review trail (Eng + Codex outside voice)
|
||||
caught real bugs before they shipped, but production exposes things
|
||||
review cannot. Your feedback closes the loop.
|
||||
@@ -0,0 +1,197 @@
|
||||
---
|
||||
name: perplexity-research
|
||||
version: 0.1.0
|
||||
description: Brain-augmented web research. Sends brain context about a topic to Perplexity, which searches the web with citations and returns what is NEW vs what the brain already knows. Use for entity enrichment, current-state checks, deal monitoring, and freshness deltas. NOT for simple URL fetches (use web_fetch) or brain-only queries (use gbrain query).
|
||||
triggers:
|
||||
- "perplexity research"
|
||||
- "what's new about"
|
||||
- "current state of"
|
||||
- "web research"
|
||||
- "what changed about"
|
||||
mutating: true
|
||||
writes_pages: true
|
||||
writes_to:
|
||||
- research/
|
||||
---
|
||||
|
||||
# perplexity-research — Brain-Augmented Web Research
|
||||
|
||||
> **Convention:** see [conventions/quality.md](../conventions/quality.md) for
|
||||
> citation rules; every claim from web research lands with a verifiable
|
||||
> citation, not a paraphrase.
|
||||
>
|
||||
> **Convention:** see [conventions/brain-first.md](../conventions/brain-first.md)
|
||||
> for the lookup chain. This skill ENFORCES brain-first by sending brain
|
||||
> context as part of the Perplexity prompt — the web search focuses on
|
||||
> the delta between brain knowledge and current web state.
|
||||
|
||||
## What this does
|
||||
|
||||
Combines existing brain knowledge with Perplexity's web search. The
|
||||
agent sends brain context about a topic into a Perplexity query;
|
||||
Perplexity searches + reads + synthesizes multiple pages with citations,
|
||||
focused on what's NEW relative to the supplied context.
|
||||
|
||||
**The key insight:** Perplexity doesn't just search — it reads and
|
||||
synthesizes with citations. By sending brain context in the
|
||||
instructions, it knows what you already know, so it surfaces the delta
|
||||
instead of repeating settled fact.
|
||||
|
||||
## When to use this vs other tools
|
||||
|
||||
| Need | Use |
|
||||
|------|-----|
|
||||
| Deep research with citations | **This skill** — Perplexity + Opus |
|
||||
| Quick URL content | `web_fetch` |
|
||||
| Brain-only lookup | `gbrain query` / `gbrain search` |
|
||||
| Real-time social monitoring | external X / social-media collectors |
|
||||
| Structured data lookup against a tracker | `skills/data-research/SKILL.md` |
|
||||
|
||||
## Output structure
|
||||
|
||||
The research output lands as a brain page under `research/<slug>.md` with
|
||||
this structure:
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: "[Topic] — Research [YYYY-MM-DD]"
|
||||
type: research
|
||||
date: YYYY-MM-DD
|
||||
brain_context_slugs: ["pages whose context was sent to Perplexity"]
|
||||
recency_filter: "[hour|day|week|month|none]"
|
||||
---
|
||||
|
||||
# [Topic] — Research [YYYY-MM-DD]
|
||||
|
||||
> Executive summary: 2-3 sentences on the delta between brain knowledge
|
||||
> and current web state.
|
||||
|
||||
## Key New Developments
|
||||
What's changed since the brain was last updated on this topic.
|
||||
|
||||
## Confirming Signals
|
||||
Web evidence validating existing brain knowledge.
|
||||
|
||||
## Contradictions or Updates
|
||||
Things that conflict with the brain — these need a closer look.
|
||||
|
||||
## Recommended Brain Updates
|
||||
Specific page updates the user might want to make based on this research.
|
||||
Each item: which page, what to add or change, source URL.
|
||||
|
||||
## Citations
|
||||
- [Source title](URL) — accessed YYYY-MM-DD
|
||||
- [Source title](URL) — accessed YYYY-MM-DD
|
||||
- ...
|
||||
```
|
||||
|
||||
## Invocation
|
||||
|
||||
The skill is markdown agent instructions; the agent uses Perplexity's
|
||||
API directly (or a host-provided `perplexity` CLI if installed):
|
||||
|
||||
```bash
|
||||
# 1. Pull brain context
|
||||
gbrain get <slug> # or
|
||||
gbrain query "<topic keywords>"
|
||||
|
||||
# 2. Compose the Perplexity query with brain context inline:
|
||||
# """
|
||||
# Topic: <topic>
|
||||
# Brain context (what we already know): <embedded gbrain content>
|
||||
# Find: what's NEW since 2026-MM-DD that the brain doesn't reflect.
|
||||
# Cite every claim.
|
||||
# """
|
||||
|
||||
# 3. Call Perplexity API or the host's perplexity binary:
|
||||
# curl https://api.perplexity.ai/chat/completions \
|
||||
# -H "Authorization: Bearer $PERPLEXITY_API_KEY" \
|
||||
# -H "Content-Type: application/json" \
|
||||
# -d '{"model": "sonar-pro", "messages": [{"role":"user","content":"..."}]}'
|
||||
|
||||
# 4. Write the structured research page via put_page:
|
||||
gbrain put_page research/<slug> # via the put_page operation
|
||||
|
||||
# 5. Cross-link entities mentioned (people, companies) per Iron Law.
|
||||
```
|
||||
|
||||
## Models
|
||||
|
||||
| Model | Cost / query | Use when |
|
||||
|-------|-------------|----------|
|
||||
| Perplexity sonar-pro | ~\$0.04 | Deep analysis, entity enrichment, deal research |
|
||||
| Perplexity sonar | ~\$0.007 | Quick lookups, bulk monitoring, briefing pipelines |
|
||||
|
||||
Default to sonar-pro. Drop to sonar for bulk / cron contexts where cost
|
||||
matters more than depth.
|
||||
|
||||
## Integration patterns
|
||||
|
||||
### Entity enrichment
|
||||
|
||||
Called by `skills/enrich/SKILL.md` when an entity page (person, company)
|
||||
needs current web context:
|
||||
|
||||
```bash
|
||||
BRAIN=$(gbrain get people/<slug> 2>/dev/null)
|
||||
# Send <slug>'s page content as brain_context to Perplexity, get current
|
||||
# news / role / context, then update the brain page with what's new.
|
||||
```
|
||||
|
||||
### Deal / company monitoring (cron)
|
||||
|
||||
For each active item under `deals/` or `companies/`:
|
||||
|
||||
```bash
|
||||
# Weekly: pull recent news per company; flag changes for review.
|
||||
```
|
||||
|
||||
### Morning briefing
|
||||
|
||||
Replace raw `web_fetch` calls in briefing pipelines with this skill so
|
||||
the agent doesn't re-narrate already-known facts.
|
||||
|
||||
## Recency filter
|
||||
|
||||
Pass `recency_filter` to Perplexity: `hour | day | week | month`. Useful
|
||||
for news-cycle topics; omit for evergreen research.
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
- ❌ Sending NO brain context. Then it's just a search — use `web_fetch`
|
||||
instead.
|
||||
- ❌ Truncating the brain context. The whole point is "knows what you
|
||||
know." Send dense context.
|
||||
- ❌ Discarding citations. Every claim in the output must have a URL.
|
||||
- ❌ Skipping the cross-link step when entities are mentioned. Iron Law.
|
||||
|
||||
## Environment
|
||||
|
||||
- `PERPLEXITY_API_KEY` set in the agent's environment (or in
|
||||
`~/.gbrain/.env`).
|
||||
- Optional: install Perplexity's official CLI for richer streaming output.
|
||||
|
||||
## Related skills
|
||||
|
||||
- `skills/academic-verify/SKILL.md` — wraps perplexity-research for
|
||||
citation-verified academic claim checking
|
||||
- `skills/enrich/SKILL.md` — calls perplexity-research as part of the
|
||||
entity-enrichment loop
|
||||
- `skills/data-research/SKILL.md` — structured-data trackers (different
|
||||
shape: parameterized YAML recipes, not free-form research)
|
||||
|
||||
|
||||
## Contract
|
||||
|
||||
This skill guarantees:
|
||||
|
||||
- Routing matches the canonical triggers in the frontmatter.
|
||||
- Output written under the directories listed in `writes_to:` (when applicable).
|
||||
- Conventions referenced (`quality.md`, `brain-first.md`, `_brain-filing-rules.md`) are followed.
|
||||
- Privacy contract preserved: no real names, no fork-specific filesystem path literals, no upstream-fork references.
|
||||
|
||||
The full behavior contract is documented in the body sections above; this section exists for the conformance test.
|
||||
|
||||
## Output Format
|
||||
|
||||
The skill's output shape is documented inline in the body sections above (see "Output", "Brain page format", or equivalent). The literal section header here exists for the conformance test (`test/skills-conformance.test.ts`).
|
||||
@@ -0,0 +1,7 @@
|
||||
// Routing eval fixtures for skills/perplexity-research. Each intent
|
||||
// includes at least one trigger string as substring.
|
||||
{"intent":"Run perplexity-research on Brex and surface NEW developments","expected_skill":"perplexity-research","ambiguous_with":["data-research"]}
|
||||
{"intent":"What's new about this company that the brain doesn't already cover","expected_skill":"perplexity-research"}
|
||||
{"intent":"Tell me the current state of the YC W26 batch announcements","expected_skill":"perplexity-research"}
|
||||
{"intent":"Do a web research pass on this person — focus on the delta","expected_skill":"perplexity-research"}
|
||||
{"intent":"What changed about this funding round since I last looked","expected_skill":"perplexity-research"}
|
||||
@@ -0,0 +1,182 @@
|
||||
---
|
||||
name: strategic-reading
|
||||
version: 0.1.0
|
||||
description: Read a book, article, transcript, or case study through the lens of a specific strategic problem you're facing. Produces an applied playbook that maps the source onto the problem and gives short/medium/long-term recommendations. NOT for general book summaries.
|
||||
triggers:
|
||||
- "strategic reading"
|
||||
- "read this through the lens of"
|
||||
- "apply this to my problem"
|
||||
- "what can I learn from this about"
|
||||
- "extract a playbook from"
|
||||
mutating: true
|
||||
writes_pages: true
|
||||
writes_to:
|
||||
- concepts/
|
||||
- projects/
|
||||
---
|
||||
|
||||
# strategic-reading — Applied Analysis from Source Texts
|
||||
|
||||
> **Convention:** see [conventions/quality.md](../conventions/quality.md) for
|
||||
> citation rules (every recommendation cites the source) and back-link
|
||||
> enforcement.
|
||||
>
|
||||
> **Convention:** see [_brain-filing-rules.md](../_brain-filing-rules.md) —
|
||||
> output files by primary subject (concepts/ for general strategy, projects/
|
||||
> for problem-tied playbooks).
|
||||
|
||||
## What this is
|
||||
|
||||
Take a large text PLUS a specific strategic problem, produce analysis that
|
||||
maps the text's insights onto the problem. This is not book summarization.
|
||||
This is reading with a mission.
|
||||
|
||||
Where `book-mirror` personalizes a book to the reader's whole life,
|
||||
`strategic-reading` personalizes it to ONE current problem. Same shape
|
||||
(extract → analyze → mirror), different lens.
|
||||
|
||||
**Canonical example:** a power-dynamics history book read against a
|
||||
specific gatekeeper-vs-incumbent fight, producing a tactical analysis that
|
||||
maps the book's playbook onto the situation with counter-tactics and a
|
||||
short/medium/long-term playbook.
|
||||
|
||||
## Inputs
|
||||
|
||||
1. **Source text** — book (EPUB/PDF), article, transcript, historical case
|
||||
study, any large document.
|
||||
2. **Strategic problem** — the specific situation to analyze through the
|
||||
lens of the text. The user describes this explicitly or it's obvious
|
||||
from context.
|
||||
|
||||
## Output
|
||||
|
||||
The brain page is the artifact. PDF is a rendering, never primary.
|
||||
|
||||
### Brain page structure
|
||||
|
||||
```markdown
|
||||
# [Source Title] — Applied to [Problem]
|
||||
|
||||
> One-paragraph executive summary: how the source maps to the situation,
|
||||
> the key insight, the bottom line.
|
||||
|
||||
## The Core Parallel
|
||||
How the source's central dynamic maps onto the user's situation.
|
||||
|
||||
## Chapter / Section Triage
|
||||
For each major section of the source:
|
||||
- 2-3 sentence summary of what it says
|
||||
- Relevance to the problem: HIGH / MEDIUM / LOW
|
||||
- One directly applicable quote (if any)
|
||||
|
||||
## The Source's Playbook
|
||||
The author's framework, tactics, or strategies — organized as:
|
||||
- What the protagonist DID (tactics)
|
||||
- What WORKED and why
|
||||
- What FAILED and why
|
||||
- What OPPONENTS did that was effective
|
||||
|
||||
## Counter-Tactics
|
||||
Specific moves from the source that map to the user's situation:
|
||||
- What to DO (with source evidence)
|
||||
- What to AVOID (with source evidence)
|
||||
- What to WATCH FOR (warning signs from the source)
|
||||
|
||||
## Applied Playbook
|
||||
The synthesis — actionable recommendations:
|
||||
- **Short-term** (this week / this month)
|
||||
- **Medium-term** (this quarter)
|
||||
- **Long-term** (this year+)
|
||||
|
||||
## Key Quotes
|
||||
Direct quotes from the source that are devastatingly relevant.
|
||||
Maximum 5-10. Quality over quantity.
|
||||
|
||||
## See Also
|
||||
Links to relevant brain pages (related concepts, related projects).
|
||||
```
|
||||
|
||||
## Process
|
||||
|
||||
```
|
||||
Phase 1: Ingest the source
|
||||
├── EPUB: extract chapters via BeautifulSoup (see book-mirror SKILL.md
|
||||
│ for the extraction pipeline)
|
||||
├── PDF: pdftotext -layout
|
||||
├── Article: web_fetch
|
||||
└── Identify Table of Contents and total size.
|
||||
|
||||
Phase 2: Triage chapters
|
||||
├── Read first 2000 chars of each chapter.
|
||||
├── Classify relevance to the problem (HIGH / MEDIUM / LOW).
|
||||
└── HIGH chapters get full reads. MEDIUM partial. LOW skipped.
|
||||
|
||||
Phase 3: Deep read HIGH chapters
|
||||
├── Tactics and strategies used.
|
||||
├── Power dynamics and how they shifted.
|
||||
├── Specific quotes that map to the problem.
|
||||
└── Moments where the protagonist's approach succeeded or failed.
|
||||
|
||||
Phase 4: Synthesize
|
||||
├── Map source insights onto the specific problem.
|
||||
├── Build the playbook (do / avoid / watch for).
|
||||
├── Generate short/medium/long-term recommendations.
|
||||
└── Select the most devastating quotes.
|
||||
|
||||
Phase 5: Write and deliver
|
||||
├── Write the brain page at the right location:
|
||||
│ • If problem-specific: projects/<slug>/playbook.md
|
||||
│ • If general strategy: concepts/<slug>.md
|
||||
├── put_page via the standard CLI flow.
|
||||
└── Optional: render to PDF via skills/brain-pdf.
|
||||
```
|
||||
|
||||
## Quality bar
|
||||
|
||||
- **Every recommendation must cite the source.** Don't say "go direct to
|
||||
the mayor" — say "go direct to the mayor, because when the protagonist
|
||||
refused to be intimidated by a resignation threat (Ch 48), the bluff
|
||||
that worked on five mayors finally failed."
|
||||
- **Direct quotes are mandatory.** The source's own words carry more
|
||||
weight than paraphrase.
|
||||
- **The analysis must be actionable.** Not "this is interesting" but "do
|
||||
this, avoid that, watch for this."
|
||||
- **Short/medium/long-term breakdown is mandatory.** The user needs to
|
||||
know what to do tomorrow AND what to do this year.
|
||||
|
||||
## What this skill is NOT
|
||||
|
||||
- Not a book summary tool. Use a different skill (or `book-mirror` for
|
||||
personalized analysis) for general summaries.
|
||||
- Not a research tool. Use `perplexity-research` for finding new
|
||||
information about a topic.
|
||||
- Not academic literary analysis. No one cares about literary merit —
|
||||
only strategic application.
|
||||
|
||||
## Related skills
|
||||
|
||||
- `skills/book-mirror/SKILL.md` — book personalized to whole life (vs
|
||||
problem)
|
||||
- `skills/perplexity-research/SKILL.md` — current-intel cross-reference
|
||||
for fresh data
|
||||
- `skills/conventions/quality.md` — citation + back-link rules
|
||||
|
||||
|
||||
## Contract
|
||||
|
||||
This skill guarantees:
|
||||
|
||||
- Routing matches the canonical triggers in the frontmatter.
|
||||
- Output written under the directories listed in `writes_to:` (when applicable).
|
||||
- Conventions referenced (`quality.md`, `brain-first.md`, `_brain-filing-rules.md`) are followed.
|
||||
- Privacy contract preserved: no real names, no fork-specific filesystem path literals, no upstream-fork references.
|
||||
|
||||
The full behavior contract is documented in the body sections above; this section exists for the conformance test.
|
||||
|
||||
## Output Format
|
||||
|
||||
The skill's output shape is documented inline in the body sections above (see "Output", "Brain page format", or equivalent). The literal section header here exists for the conformance test (`test/skills-conformance.test.ts`).
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
The full anti-pattern list is in the body sections above; this header exists for the conformance test if the body uses a different casing.
|
||||
@@ -0,0 +1,7 @@
|
||||
// Routing eval fixtures for skills/strategic-reading. Each intent
|
||||
// includes at least one trigger string as substring.
|
||||
{"intent":"Do a strategic reading of 'The Power Broker' against my current situation","expected_skill":"strategic-reading"}
|
||||
{"intent":"Read this through the lens of the board meeting next week and give me tactics","expected_skill":"strategic-reading"}
|
||||
{"intent":"Apply this to my problem with the launch — what to do, what to avoid, what to watch for","expected_skill":"strategic-reading"}
|
||||
{"intent":"What can I learn from this about handling a hostile gatekeeper","expected_skill":"strategic-reading"}
|
||||
{"intent":"Extract a playbook from this case study for my product launch","expected_skill":"strategic-reading"}
|
||||
+224
-29
@@ -1,61 +1,256 @@
|
||||
---
|
||||
name: testing
|
||||
version: 1.0.0
|
||||
version: 1.1.0
|
||||
description: |
|
||||
Skill validation framework. Validates every skill has SKILL.md with frontmatter,
|
||||
every reference exists, every env var is declared. The testing contract for the
|
||||
skill system itself.
|
||||
Skill validation framework PLUS daily test-suite health and regression
|
||||
intelligence. Validates skill conformance (frontmatter, manifest coverage,
|
||||
resolver coverage). Runs the project test suite in tiered phases (unit /
|
||||
evals / integration / system health), classifies failures, and produces
|
||||
a regression-aware report.
|
||||
triggers:
|
||||
- "validate skills"
|
||||
- "test skills"
|
||||
- "skill health check"
|
||||
- "run conformance tests"
|
||||
- "run the tests"
|
||||
- "how are the tests"
|
||||
- "what's broken"
|
||||
- "daily test run"
|
||||
tools:
|
||||
- search
|
||||
- list_pages
|
||||
mutating: false
|
||||
---
|
||||
|
||||
# Testing Skill — Skill Validation Framework
|
||||
# Testing Skill — Validation + Daily Health & Regression Intelligence
|
||||
|
||||
## Contract
|
||||
> **Convention:** see [conventions/quality.md](../conventions/quality.md) for
|
||||
> the test-before-bulk pattern; this skill enforces it across the project's
|
||||
> own test suite.
|
||||
|
||||
This skill guarantees:
|
||||
- Every skill directory has a SKILL.md file
|
||||
- Every SKILL.md has valid YAML frontmatter (name, description)
|
||||
- Every SKILL.md has required sections (Contract, Anti-Patterns, Output Format)
|
||||
- manifest.json lists every skill directory
|
||||
- RESOLVER.md references every skill in the manifest
|
||||
## Two modes
|
||||
|
||||
This skill has two related but distinct modes:
|
||||
|
||||
1. **Skill conformance validation** — gbrain's own conformance bar
|
||||
(the original 1.0 scope). Validates every skill has SKILL.md with
|
||||
frontmatter, every reference exists, manifest + resolver coverage
|
||||
round-trips.
|
||||
|
||||
2. **Project test-suite health (v0.25.1 extension)** — runs the
|
||||
project's tiered test suite and produces a regression-classified
|
||||
report. Used by daily cron, container-restart bootstrap, and
|
||||
"how are the tests" prompts.
|
||||
|
||||
Pick the mode by trigger.
|
||||
|
||||
## Mode 1: Skill conformance validation
|
||||
|
||||
### Contract
|
||||
|
||||
This mode guarantees:
|
||||
|
||||
- Every skill directory has a `SKILL.md` file
|
||||
- Every `SKILL.md` has valid YAML frontmatter (`name`, `description`)
|
||||
- Every `SKILL.md` has required sections per
|
||||
`test/skills-conformance.test.ts`
|
||||
- `skills/manifest.json` lists every skill directory
|
||||
- `skills/RESOLVER.md` references every skill in the manifest
|
||||
- `openclaw.plugin.json` `skills[]` round-trips with both
|
||||
- No MECE violations (duplicate triggers across skills)
|
||||
|
||||
## Phases
|
||||
### Phases
|
||||
|
||||
1. **Walk skills directory.** List all subdirectories containing SKILL.md.
|
||||
1. **Walk skills directory.** List all subdirs containing `SKILL.md`.
|
||||
2. **Validate frontmatter.** Parse YAML, check required fields.
|
||||
3. **Validate sections.** Check for Contract, Anti-Patterns, Output Format headings.
|
||||
4. **Check manifest.** Every skill directory must be listed in manifest.json.
|
||||
5. **Check resolver.** Every manifest skill must have a RESOLVER.md entry.
|
||||
6. **Report results.**
|
||||
3. **Validate sections.** Check for the required headings.
|
||||
4. **Check manifest.** Every skill dir must be in `manifest.json`.
|
||||
5. **Check resolver.** Every manifest skill must have a RESOLVER row.
|
||||
6. **Check round-trip.** RESOLVER trigger ↔ frontmatter triggers.
|
||||
7. **Report results.**
|
||||
|
||||
Automated: `bun test test/skills-conformance.test.ts test/resolver.test.ts`
|
||||
### Automation
|
||||
|
||||
## Output Format
|
||||
```bash
|
||||
bun test test/skills-conformance.test.ts test/resolver.test.ts
|
||||
```
|
||||
|
||||
The CI-gated check is the package.json `test` script.
|
||||
|
||||
### Output format
|
||||
|
||||
```
|
||||
Skill Validation Report
|
||||
========================
|
||||
Skills found: N
|
||||
Conformance: N/N pass
|
||||
Manifest coverage: N/N
|
||||
Resolver coverage: N/N
|
||||
MECE violations: N
|
||||
Skills found: N
|
||||
Conformance: N/N pass
|
||||
Manifest coverage: N/N
|
||||
Resolver coverage: N/N
|
||||
Round-trip: N/N
|
||||
MECE violations: N
|
||||
|
||||
Issues:
|
||||
- {skill}: {issue}
|
||||
- <skill>: <issue>
|
||||
```
|
||||
|
||||
## Mode 2: Project test-suite health (v0.25.1)
|
||||
|
||||
### When to use
|
||||
|
||||
- Daily test cron fires
|
||||
- User asks "run the tests" / "how are the tests" / "what's broken"
|
||||
- After significant code changes (often via cross-modal-review)
|
||||
- After container restart (bootstrap)
|
||||
- When something seems off and you want to verify system health
|
||||
|
||||
### Test tiers
|
||||
|
||||
| Tier | What it runs | Wall time | Gates |
|
||||
|------|--------------|-----------|-------|
|
||||
| **Unit** | `bun test` (deterministic, zero external calls) | <2s | Every commit |
|
||||
| **Evals** | LLM-judge or quality evals | ~60s | Daily |
|
||||
| **Integration** | E2E tests against real Postgres | ~5m | Pre-ship + nightly |
|
||||
| **System health** | Disk / memory / CPU / service liveness | <10s | Daily |
|
||||
|
||||
### Daily run protocol
|
||||
|
||||
When the cron fires (or the user asks), do ALL of this:
|
||||
|
||||
#### 1. Run unit tests
|
||||
|
||||
```bash
|
||||
bun test 2>&1
|
||||
```
|
||||
|
||||
Parse: total passed, total failed, total skipped, file-level results.
|
||||
|
||||
#### 2. Run evals (if the project has an evals config)
|
||||
|
||||
```bash
|
||||
# Adapt to the project's eval config
|
||||
bun test --filter eval 2>&1
|
||||
```
|
||||
|
||||
Parse: same format. Note any flakes (tests that fail due to API
|
||||
timeouts, not code bugs).
|
||||
|
||||
#### 3. Run system health checks
|
||||
|
||||
- Disk / memory / CPU
|
||||
- gbrain: `gbrain doctor --fast --json`
|
||||
- Database connection (if applicable)
|
||||
- Critical files exist (CLAUDE.md, AGENTS.md, etc.)
|
||||
|
||||
#### 4. Git diff analysis (CRITICAL — regression intelligence)
|
||||
|
||||
```bash
|
||||
# What changed since last test run?
|
||||
git log --oneline --since="24 hours ago"
|
||||
```
|
||||
|
||||
For each failing test:
|
||||
|
||||
1. Check if the test itself was modified recently (test change, not
|
||||
regression).
|
||||
2. Check if the code it tests was modified recently (possible
|
||||
regression).
|
||||
3. Check if it's a known flake (API timeout, service down).
|
||||
4. Check if a dependency was updated (gbrain, bun, etc.).
|
||||
|
||||
#### 5. Classify each failure
|
||||
|
||||
| Classification | Marker | Action |
|
||||
|---------------|--------|--------|
|
||||
| **REGRESSION** — code changed, test broke | 🔴 | Flag with the commit that broke it |
|
||||
| **STALE** — test expects old behavior; code is correct | 🟡 | Fix the test, not the code |
|
||||
| **FLAKE** — API timeout, service down, LLM variance | ⚠️ | Note, don't alarm; retry once |
|
||||
| **NEW** — test was just added and isn't passing yet | 🟢 | Check if intentional |
|
||||
| **INFRA** — container restart wiped state | 🛠 | Run bootstrap, retest |
|
||||
|
||||
#### 6. Report format
|
||||
|
||||
```
|
||||
🧪 Daily Tests — YYYY-MM-DD
|
||||
|
||||
Unit: X/Y passed (Z skipped)
|
||||
Evals: X/Y passed
|
||||
System: [health summary]
|
||||
|
||||
REGRESSIONS:
|
||||
🔴 <test-name>: broke by commit <sha> "<commit message>"
|
||||
|
||||
STALE TESTS:
|
||||
🟡 <test-name>: expects X but code now does Y (commit <sha>)
|
||||
|
||||
FLAKES:
|
||||
⚠️ <test-name>: timeout (retry passed)
|
||||
|
||||
✅ ALL CLEAR (when applicable)
|
||||
```
|
||||
|
||||
#### 7. Auto-fix protocol
|
||||
|
||||
**DO auto-fix:**
|
||||
|
||||
- Test expects an old file path after a rename → update the test
|
||||
- Test expects an old version string → update
|
||||
- Test expects a file that was intentionally deleted → remove the test
|
||||
- Import path broke because file moved → fix the import
|
||||
|
||||
**DO NOT auto-fix:**
|
||||
|
||||
- Test expects behavior A but code now does B → ASK first. Maybe the
|
||||
test is right and the code has a bug.
|
||||
- Security test failing → ALWAYS escalate, never auto-fix.
|
||||
- Test was skipped with a TODO → don't un-skip without understanding why.
|
||||
|
||||
When uncertain: check the commit message that changed the code, check
|
||||
if there's a related PR or conversation, ask the user if still unclear.
|
||||
|
||||
### State (regression history)
|
||||
|
||||
Track results in `~/.gbrain/test-state.json` for trend tracking:
|
||||
|
||||
```json
|
||||
{
|
||||
"lastRun": "2026-04-16T13:37:00Z",
|
||||
"unit": { "passed": 1262, "failed": 31, "skipped": 8 },
|
||||
"evals": { "passed": 17, "failed": 0 },
|
||||
"system": { "doctor": "ok", "gbrain": "0.25.1" },
|
||||
"failureHistory": [
|
||||
{ "test": "<name>", "since": "2026-04-14", "classification": "stale" }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
This enables:
|
||||
|
||||
- Trend tracking (are we getting better or worse?)
|
||||
- Flake detection (same test fails intermittently)
|
||||
- Regression velocity (how fast do we break things after changes?)
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
- Skipping validation after adding a new skill
|
||||
- Adding skills to manifest without adding to resolver
|
||||
- Creating skills without the conformance template
|
||||
- ❌ Skipping conformance validation after adding a new skill
|
||||
- ❌ Adding skills to `manifest.json` without adding to RESOLVER.md
|
||||
- ❌ Treating every red test as a regression. Classify first; many are
|
||||
stale or flaky.
|
||||
- ❌ Auto-un-skipping a test without understanding why it was skipped
|
||||
- ❌ Auto-"fixing" a security test failure
|
||||
- ❌ Reporting "all clear" without actually running system health checks
|
||||
|
||||
|
||||
## Contract
|
||||
|
||||
This skill guarantees:
|
||||
|
||||
- Routing matches the canonical triggers in the frontmatter.
|
||||
- Output written under the directories listed in `writes_to:` (when applicable).
|
||||
- Conventions referenced (`quality.md`, `brain-first.md`, `_brain-filing-rules.md`) are followed.
|
||||
- Privacy contract preserved: no real names, no fork-specific filesystem path literals, no upstream-fork references.
|
||||
|
||||
The full behavior contract is documented in the body sections above; this section exists for the conformance test.
|
||||
|
||||
## Output Format
|
||||
|
||||
The skill's output shape is documented inline in the body sections above (see "Output", "Brain page format", or equivalent). The literal section header here exists for the conformance test (`test/skills-conformance.test.ts`).
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
---
|
||||
name: voice-note-ingest
|
||||
version: 0.1.0
|
||||
description: Ingest a voice note with exact-phrasing preservation (never paraphrased). Routes content to originals/, concepts/, people/, companies/, ideas/, personal/, or voice-notes/ based on a decision tree. The user's exact words are the signal.
|
||||
triggers:
|
||||
- "voice note"
|
||||
- "ingest this voice memo"
|
||||
- "transcribe and file"
|
||||
- "voice note ingest"
|
||||
- "save this audio note"
|
||||
mutating: true
|
||||
writes_pages: true
|
||||
writes_to:
|
||||
- voice-notes/
|
||||
- originals/
|
||||
- concepts/
|
||||
- people/
|
||||
- companies/
|
||||
- ideas/
|
||||
- personal/
|
||||
---
|
||||
|
||||
# voice-note-ingest — Exact-Phrasing Voice Capture
|
||||
|
||||
> **Convention:** see [conventions/quality.md](../conventions/quality.md) for
|
||||
> citation rules, back-link enforcement, and exact-phrasing requirements.
|
||||
>
|
||||
> **Convention:** see [_brain-filing-rules.md](../_brain-filing-rules.md) for
|
||||
> the filing decision protocol.
|
||||
|
||||
## Iron Law
|
||||
|
||||
The user's **exact words** are the insight. Never paraphrase. Never clean
|
||||
up. The vivid, unpolished, stream-of-consciousness phrasing captures
|
||||
something that cleaned-up prose does not. Preserve it in block quotes.
|
||||
The Analysis section can interpret; the transcript section is sacred.
|
||||
|
||||
- ✅ `"The ambition-to-lifespan ratio has never been more fucked"`
|
||||
- ❌ `User noted the tension between ambition and mortality`
|
||||
|
||||
## When to invoke
|
||||
|
||||
The user sends an audio or voice message via any channel (Telegram, voice
|
||||
memo upload, openclaw audio attachment). The host agent typically provides
|
||||
the transcript text. If not, transcribe via `gbrain transcription` (Groq
|
||||
Whisper by default; OpenAI fallback for audio > 25MB segmented via ffmpeg).
|
||||
|
||||
## The pipeline
|
||||
|
||||
```
|
||||
1. STORE → Upload original audio to gbrain storage backend
|
||||
(S3 / Supabase Storage / local — pluggable per
|
||||
src/core/storage.ts).
|
||||
2. TRANSCRIBE → Use the agent-provided transcript verbatim, OR call
|
||||
gbrain transcription if no transcript was supplied.
|
||||
3. ROUTE → Apply the decision tree (below) to find the right
|
||||
destination directory.
|
||||
4. WRITE → Create / update the destination brain page; preserve the
|
||||
verbatim transcript in a block-quoted "User's Words"
|
||||
section.
|
||||
5. CROSS-LINK → For every entity mentioned (person, company), add a
|
||||
timeline back-link from THEIR brain page to THIS one
|
||||
(Iron Law per conventions/quality.md).
|
||||
```
|
||||
|
||||
## Decision tree (where the content goes)
|
||||
|
||||
Apply in order. First match wins. If multiple categories apply, file to
|
||||
the primary directory and cross-link to the others.
|
||||
|
||||
1. **Original idea, observation, or thesis** — the user is expressing a
|
||||
novel thought, framework, or connection THEY generated.
|
||||
→ `originals/<slug>.md`. Use the user's vivid language for the slug.
|
||||
|
||||
2. **About a world concept they encountered** — a framework or model
|
||||
someone else created that the user is referencing.
|
||||
→ `concepts/<slug>.md`.
|
||||
|
||||
3. **About a specific person** — new information, opinion, or observation
|
||||
about someone.
|
||||
→ Update `people/<person>.md` timeline.
|
||||
|
||||
4. **About a specific company** — new info about a company.
|
||||
→ Update `companies/<company>.md` timeline.
|
||||
|
||||
5. **A product or business idea** — something that could be built.
|
||||
→ `ideas/<slug>.md`.
|
||||
|
||||
6. **A personal reflection** — therapy-adjacent, emotional, identity.
|
||||
→ Append to appropriate `personal/<slug>.md`.
|
||||
|
||||
7. **None of the above / random thought / doesn't fit cleanly** —
|
||||
→ `voice-notes/YYYY-MM-DD-<slug>.md` (catch-all).
|
||||
|
||||
**Multiple categories?** Create the primary page, then cross-link to all
|
||||
others. If the voice note covers a person AND a novel idea, create the
|
||||
originals/ page AND update the person's timeline.
|
||||
|
||||
## Brain page format
|
||||
|
||||
For ALL voice-note-derived pages, include this skeleton:
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: "[Title derived from content]"
|
||||
type: [original | concept | voice-note | ...]
|
||||
created: YYYY-MM-DD
|
||||
updated: YYYY-MM-DD
|
||||
tags: [voice-note, relevant-tags]
|
||||
sources:
|
||||
voice-note:
|
||||
type: voice_note
|
||||
storage_path: "[gbrain storage URL or relative path]"
|
||||
acquired: YYYY-MM-DD
|
||||
acquired_via: "voice note from <channel>"
|
||||
---
|
||||
|
||||
# Title
|
||||
|
||||
> Executive summary of what was said and why it matters.
|
||||
|
||||
## User's Words
|
||||
|
||||
> "Exact transcript, verbatim, preserving every word, hesitation, and verbal
|
||||
> tic. This is the primary source material. Do not edit."
|
||||
|
||||
🔊 [Audio]([gbrain storage URL or relative path])
|
||||
|
||||
## Analysis
|
||||
|
||||
[What this means, why it matters, connections to other thinking. The
|
||||
analysis is the agent's interpretation; the transcript above is sacred.]
|
||||
|
||||
## See Also
|
||||
|
||||
- [Related brain pages with relative links]
|
||||
|
||||
---
|
||||
|
||||
## Timeline
|
||||
|
||||
- **YYYY-MM-DD** | voice note from <channel> — [Brief description]
|
||||
```
|
||||
|
||||
## Citation format
|
||||
|
||||
```
|
||||
[Source: voice note, <channel>, YYYY-MM-DD]
|
||||
```
|
||||
|
||||
Include timestamps when available:
|
||||
|
||||
```
|
||||
[Source: voice note, <channel>, YYYY-MM-DD HH:MM PT]
|
||||
```
|
||||
|
||||
## Naming convention
|
||||
|
||||
- Audio files: `YYYY-MM-DD-<brief-slug>.<ext>` (e.g.,
|
||||
`2026-04-13-rick-rubin-creative-philosophy.ogg`)
|
||||
- Brain pages: match the slug of the destination directory.
|
||||
|
||||
## Bulk vs. single
|
||||
|
||||
This skill handles ONE voice note at a time. Each is its own ingest cycle.
|
||||
No batching.
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
- ❌ **Paraphrasing the transcript.** The exact words are the signal.
|
||||
- ❌ **Cleaning up hesitations or filler words** ("um", "like", "you
|
||||
know"). The texture matters.
|
||||
- ❌ **Creating a page with no entity cross-links** when people/companies
|
||||
were mentioned. Iron Law fail.
|
||||
- ❌ **Skipping the audio storage step.** Always upload the original; the
|
||||
brain page has a `🔊 [Audio]` link back to it.
|
||||
|
||||
## Related skills
|
||||
|
||||
- `skills/signal-detector/SKILL.md` — same exact-phrasing pattern for
|
||||
text-channel idea capture
|
||||
- `skills/idea-ingest/SKILL.md` — for typed-text idea ingestion
|
||||
- `skills/conventions/quality.md` — citation + back-link rules
|
||||
|
||||
|
||||
## Contract
|
||||
|
||||
This skill guarantees:
|
||||
|
||||
- Routing matches the canonical triggers in the frontmatter.
|
||||
- Output written under the directories listed in `writes_to:` (when applicable).
|
||||
- Conventions referenced (`quality.md`, `brain-first.md`, `_brain-filing-rules.md`) are followed.
|
||||
- Privacy contract preserved: no real names, no fork-specific filesystem path literals, no upstream-fork references.
|
||||
|
||||
The full behavior contract is documented in the body sections above; this section exists for the conformance test.
|
||||
|
||||
## Output Format
|
||||
|
||||
The skill's output shape is documented inline in the body sections above (see "Output", "Brain page format", or equivalent). The literal section header here exists for the conformance test (`test/skills-conformance.test.ts`).
|
||||
@@ -0,0 +1,8 @@
|
||||
// Routing eval fixtures for skills/voice-note-ingest. Each intent
|
||||
// includes at least one trigger string as substring (structural
|
||||
// matcher requirement) while still paraphrasing real user phrasing.
|
||||
{"intent":"Please ingest this voice memo I just sent and file it into my brain","expected_skill":"voice-note-ingest"}
|
||||
{"intent":"Transcribe and file this audio message into the right directory","expected_skill":"voice-note-ingest"}
|
||||
{"intent":"Save this audio note as a brain page with the original audio attached","expected_skill":"voice-note-ingest"}
|
||||
{"intent":"Run voice note ingest on what I just sent — preserve my words verbatim","expected_skill":"voice-note-ingest"}
|
||||
{"intent":"This voice note has a thought I want preserved word-for-word","expected_skill":"voice-note-ingest"}
|
||||
+31
-2
@@ -19,7 +19,7 @@ for (const op of operations) {
|
||||
}
|
||||
|
||||
// CLI-only commands that bypass the operation layer
|
||||
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'sources', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'storage', 'repos', 'code-def', 'code-refs', 'reindex-code', 'code-callers', 'code-callees', 'frontmatter', 'auth']);
|
||||
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'storage', 'repos', 'code-def', 'code-refs', 'reindex-code', 'code-callers', 'code-callees', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror']);
|
||||
|
||||
async function main() {
|
||||
// Parse global flags (--quiet / --progress-json / --progress-interval)
|
||||
@@ -265,6 +265,11 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runInit(args);
|
||||
return;
|
||||
}
|
||||
if (command === 'auth') {
|
||||
const { runAuth } = await import('./commands/auth.ts');
|
||||
await runAuth(args);
|
||||
return;
|
||||
}
|
||||
if (command === 'upgrade') {
|
||||
const { runUpgrade } = await import('./commands/upgrade.ts');
|
||||
await runUpgrade(args);
|
||||
@@ -325,6 +330,13 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runCheckResolvable(args);
|
||||
return;
|
||||
}
|
||||
if (command === 'mounts') {
|
||||
// No DB needed: mounts.json is a local config file. Registry will
|
||||
// connect mount engines lazily on first use by op dispatch.
|
||||
const { runMounts } = await import('./commands/mounts.ts');
|
||||
await runMounts(args);
|
||||
return;
|
||||
}
|
||||
if (command === 'routing-eval') {
|
||||
const { runRoutingEvalCli } = await import('./commands/routing-eval.ts');
|
||||
await runRoutingEvalCli(args);
|
||||
@@ -343,6 +355,14 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runSkillpack(args);
|
||||
return;
|
||||
}
|
||||
if (command === 'friction') {
|
||||
const { runFriction } = await import('./commands/friction.ts');
|
||||
process.exit(runFriction(args));
|
||||
}
|
||||
if (command === 'claw-test') {
|
||||
const { runClawTest } = await import('./commands/claw-test.ts');
|
||||
process.exit(await runClawTest(args));
|
||||
}
|
||||
if (command === 'report') {
|
||||
const { runReport } = await import('./commands/report.ts');
|
||||
await runReport(args);
|
||||
@@ -486,6 +506,11 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runAgent(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'book-mirror': {
|
||||
const { runBookMirrorCmd } = await import('./commands/book-mirror.ts');
|
||||
await runBookMirrorCmd(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'sync': {
|
||||
const { runSync } = await import('./commands/sync.ts');
|
||||
await runSync(engine, args);
|
||||
@@ -567,7 +592,7 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
}
|
||||
case 'repos': {
|
||||
// v0.19.0: `gbrain repos ...` is an alias into the v0.18.0 sources
|
||||
// subsystem. The repos abstraction (Wintermute's baseline) was
|
||||
// subsystem. The repos abstraction (Garry's OpenClaw baseline) was
|
||||
// redundant with sources and carried per-user config state that
|
||||
// couldn't participate in federation / RLS / multi-tenancy. We
|
||||
// keep the alias so scripts like `gbrain repos add .` keep
|
||||
@@ -736,6 +761,10 @@ ADMIN
|
||||
storage status [--repo <path>] Storage tier status and health
|
||||
[--json] (git-tracked vs supabase-only)
|
||||
serve MCP server (stdio)
|
||||
serve --http [--port N] HTTP MCP server with OAuth 2.1
|
||||
--token-ttl N Access token TTL in seconds (default: 3600)
|
||||
--enable-dcr Enable Dynamic Client Registration
|
||||
--public-url URL Public issuer URL (required behind proxy/tunnel)
|
||||
call <tool> '<json>' Raw tool invocation
|
||||
version Version info
|
||||
--tools-json Tool discovery (JSON)
|
||||
|
||||
+69
-4
@@ -225,6 +225,65 @@ async function test(url: string, token: string) {
|
||||
console.log(`\n🧠 Your brain is live! (${elapsed}s)`);
|
||||
}
|
||||
|
||||
async function revokeClient(clientId: string) {
|
||||
if (!clientId) {
|
||||
console.error('Usage: auth revoke-client <client_id>');
|
||||
process.exit(1);
|
||||
}
|
||||
const sql = postgres(getDatabaseUrl(true)!);
|
||||
try {
|
||||
// Atomic single-statement delete: no race window between count + delete.
|
||||
// Postgres cascades to oauth_tokens and oauth_codes (FK ON DELETE CASCADE
|
||||
// declared in src/schema.sql:370,382) before the transaction commits.
|
||||
const rows = await sql`
|
||||
DELETE FROM oauth_clients WHERE client_id = ${clientId}
|
||||
RETURNING client_id, client_name
|
||||
`;
|
||||
if (rows.length === 0) {
|
||||
console.error(`No client found with id "${clientId}"`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`OAuth client revoked: "${rows[0].client_name}" (${clientId})`);
|
||||
console.log('Tokens and authorization codes purged via cascade.');
|
||||
} catch (e: any) {
|
||||
console.error('Error:', e.message);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
}
|
||||
|
||||
async function registerClient(name: string, args: string[]) {
|
||||
if (!name) { console.error('Usage: auth register-client <name> [--grant-types G] [--scopes S]'); process.exit(1); }
|
||||
const grantsIdx = args.indexOf('--grant-types');
|
||||
const scopesIdx = args.indexOf('--scopes');
|
||||
const grantTypes = grantsIdx >= 0 && args[grantsIdx + 1]
|
||||
? args[grantsIdx + 1].split(',').map(s => s.trim()).filter(Boolean)
|
||||
: ['client_credentials'];
|
||||
const scopes = scopesIdx >= 0 && args[scopesIdx + 1] ? args[scopesIdx + 1] : 'read';
|
||||
|
||||
const sql = postgres(getDatabaseUrl(true)!);
|
||||
try {
|
||||
const { GBrainOAuthProvider } = await import('../core/oauth-provider.ts');
|
||||
const provider = new GBrainOAuthProvider({ sql: sql as any });
|
||||
const { clientId, clientSecret } = await provider.registerClientManual(
|
||||
name, grantTypes, scopes, [],
|
||||
);
|
||||
console.log(`OAuth client registered: "${name}"\n`);
|
||||
console.log(` Client ID: ${clientId}`);
|
||||
console.log(` Client Secret: ${clientSecret}\n`);
|
||||
console.log(` Grant types: ${grantTypes.join(', ')}`);
|
||||
console.log(` Scopes: ${scopes}\n`);
|
||||
console.log('Save the client secret — it will not be shown again.');
|
||||
console.log(`Revoke with: gbrain auth revoke-client "${clientId}"`);
|
||||
} catch (e: any) {
|
||||
console.error('Error:', e.message);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Entry point for the `gbrain auth` CLI subcommand. Also reused by the
|
||||
* direct-script path (see bottom of file) so `bun run src/commands/auth.ts`
|
||||
@@ -236,6 +295,8 @@ export async function runAuth(args: string[]): Promise<void> {
|
||||
case 'create': await create(rest[0]); return;
|
||||
case 'list': await list(); return;
|
||||
case 'revoke': await revoke(rest[0]); return;
|
||||
case 'register-client': await registerClient(rest[0], rest.slice(1)); return;
|
||||
case 'revoke-client': await revokeClient(rest[0]); return;
|
||||
case 'test': {
|
||||
const tokenIdx = rest.indexOf('--token');
|
||||
const url = rest.find(a => !a.startsWith('--') && a !== rest[tokenIdx + 1]);
|
||||
@@ -247,10 +308,14 @@ export async function runAuth(args: string[]): Promise<void> {
|
||||
console.log(`GBrain Token Management
|
||||
|
||||
Usage:
|
||||
gbrain auth create <name> Create a new access token
|
||||
gbrain auth list List all tokens
|
||||
gbrain auth revoke <name> Revoke a token
|
||||
gbrain auth test <url> --token <t> Smoke-test a remote MCP server
|
||||
gbrain auth create <name> Create a legacy bearer token
|
||||
gbrain auth list List all tokens
|
||||
gbrain auth revoke <name> Revoke a legacy token
|
||||
gbrain auth register-client <name> [options] Register an OAuth 2.1 client
|
||||
--grant-types <client_credentials,authorization_code> (default: client_credentials)
|
||||
--scopes "<read write admin>" (default: read)
|
||||
gbrain auth revoke-client <client_id> Hard-delete an OAuth 2.1 client (cascades to tokens + codes)
|
||||
gbrain auth test <url> --token <token> Smoke-test a remote MCP server
|
||||
`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,531 @@
|
||||
/**
|
||||
* `gbrain book-mirror` — flagship of the v0.25.1 skills wave.
|
||||
*
|
||||
* Takes pre-extracted chapter text + context, fans out N read-only Opus
|
||||
* subagents (one per chapter), waits for all to complete, assembles the
|
||||
* two-column personalized analysis, and writes ONE put_page under
|
||||
* `media/books/<slug>-personalized.md` using the operator-trust path.
|
||||
*
|
||||
* Trust contract (D2/α + codex HIGH-1 fix):
|
||||
* - Subagents have allowed_tools: ['get_page', 'search'] only — they
|
||||
* can READ the brain, but they CANNOT call put_page. They produce
|
||||
* markdown analysis text via their final_message; the CLI reads
|
||||
* job.result and assembles the final page itself.
|
||||
* - The CLI calls put_page once at the end with operator-level trust
|
||||
* (no viaSubagent flag), so the subagent namespace check doesn't
|
||||
* apply. Untrusted EPUB content cannot prompt-inject any people/*
|
||||
* page because subagents lack write access entirely.
|
||||
*
|
||||
* The skill (skills/book-mirror/SKILL.md) handles EPUB/PDF extraction
|
||||
* via the agent's shell + python access (BeautifulSoup4, pdftotext) and
|
||||
* invokes this CLI with --chapters-dir pointing at the extracted text.
|
||||
* Separation of concerns: skill prepares inputs, CLI is the trusted
|
||||
* runtime.
|
||||
*
|
||||
* Cost: a 20-chapter book at Opus pricing is ~$6/run. The CLI prints an
|
||||
* estimate before launching and prompts for confirmation unless
|
||||
* --no-confirm is passed.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { MinionQueue } from '../core/minions/queue.ts';
|
||||
import { waitForCompletion, TimeoutError } from '../core/minions/wait-for-completion.ts';
|
||||
import type { MinionJobInput, SubagentHandlerData } from '../core/minions/types.ts';
|
||||
import { operations } from '../core/operations.ts';
|
||||
import { loadConfig } from '../core/config.ts';
|
||||
import { getCliOptions } from '../core/cli-options.ts';
|
||||
|
||||
const COST_PER_CHAPTER_OPUS = 0.30; // rough; depends on chapter length
|
||||
const COST_PER_CHAPTER_SONNET = 0.06;
|
||||
const DEFAULT_MAX_TURNS = 10;
|
||||
const DEFAULT_WORKERS = 4; // queue concurrency hint; rate-leases enforce real cap
|
||||
|
||||
interface BookMirrorFlags {
|
||||
chaptersDir?: string;
|
||||
contextFile?: string;
|
||||
slug?: string;
|
||||
title?: string;
|
||||
author?: string;
|
||||
model: string;
|
||||
maxTurns: number;
|
||||
timeoutMs?: number;
|
||||
noConfirm: boolean;
|
||||
follow: boolean;
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
interface ChapterEntry {
|
||||
index: number;
|
||||
filename: string;
|
||||
fullPath: string;
|
||||
text: string;
|
||||
wordCount: number;
|
||||
}
|
||||
|
||||
// ── arg parsing ────────────────────────────────────────────
|
||||
|
||||
function parseFlag(args: string[], flag: string): string | undefined {
|
||||
const i = args.indexOf(flag);
|
||||
return i >= 0 && i + 1 < args.length ? args[i + 1] : undefined;
|
||||
}
|
||||
|
||||
function hasFlag(args: string[], flag: string): boolean {
|
||||
return args.includes(flag);
|
||||
}
|
||||
|
||||
function parseFlags(args: string[]): BookMirrorFlags {
|
||||
if (hasFlag(args, '--help') || hasFlag(args, '-h')) {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const chaptersDir = parseFlag(args, '--chapters-dir');
|
||||
const contextFile = parseFlag(args, '--context-file');
|
||||
const slug = parseFlag(args, '--slug');
|
||||
const title = parseFlag(args, '--title');
|
||||
const author = parseFlag(args, '--author');
|
||||
const model = parseFlag(args, '--model') ?? 'claude-opus-4-7';
|
||||
const maxTurnsStr = parseFlag(args, '--max-turns');
|
||||
const timeoutMsStr = parseFlag(args, '--timeout-ms');
|
||||
|
||||
return {
|
||||
chaptersDir,
|
||||
contextFile,
|
||||
slug,
|
||||
title,
|
||||
author,
|
||||
model,
|
||||
maxTurns: maxTurnsStr ? parseInt(maxTurnsStr, 10) : DEFAULT_MAX_TURNS,
|
||||
timeoutMs: timeoutMsStr ? parseInt(timeoutMsStr, 10) : undefined,
|
||||
noConfirm: hasFlag(args, '--no-confirm') || hasFlag(args, '--yes'),
|
||||
follow: process.stdout.isTTY === true && !hasFlag(args, '--no-follow'),
|
||||
dryRun: hasFlag(args, '--dry-run'),
|
||||
};
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
console.log(`gbrain book-mirror — personalized chapter-by-chapter book analysis
|
||||
|
||||
USAGE
|
||||
gbrain book-mirror --chapters-dir <path> --slug <slug> [flags]
|
||||
|
||||
REQUIRED
|
||||
--chapters-dir <path> Directory containing chapter text files (.txt).
|
||||
Files sort alphabetically; chapter order = sort order.
|
||||
The skill (skills/book-mirror/SKILL.md) handles EPUB
|
||||
and PDF extraction; this CLI takes pre-extracted
|
||||
chapter text as its input contract.
|
||||
--slug <slug> Brain page slug (kebab-case, no leading slash).
|
||||
Output lands at media/books/<slug>-personalized.md.
|
||||
|
||||
OPTIONAL
|
||||
--context-file <path> Path to a context pack (USER.md + SOUL.md + memory
|
||||
excerpts + entity searches). Embedded in every
|
||||
child subagent's prompt. The skill prepares this.
|
||||
--title "<title>" Book title (used in the assembled page header).
|
||||
Defaults to slug if omitted.
|
||||
--author "<author>" Book author (used in frontmatter + page header).
|
||||
--model <id> Anthropic model id for chapter analysis.
|
||||
Default: claude-opus-4-7. Sonnet works but the
|
||||
right-column quality drops.
|
||||
--max-turns <n> Per-chapter subagent turn budget. Default ${DEFAULT_MAX_TURNS}.
|
||||
--timeout-ms <n> Per-chapter wall-clock timeout.
|
||||
--no-confirm / --yes Skip the cost-estimate confirmation prompt.
|
||||
--no-follow Submit and exit; don't tail children.
|
||||
--dry-run Validate inputs + print plan; submit nothing.
|
||||
|
||||
TRUST CONTRACT (read this)
|
||||
Each chapter is analyzed by a separate subagent with allowed_tools
|
||||
restricted to ['get_page', 'search'] — read-only. Subagents return
|
||||
markdown analysis text in their final message. THIS CLI assembles all
|
||||
child outputs and writes one put_page under media/books/<slug>-personalized.md
|
||||
with operator trust. Subagents NEVER call put_page; untrusted book
|
||||
content cannot prompt-inject any people/* page.
|
||||
|
||||
See src/commands/book-mirror.ts top-of-file comment for the full
|
||||
rationale (codex HIGH-1 fix vs the v0.25.1 plan's earlier draft).
|
||||
|
||||
COST
|
||||
~\$${COST_PER_CHAPTER_OPUS.toFixed(2)} per chapter at Opus, ~\$${COST_PER_CHAPTER_SONNET.toFixed(2)} at Sonnet. A 20-chapter book
|
||||
is ~\$${(20 * COST_PER_CHAPTER_OPUS).toFixed(2)} at Opus. The CLI prints an estimate before launching.
|
||||
|
||||
EXAMPLES
|
||||
# After the skill extracts chapters to /tmp/books/<slug>/chapters/:
|
||||
gbrain book-mirror \\
|
||||
--chapters-dir /tmp/books/this-book/chapters \\
|
||||
--context-file /tmp/books/this-book/context.md \\
|
||||
--slug this-book \\
|
||||
--title "This Book Title" \\
|
||||
--author "Some Author"
|
||||
|
||||
# Dry run (no subagent submission, just plan):
|
||||
gbrain book-mirror --chapters-dir ./chapters --slug test --dry-run
|
||||
`);
|
||||
}
|
||||
|
||||
// ── chapter loading ────────────────────────────────────────
|
||||
|
||||
function loadChapters(dir: string): ChapterEntry[] {
|
||||
if (!fs.existsSync(dir)) {
|
||||
throw new Error(`--chapters-dir not found: ${dir}`);
|
||||
}
|
||||
const stat = fs.statSync(dir);
|
||||
if (!stat.isDirectory()) {
|
||||
throw new Error(`--chapters-dir is not a directory: ${dir}`);
|
||||
}
|
||||
const files = fs.readdirSync(dir)
|
||||
.filter(f => f.endsWith('.txt'))
|
||||
.sort();
|
||||
if (files.length === 0) {
|
||||
throw new Error(`No .txt files in --chapters-dir: ${dir}`);
|
||||
}
|
||||
const chapters: ChapterEntry[] = [];
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const filename = files[i]!;
|
||||
const fullPath = path.join(dir, filename);
|
||||
const text = fs.readFileSync(fullPath, 'utf8');
|
||||
const wordCount = text.split(/\s+/).filter(Boolean).length;
|
||||
chapters.push({
|
||||
index: i + 1,
|
||||
filename,
|
||||
fullPath,
|
||||
text,
|
||||
wordCount,
|
||||
});
|
||||
}
|
||||
return chapters;
|
||||
}
|
||||
|
||||
// ── cost confirm ───────────────────────────────────────────
|
||||
|
||||
function estimateCost(chapters: ChapterEntry[], model: string): number {
|
||||
const perChapter = model.includes('opus') ? COST_PER_CHAPTER_OPUS : COST_PER_CHAPTER_SONNET;
|
||||
return chapters.length * perChapter;
|
||||
}
|
||||
|
||||
async function confirmInteractive(estimateUsd: number, chapters: number): Promise<boolean> {
|
||||
if (process.stdin.isTTY !== true) {
|
||||
// Non-TTY: refuse to spend without an explicit --yes / --no-confirm.
|
||||
process.stderr.write(
|
||||
`gbrain book-mirror: refusing to spend ~$${estimateUsd.toFixed(2)} on ${chapters} chapters from a non-TTY context. ` +
|
||||
`Pass --yes to confirm.\n`
|
||||
);
|
||||
return false;
|
||||
}
|
||||
process.stderr.write(
|
||||
`\nThis will spawn ${chapters} subagent jobs at ~$${(estimateUsd / chapters).toFixed(2)} each = ~$${estimateUsd.toFixed(2)} total.\n` +
|
||||
`Continue? [y/N] `
|
||||
);
|
||||
return new Promise(resolve => {
|
||||
process.stdin.setEncoding('utf8');
|
||||
process.stdin.once('data', (chunk) => {
|
||||
const reply = chunk.toString().trim().toLowerCase();
|
||||
resolve(reply === 'y' || reply === 'yes');
|
||||
process.stdin.pause();
|
||||
});
|
||||
process.stdin.resume();
|
||||
});
|
||||
}
|
||||
|
||||
// ── prompt assembly ────────────────────────────────────────
|
||||
|
||||
function buildChapterPrompt(
|
||||
chapter: ChapterEntry,
|
||||
totalChapters: number,
|
||||
bookTitle: string,
|
||||
bookAuthor: string | undefined,
|
||||
contextPack: string | undefined,
|
||||
): string {
|
||||
const authorLine = bookAuthor ? ` by ${bookAuthor}` : '';
|
||||
const contextSection = contextPack
|
||||
? `\n\n## READER CONTEXT\n\n${contextPack}\n\n`
|
||||
: '\n\n## READER CONTEXT\n\n(No context pack supplied; right column will be limited to brain-search-discoverable content.)\n\n';
|
||||
|
||||
return `You are analyzing one chapter of "${bookTitle}"${authorLine} for the user.
|
||||
|
||||
Your output is a markdown two-column table where the LEFT column preserves the chapter's actual content (stories, frameworks, statistics, named examples) and the RIGHT column maps each idea to the user's actual life using their words, situations, and patterns from the brain.
|
||||
|
||||
This is chapter ${chapter.index} of ${totalChapters}.
|
||||
|
||||
## CHAPTER ${chapter.index} TEXT (full, do not summarize this away)
|
||||
|
||||
${chapter.text}
|
||||
${contextSection}
|
||||
|
||||
## OUTPUT
|
||||
|
||||
Return ONLY a single markdown section in this exact shape:
|
||||
|
||||
\`\`\`
|
||||
## Chapter ${chapter.index}: [Title from the chapter — extract or infer]
|
||||
|
||||
### Key Ideas
|
||||
[2-4 sentence thesis of the chapter — what the author is actually arguing.]
|
||||
|
||||
| What the Author Says | How This Applies to You |
|
||||
|---|---|
|
||||
| [Detailed paragraph: a section/argument from the chapter, preserving stories, stats, frameworks, named examples. Use \`<br><br>\` for paragraph breaks within the cell.] | [Specific personal connection: name dates, people, exact quotes from the user, real situations. Same \`<br><br>\` for breaks.] |
|
||||
| [Next section] | [Next mirror] |
|
||||
| [4-10 rows depending on chapter density] | |
|
||||
\`\`\`
|
||||
|
||||
## RULES
|
||||
|
||||
- LEFT column: preserve stories, stats, frameworks. Don't summarize away the texture.
|
||||
- RIGHT column: use the user's actual words from READER CONTEXT. Name specific people, dates, situations. Read like a therapist who knows them.
|
||||
- 4-10 rows per chapter. If a section honestly doesn't apply, write \`*This section is less directly relevant because [specific reason].*\` Don't force connections.
|
||||
- Never generic ("This might apply if you've ever felt..."). Never sycophantic. Never preach.
|
||||
- Use \`<br><br>\` for paragraph breaks inside table cells, not literal newlines.
|
||||
|
||||
You have ${DEFAULT_MAX_TURNS} turns and read-only tools (get_page, search). You CANNOT call put_page — your output is the markdown text in your final message. The CLI assembles all chapters and writes the brain page.
|
||||
|
||||
When done, your final message should contain ONLY the \`## Chapter ${chapter.index}: ...\` section above. No preamble, no postscript, no commentary.`;
|
||||
}
|
||||
|
||||
function buildAssembledPage(opts: {
|
||||
slug: string;
|
||||
title: string;
|
||||
author: string | undefined;
|
||||
contextPack: string | undefined;
|
||||
chapterAnalyses: Array<{ index: number; result: string; failed: boolean; error?: string }>;
|
||||
}): string {
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
const authorLine = opts.author ? `\nauthor: "${opts.author}"` : '';
|
||||
const contextSummary = opts.contextPack
|
||||
? opts.contextPack.split('\n').slice(0, 3).join(' ').slice(0, 200)
|
||||
: 'No reader-context pack supplied.';
|
||||
|
||||
const frontmatter = `---
|
||||
title: "${opts.title} — Personalized"
|
||||
type: book-analysis${authorLine}
|
||||
date: ${today}
|
||||
context: "${contextSummary.replace(/"/g, '\\"')}"
|
||||
tags: [book, personalized, two-column]
|
||||
---`;
|
||||
|
||||
const intro = `# ${opts.title} — Personalized
|
||||
|
||||
## What this is
|
||||
|
||||
A chapter-by-chapter personalized analysis of *${opts.title}*${opts.author ? ` by ${opts.author}` : ''}. Each chapter is summarized in detail on the left and mirrored to the reader's actual life on the right, drawing on brain context.
|
||||
|
||||
This page was generated by \`gbrain book-mirror\`. Each chapter analysis came from a separate read-only subagent that had access to the chapter text and a reader-context pack but no write tools — so the brain wasn't modified during the per-chapter analysis. This page is the only artifact written.
|
||||
|
||||
`;
|
||||
|
||||
const failedSection = opts.chapterAnalyses
|
||||
.filter(a => a.failed)
|
||||
.map(a => `> Chapter ${a.index}: analysis failed (${a.error ?? 'unknown error'}). Re-run \`gbrain book-mirror\` to retry; idempotent on the same inputs.`)
|
||||
.join('\n\n');
|
||||
|
||||
const failedHeader = failedSection
|
||||
? `\n\n## Failed chapters (${opts.chapterAnalyses.filter(a => a.failed).length})\n\n${failedSection}\n\n---\n`
|
||||
: '';
|
||||
|
||||
const completed = opts.chapterAnalyses
|
||||
.filter(a => !a.failed)
|
||||
.sort((a, b) => a.index - b.index)
|
||||
.map(a => a.result.trim())
|
||||
.join('\n\n---\n\n');
|
||||
|
||||
return `${frontmatter}\n\n${intro}${failedHeader}\n${completed}\n`;
|
||||
}
|
||||
|
||||
// ── main entry ─────────────────────────────────────────────
|
||||
|
||||
export async function runBookMirrorCmd(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const flags = parseFlags(args);
|
||||
|
||||
if (!flags.chaptersDir) {
|
||||
console.error('gbrain book-mirror: --chapters-dir is required. Run with --help.');
|
||||
process.exit(2);
|
||||
}
|
||||
if (!flags.slug) {
|
||||
console.error('gbrain book-mirror: --slug is required. Run with --help.');
|
||||
process.exit(2);
|
||||
}
|
||||
if (!/^[a-z0-9][a-z0-9-]*$/i.test(flags.slug)) {
|
||||
console.error(`gbrain book-mirror: invalid --slug "${flags.slug}". Use kebab-case (a-z, 0-9, hyphens).`);
|
||||
process.exit(2);
|
||||
}
|
||||
if (flags.contextFile && !fs.existsSync(flags.contextFile)) {
|
||||
console.error(`gbrain book-mirror: --context-file not found: ${flags.contextFile}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// Load chapter files.
|
||||
let chapters: ChapterEntry[];
|
||||
try {
|
||||
chapters = loadChapters(flags.chaptersDir);
|
||||
} catch (e) {
|
||||
console.error(`gbrain book-mirror: ${e instanceof Error ? e.message : String(e)}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const contextPack = flags.contextFile ? fs.readFileSync(flags.contextFile, 'utf8') : undefined;
|
||||
const bookTitle = flags.title ?? flags.slug;
|
||||
const targetSlug = `media/books/${flags.slug}-personalized`;
|
||||
|
||||
process.stderr.write(
|
||||
`\ngbrain book-mirror — plan\n` +
|
||||
` slug: ${flags.slug}\n` +
|
||||
` output: ${targetSlug}\n` +
|
||||
` chapters: ${chapters.length} (from ${flags.chaptersDir})\n` +
|
||||
` context: ${flags.contextFile ?? '(none)'}\n` +
|
||||
` model: ${flags.model}\n` +
|
||||
` max_turns: ${flags.maxTurns}\n`
|
||||
);
|
||||
|
||||
const estimateUsd = estimateCost(chapters, flags.model);
|
||||
process.stderr.write(` est. cost: ~$${estimateUsd.toFixed(2)} (${chapters.length} subagents)\n\n`);
|
||||
|
||||
if (flags.dryRun) {
|
||||
process.stderr.write(`gbrain book-mirror: --dry-run — exiting without submission.\n`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!flags.noConfirm) {
|
||||
const ok = await confirmInteractive(estimateUsd, chapters.length);
|
||||
if (!ok) {
|
||||
process.stderr.write(`gbrain book-mirror: cancelled by user.\n`);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Submit fan-out: N children, no aggregator. Each child gets read-only
|
||||
// tools so the codex HIGH-1 prompt-injection vector is closed at the
|
||||
// tool-allowlist layer rather than at allowedSlugPrefixes scope.
|
||||
const queue = new MinionQueue(engine);
|
||||
const childIds: number[] = [];
|
||||
for (const ch of chapters) {
|
||||
const data: SubagentHandlerData = {
|
||||
prompt: buildChapterPrompt(ch, chapters.length, bookTitle, flags.author, contextPack),
|
||||
model: flags.model,
|
||||
max_turns: flags.maxTurns,
|
||||
// CODEX HIGH-1 FIX: read-only tool allowlist. Subagents cannot call
|
||||
// put_page or any mutating op. Their only output is final_message text.
|
||||
allowed_tools: ['get_page', 'search'],
|
||||
};
|
||||
const submitOpts: Partial<MinionJobInput> = {
|
||||
max_stalled: 3,
|
||||
// Loose idempotency: same chapter file + slug → same idempotency key,
|
||||
// so re-running the CLI on identical input dedups against the queue.
|
||||
idempotency_key: `book-mirror:${flags.slug}:ch-${ch.index}`,
|
||||
};
|
||||
if (flags.timeoutMs) submitOpts.timeout_ms = flags.timeoutMs;
|
||||
const job = await queue.add(
|
||||
'subagent',
|
||||
data as unknown as Record<string, unknown>,
|
||||
submitOpts,
|
||||
{ allowProtectedSubmit: true },
|
||||
);
|
||||
childIds.push(job.id);
|
||||
}
|
||||
|
||||
process.stderr.write(
|
||||
`submitted: ${childIds.length} subagent jobs (${childIds[0]}..${childIds[childIds.length - 1]})\n`
|
||||
);
|
||||
|
||||
if (!flags.follow) {
|
||||
process.stdout.write(JSON.stringify({ child_ids: childIds, slug: targetSlug }) + '\n');
|
||||
process.stderr.write(
|
||||
`gbrain book-mirror: detached. Run \`gbrain jobs get <id>\` per child, then re-run with same args once all are complete.\n`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Wait for every child. Order doesn't matter for the wait, but it does
|
||||
// matter for the assembly — we sort by chapter index in buildAssembledPage.
|
||||
process.stderr.write(`waiting for all ${childIds.length} chapters to complete...\n`);
|
||||
const analyses: Array<{ index: number; result: string; failed: boolean; error?: string }> = [];
|
||||
for (let i = 0; i < childIds.length; i++) {
|
||||
const childId = childIds[i]!;
|
||||
const chapterIndex = chapters[i]!.index;
|
||||
try {
|
||||
const job = await waitForCompletion(queue, childId, {
|
||||
timeoutMs: flags.timeoutMs ?? 30 * 60 * 1000, // 30 min per child
|
||||
pollMs: 1000,
|
||||
});
|
||||
if (job.status === 'completed' && job.result && typeof job.result === 'object') {
|
||||
const result = (job.result as { result?: string }).result ?? '';
|
||||
analyses.push({ index: chapterIndex, result, failed: false });
|
||||
process.stderr.write(` chapter ${chapterIndex}: complete (job ${childId})\n`);
|
||||
} else {
|
||||
analyses.push({
|
||||
index: chapterIndex,
|
||||
result: '',
|
||||
failed: true,
|
||||
error: `job ${childId} status=${job.status}`,
|
||||
});
|
||||
process.stderr.write(` chapter ${chapterIndex}: FAILED (job ${childId} status=${job.status})\n`);
|
||||
}
|
||||
} catch (e) {
|
||||
const msg = e instanceof TimeoutError
|
||||
? `timeout after ${e.elapsedMs}ms`
|
||||
: (e instanceof Error ? e.message : String(e));
|
||||
analyses.push({ index: chapterIndex, result: '', failed: true, error: msg });
|
||||
process.stderr.write(` chapter ${chapterIndex}: ERROR — ${msg}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
const failed = analyses.filter(a => a.failed).length;
|
||||
const completed = analyses.length - failed;
|
||||
process.stderr.write(
|
||||
`\nassembled: ${completed} chapters successful, ${failed} failed.\n`
|
||||
);
|
||||
|
||||
if (completed === 0) {
|
||||
console.error(`gbrain book-mirror: every chapter failed. Not writing the brain page. Re-run after diagnosing.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Assemble the final page.
|
||||
const assembled = buildAssembledPage({
|
||||
slug: flags.slug!,
|
||||
title: bookTitle,
|
||||
author: flags.author,
|
||||
contextPack,
|
||||
chapterAnalyses: analyses,
|
||||
});
|
||||
|
||||
// Operator-trust put_page — viaSubagent is NOT set, so the namespace
|
||||
// check doesn't fire. The CLI is the trusted writer.
|
||||
const putPageOp = operations.find(op => op.name === 'put_page');
|
||||
if (!putPageOp) {
|
||||
throw new Error('internal: put_page operation not registered');
|
||||
}
|
||||
|
||||
await putPageOp.handler(
|
||||
{
|
||||
engine,
|
||||
config: loadConfig() || { engine: 'postgres' },
|
||||
logger: { info: console.log, warn: console.warn, error: console.error },
|
||||
dryRun: false,
|
||||
remote: false, // local CLI caller — operator trust path
|
||||
cliOpts: getCliOptions(),
|
||||
// viaSubagent intentionally omitted — operator trust path.
|
||||
// allowedSlugPrefixes intentionally omitted — operator can write anywhere.
|
||||
},
|
||||
{
|
||||
slug: targetSlug,
|
||||
content: assembled,
|
||||
},
|
||||
);
|
||||
|
||||
process.stderr.write(`\nwrote: ${targetSlug} (${chapters.length} chapter sections, ${assembled.length} bytes)\n`);
|
||||
process.stdout.write(JSON.stringify({
|
||||
slug: targetSlug,
|
||||
chapters_total: chapters.length,
|
||||
chapters_completed: completed,
|
||||
chapters_failed: failed,
|
||||
}) + '\n');
|
||||
|
||||
if (failed > 0) {
|
||||
process.stderr.write(
|
||||
`\ngbrain book-mirror: ${failed} chapter(s) failed. The page was written with the completed chapters; run again to retry the failed ones (idempotency keys dedupe successful chapters).\n`
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
@@ -57,11 +57,11 @@ export interface Flags {
|
||||
skillsDir: string | null;
|
||||
}
|
||||
|
||||
// Check 5 (trigger_routing_eval) landed in v0.17 (W2). Check 6
|
||||
// (brain_filing) landed in v0.17 (W3). Array is now empty; the
|
||||
// export stays as a stable public field of the --json envelope so
|
||||
// downstream consumers that check `.deferred[]` keep working.
|
||||
// Future deferred checks get appended here.
|
||||
// Check 5 (trigger_routing_eval) and Check 6 (brain_filing) both
|
||||
// shipped as real implementations in v0.19 (W2 + W3). Array is now
|
||||
// empty; the export stays as a stable public field of the --json
|
||||
// envelope so downstream consumers that check `.deferred[]` keep
|
||||
// working. Future deferred checks get appended here.
|
||||
export const DEFERRED: DeferredCheck[] = [];
|
||||
|
||||
const HELP_TEXT = `gbrain check-resolvable [options]
|
||||
@@ -83,13 +83,13 @@ Exit codes:
|
||||
0 clean (no errors; no warnings unless --strict)
|
||||
1 errors present, OR (with --strict) warnings present
|
||||
|
||||
Check 5 (trigger routing eval) lands in v0.17 via W2: any
|
||||
Check 5 (trigger routing eval) runs via W2: any
|
||||
skills/<name>/routing-eval.jsonl fixtures are evaluated and routing
|
||||
gaps surface as warnings.
|
||||
|
||||
Check 6 (brain filing) lands in v0.17 via W3: skills with
|
||||
writes_pages: true are audited against skills/_brain-filing-rules.json.
|
||||
No checks are deferred as of v0.17.
|
||||
Check 6 (brain filing) runs via W3: skills with writes_pages: true
|
||||
are audited against skills/_brain-filing-rules.json. No checks are
|
||||
currently deferred.
|
||||
`;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,424 @@
|
||||
/**
|
||||
* gbrain claw-test — end-to-end "fresh user" test harness.
|
||||
*
|
||||
* Two tiers:
|
||||
* gbrain claw-test — scripted (no LLM, CI gate)
|
||||
* gbrain claw-test --live --agent openclaw — real agent, friction discovery
|
||||
*
|
||||
* Phases (scripted mode):
|
||||
* setup → install_brain → import → query → extract → verify → render
|
||||
*
|
||||
* The harness sets GBRAIN_HOME=<tempdir> so the run is hermetic. Each child
|
||||
* gbrain invocation runs with --progress-json and the harness captures stderr
|
||||
* to assert expected_phases from scenario.json fired.
|
||||
*
|
||||
* See ~/.claude/plans/system-instruction-you-are-working-noble-biscuit.md
|
||||
* for the full design rationale (D1–D23 decisions).
|
||||
*/
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
import { mkdtempSync, writeFileSync, mkdirSync, rmSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { randomBytes } from 'crypto';
|
||||
import { logFriction, frictionDir } from '../core/friction.ts';
|
||||
import { loadScenario, listScenarios, readBrief, type ScenarioConfig } from '../core/claw-test/scenarios.ts';
|
||||
import { parseProgressEvents, verifyExpectedPhases } from '../core/claw-test/progress-tail.ts';
|
||||
import { resolveAgentRunner, listRegisteredAgents, registerAgentRunner } from '../core/claw-test/agent-runner.ts';
|
||||
import { OpenClawRunner } from '../core/claw-test/runners/openclaw.ts';
|
||||
import { createTranscriptSink } from '../core/claw-test/transcript-capture.ts';
|
||||
|
||||
// Ensure built-in runners are registered.
|
||||
registerAgentRunner('openclaw', () => new OpenClawRunner());
|
||||
|
||||
interface HarnessOpts {
|
||||
scenario: string;
|
||||
live: boolean;
|
||||
agent: string;
|
||||
keepTempdir: boolean;
|
||||
listAgents: boolean;
|
||||
help: boolean;
|
||||
/** Path to the gbrain binary used to invoke child commands. Defaults to argv[0]. */
|
||||
gbrainBin?: string;
|
||||
}
|
||||
|
||||
interface PhaseOutcome {
|
||||
phase: string;
|
||||
exitCode: number;
|
||||
durationMs: number;
|
||||
stderrEvents: number;
|
||||
stdoutTail: string;
|
||||
stderrTail: string;
|
||||
}
|
||||
|
||||
const TAIL_BYTES = 4_096;
|
||||
const SUBPROCESS_TIMEOUT_MS = 5 * 60_000; // 5 minutes per phase
|
||||
|
||||
export async function runClawTest(args: string[]): Promise<number> {
|
||||
const opts = parseArgs(args);
|
||||
|
||||
if (opts.help) {
|
||||
printHelp();
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (opts.listAgents) {
|
||||
return cmdListAgents();
|
||||
}
|
||||
|
||||
let scenario: ScenarioConfig;
|
||||
try {
|
||||
scenario = loadScenario(opts.scenario);
|
||||
} catch (e) {
|
||||
console.error(`scenario load failed: ${e instanceof Error ? e.message : String(e)}`);
|
||||
const available = listScenarios();
|
||||
if (available.length) console.error(`available scenarios: ${available.join(', ')}`);
|
||||
return 2;
|
||||
}
|
||||
|
||||
const runId = newRunId(opts.agent);
|
||||
const runRoot = mkdtempSync(join(tmpdir(), `claw-test-${runId}-`));
|
||||
const gbrainHome = runRoot; // configDir() appends '.gbrain' itself
|
||||
const transcriptPath = join(runRoot, 'transcript.jsonl');
|
||||
console.log(`run-id: ${runId}`);
|
||||
console.log(`tempdir: ${runRoot}`);
|
||||
|
||||
// SIGINT/SIGTERM finalization (D11)
|
||||
let interrupted = false;
|
||||
const onSignal = () => {
|
||||
interrupted = true;
|
||||
try {
|
||||
logFriction({
|
||||
runId,
|
||||
phase: 'harness',
|
||||
message: 'run interrupted by signal',
|
||||
kind: 'interrupted',
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
});
|
||||
} catch { /* best effort */ }
|
||||
};
|
||||
process.once('SIGINT', onSignal);
|
||||
process.once('SIGTERM', onSignal);
|
||||
|
||||
let exitCode = 0;
|
||||
try {
|
||||
if (opts.live) {
|
||||
exitCode = await runLive(opts, scenario, { runId, runRoot, gbrainHome, transcriptPath });
|
||||
} else {
|
||||
exitCode = await runScripted(opts, scenario, { runId, runRoot, gbrainHome });
|
||||
}
|
||||
} finally {
|
||||
process.off('SIGINT', onSignal);
|
||||
process.off('SIGTERM', onSignal);
|
||||
if (!opts.keepTempdir && !interrupted) {
|
||||
try { rmSync(runRoot, { recursive: true, force: true }); } catch { /* best effort */ }
|
||||
} else {
|
||||
console.log(`tempdir kept at: ${runRoot}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Always render at the end so the operator can immediately see the report.
|
||||
console.log('---');
|
||||
console.log(`friction log: ${join(frictionDir(), runId + '.jsonl')}`);
|
||||
console.log(`render report: gbrain friction render --run-id ${runId}`);
|
||||
|
||||
if (interrupted) return 130;
|
||||
return exitCode;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scripted mode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function runScripted(
|
||||
opts: HarnessOpts,
|
||||
scenario: ScenarioConfig,
|
||||
ctx: { runId: string; runRoot: string; gbrainHome: string },
|
||||
): Promise<number> {
|
||||
const childEnv: Record<string, string> = {
|
||||
...process.env as Record<string, string>,
|
||||
GBRAIN_HOME: ctx.gbrainHome,
|
||||
GBRAIN_FRICTION_RUN_ID: ctx.runId,
|
||||
};
|
||||
|
||||
const phases: { name: string; argv: string[] }[] = [];
|
||||
// Phase 2: install_brain
|
||||
phases.push({ name: 'install_brain', argv: ['init', '--pglite'] });
|
||||
|
||||
// Phase 3: import (only when scenario has a brain dir)
|
||||
if (scenario.brainRelative) {
|
||||
const brainDir = join(scenario.dir, scenario.brainRelative);
|
||||
phases.push({ name: 'import', argv: ['import', brainDir, '--no-embed', '--progress-json'] });
|
||||
}
|
||||
|
||||
// Phase 4: query (best-effort sanity)
|
||||
phases.push({ name: 'query', argv: ['query', 'the'] });
|
||||
|
||||
// Phase 5: extract (positional argument is required: 'all' covers links + timeline)
|
||||
phases.push({ name: 'extract', argv: ['extract', 'all', '--source', 'fs', '--progress-json'] });
|
||||
|
||||
// Phase 6: verify
|
||||
phases.push({ name: 'verify', argv: ['doctor', '--json', '--progress-json'] });
|
||||
|
||||
// Pre-phase: upgrade scenario seeds the database
|
||||
if (scenario.kind === 'upgrade' && scenario.seedRelative) {
|
||||
const seedSql = join(scenario.dir, scenario.seedRelative, 'dump.sql');
|
||||
if (existsSync(seedSql)) {
|
||||
const dbPath = join(ctx.gbrainHome, '.gbrain', 'brain.pglite');
|
||||
mkdirSync(join(ctx.gbrainHome, '.gbrain'), { recursive: true });
|
||||
const { seedPgliteFromFile } = await import('../core/claw-test/seed-pglite.ts');
|
||||
try {
|
||||
await seedPgliteFromFile({ dbPath, sqlPath: seedSql });
|
||||
console.log(`[seed] replayed ${seedSql} → ${dbPath}`);
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
logFriction({
|
||||
runId: ctx.runId,
|
||||
phase: 'seed',
|
||||
message: `seed replay failed: ${msg}`,
|
||||
severity: 'blocker',
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
});
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const allStderr: string[] = [];
|
||||
const outcomes: PhaseOutcome[] = [];
|
||||
for (const phase of phases) {
|
||||
const outcome = await invokeGbrain(opts.gbrainBin ?? 'gbrain', phase.argv, ctx.runRoot, childEnv);
|
||||
outcome.phase = phase.name;
|
||||
outcomes.push(outcome);
|
||||
allStderr.push(outcome.stderrTail);
|
||||
if (outcome.exitCode !== 0) {
|
||||
logFriction({
|
||||
runId: ctx.runId,
|
||||
phase: phase.name,
|
||||
message: `command failed (exit ${outcome.exitCode}): gbrain ${phase.argv.join(' ')}`,
|
||||
severity: 'error',
|
||||
hint: outcome.stderrTail.trim().slice(0, 500),
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
});
|
||||
return 1;
|
||||
} else {
|
||||
logFriction({
|
||||
runId: ctx.runId,
|
||||
phase: phase.name,
|
||||
message: `phase complete in ${outcome.durationMs}ms`,
|
||||
kind: 'phase-marker',
|
||||
marker: 'end',
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Phase verification: collect all events from every captured stderr and assert coverage.
|
||||
const events = allStderr.flatMap(parseProgressEvents);
|
||||
const missing = verifyExpectedPhases(events, scenario.expectedPhases);
|
||||
if (missing.length) {
|
||||
for (const phaseName of missing) {
|
||||
logFriction({
|
||||
runId: ctx.runId,
|
||||
phase: phaseName,
|
||||
message: `expected progress event for "${phaseName}" never fired`,
|
||||
severity: 'blocker',
|
||||
hint: 'either the command did not run or it did not emit progress events; check phase log above',
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
});
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Live mode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function runLive(
|
||||
opts: HarnessOpts,
|
||||
scenario: ScenarioConfig,
|
||||
ctx: { runId: string; runRoot: string; gbrainHome: string; transcriptPath: string },
|
||||
): Promise<number> {
|
||||
let runner;
|
||||
try {
|
||||
runner = resolveAgentRunner(opts.agent);
|
||||
} catch (e) {
|
||||
console.error(e instanceof Error ? e.message : String(e));
|
||||
return 2;
|
||||
}
|
||||
|
||||
const detected = await runner.detect();
|
||||
if (!detected.available) {
|
||||
console.error(`agent "${opts.agent}" not available: ${detected.reason ?? 'unknown'}`);
|
||||
logFriction({
|
||||
runId: ctx.runId,
|
||||
phase: 'agent_detect',
|
||||
message: `agent ${opts.agent} not available: ${detected.reason ?? 'unknown'}`,
|
||||
severity: 'blocker',
|
||||
hint: opts.agent === 'openclaw' ? 'install openclaw or set OPENCLAW_BIN' : undefined,
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
});
|
||||
return 2;
|
||||
}
|
||||
|
||||
const sink = createTranscriptSink(ctx.transcriptPath);
|
||||
const env: Record<string, string> = {
|
||||
GBRAIN_HOME: ctx.gbrainHome,
|
||||
GBRAIN_FRICTION_RUN_ID: ctx.runId,
|
||||
};
|
||||
|
||||
const brief = readBrief(scenario);
|
||||
let result;
|
||||
try {
|
||||
result = await runner.invoke({
|
||||
cwd: ctx.runRoot,
|
||||
brief,
|
||||
env,
|
||||
timeoutMs: SUBPROCESS_TIMEOUT_MS,
|
||||
transcriptSink: sink,
|
||||
});
|
||||
} finally {
|
||||
await sink.close();
|
||||
}
|
||||
|
||||
if (result.exitCode !== 0) {
|
||||
logFriction({
|
||||
runId: ctx.runId,
|
||||
phase: 'agent_invoke',
|
||||
message: `agent exited with code ${result.exitCode} after ${result.durationMs}ms`,
|
||||
severity: 'error',
|
||||
source: 'harness',
|
||||
agent: opts.agent,
|
||||
});
|
||||
return result.exitCode;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Subprocess helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function invokeGbrain(
|
||||
bin: string,
|
||||
argv: string[],
|
||||
cwd: string,
|
||||
env: Record<string, string>,
|
||||
): Promise<PhaseOutcome> {
|
||||
return new Promise((resolve) => {
|
||||
const start = Date.now();
|
||||
const child = spawn(bin, argv, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], shell: false });
|
||||
const stdout: Buffer[] = [];
|
||||
const stderr: Buffer[] = [];
|
||||
child.stdout?.on('data', (b: Buffer) => stdout.push(b));
|
||||
child.stderr?.on('data', (b: Buffer) => stderr.push(b));
|
||||
child.on('error', (err) => {
|
||||
const stderrJoined = Buffer.concat(stderr).toString('utf-8') + '\nspawn error: ' + err.message;
|
||||
resolve({
|
||||
phase: '',
|
||||
exitCode: 127,
|
||||
durationMs: Date.now() - start,
|
||||
stderrEvents: 0,
|
||||
stdoutTail: tailOf(Buffer.concat(stdout).toString('utf-8')),
|
||||
stderrTail: tailOf(stderrJoined),
|
||||
});
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
const stderrText = Buffer.concat(stderr).toString('utf-8');
|
||||
resolve({
|
||||
phase: '',
|
||||
exitCode: typeof code === 'number' ? code : 1,
|
||||
durationMs: Date.now() - start,
|
||||
stderrEvents: parseProgressEvents(stderrText).length,
|
||||
stdoutTail: tailOf(Buffer.concat(stdout).toString('utf-8')),
|
||||
stderrTail: stderrText,
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function tailOf(s: string): string {
|
||||
if (s.length <= TAIL_BYTES) return s;
|
||||
return s.slice(-TAIL_BYTES);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Argv parsing + helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function parseArgs(args: string[]): HarnessOpts {
|
||||
const out: HarnessOpts = {
|
||||
scenario: 'fresh-install',
|
||||
live: false,
|
||||
agent: 'openclaw',
|
||||
keepTempdir: false,
|
||||
listAgents: false,
|
||||
help: args.includes('--help') || args.includes('-h'),
|
||||
gbrainBin: process.env.GBRAIN_BIN_OVERRIDE || process.execPath,
|
||||
};
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const a = args[i];
|
||||
if (a === '--live') out.live = true;
|
||||
else if (a === '--keep-tempdir') out.keepTempdir = true;
|
||||
else if (a === '--list-agents') out.listAgents = true;
|
||||
else if (a === '--scenario') out.scenario = args[++i] ?? out.scenario;
|
||||
else if (a === '--agent') out.agent = args[++i] ?? out.agent;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function newRunId(agent: string): string {
|
||||
const now = new Date();
|
||||
const ts = now.toISOString().replace(/[-:]/g, '').replace(/\..*/, '').replace('T', '-');
|
||||
const suf = randomBytes(4).toString('hex');
|
||||
return `claw-test-${ts}-${agent}-${suf}`;
|
||||
}
|
||||
|
||||
function cmdListAgents(): number {
|
||||
const names = listRegisteredAgents();
|
||||
if (!names.length) {
|
||||
console.log('no agents registered');
|
||||
return 0;
|
||||
}
|
||||
for (const name of names) {
|
||||
try {
|
||||
const runner = resolveAgentRunner(name);
|
||||
runner.detect().then((d) => {
|
||||
const status = d.available ? `available at ${d.binPath}` : `unavailable: ${d.reason}`;
|
||||
console.log(`${name}: ${status}`);
|
||||
}).catch(() => { /* best effort */ });
|
||||
} catch {
|
||||
console.log(`${name}: (factory error)`);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`gbrain claw-test — end-to-end claw-setup friction harness
|
||||
|
||||
Usage:
|
||||
gbrain claw-test [--scenario <name>] [--live --agent <name>] [--keep-tempdir]
|
||||
gbrain claw-test --list-agents
|
||||
|
||||
Defaults:
|
||||
--scenario fresh-install
|
||||
--agent openclaw (live mode only)
|
||||
|
||||
Scripted mode runs canonical commands without an LLM (CI gate).
|
||||
Live mode spawns a real agent and lets it drive (~5–10 min, costs tokens).
|
||||
|
||||
Examples:
|
||||
gbrain claw-test --scenario fresh-install
|
||||
gbrain claw-test --scenario upgrade-from-v0.18 --keep-tempdir
|
||||
gbrain claw-test --live --agent openclaw`);
|
||||
}
|
||||
@@ -719,6 +719,58 @@ export async function runDoctor(engine: BrainEngine | null, args: string[], dbSo
|
||||
fmHb();
|
||||
}
|
||||
|
||||
// 11a-bis. Eval-capture health (v0.25.0). Capture is a fire-and-forget
|
||||
// side-effect that logs failures to a persistent table so this check
|
||||
// can see drops cross-process (the MCP server captures; `gbrain doctor`
|
||||
// runs in a separate process). Counts failures in the last 24h and
|
||||
// warns when non-zero. Pre-v31 brains: the table doesn't exist yet;
|
||||
// swallow the error and report skipped.
|
||||
progress.heartbeat('eval_capture');
|
||||
try {
|
||||
const since = new Date(Date.now() - 24 * 3600 * 1000);
|
||||
const failures = await engine.listEvalCaptureFailures({ since });
|
||||
if (failures.length === 0) {
|
||||
checks.push({ name: 'eval_capture', status: 'ok', message: 'No capture failures in the last 24h' });
|
||||
} else {
|
||||
const byReason = new Map<string, number>();
|
||||
for (const f of failures) {
|
||||
byReason.set(f.reason, (byReason.get(f.reason) ?? 0) + 1);
|
||||
}
|
||||
const breakdown = [...byReason.entries()]
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.map(([r, n]) => `${n} ${r}`)
|
||||
.join(', ');
|
||||
checks.push({
|
||||
name: 'eval_capture',
|
||||
status: 'warn',
|
||||
message: `${failures.length} capture failure(s) in the last 24h (${breakdown}). ` +
|
||||
`If you care about replay fidelity, investigate. If not, set eval.capture: false ` +
|
||||
`in ~/.gbrain/config.json to silence.`,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
// Distinguish "table doesn't exist yet" (pre-v31, ok skip) from real
|
||||
// problems like RLS denying SELECT — the latter masks the very condition
|
||||
// this check is supposed to surface (capture INSERTs almost certainly
|
||||
// also fail).
|
||||
const code = (err as { code?: string } | null)?.code;
|
||||
if (code === '42P01') {
|
||||
checks.push({ name: 'eval_capture', status: 'ok', message: 'Skipped (eval_capture_failures table unavailable — apply migrations or upgrade)' });
|
||||
} else if (code === '42501') {
|
||||
checks.push({
|
||||
name: 'eval_capture',
|
||||
status: 'warn',
|
||||
message: 'RLS denies SELECT on eval_capture_failures. Capture INSERTs are almost certainly failing too. Run as a role with BYPASSRLS or grant SELECT on this table.',
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: 'eval_capture',
|
||||
status: 'warn',
|
||||
message: `Could not read eval_capture_failures: ${(err as Error)?.message ?? String(err)}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 11b. Queue health (v0.19.1 queue-resilience wave).
|
||||
// Postgres-only because PGLite has no multi-process worker surface. Two
|
||||
// subchecks, both cheap (single SELECT each, status-index-covered):
|
||||
@@ -774,6 +826,30 @@ export async function runDoctor(engine: BrainEngine | null, args: string[], dbSo
|
||||
ORDER BY depth DESC
|
||||
LIMIT 5
|
||||
`;
|
||||
// Subcheck 3 (v0.22.14): RSS-watchdog kills in the last 24h. Bare workers
|
||||
// newly default to --max-rss 2048 (was 0); operators who run large embed
|
||||
// or import jobs may see kills that didn't happen pre-v0.22.14. We surface
|
||||
// a hint when this signature appears so the upgrade path is obvious.
|
||||
// Signature: when the watchdog trips, gracefulShutdown('watchdog') aborts
|
||||
// in-flight jobs with `new Error('watchdog')`. The worker's failJob path
|
||||
// (worker.ts:660-664) writes `error_text = 'aborted: watchdog'` for any
|
||||
// job in-flight at the moment of the kill.
|
||||
//
|
||||
// We deliberately DO NOT do a loose `ILIKE '%watchdog%'`:
|
||||
// 1. Parent jobs that inherit `on_child_fail='fail_parent'` get
|
||||
// `"child job N failed: aborted: watchdog"` — counting that
|
||||
// double-counts (child + parent) for one watchdog event.
|
||||
// 2. Any user error_text containing the word "watchdog" matches.
|
||||
// Match the exact prefix `'aborted: watchdog'` to scope this purely to
|
||||
// the worker's own kill signature.
|
||||
const rssKillRows: Array<{ cnt: number }> = await sql`
|
||||
SELECT count(*)::int AS cnt
|
||||
FROM minion_jobs
|
||||
WHERE status IN ('dead', 'failed')
|
||||
AND finished_at > now() - interval '24 hours'
|
||||
AND error_text = 'aborted: watchdog'
|
||||
`;
|
||||
const rssKillCount = rssKillRows[0]?.cnt ?? 0;
|
||||
|
||||
const problems: string[] = [];
|
||||
if (stalledRows.length > 0) {
|
||||
@@ -794,6 +870,14 @@ export async function runDoctor(engine: BrainEngine | null, args: string[], dbSo
|
||||
`Fix: set maxWaiting on the submitter (or raise GBRAIN_QUEUE_WAITING_THRESHOLD).`
|
||||
);
|
||||
}
|
||||
if (rssKillCount > 0) {
|
||||
problems.push(
|
||||
`${rssKillCount} job(s) dead-lettered for RSS-watchdog memory-limit kills in last 24h. ` +
|
||||
`v0.22.14 changed the bare-worker --max-rss default from 0 (off) to 2048 MB. ` +
|
||||
`Fix: raise the limit (e.g. \`gbrain jobs work --max-rss 4096\`) or opt out (\`--max-rss 0\`). ` +
|
||||
`See skills/migrations/v0.22.14.md.`
|
||||
);
|
||||
}
|
||||
|
||||
if (problems.length === 0) {
|
||||
checks.push({
|
||||
|
||||
+100
-6
@@ -39,12 +39,28 @@ interface DreamArgs {
|
||||
phase: CyclePhase | null;
|
||||
dir: string | null;
|
||||
help: boolean;
|
||||
/** v0.21: ad-hoc transcript file path; implies --phase synthesize. */
|
||||
inputFile: string | null;
|
||||
/** v0.21: restrict synthesize to a single date (YYYY-MM-DD). */
|
||||
date: string | null;
|
||||
/** v0.21: backfill range start (YYYY-MM-DD). */
|
||||
from: string | null;
|
||||
/** v0.21: backfill range end (YYYY-MM-DD). */
|
||||
to: string | null;
|
||||
/**
|
||||
* v0.23.2: disable the synthesize phase's self-consumption guard.
|
||||
* Long-form flag name to discourage casual use; loud stderr warning fires when set.
|
||||
* Never auto-applied for --input (codex finding #3).
|
||||
*/
|
||||
bypassDreamGuard: boolean;
|
||||
}
|
||||
|
||||
const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
|
||||
|
||||
function parseArgs(args: string[]): DreamArgs {
|
||||
const phaseIdx = args.indexOf('--phase');
|
||||
const rawPhase = phaseIdx !== -1 ? args[phaseIdx + 1] : null;
|
||||
const phase = rawPhase && (ALL_PHASES as string[]).includes(rawPhase)
|
||||
let phase = rawPhase && (ALL_PHASES as string[]).includes(rawPhase)
|
||||
? (rawPhase as CyclePhase)
|
||||
: null;
|
||||
if (rawPhase && !phase) {
|
||||
@@ -55,6 +71,44 @@ function parseArgs(args: string[]): DreamArgs {
|
||||
const dirIdx = args.indexOf('--dir');
|
||||
const dir = dirIdx !== -1 ? args[dirIdx + 1] : null;
|
||||
|
||||
const inputIdx = args.indexOf('--input');
|
||||
const inputFile = inputIdx !== -1 ? args[inputIdx + 1] ?? null : null;
|
||||
|
||||
const dateIdx = args.indexOf('--date');
|
||||
const date = dateIdx !== -1 ? args[dateIdx + 1] ?? null : null;
|
||||
if (date && !ISO_DATE_RE.test(date)) {
|
||||
console.error(`--date must be YYYY-MM-DD; got "${date}"`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const fromIdx = args.indexOf('--from');
|
||||
const from = fromIdx !== -1 ? args[fromIdx + 1] ?? null : null;
|
||||
if (from && !ISO_DATE_RE.test(from)) {
|
||||
console.error(`--from must be YYYY-MM-DD; got "${from}"`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const toIdx = args.indexOf('--to');
|
||||
const to = toIdx !== -1 ? args[toIdx + 1] ?? null : null;
|
||||
if (to && !ISO_DATE_RE.test(to)) {
|
||||
console.error(`--to must be YYYY-MM-DD; got "${to}"`);
|
||||
process.exit(2);
|
||||
}
|
||||
if (from && to && from > to) {
|
||||
console.error(`--from (${from}) is after --to (${to}); empty range`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// --input + --date / --from / --to is incoherent: --input is a single
|
||||
// file, the date filters scan a directory.
|
||||
if (inputFile && (date || from || to)) {
|
||||
console.error('--input cannot be combined with --date / --from / --to');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// --input implies --phase synthesize.
|
||||
if (inputFile && !phase) phase = 'synthesize';
|
||||
|
||||
return {
|
||||
json: args.includes('--json'),
|
||||
dryRun: args.includes('--dry-run'),
|
||||
@@ -62,6 +116,11 @@ function parseArgs(args: string[]): DreamArgs {
|
||||
phase,
|
||||
dir,
|
||||
help: args.includes('--help') || args.includes('-h'),
|
||||
inputFile,
|
||||
date,
|
||||
from,
|
||||
to,
|
||||
bypassDreamGuard: args.includes('--unsafe-bypass-dream-guard'),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -104,23 +163,49 @@ async function resolveBrainDir(
|
||||
function printHelp() {
|
||||
console.log(`Usage: gbrain dream [options]
|
||||
|
||||
Run one brain maintenance cycle: lint, backlinks, orphan sweep, sync,
|
||||
extract, and embed. Designed for cron (exits when done).
|
||||
Run one brain maintenance cycle. Eight phases:
|
||||
lint -> backlinks -> sync -> synthesize -> extract -> patterns -> embed -> orphans
|
||||
|
||||
The synthesize + patterns phases (v0.21) consolidate yesterday's
|
||||
conversation transcripts into reflections, originals, and cross-session
|
||||
pattern pages. Designed for cron (exits when done).
|
||||
|
||||
Options:
|
||||
--dry-run Preview all fixes without writing (fs or DB)
|
||||
--dry-run Preview all fixes without writing. Note: synthesize
|
||||
runs the cheap Haiku significance filter (caches
|
||||
verdicts), but skips the Sonnet synthesis pass.
|
||||
"--dry-run" does NOT mean "zero LLM calls."
|
||||
--json Emit the CycleReport as JSON (agent-readable)
|
||||
--phase <name> Run a single phase: ${ALL_PHASES.join(' | ')}
|
||||
--pull git pull the brain repo before syncing (default: no pull)
|
||||
--dir <path> Brain directory (default: configured brain)
|
||||
|
||||
--input <file> Synthesize a specific transcript file (implies
|
||||
--phase synthesize). Bypasses corpus-dir scan.
|
||||
--date YYYY-MM-DD Synthesize transcripts dated for one specific day.
|
||||
--from YYYY-MM-DD Backfill range start (use with --to).
|
||||
--to YYYY-MM-DD Backfill range end.
|
||||
|
||||
--unsafe-bypass-dream-guard
|
||||
Disable the self-consumption guard. Use only when you
|
||||
know the input file is NOT dream-cycle output but the
|
||||
guard is firing. Loud stderr warning + cost reminder
|
||||
fires every run.
|
||||
|
||||
--help, -h Show this help
|
||||
|
||||
Examples:
|
||||
gbrain dream
|
||||
gbrain dream --dry-run --json
|
||||
gbrain dream --phase lint
|
||||
gbrain dream --phase synthesize --input ~/transcripts/2026-04-25.txt
|
||||
gbrain dream --phase synthesize --from 2026-04-01 --to 2026-04-25
|
||||
0 2 * * * gbrain dream --json # nightly via cron
|
||||
|
||||
Configure synthesize:
|
||||
gbrain config set dream.synthesize.session_corpus_dir /path/to/transcripts
|
||||
gbrain config set dream.synthesize.enabled true
|
||||
|
||||
Related:
|
||||
gbrain autopilot --install # continuous maintenance as a daemon
|
||||
gbrain autopilot # same maintenance cycle, scheduled
|
||||
@@ -165,10 +250,14 @@ function printHuman(report: CycleReport) {
|
||||
const t = report.totals;
|
||||
const hasTotals =
|
||||
t.lint_fixes > 0 || t.backlinks_added > 0 || t.pages_synced > 0 ||
|
||||
t.pages_extracted > 0 || t.pages_embedded > 0 || t.orphans_found > 0;
|
||||
t.pages_extracted > 0 || t.pages_embedded > 0 || t.orphans_found > 0 ||
|
||||
t.transcripts_processed > 0 || t.synth_pages_written > 0 || t.patterns_written > 0;
|
||||
if (hasTotals) {
|
||||
console.log(
|
||||
` totals: lint=${t.lint_fixes} backlinks=${t.backlinks_added} synced=${t.pages_synced} extracted=${t.pages_extracted} embedded=${t.pages_embedded} orphans=${t.orphans_found}`,
|
||||
` totals: lint=${t.lint_fixes} backlinks=${t.backlinks_added} synced=${t.pages_synced} ` +
|
||||
`extracted=${t.pages_extracted} embedded=${t.pages_embedded} orphans=${t.orphans_found} ` +
|
||||
`synth_transcripts=${t.transcripts_processed} synth_pages=${t.synth_pages_written} ` +
|
||||
`patterns=${t.patterns_written}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -191,6 +280,11 @@ export async function runDream(engine: BrainEngine | null, args: string[]): Prom
|
||||
dryRun: opts.dryRun,
|
||||
pull: opts.pull,
|
||||
phases,
|
||||
synthInputFile: opts.inputFile ?? undefined,
|
||||
synthDate: opts.date ?? undefined,
|
||||
synthFrom: opts.from ?? undefined,
|
||||
synthTo: opts.to ?? undefined,
|
||||
synthBypassDreamGuard: opts.bypassDreamGuard,
|
||||
});
|
||||
|
||||
if (opts.json) {
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
/**
|
||||
* gbrain eval export — stream captured eval_candidates rows as NDJSON (v0.21.0).
|
||||
*
|
||||
* Consumer: sibling gbrain-evals repo, which imports the NDJSON as a
|
||||
* BrainBench-Real fixture alongside the fictional amara-life corpus.
|
||||
*
|
||||
* Output contract (stable from v0.21.0 — schema_version:1 on every row):
|
||||
* { "schema_version": 1, "id": N, "tool_name": "query"|"search", ... }\n
|
||||
*
|
||||
* The schema_version prefix lets gbrain-evals detect format drift and
|
||||
* warn on unknown versions instead of silently misparsing.
|
||||
*
|
||||
* Usage:
|
||||
* gbrain eval export [--since 7d] [--limit N] [--tool query|search] > rows.ndjson
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import type { EvalCandidate } from '../core/types.ts';
|
||||
|
||||
const SCHEMA_VERSION = 1;
|
||||
|
||||
interface ExportOpts {
|
||||
help?: boolean;
|
||||
since?: Date;
|
||||
limit?: number;
|
||||
tool?: 'query' | 'search';
|
||||
}
|
||||
|
||||
function parseDurationToMs(s: string): number | null {
|
||||
// Accepts "30d", "7d", "1h", "90m", "3600s". Same shape as gbrain eval prune + jobs prune.
|
||||
const m = s.match(/^(\d+)\s*(ms|s|m|h|d)$/);
|
||||
if (!m) return null;
|
||||
const n = parseInt(m[1]!, 10);
|
||||
const unit = m[2]!;
|
||||
const mults: Record<string, number> = {
|
||||
ms: 1,
|
||||
s: 1000,
|
||||
m: 60_000,
|
||||
h: 3_600_000,
|
||||
d: 86_400_000,
|
||||
};
|
||||
return n * mults[unit]!;
|
||||
}
|
||||
|
||||
function parseArgs(args: string[]): ExportOpts {
|
||||
const opts: ExportOpts = {};
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i]!;
|
||||
const next = args[i + 1];
|
||||
switch (arg) {
|
||||
case '--help':
|
||||
case '-h':
|
||||
opts.help = true;
|
||||
break;
|
||||
case '--since': {
|
||||
if (!next) break;
|
||||
const ms = parseDurationToMs(next);
|
||||
if (ms !== null) {
|
||||
opts.since = new Date(Date.now() - ms);
|
||||
} else {
|
||||
console.error(`Invalid --since value: ${next} (use like 7d, 1h, 30m)`);
|
||||
process.exit(1);
|
||||
}
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
case '--limit':
|
||||
if (next) opts.limit = parseInt(next, 10);
|
||||
i++;
|
||||
break;
|
||||
case '--tool':
|
||||
if (next === 'query' || next === 'search') {
|
||||
opts.tool = next;
|
||||
} else if (next) {
|
||||
console.error(`Invalid --tool value: ${next} (use 'query' or 'search')`);
|
||||
process.exit(1);
|
||||
}
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
console.error(`gbrain eval export — emit captured eval_candidates as NDJSON to stdout
|
||||
|
||||
USAGE:
|
||||
gbrain eval export [--since DUR] [--limit N] [--tool query|search]
|
||||
|
||||
FLAGS:
|
||||
--since DUR Only rows created within DUR (e.g. 7d, 1h, 30m). Default: all.
|
||||
--limit N Cap rows returned. Default: 1000. Max: 100000.
|
||||
--tool X Filter to a specific tool ('query' or 'search'). Default: both.
|
||||
--help, -h Show this help.
|
||||
|
||||
OUTPUT:
|
||||
One JSON object per line on stdout. Every row begins with
|
||||
"schema_version": 1 so downstream consumers (gbrain-evals) can
|
||||
detect format changes.
|
||||
|
||||
EXAMPLES:
|
||||
gbrain eval export > rows.ndjson
|
||||
gbrain eval export --since 7d --tool query | jq '.query'
|
||||
gbrain eval export --limit 100 | head
|
||||
`);
|
||||
}
|
||||
|
||||
export async function runEvalExport(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const opts = parseArgs(args);
|
||||
if (opts.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
// Progress to stderr (stdout is reserved for NDJSON data).
|
||||
const { createProgress, startHeartbeat } = await import('../core/progress.ts');
|
||||
const { getCliOptions, cliOptsToProgressOptions } = await import('../core/cli-options.ts');
|
||||
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
|
||||
|
||||
progress.start('eval.export');
|
||||
const stopHeartbeat = startHeartbeat(progress, 'reading eval_candidates');
|
||||
let rows: EvalCandidate[];
|
||||
try {
|
||||
rows = await engine.listEvalCandidates({
|
||||
since: opts.since,
|
||||
limit: opts.limit,
|
||||
tool: opts.tool,
|
||||
});
|
||||
} finally {
|
||||
stopHeartbeat();
|
||||
}
|
||||
|
||||
// Emit NDJSON to stdout. EPIPE-safe: if the downstream process
|
||||
// (e.g. `| head`) closes its end early, we abort cleanly without a
|
||||
// stack trace. Matches src/core/progress.ts EPIPE handling precedent.
|
||||
const stdout = process.stdout;
|
||||
const abortOnEpipe = (err: NodeJS.ErrnoException) => {
|
||||
if (err.code === 'EPIPE') process.exit(0);
|
||||
};
|
||||
stdout.on('error', abortOnEpipe);
|
||||
|
||||
let written = 0;
|
||||
for (const row of rows) {
|
||||
// Prefix every line with schema_version:1 so gbrain-evals can detect
|
||||
// schema drift before parsing the rest of the fields.
|
||||
const line = JSON.stringify({ schema_version: SCHEMA_VERSION, ...row });
|
||||
if (!stdout.write(line + '\n')) {
|
||||
// Backpressure: wait for drain before continuing.
|
||||
await new Promise(r => stdout.once('drain', r));
|
||||
}
|
||||
written++;
|
||||
progress.tick();
|
||||
}
|
||||
|
||||
stdout.off('error', abortOnEpipe);
|
||||
progress.finish();
|
||||
console.error(`exported ${written} row(s)`);
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* gbrain eval prune — delete old eval_candidates rows (v0.21.0).
|
||||
*
|
||||
* Retention is unlimited by default (matches ingest_log precedent).
|
||||
* This command is the explicit cleanup; pairs with `gbrain eval export`
|
||||
* (snapshot first, then prune if you want to reset).
|
||||
*
|
||||
* Usage:
|
||||
* gbrain eval prune --older-than 30d
|
||||
* gbrain eval prune --older-than 1h --dry-run
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
|
||||
interface PruneOpts {
|
||||
help?: boolean;
|
||||
olderThanMs?: number;
|
||||
dryRun?: boolean;
|
||||
}
|
||||
|
||||
function parseDurationToMs(s: string): number | null {
|
||||
const m = s.match(/^(\d+)\s*(ms|s|m|h|d)$/);
|
||||
if (!m) return null;
|
||||
const n = parseInt(m[1]!, 10);
|
||||
const unit = m[2]!;
|
||||
const mults: Record<string, number> = {
|
||||
ms: 1,
|
||||
s: 1000,
|
||||
m: 60_000,
|
||||
h: 3_600_000,
|
||||
d: 86_400_000,
|
||||
};
|
||||
return n * mults[unit]!;
|
||||
}
|
||||
|
||||
function parseArgs(args: string[]): PruneOpts {
|
||||
const opts: PruneOpts = {};
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i]!;
|
||||
const next = args[i + 1];
|
||||
switch (arg) {
|
||||
case '--help':
|
||||
case '-h':
|
||||
opts.help = true;
|
||||
break;
|
||||
case '--older-than': {
|
||||
if (!next) break;
|
||||
const ms = parseDurationToMs(next);
|
||||
if (ms === null) {
|
||||
console.error(`Invalid --older-than value: ${next} (use like 30d, 1h, 90m)`);
|
||||
process.exit(1);
|
||||
}
|
||||
opts.olderThanMs = ms;
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
case '--dry-run':
|
||||
opts.dryRun = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
console.error(`gbrain eval prune — delete old eval_candidates rows
|
||||
|
||||
USAGE:
|
||||
gbrain eval prune --older-than DUR [--dry-run]
|
||||
|
||||
FLAGS:
|
||||
--older-than DUR Delete rows created before now() - DUR (e.g. 30d, 7d, 1h).
|
||||
Required — this command never deletes without a window.
|
||||
--dry-run Report what would be deleted; don't actually delete.
|
||||
--help, -h Show this help.
|
||||
|
||||
EXAMPLES:
|
||||
gbrain eval prune --older-than 30d
|
||||
gbrain eval prune --older-than 90d --dry-run
|
||||
`);
|
||||
}
|
||||
|
||||
export async function runEvalPrune(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const opts = parseArgs(args);
|
||||
if (opts.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
if (!opts.olderThanMs) {
|
||||
console.error('Error: --older-than is required\n');
|
||||
printHelp();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const cutoff = new Date(Date.now() - opts.olderThanMs);
|
||||
|
||||
if (opts.dryRun) {
|
||||
// Snapshot-count the rows we *would* delete — the list call caps at
|
||||
// 100k which matches the export ceiling, so larger windows get a
|
||||
// floor-estimate that's still useful signal.
|
||||
const rows = await engine.listEvalCandidates({
|
||||
since: new Date(0),
|
||||
limit: 100_000,
|
||||
});
|
||||
const wouldDelete = rows.filter(r => new Date(r.created_at) < cutoff).length;
|
||||
console.log(`[dry-run] would delete ${wouldDelete} row(s) created before ${cutoff.toISOString()}`);
|
||||
if (rows.length === 100_000) {
|
||||
console.log('[dry-run] (count may be undercounted — the scan hit the 100k row limit)');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const deleted = await engine.deleteEvalCandidatesBefore(cutoff);
|
||||
console.log(`deleted ${deleted} row(s) created before ${cutoff.toISOString()}`);
|
||||
}
|
||||
@@ -0,0 +1,424 @@
|
||||
/**
|
||||
* gbrain eval replay — replay captured eval_candidates against current brain (v0.25.0).
|
||||
*
|
||||
* The contributor-facing half of BrainBench-Real:
|
||||
*
|
||||
* 1. capture some real traffic (default-on, lands in eval_candidates)
|
||||
* 2. snapshot it (gbrain eval export --since 7d > baseline.ndjson)
|
||||
* 3. make a code change (tune RRF_K, edit hybrid.ts, swap an embed model)
|
||||
* 4. replay against the snapshot (gbrain eval replay --against baseline.ndjson)
|
||||
*
|
||||
* Outputs three numbers a contributor can read at a glance:
|
||||
*
|
||||
* - mean Jaccard@k between captured retrieved_slugs and current run's slugs
|
||||
* - top-1 stability rate (was the #1 result the same?)
|
||||
* - mean latency delta (current - captured), positive = slower now
|
||||
*
|
||||
* Best-effort by design. Replay is NOT pure — your brain has more pages than
|
||||
* when the capture was taken, embeddings may have drifted, and the OPENAI key
|
||||
* may be different. The metrics describe "did this change hurt retrieval on
|
||||
* the queries you actually serve" not "do these match the baseline byte for
|
||||
* byte." Use it before merging anything that touches src/core/search/ or the
|
||||
* query/search op handlers.
|
||||
*
|
||||
* Usage:
|
||||
* gbrain eval replay --against captured.ndjson [--limit N] [--json]
|
||||
* [--top-regressions K] [--verbose]
|
||||
*/
|
||||
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import type { SearchResult } from '../core/types.ts';
|
||||
import { hybridSearch } from '../core/search/hybrid.ts';
|
||||
|
||||
interface ReplayOpts {
|
||||
help?: boolean;
|
||||
against?: string;
|
||||
limit?: number;
|
||||
json?: boolean;
|
||||
verbose?: boolean;
|
||||
topRegressions?: number;
|
||||
}
|
||||
|
||||
interface RowResult {
|
||||
/** Captured row's id, for back-referencing into the source NDJSON. */
|
||||
id: number;
|
||||
tool_name: 'query' | 'search';
|
||||
query: string;
|
||||
/** Set-overlap score in [0, 1]. 1.0 = identical retrieved set. */
|
||||
jaccard: number;
|
||||
/** True when current top result matches captured top result. */
|
||||
top1Match: boolean;
|
||||
/** Captured retrieved_slugs (as-is from NDJSON). */
|
||||
captured_slugs: string[];
|
||||
/** Current run's slugs (deduped, in result order). */
|
||||
current_slugs: string[];
|
||||
/** Wall-clock latency (ms) of the current re-run. */
|
||||
current_latency_ms: number;
|
||||
/** latency delta = current - captured. Positive = slower now. */
|
||||
latency_delta_ms: number;
|
||||
/** True if the row was skipped (e.g. captured query was empty). */
|
||||
skipped?: boolean;
|
||||
/** Reason the row was skipped, if any. */
|
||||
skip_reason?: string;
|
||||
/** True if the row threw during replay; current_slugs is empty. */
|
||||
errored?: boolean;
|
||||
error_message?: string;
|
||||
}
|
||||
|
||||
function parseArgs(args: string[]): ReplayOpts {
|
||||
const opts: ReplayOpts = {};
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i]!;
|
||||
const next = args[i + 1];
|
||||
switch (arg) {
|
||||
case '--help':
|
||||
case '-h':
|
||||
opts.help = true;
|
||||
break;
|
||||
case '--against':
|
||||
if (!next) break;
|
||||
opts.against = next;
|
||||
i++;
|
||||
break;
|
||||
case '--limit':
|
||||
if (!next) break;
|
||||
opts.limit = parseInt(next, 10);
|
||||
i++;
|
||||
break;
|
||||
case '--json':
|
||||
opts.json = true;
|
||||
break;
|
||||
case '--verbose':
|
||||
opts.verbose = true;
|
||||
break;
|
||||
case '--top-regressions':
|
||||
if (!next) break;
|
||||
opts.topRegressions = parseInt(next, 10);
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
console.error(`gbrain eval replay — replay captured queries against current brain
|
||||
|
||||
USAGE:
|
||||
gbrain eval replay --against FILE.ndjson [flags]
|
||||
|
||||
FLAGS:
|
||||
--against FILE NDJSON file from \`gbrain eval export\` (required).
|
||||
--limit N Replay at most N rows (default: replay all).
|
||||
Each row hits OpenAI once for query embedding —
|
||||
cap aggressively when iterating locally.
|
||||
--top-regressions K Print the K rows with the worst Jaccard scores.
|
||||
Default 5 in human mode, 0 in --json.
|
||||
--json Emit one JSON object on stdout instead of a table.
|
||||
Stable shape for CI consumption.
|
||||
--verbose Include every row's per-row diff (large output).
|
||||
--help, -h Show this help.
|
||||
|
||||
OUTPUT (human mode):
|
||||
Replayed N captured queries (M skipped, K errored)
|
||||
Mean Jaccard@k: 0.873
|
||||
Top-1 stability: 87% (N=87 / 100)
|
||||
Mean latency Δ: +12ms (current slower)
|
||||
|
||||
Top 5 regressions:
|
||||
0.20 "find every reference to widget-co" captured=12 current=3
|
||||
...
|
||||
|
||||
EXIT CODE:
|
||||
0 — replay completed (regardless of regression magnitude).
|
||||
1 — invalid args, --against not found, or NDJSON parse failure.
|
||||
|
||||
NOTES:
|
||||
Replay is best-effort. Your brain has more pages than when the snapshot
|
||||
was taken; embeddings may have drifted; OPENAI_API_KEY may be different.
|
||||
Use the metrics to spot regressions on REAL queries, not as a hash check.
|
||||
`);
|
||||
}
|
||||
|
||||
interface CapturedRow {
|
||||
schema_version: number;
|
||||
id: number;
|
||||
tool_name: 'query' | 'search';
|
||||
query: string;
|
||||
retrieved_slugs: string[];
|
||||
retrieved_chunk_ids?: number[];
|
||||
source_ids?: string[];
|
||||
expand_enabled?: boolean | null;
|
||||
detail?: 'low' | 'medium' | 'high' | null;
|
||||
detail_resolved?: 'low' | 'medium' | 'high' | null;
|
||||
vector_enabled?: boolean;
|
||||
expansion_applied?: boolean;
|
||||
latency_ms: number;
|
||||
remote?: boolean;
|
||||
job_id?: number | null;
|
||||
subagent_id?: number | null;
|
||||
created_at?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse NDJSON. One object per non-blank line. Single bad line throws — it's
|
||||
* a corrupt export and silently dropping rows would mask real bugs.
|
||||
*/
|
||||
function parseNdjson(content: string): CapturedRow[] {
|
||||
const lines = content.split('\n');
|
||||
const rows: CapturedRow[] = [];
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i]!.trim();
|
||||
if (!line) continue;
|
||||
let row: CapturedRow;
|
||||
try {
|
||||
row = JSON.parse(line);
|
||||
} catch (err) {
|
||||
throw new Error(`NDJSON parse error on line ${i + 1}: ${(err as Error).message}`);
|
||||
}
|
||||
if (typeof row.schema_version !== 'number') {
|
||||
throw new Error(`Line ${i + 1} missing schema_version — not from \`gbrain eval export\`?`);
|
||||
}
|
||||
if (row.schema_version !== 1) {
|
||||
throw new Error(
|
||||
`Line ${i + 1} has schema_version=${row.schema_version}; this replay only supports v1. ` +
|
||||
`Upgrade gbrain or re-export.`,
|
||||
);
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set-Jaccard between two slug arrays. Order ignored, dupes collapsed.
|
||||
* Both empty → 1.0 (identical empty sets, no information lost).
|
||||
*/
|
||||
function jaccardSlugs(a: string[], b: string[]): number {
|
||||
const setA = new Set(a);
|
||||
const setB = new Set(b);
|
||||
if (setA.size === 0 && setB.size === 0) return 1.0;
|
||||
let intersection = 0;
|
||||
for (const s of setA) if (setB.has(s)) intersection++;
|
||||
const union = setA.size + setB.size - intersection;
|
||||
return union === 0 ? 1.0 : intersection / union;
|
||||
}
|
||||
|
||||
async function replayRow(engine: BrainEngine, row: CapturedRow): Promise<RowResult> {
|
||||
const captured_slugs = row.retrieved_slugs ?? [];
|
||||
const startedAt = Date.now();
|
||||
|
||||
// Default replay limit matches hybridSearch's default (20).
|
||||
const limit = Math.max(captured_slugs.length, 20);
|
||||
|
||||
// search → bare keyword path. query → hybrid path (vector + keyword + RRF).
|
||||
// detail and expansion are threaded in from the captured row so the same
|
||||
// logic runs that produced the original retrieval.
|
||||
let current: SearchResult[];
|
||||
try {
|
||||
if (row.tool_name === 'search') {
|
||||
const dedupedRaw = await engine.searchKeyword(row.query, { limit });
|
||||
current = dedupedRaw;
|
||||
} else {
|
||||
current = await hybridSearch(engine, row.query, {
|
||||
limit,
|
||||
detail: row.detail ?? undefined,
|
||||
expansion: row.expand_enabled ?? false,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
return {
|
||||
id: row.id,
|
||||
tool_name: row.tool_name,
|
||||
query: row.query,
|
||||
jaccard: 0,
|
||||
top1Match: false,
|
||||
captured_slugs,
|
||||
current_slugs: [],
|
||||
current_latency_ms: Date.now() - startedAt,
|
||||
latency_delta_ms: Date.now() - startedAt - row.latency_ms,
|
||||
errored: true,
|
||||
error_message: (err as Error).message ?? String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const current_latency_ms = Date.now() - startedAt;
|
||||
// Dedup slugs while preserving order — same convention as search results.
|
||||
const seen = new Set<string>();
|
||||
const current_slugs: string[] = [];
|
||||
for (const r of current) {
|
||||
if (!seen.has(r.slug)) {
|
||||
seen.add(r.slug);
|
||||
current_slugs.push(r.slug);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
id: row.id,
|
||||
tool_name: row.tool_name,
|
||||
query: row.query,
|
||||
jaccard: jaccardSlugs(captured_slugs, current_slugs),
|
||||
top1Match: captured_slugs[0] !== undefined && current_slugs[0] === captured_slugs[0],
|
||||
captured_slugs,
|
||||
current_slugs,
|
||||
current_latency_ms,
|
||||
latency_delta_ms: current_latency_ms - row.latency_ms,
|
||||
};
|
||||
}
|
||||
|
||||
interface ReplaySummary {
|
||||
rows_total: number;
|
||||
rows_replayed: number;
|
||||
rows_skipped: number;
|
||||
rows_errored: number;
|
||||
/** Mean Jaccard across non-skipped, non-errored rows. */
|
||||
mean_jaccard: number;
|
||||
top1_stability_rate: number;
|
||||
mean_latency_delta_ms: number;
|
||||
/** Rows where current latency is more than 2x captured (regression alarm). */
|
||||
rows_over_2x_latency: number;
|
||||
}
|
||||
|
||||
function summarize(results: RowResult[]): ReplaySummary {
|
||||
const eligible = results.filter(r => !r.skipped && !r.errored);
|
||||
const meanJaccard = eligible.length === 0
|
||||
? 0
|
||||
: eligible.reduce((a, r) => a + r.jaccard, 0) / eligible.length;
|
||||
const top1Rate = eligible.length === 0
|
||||
? 0
|
||||
: eligible.filter(r => r.top1Match).length / eligible.length;
|
||||
const meanLatencyDelta = eligible.length === 0
|
||||
? 0
|
||||
: eligible.reduce((a, r) => a + r.latency_delta_ms, 0) / eligible.length;
|
||||
const over2x = eligible.filter(r => {
|
||||
const captured = results.find(x => x.id === r.id);
|
||||
return captured && captured.current_latency_ms > 2 * (captured.current_latency_ms - captured.latency_delta_ms);
|
||||
}).length;
|
||||
|
||||
return {
|
||||
rows_total: results.length,
|
||||
rows_replayed: eligible.length,
|
||||
rows_skipped: results.filter(r => r.skipped).length,
|
||||
rows_errored: results.filter(r => r.errored).length,
|
||||
mean_jaccard: meanJaccard,
|
||||
top1_stability_rate: top1Rate,
|
||||
mean_latency_delta_ms: meanLatencyDelta,
|
||||
rows_over_2x_latency: over2x,
|
||||
};
|
||||
}
|
||||
|
||||
function printHumanSummary(summary: ReplaySummary, results: RowResult[], topRegressions: number): void {
|
||||
const total = summary.rows_total;
|
||||
const eligible = summary.rows_replayed;
|
||||
console.log(`Replayed ${eligible} of ${total} captured queries (${summary.rows_skipped} skipped, ${summary.rows_errored} errored)`);
|
||||
console.log(`Mean Jaccard@k: ${summary.mean_jaccard.toFixed(3)}`);
|
||||
console.log(`Top-1 stability: ${(summary.top1_stability_rate * 100).toFixed(1)}%`);
|
||||
const sign = summary.mean_latency_delta_ms >= 0 ? '+' : '';
|
||||
console.log(`Mean latency Δ: ${sign}${summary.mean_latency_delta_ms.toFixed(0)}ms (current vs captured)`);
|
||||
if (summary.rows_over_2x_latency > 0) {
|
||||
console.log(`⚠ ${summary.rows_over_2x_latency} row(s) ran more than 2× slower than captured`);
|
||||
}
|
||||
|
||||
if (topRegressions > 0) {
|
||||
const sorted = [...results]
|
||||
.filter(r => !r.skipped && !r.errored)
|
||||
.sort((a, b) => a.jaccard - b.jaccard)
|
||||
.slice(0, topRegressions);
|
||||
if (sorted.length > 0 && sorted[0]!.jaccard < 1.0) {
|
||||
console.log(`\nTop ${sorted.length} regression(s):`);
|
||||
for (const r of sorted) {
|
||||
const truncQuery = r.query.length > 60 ? r.query.slice(0, 57) + '...' : r.query;
|
||||
console.log(
|
||||
` jaccard=${r.jaccard.toFixed(2)} captured=${r.captured_slugs.length} current=${r.current_slugs.length} ` +
|
||||
`"${truncQuery}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (summary.rows_errored > 0) {
|
||||
const errors = results.filter(r => r.errored).slice(0, 3);
|
||||
console.log(`\n${summary.rows_errored} row(s) errored. First ${errors.length}:`);
|
||||
for (const r of errors) {
|
||||
const truncQuery = r.query.length > 60 ? r.query.slice(0, 57) + '...' : r.query;
|
||||
console.log(` id=${r.id} "${truncQuery}" ${r.error_message ?? ''}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function runEvalReplay(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const opts = parseArgs(args);
|
||||
if (opts.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
if (!opts.against) {
|
||||
console.error('Error: --against FILE.ndjson is required\n');
|
||||
printHelp();
|
||||
process.exit(1);
|
||||
}
|
||||
if (!existsSync(opts.against)) {
|
||||
console.error(`Error: file not found: ${opts.against}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let rows: CapturedRow[];
|
||||
try {
|
||||
const content = readFileSync(opts.against, 'utf-8');
|
||||
rows = parseNdjson(content);
|
||||
} catch (err) {
|
||||
console.error(`Error: ${(err as Error).message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (rows.length === 0) {
|
||||
console.error(`Error: ${opts.against} is empty (no NDJSON rows)`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const capped = opts.limit && opts.limit > 0 ? rows.slice(0, opts.limit) : rows;
|
||||
if (!opts.json) {
|
||||
console.error(
|
||||
`Replaying ${capped.length}${capped.length < rows.length ? ` of ${rows.length}` : ''} captured queries…`,
|
||||
);
|
||||
}
|
||||
|
||||
const results: RowResult[] = [];
|
||||
for (const row of capped) {
|
||||
if (!row.query || row.query.length === 0) {
|
||||
results.push({
|
||||
id: row.id,
|
||||
tool_name: row.tool_name,
|
||||
query: row.query ?? '',
|
||||
jaccard: 0,
|
||||
top1Match: false,
|
||||
captured_slugs: row.retrieved_slugs ?? [],
|
||||
current_slugs: [],
|
||||
current_latency_ms: 0,
|
||||
latency_delta_ms: 0,
|
||||
skipped: true,
|
||||
skip_reason: 'empty query',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const r = await replayRow(engine, row);
|
||||
results.push(r);
|
||||
if (!opts.json && results.length % 25 === 0) {
|
||||
process.stderr.write(` ...${results.length}/${capped.length}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
const summary = summarize(results);
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify({
|
||||
schema_version: 1,
|
||||
summary,
|
||||
results: opts.verbose ? results : undefined,
|
||||
}, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
const topN = opts.topRegressions ?? 5;
|
||||
printHumanSummary(summary, results, topN);
|
||||
}
|
||||
@@ -21,6 +21,23 @@ import {
|
||||
} from '../core/search/eval.ts';
|
||||
|
||||
export async function runEvalCommand(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
// v0.25.0 — sub-subcommand dispatch. Bare `gbrain eval --qrels ...`
|
||||
// falls through to the legacy IR-metrics flow so existing callers
|
||||
// don't break.
|
||||
const sub = args[0];
|
||||
if (sub === 'export') {
|
||||
const { runEvalExport } = await import('./eval-export.ts');
|
||||
return runEvalExport(engine, args.slice(1));
|
||||
}
|
||||
if (sub === 'prune') {
|
||||
const { runEvalPrune } = await import('./eval-prune.ts');
|
||||
return runEvalPrune(engine, args.slice(1));
|
||||
}
|
||||
if (sub === 'replay') {
|
||||
const { runEvalReplay } = await import('./eval-replay.ts');
|
||||
return runEvalReplay(engine, args.slice(1));
|
||||
}
|
||||
|
||||
const opts = parseArgs(args);
|
||||
|
||||
if (opts.help) {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user