mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-16 18:02:30 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3ece117449 |
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/bin/sh
|
||||
# .agents/gbrain-launcher — MCP-server launcher for the gbrain Codex and
|
||||
# Claude Code plugins. Unix-only (macOS/Linux): needs /bin/sh, executable
|
||||
# bits, and `command -v`. Windows support is a filed follow-up.
|
||||
#
|
||||
# Resolves the gbrain binary (the plugin snapshot cannot ship it — the CLI
|
||||
# installs separately), then execs it with the argv the plugin manifest
|
||||
# pinned. Resolution order:
|
||||
# 1. $GBRAIN_BIN explicit override (must be executable)
|
||||
# 2. `gbrain` on PATH
|
||||
# 3. ~/.bun/bin/gbrain the sanctioned global-install location
|
||||
#
|
||||
# GBRAIN_SURFACE: when set and argv[0] is `serve`, replaces the value of an
|
||||
# existing `--surface <x>` pair, or appends `--surface $GBRAIN_SURFACE` if
|
||||
# the pair is absent — so a user can widen (full) or narrow (verbs) this
|
||||
# machine's plugin surface without editing the plugin snapshot.
|
||||
#
|
||||
# No auto-install by design: an MCP server start must never run a network
|
||||
# install. On a miss this exits 127 with the recovery path on stderr; the
|
||||
# bundled `setup` skill walks the install interactively.
|
||||
|
||||
set -eu
|
||||
|
||||
resolve_bin() {
|
||||
if [ -n "${GBRAIN_BIN:-}" ]; then
|
||||
if [ ! -x "$GBRAIN_BIN" ]; then
|
||||
echo "gbrain-launcher: GBRAIN_BIN='$GBRAIN_BIN' is not an executable file" >&2
|
||||
exit 127
|
||||
fi
|
||||
printf '%s' "$GBRAIN_BIN"
|
||||
return
|
||||
fi
|
||||
# ~/.bun/bin (the sanctioned global-install location) is preferred OVER a
|
||||
# bare PATH lookup: a hostile repo that prepends node_modules/.bin with a
|
||||
# fake `gbrain` must not win over the real install. GBRAIN_BIN (above) is
|
||||
# the explicit escape hatch for a gbrain living elsewhere.
|
||||
if [ -x "${HOME:-}/.bun/bin/gbrain" ]; then
|
||||
printf '%s' "$HOME/.bun/bin/gbrain"
|
||||
return
|
||||
fi
|
||||
if command -v gbrain >/dev/null 2>&1; then
|
||||
command -v gbrain
|
||||
return
|
||||
fi
|
||||
echo "gbrain-launcher: gbrain binary not found." >&2
|
||||
echo " install: bun install -g github:garrytan/gbrain#latest-stable" >&2
|
||||
echo " (the npm package named 'gbrain' is unrelated - do not npm install it)" >&2
|
||||
echo " then run the bundled 'setup' skill to initialize your brain," >&2
|
||||
echo " or set GBRAIN_BIN to an absolute gbrain binary path." >&2
|
||||
exit 127
|
||||
}
|
||||
|
||||
BIN="$(resolve_bin)"
|
||||
echo "gbrain-launcher: using $BIN" >&2
|
||||
|
||||
# Surface override — only for `serve` invocations. Rebuilds the positional
|
||||
# params in place (rotate-through-sentinel idiom; no eval, no word-splitting
|
||||
# hazards): replace the value of an existing `--surface <x>` pair, or append
|
||||
# the pair when absent.
|
||||
if [ -n "${GBRAIN_SURFACE:-}" ] && [ "${1:-}" = "serve" ]; then
|
||||
replaced=0
|
||||
expect_value=0
|
||||
set -- "$@" "__gbrain_end__"
|
||||
while [ "$1" != "__gbrain_end__" ]; do
|
||||
a="$1"
|
||||
shift
|
||||
if [ "$expect_value" = 1 ]; then
|
||||
expect_value=0
|
||||
replaced=1
|
||||
set -- "$@" "$GBRAIN_SURFACE"
|
||||
continue
|
||||
fi
|
||||
if [ "$a" = "--surface" ]; then
|
||||
expect_value=1
|
||||
fi
|
||||
set -- "$@" "$a"
|
||||
done
|
||||
shift
|
||||
if [ "$replaced" = 0 ]; then
|
||||
set -- "$@" "--surface" "$GBRAIN_SURFACE"
|
||||
fi
|
||||
fi
|
||||
|
||||
exec "$BIN" "$@"
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"interface": { "displayName": "GBrain" },
|
||||
"plugins": [
|
||||
{
|
||||
"name": "gbrain",
|
||||
"source": { "source": "local", "path": "./" },
|
||||
"policy": { "installation": "AVAILABLE", "authentication": "ON_USE" },
|
||||
"category": "Productivity"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"description": "GBrain — a persistent knowledge brain for your coding agent: hybrid search, synthesis, and cross-session memory.",
|
||||
"owner": { "name": "Garry Tan" },
|
||||
"plugins": [
|
||||
{
|
||||
"name": "gbrain",
|
||||
"source": "./",
|
||||
"description": "Personal knowledge brain for your coding agent — hybrid search, synthesis, and durable cross-session memory, plus a curated brain-first skill set.",
|
||||
"category": "productivity"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.46.11.0",
|
||||
"description": "Personal knowledge brain for your coding agent — hybrid search, synthesis, graph traversal, and durable cross-session memory over Postgres/PGLite with pgvector, plus a curated brain-first skill set.",
|
||||
"author": {
|
||||
"name": "Garry Tan",
|
||||
"url": "https://github.com/garrytan"
|
||||
},
|
||||
"homepage": "https://github.com/garrytan/gbrain",
|
||||
"repository": "https://github.com/garrytan/gbrain",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"memory",
|
||||
"knowledge-base",
|
||||
"mcp",
|
||||
"search",
|
||||
"agent",
|
||||
"brain",
|
||||
"pgvector"
|
||||
],
|
||||
"skills": "./plugin/skills/",
|
||||
"mcpServers": {
|
||||
"gbrain": {
|
||||
"command": "${CLAUDE_PLUGIN_ROOT}/.agents/gbrain-launcher",
|
||||
"args": [
|
||||
"serve",
|
||||
"--surface",
|
||||
"starter",
|
||||
"--source-guard"
|
||||
],
|
||||
"cwd": "${CLAUDE_PLUGIN_ROOT}"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"gbrain": {
|
||||
"command": "./.agents/gbrain-launcher",
|
||||
"args": [
|
||||
"serve",
|
||||
"--surface",
|
||||
"starter",
|
||||
"--source-guard"
|
||||
],
|
||||
"cwd": ".",
|
||||
"env_vars": [
|
||||
"ANTHROPIC_API_KEY",
|
||||
"ANTHROPIC_BASE_URL",
|
||||
"AZURE_OPENAI_API_KEY",
|
||||
"DASHSCOPE_API_KEY",
|
||||
"DATABASE_URL",
|
||||
"DEEPGRAM_API_KEY",
|
||||
"DEEPSEEK_API_KEY",
|
||||
"GBRAIN_BIN",
|
||||
"GBRAIN_BRAIN_ID",
|
||||
"GBRAIN_CHAT_FALLBACK_CHAIN",
|
||||
"GBRAIN_CHAT_MODEL",
|
||||
"GBRAIN_DATABASE_URL",
|
||||
"GBRAIN_EMBEDDING_DIMENSIONS",
|
||||
"GBRAIN_EMBEDDING_IMAGE_OCR",
|
||||
"GBRAIN_EMBEDDING_IMAGE_OCR_MODEL",
|
||||
"GBRAIN_EMBEDDING_MODEL",
|
||||
"GBRAIN_EMBEDDING_MULTIMODAL",
|
||||
"GBRAIN_EMBEDDING_MULTIMODAL_MODEL",
|
||||
"GBRAIN_EXPANSION_MODEL",
|
||||
"GBRAIN_HOME",
|
||||
"GBRAIN_MAX_MARKUP_RATIO",
|
||||
"GBRAIN_MCP_FORCE_SURFACE",
|
||||
"GBRAIN_NO_JUNK_PATTERNS",
|
||||
"GBRAIN_NO_SANITY",
|
||||
"GBRAIN_PAGE_BLOCK_BYTES",
|
||||
"GBRAIN_PAGE_WARN_BYTES",
|
||||
"GBRAIN_REMOTE_CLIENT_SECRET",
|
||||
"GBRAIN_RETRIEVAL_REFLEX",
|
||||
"GBRAIN_RETRIEVAL_REFLEX_WINDOW_TURNS",
|
||||
"GBRAIN_SOURCE",
|
||||
"GBRAIN_SURFACE",
|
||||
"GEMINI_API_KEY",
|
||||
"GOOGLE_GENERATIVE_AI_API_KEY",
|
||||
"GROQ_API_KEY",
|
||||
"HOME",
|
||||
"LITELLM_API_KEY",
|
||||
"LITELLM_BASE_URL",
|
||||
"LLAMA_SERVER_API_KEY",
|
||||
"LLAMA_SERVER_BASE_URL",
|
||||
"LLAMA_SERVER_RERANKER_API_KEY",
|
||||
"LLAMA_SERVER_RERANKER_BASE_URL",
|
||||
"LMSTUDIO_BASE_URL",
|
||||
"MINIMAX_API_KEY",
|
||||
"MISTRAL_API_KEY",
|
||||
"MOONSHOT_API_KEY",
|
||||
"NVIDIA_API_KEY",
|
||||
"OLLAMA_API_KEY",
|
||||
"OLLAMA_BASE_URL",
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_BASE_URL",
|
||||
"OPENROUTER_API_KEY",
|
||||
"OPENROUTER_BASE_URL",
|
||||
"PATH",
|
||||
"PERPLEXITY_API_KEY",
|
||||
"PPLX_API_KEY",
|
||||
"TOGETHER_API_KEY",
|
||||
"VOYAGE_API_KEY",
|
||||
"ZEROENTROPY_API_KEY",
|
||||
"ZHIPUAI_API_KEY"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.46.11.0",
|
||||
"description": "Personal knowledge brain for your coding agent — hybrid search, synthesis, graph traversal, and durable cross-session memory over Postgres/PGLite with pgvector, plus a curated brain-first skill set.",
|
||||
"author": {
|
||||
"name": "Garry Tan",
|
||||
"url": "https://github.com/garrytan"
|
||||
},
|
||||
"homepage": "https://github.com/garrytan/gbrain",
|
||||
"repository": "https://github.com/garrytan/gbrain",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"memory",
|
||||
"knowledge-base",
|
||||
"mcp",
|
||||
"search",
|
||||
"agent",
|
||||
"brain",
|
||||
"pgvector"
|
||||
],
|
||||
"skills": "./plugin/skills/",
|
||||
"mcpServers": "./.codex-plugin/mcp.json",
|
||||
"interface": {
|
||||
"displayName": "GBrain",
|
||||
"shortDescription": "Give your agent a persistent brain: search, synthesis, memory",
|
||||
"longDescription": "GBrain wires a personal knowledge brain into every session: hybrid keyword+vector search, entity graph traversal, synthesis, and memory your agent writes itself — served on the starter MCP surface (the seven memory verbs plus the daily-driver brain ops). Bundles the curated brain-first skill set: setup (walks install + gbrain init), cold-start day-one brain filling, ingest, query, briefing, upgrade, and more. Requires the gbrain CLI (bun install -g github:garrytan/gbrain#latest-stable) and a brain (gbrain init); the bundled setup skill walks the rest. Unix (macOS/Linux) only.",
|
||||
"developerName": "Garry Tan",
|
||||
"category": "Productivity",
|
||||
"capabilities": [
|
||||
"Interactive",
|
||||
"Write"
|
||||
],
|
||||
"websiteURL": "https://github.com/garrytan/gbrain",
|
||||
"defaultPrompt": [
|
||||
"Search my brain, recall context across sessions, and write new memory as we work"
|
||||
],
|
||||
"brandColor": "#1F6F5C"
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
# GBrain E2E Test Configuration
|
||||
# Copy to .env.testing and fill in real values
|
||||
#
|
||||
# Tier 1 (required for E2E tests)
|
||||
# Option A: Local Docker Postgres (default)
|
||||
DATABASE_URL=postgresql://postgres:postgres@localhost:5433/gbrain_test
|
||||
# Option B: Real Supabase instance (tests the actual production path)
|
||||
# DATABASE_URL=postgresql://postgres.[project-ref]:[password]@aws-0-us-east-1.pooler.supabase.com:6543/postgres
|
||||
|
||||
# Tier 2 (required for skill tests, optional for mechanical tests)
|
||||
OPENAI_API_KEY=sk-...
|
||||
ANTHROPIC_API_KEY=sk-ant-...
|
||||
@@ -1,27 +0,0 @@
|
||||
---
|
||||
name: Bug Report
|
||||
about: Something isn't working
|
||||
labels: bug
|
||||
---
|
||||
|
||||
**What happened?**
|
||||
|
||||
|
||||
**What did you expect?**
|
||||
|
||||
|
||||
**Steps to reproduce**
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
**Environment**
|
||||
- gbrain version: (`gbrain version`)
|
||||
- OS:
|
||||
- Bun version: (`bun --version`)
|
||||
- Database: Supabase / self-hosted Postgres
|
||||
|
||||
**`gbrain doctor --json` output**
|
||||
```json
|
||||
(paste output here)
|
||||
```
|
||||
@@ -1,14 +0,0 @@
|
||||
---
|
||||
name: Feature Request
|
||||
about: Suggest an improvement
|
||||
labels: enhancement
|
||||
---
|
||||
|
||||
**What problem does this solve?**
|
||||
|
||||
|
||||
**What does the solution look like?**
|
||||
|
||||
|
||||
**Alternatives considered**
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
<!--
|
||||
Tier 5.5 Externally-Authored Query Submission template
|
||||
See eval/CONTRIBUTING.md for the full workflow.
|
||||
-->
|
||||
|
||||
## Summary
|
||||
|
||||
Submitting **N** Tier 5.5 queries for BrainBench.
|
||||
|
||||
- Author handle: `@your-handle`
|
||||
- File location: `eval/external-authors/your-handle/queries.json`
|
||||
- Queries authored fresh (not copy-pasted from a model output)
|
||||
- Slugs verified against `eval/data/world-v1/` (via `bun run eval:world:view`)
|
||||
|
||||
## Checklist
|
||||
|
||||
- [ ] `bun run eval:query:validate eval/external-authors/your-handle/queries.json` passes
|
||||
- [ ] At least 20 queries
|
||||
- [ ] Each query has either `gold.relevant` (with real slugs) or `gold.expected_abstention: true`
|
||||
- [ ] Temporal queries have `as_of_date` set (`corpus-end` | `per-source` | ISO-8601)
|
||||
- [ ] Phrasing is varied (not all the same template)
|
||||
- [ ] `author` field matches my handle
|
||||
|
||||
## Phrasing variety (optional self-audit)
|
||||
|
||||
Tick the styles represented in your batch:
|
||||
|
||||
- [ ] Full sentence questions
|
||||
- [ ] Fragment-style ("crypto founder Goldman Sachs background")
|
||||
- [ ] Comparison ("X vs Y")
|
||||
- [ ] Follow-up ("And who else...")
|
||||
- [ ] Imperative ("Pull up Alice Davis")
|
||||
- [ ] Trait-based ("the demanding engineering leader")
|
||||
- [ ] Abstention bait (answer is "not in corpus")
|
||||
|
||||
## Notes to reviewer
|
||||
|
||||
Anything worth flagging — ambiguous cases, corpus gaps you found, specific
|
||||
phrasings you were uncertain about.
|
||||
@@ -1,95 +0,0 @@
|
||||
name: E2E Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
pull_request:
|
||||
branches: [master]
|
||||
schedule:
|
||||
- cron: '0 6 * * *' # Nightly at 6am UTC
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
tier1:
|
||||
name: Tier 1 (Mechanical)
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: latest
|
||||
- run: bun install
|
||||
- name: Run Tier 1 E2E tests
|
||||
run: bun test test/e2e/mechanical.test.ts test/e2e/mcp.test.ts
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gbrain_test
|
||||
|
||||
tier2:
|
||||
name: Tier 2 (LLM Skills)
|
||||
runs-on: ubuntu-latest
|
||||
# Runs on every push/PR now (promoted from schedule-only in v0.19.0).
|
||||
# Tier 1 must pass first; Tier 2 uses OPENAI_API_KEY + ANTHROPIC_API_KEY
|
||||
# from repo/org secrets. Nightly + manual triggers still supported via
|
||||
# the workflow-level `on:` list.
|
||||
needs: tier1
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: latest
|
||||
- run: bun install
|
||||
- name: Install OpenClaw
|
||||
run: npm install -g openclaw@2026.4.9
|
||||
- name: Configure OpenClaw MCP
|
||||
run: |
|
||||
mkdir -p ~/.openclaw
|
||||
cat > ~/.openclaw/config.json << 'EOF'
|
||||
{
|
||||
"mcpServers": {
|
||||
"gbrain": {
|
||||
"command": "bun",
|
||||
"args": ["run", "src/cli.ts", "serve"],
|
||||
"env": {
|
||||
"DATABASE_URL": "${{ env.DATABASE_URL }}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
- name: Run Tier 2 skill tests
|
||||
run: bun test test/e2e/skills.test.ts
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gbrain_test
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
@@ -1,48 +0,0 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ['v*']
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- os: macos-latest
|
||||
target: bun-darwin-arm64
|
||||
artifact: gbrain-darwin-arm64
|
||||
- os: ubuntu-latest
|
||||
target: bun-linux-x64
|
||||
artifact: gbrain-linux-x64
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: latest
|
||||
- run: bun install
|
||||
- run: bun test
|
||||
- run: bun build --compile --target=${{ matrix.target }} --outfile bin/${{ matrix.artifact }} src/cli.ts
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: ${{ matrix.artifact }}
|
||||
path: bin/${{ matrix.artifact }}
|
||||
|
||||
release:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
path: artifacts
|
||||
- name: Create release
|
||||
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
|
||||
with:
|
||||
files: |
|
||||
artifacts/gbrain-darwin-arm64/gbrain-darwin-arm64
|
||||
artifacts/gbrain-linux-x64/gbrain-linux-x64
|
||||
generate_release_notes: true
|
||||
@@ -1,42 +0,0 @@
|
||||
name: Test
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
pull_request:
|
||||
branches: [master]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
gitleaks:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: gitleaks/gitleaks-action@dcedce43c6f43de0b836d1fe38946645c9c638dc # v2
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test:
|
||||
# ubuntu-latest is free 2-core/7GB. Larger runners (16-cores, etc.) require
|
||||
# a provisioned runner pool in repo settings. Falling back to default keeps
|
||||
# the matrix shard speedup (~5-6x via parallelism) at zero cost.
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4]
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: latest
|
||||
- run: bun install
|
||||
- name: Pre-test gates (shard 1 only — they're not test files)
|
||||
if: matrix.shard == 1
|
||||
run: scripts/check-jsonb-pattern.sh && scripts/check-progress-to-stdout.sh && scripts/check-wasm-embedded.sh && bun run typecheck
|
||||
- name: Run test shard ${{ matrix.shard }}/4
|
||||
run: scripts/test-shard.sh ${{ matrix.shard }} 4
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
node_modules/
|
||||
bin/
|
||||
.DS_Store
|
||||
*.log
|
||||
.env
|
||||
.env.*
|
||||
!.env.*.example
|
||||
# Bun --compile temp artifacts. Each build emits a new hash-named .bun-build
|
||||
# file in cwd; glob catches all of them.
|
||||
*.bun-build
|
||||
.gstack/
|
||||
supabase/.temp/
|
||||
.claude/skills/
|
||||
.idea
|
||||
eval/reports/
|
||||
eval/data/world-v1/world.html
|
||||
|
||||
# BrainBench amara-life-v1 Opus cache (regenerate via eval:generate-amara-life)
|
||||
eval/data/amara-life-v1/_cache/
|
||||
|
||||
# claw-test E2E build cache (shim + scratch outputs)
|
||||
test/.cache/
|
||||
|
||||
.claude/
|
||||
export/
|
||||
|
||||
# Tier 3 PGLite snapshot fixture (built on demand by build:pglite-snapshot)
|
||||
test/fixtures/pglite-snapshot.tar
|
||||
test/fixtures/pglite-snapshot.version
|
||||
@@ -1,11 +0,0 @@
|
||||
title = "GBrain gitleaks config"
|
||||
|
||||
[allowlist]
|
||||
paths = [
|
||||
'''.env\.testing\.example''',
|
||||
'''.env\.example''',
|
||||
'''test/''',
|
||||
'''skills/''',
|
||||
'''.claude/skills/''',
|
||||
'''GBRAIN_SKILLPACK\.md''',
|
||||
]
|
||||
@@ -1,71 +0,0 @@
|
||||
# Agents working on GBrain
|
||||
|
||||
This is your install + operating protocol. Claude Code reads `./CLAUDE.md` automatically.
|
||||
Everyone else (Codex, Cursor, OpenClaw, Aider, Continue, or an LLM fetching via URL):
|
||||
start here.
|
||||
|
||||
## Install (5 min)
|
||||
|
||||
1. Clone: `git clone https://github.com/garrytan/gbrain ~/gbrain && cd ~/gbrain`
|
||||
2. Install: `bun install`
|
||||
3. Init the brain: `gbrain init` (defaults to PGLite, zero-config). For 1000+ files or
|
||||
multi-machine sync, init suggests Postgres + pgvector via Supabase.
|
||||
4. Read [`./INSTALL_FOR_AGENTS.md`](./INSTALL_FOR_AGENTS.md) for the full 9-step flow
|
||||
(API keys, identity, cron, verification).
|
||||
|
||||
## Read this order
|
||||
|
||||
1. `./AGENTS.md` (this file) — install + operating protocol.
|
||||
2. [`./CLAUDE.md`](./CLAUDE.md) — architecture reference, key files, trust boundaries,
|
||||
test layout.
|
||||
3. [`./skills/RESOLVER.md`](./skills/RESOLVER.md) — skill dispatcher. Read before any task.
|
||||
|
||||
## Trust boundary (critical)
|
||||
|
||||
GBrain distinguishes **trusted local CLI callers** (`OperationContext.remote = false`,
|
||||
set by `src/cli.ts`) from **untrusted agent-facing callers** (`remote = true`, set by
|
||||
`src/mcp/server.ts`). Security-sensitive operations like `file_upload` tighten filesystem
|
||||
confinement when `remote = true` and default to strict behavior when unset. If you are
|
||||
writing or reviewing an operation, consult `src/core/operations.ts` for the contract.
|
||||
|
||||
## Common tasks
|
||||
|
||||
- **Configure:** [`docs/ENGINES.md`](./docs/ENGINES.md),
|
||||
[`docs/guides/live-sync.md`](./docs/guides/live-sync.md),
|
||||
[`docs/mcp/DEPLOY.md`](./docs/mcp/DEPLOY.md).
|
||||
- **Debug:** [`docs/GBRAIN_VERIFY.md`](./docs/GBRAIN_VERIFY.md),
|
||||
[`docs/guides/minions-fix.md`](./docs/guides/minions-fix.md), `gbrain doctor --fix`.
|
||||
- **Migrate:** [`docs/UPGRADING_DOWNSTREAM_AGENTS.md`](./docs/UPGRADING_DOWNSTREAM_AGENTS.md),
|
||||
[`skills/migrations/`](./skills/migrations/), `gbrain apply-migrations`.
|
||||
- **Eval retrieval changes:** capture is off by default. To benchmark a
|
||||
retrieval change against real captured queries, set
|
||||
`GBRAIN_CONTRIBUTOR_MODE=1`, then `gbrain eval export --since 7d > base.ndjson`
|
||||
and `gbrain eval replay --against base.ndjson`. Full guide:
|
||||
[`docs/eval-bench.md`](./docs/eval-bench.md).
|
||||
- **Everything else:** [`./llms.txt`](./llms.txt) is the full documentation map.
|
||||
[`./llms-full.txt`](./llms-full.txt) is the same map with core docs inlined for
|
||||
single-fetch ingestion.
|
||||
|
||||
## Before shipping
|
||||
|
||||
Easiest path: `bun run ci:local` runs the full CI gate inside Docker (gitleaks,
|
||||
unit tests with `DATABASE_URL` unset, then all 29 E2E files sequentially against a
|
||||
fresh pgvector container) and tears down. Use `bun run ci:local:diff` for the
|
||||
diff-aware subset during fast iteration on a focused branch. Requires Docker
|
||||
(Docker Desktop / OrbStack / Colima) and `gitleaks` (`brew install gitleaks`).
|
||||
|
||||
Manual path: `bun test` plus the E2E lifecycle described in `./CLAUDE.md` (spin
|
||||
up the test Postgres container, run `bun run test:e2e`, tear it down).
|
||||
|
||||
Ship via the `/ship` skill, not by hand.
|
||||
|
||||
## Privacy
|
||||
|
||||
Never commit real names of people, companies, or funds into public artifacts. See the
|
||||
Privacy rule in `./CLAUDE.md`. GBrain pages reference real contacts; public docs must
|
||||
use generic placeholders (`alice-example`, `acme-example`, `fund-a`).
|
||||
|
||||
## Forks
|
||||
|
||||
If you are a fork, regenerate `llms.txt` + `llms-full.txt` with your own URL base before
|
||||
publishing: `LLMS_REPO_BASE=https://raw.githubusercontent.com/your-org/your-fork/main bun run build:llms`.
|
||||
-4584
File diff suppressed because it is too large
Load Diff
@@ -1,988 +0,0 @@
|
||||
# CLAUDE.md
|
||||
|
||||
GBrain is a personal knowledge brain and GStack mod for agent platforms. Pluggable
|
||||
engines: PGLite (embedded Postgres via WASM, zero-config default) or Postgres + pgvector
|
||||
+ hybrid search in a managed Supabase instance. `gbrain init` defaults to PGLite;
|
||||
suggests Supabase for 1000+ files. GStack teaches agents how to code. GBrain teaches
|
||||
agents everything else: brain ops, signal detection, content ingestion, enrichment,
|
||||
cron scheduling, reports, identity, and access control.
|
||||
|
||||
## Architecture
|
||||
|
||||
Contract-first: `src/core/operations.ts` defines ~41 shared operations (adds `find_orphans` in v0.12.3). CLI and MCP
|
||||
server are both generated from this single source. Engine factory (`src/core/engine-factory.ts`)
|
||||
dynamically imports the configured engine (`'pglite'` or `'postgres'`). Skills are fat
|
||||
markdown files (tool-agnostic, work with both CLI and plugin contexts).
|
||||
|
||||
**Trust boundary:** `OperationContext.remote` distinguishes trusted local CLI callers
|
||||
(`remote: false` set by `src/cli.ts`) from untrusted agent-facing callers
|
||||
(`remote: true` set by `src/mcp/server.ts`). Security-sensitive operations like
|
||||
`file_upload` tighten filesystem confinement when `remote=true` and default to
|
||||
strict behavior when unset.
|
||||
|
||||
## Key files
|
||||
|
||||
- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Also exports upload validators: `validateUploadPath`, `validatePageSlug`, `validateFilename`, plus `matchesSlugAllowList(slug, prefixes)` (v0.23 glob matcher: `<prefix>/*` matches recursive children; bare `<prefix>` matches exact only). `OperationContext.remote` flags untrusted callers; `OperationContext.allowedSlugPrefixes` (v0.23) is the trusted-workspace allow-list set by the dream cycle. `put_page` enforces: when `viaSubagent` and `allowedSlugPrefixes` is set, slug must match the allow-list; else the legacy `wiki/agents/<id>/...` namespace check applies. Auto-link enabled for trusted-workspace writes (skipped only when `remote=true && !trustedWorkspace`).
|
||||
- `src/core/engine.ts` — Pluggable engine interface (BrainEngine). `clampSearchLimit(limit, default, cap)` takes an explicit cap so per-operation caps can be tighter than `MAX_SEARCH_LIMIT`. Exports `LinkBatchInput` / `TimelineBatchInput` for the v0.12.1 bulk-insert API (`addLinksBatch` / `addTimelineEntriesBatch`). As of v0.13.1, `BrainEngine` has a `readonly kind: 'postgres' | 'pglite'` discriminator so migrations (`src/core/migrate.ts`) and other consumers can branch on engine without `instanceof` + dynamic imports.
|
||||
- `src/core/engine-factory.ts` — Engine factory with dynamic imports (`'pglite'` | `'postgres'`)
|
||||
- `src/core/pglite-engine.ts` — PGLite (embedded Postgres 17.5 via WASM) implementation, all 40 BrainEngine methods. `addLinksBatch` / `addTimelineEntriesBatch` use multi-row `unnest()` with manual `$N` placeholders. As of v0.13.1, `connect()` wraps `PGlite.create()` in a try/catch that emits an actionable error naming the macOS 26.3 WASM bug (#223) and pointing at `gbrain doctor`; the lock is released on failure so the next process can retry cleanly. v0.22.0: `searchKeyword` and `searchKeywordChunks` multiply `ts_rank` by the source-factor CASE expression at the chunk-grain level; `searchVector` becomes a two-stage CTE — inner CTE keeps `ORDER BY cc.embedding <=> vec` so HNSW stays usable, outer SELECT re-ranks by `raw_score * source_factor`. Inner LIMIT scales with offset to preserve pagination contract. As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL — probes for the specific forward-referenced state the embedded schema blob needs (`pages.source_id`, `links.link_source`, `links.origin_page_id`, `content_chunks.symbol_name`, `content_chunks.language`, `sources` FK target table) and adds only what's missing. Closes the upgrade-wedge bug class that bit users 10+ times across 6 schema versions over 2 years (#239/#243/#266/#357/#366/#374/#375/#378/#395/#396). No-op on fresh installs and modern brains.
|
||||
- `src/core/pglite-schema.ts` — PGLite-specific DDL (pgvector, pg_trgm, triggers)
|
||||
- `src/core/postgres-engine.ts` — Postgres + pgvector implementation (Supabase / self-hosted). `addLinksBatch` / `addTimelineEntriesBatch` use `INSERT ... SELECT FROM unnest($1::text[], ...) JOIN pages ON CONFLICT DO NOTHING RETURNING 1` — 4-5 array params regardless of batch size, sidesteps the 65535-parameter cap. As of v0.12.3, `searchKeyword` / `searchVector` scope `statement_timeout` via `sql.begin` + `SET LOCAL` so the GUC dies with the transaction instead of leaking across the pooled postgres.js connection (contributed by @garagon). `getEmbeddingsByChunkIds` uses `tryParseEmbedding` so one corrupt row skips+warns instead of killing the query. v0.22.0: `searchKeyword`, `searchKeywordChunks`, and `searchVector` apply source-aware ranking by inlining the source-factor CASE and `NOT (col LIKE …)` hard-exclude clause from `src/core/search/sql-ranking.ts`. `searchVector` switches to a two-stage CTE (HNSW-safe inner ORDER BY, source-boost re-rank in the outer SELECT) and carries `p.source_id` through inner→outer for v0.18 multi-source callers. v0.22.1 (#406): `_savedConfig` retains the connect config; `reconnect()` tears down + recreates the pool from saved config (called by supervisor watchdog after 3 consecutive health-check failures). `executeRaw` is a single-statement passthrough — no per-call retry (D3 dropped that as unsound for non-idempotent statements; recovery is supervisor-driven). v0.22.1 (#363, contributed by @orendi84): `connect()` applies `resolveSessionTimeouts()` from `db.ts` as connection-time startup parameters (`statement_timeout`, `idle_in_transaction_session_timeout`) so orphan pgbouncer backends can't hold locks for hours. v0.22.1 (#409, contributed by @atrevino47): `countStaleChunks()` + `listStaleChunks()` server-side-filter on `embedding IS NULL` for `embed --stale`, eliminating ~76 MB/call client-side pull on a fully-embedded brain; `upsertChunks()` resets both `embedding` AND `embedded_at` to NULL when chunk_text changes without a new embedding (consistency). As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL on the same forward-reference probe set as the PGLite engine, so old Postgres brains pinned at v0.13/v0.18/v0.19 walk forward cleanly instead of wedging on `column "..." does not exist`.
|
||||
- `src/core/utils.ts` — Shared SQL utilities extracted from postgres-engine.ts. Exports `parseEmbedding(value)` (throws on unknown input, used by migration + ingest paths where data integrity matters) and as of v0.12.3 `tryParseEmbedding(value)` (returns `null` + warns once per process, used by search/rescore paths where availability matters more than strictness).
|
||||
- `src/core/db.ts` — Connection management, schema initialization. v0.22.1 (#363, contributed by @orendi84): `resolveSessionTimeouts()` returns `statement_timeout` + `idle_in_transaction_session_timeout` (defaults: 5min each, env-overridable via `GBRAIN_STATEMENT_TIMEOUT` / `GBRAIN_IDLE_TX_TIMEOUT` / `GBRAIN_CLIENT_CHECK_INTERVAL`). Both `connect()` (module singleton) and `PostgresEngine.connect()` (worker pool) consume the result via postgres.js's `connection` option, sending GUCs as startup parameters that survive PgBouncer transaction mode (unlike the prior `setSessionDefaults` post-pool SET, kept as a back-compat no-op shim).
|
||||
- `src/commands/migrate-engine.ts` — Bidirectional engine migration (`gbrain migrate --to supabase/pglite`)
|
||||
- `src/core/import-file.ts` — importFromFile + importFromContent (chunk + embed + tags)
|
||||
- `src/core/sync.ts` — Pure sync functions (manifest parsing, filtering, slug conversion). v0.22.12 (#500, foundation by @wintermute via #501): `classifyErrorCode(errorMsg)` regex-based classifier with 12 codes (`SLUG_MISMATCH`, `YAML_PARSE`, `YAML_DUPLICATE_KEY`, `MISSING_OPEN`, `MISSING_CLOSE`, `NESTED_QUOTES`, `EMPTY_FRONTMATTER`, `NULL_BYTES`, `INVALID_UTF8`, `STATEMENT_TIMEOUT`, `FILE_TOO_LARGE`, `SYMLINK_NOT_ALLOWED`) plus `UNKNOWN` fallback. `summarizeFailuresByCode(failures)` returns sorted `[{code, count}]`. `code?` optional field on `SyncFailure`; backfilled at ack time on pre-v0.22.12 entries. `acknowledgeSyncFailures()` returns `AcknowledgeResult { count, summary }`. Three regexes (`MISSING_OPEN`, `MISSING_CLOSE`, `EMPTY_FRONTMATTER`) broadened to match actual `markdown.ts:159-244` validator message strings, not just the literal code-name prefix. `FILE_TOO_LARGE` covers all three production size sites in `import-file.ts:199, 352, 401`; `SYMLINK_NOT_ALLOWED` covers the rejection at `:347`. Closes the silent-skip pattern that motivated #500.
|
||||
- `src/core/storage.ts` — Pluggable storage interface (S3, Supabase Storage, local)
|
||||
- `src/core/storage-config.ts` (v0.22.11) — Storage tiering: `loadStorageConfig` reads `gbrain.yml`, normalizes deprecated keys (`git_tracked` / `supabase_only`) to canonical (`db_tracked` / `db_only`) with once-per-process deprecation warning, and runs `normalizeAndValidateStorageConfig` (auto-fixes missing trailing `/`, throws `StorageConfigError` on tier overlap). Path-segment matcher: `media/x/` does NOT match `media/xerox/foo`. Replaces gray-matter (broken on delimiter-less YAML) with a dedicated parser for the `gbrain.yml` shape.
|
||||
- `src/core/disk-walk.ts` (v0.22.11) — `walkBrainRepo(repoPath)` returns `Map<slug, {size, mtimeMs}>` from one recursive `readdirSync`. Skips dot-dirs, `node_modules`, non-`.md` files. Used by `gbrain storage status` to replace per-page `existsSync + statSync` (~400K syscalls on 200K-page brains → tens).
|
||||
- `src/commands/storage.ts` (v0.22.11) — `gbrain storage status [--repo P] [--json]`. Split into pure data (`getStorageStatus`) + JSON formatter + human formatter (ASCII-only per D10) matching the `orphans.ts` pattern. `PageCountsByTier` and `DiskUsageByTier` are distinct nominal types so swaps fail at compile time.
|
||||
- `gbrain.yml` (brain repo root, v0.22.11) — Optional storage tiering config. Top-level `storage:` section with `db_tracked:` and `db_only:` array-valued keys. `gbrain sync` auto-manages `.gitignore` for `db_only` paths on successful sync (skips on dry-run, blocked-by-failures, submodule context, or `GBRAIN_NO_GITIGNORE=1`). `gbrain export --restore-only [--repo P] [--type T] [--slug-prefix S]` repopulates missing `db_only` files from the database.
|
||||
- `src/core/supabase-admin.ts` — Supabase admin API (project discovery, pgvector check)
|
||||
- `src/core/file-resolver.ts` — File resolution with fallback chain (local -> .redirect.yaml -> .redirect -> .supabase)
|
||||
- `src/core/chunkers/` — 3-tier chunking (recursive, semantic, LLM-guided). v0.19.0 adds `code.ts` — tree-sitter-based semantic chunker for 29 languages with embedded-asset WASMs (`src/assets/wasm/`), `@dqbd/tiktoken` cl100k_base tokenizer, small-sibling merging. `CHUNKER_VERSION` constant folded into `importCodeFile`'s `content_hash` so chunker shape changes force clean re-chunks across releases.
|
||||
- `src/core/errors.ts` (v0.19.0) — `StructuredAgentError` + `buildError` + `serializeError`. Every new v0.19.0 agent-facing surface (code-def, code-refs, usage errors) uses this envelope; matches v0.17.0 `CycleReport.PhaseResult.error` shape.
|
||||
- `src/assets/wasm/` (v0.19.0) — 36 tree-sitter grammar WASMs + tree-sitter runtime. Committed to the repo so `bun --compile` embeds them deterministically via `import path from ... with { type: 'file' }`. The CI guard `scripts/check-wasm-embedded.sh` fails the build if the compiled binary ever silently falls through to recursive chunks.
|
||||
- `src/commands/code-def.ts` + `src/commands/code-refs.ts` (v0.19.0) — symbol definition + references lookup. Query `content_chunks.symbol_name` or chunk_text ILIKE with `page_kind='code'` filter. Auto-JSON when stdout is not a TTY (gh-CLI convention). Bypass the standard `searchKeyword` `DISTINCT ON (slug)` collapse so multiple call-sites from the same file surface.
|
||||
- `src/core/search/` — Hybrid search: vector + keyword + RRF + multi-query expansion + dedup. As of v0.22.0, `searchKeyword` / `searchKeywordChunks` / `searchVector` apply source-aware ranking at the SQL layer (curated content like `originals/`, `concepts/`, `writing/` outranks bulk content like `wintermute/chat/`, `daily/`, `media/x/`). `searchVector` uses a two-stage CTE so source-boost re-ranking doesn't kill the HNSW index. Hard-exclude prefixes (`test/`, `archive/`, `attachments/`, `.raw/` by default) filter at retrieval, not post-rank. Both gates honor `detail !== 'high'` so temporal queries surface chat pages normally.
|
||||
- `src/core/search/intent.ts` — Query intent classifier (entity/temporal/event/general → auto-selects detail level)
|
||||
- `src/core/search/eval.ts` — Retrieval eval harness: P@k, R@k, MRR, nDCG@k metrics + runEval() orchestrator
|
||||
- `src/core/search/source-boost.ts` (v0.22.0) — Source-type boost map keyed by slug prefix. `DEFAULT_SOURCE_BOOSTS` (originals/ 1.5, concepts/ 1.3, writing/ 1.4, people/companies/deals/ 1.2, daily/ 0.8, media/x/ 0.7, wintermute/chat/ 0.5) and `DEFAULT_HARD_EXCLUDES` (test/, archive/, attachments/, .raw/). `parseSourceBoostEnv` / `parseHardExcludesEnv` parse comma-separated `prefix:factor` pairs from `GBRAIN_SOURCE_BOOST` / `GBRAIN_SEARCH_EXCLUDE` env vars. `resolveBoostMap` and `resolveHardExcludes` merge defaults + env + caller `SearchOpts.exclude_slug_prefixes`/`include_slug_prefixes`.
|
||||
- `src/core/search/sql-ranking.ts` (v0.22.0) — Pure SQL string builders. `buildSourceFactorCase(slugColumn, boostMap, detail)` emits a CASE expression with longest-prefix-match wins (returns literal `'1.0'` when `detail === 'high'` for temporal-bypass parity with COMPILED_TRUTH_BOOST). `buildHardExcludeClause(slugColumn, prefixes)` emits `NOT (col LIKE 'p1%' OR col LIKE 'p2%')` — OR-chain wrapped in NOT, NOT `NOT LIKE ALL/ANY` (those quantifiers don't express set-exclusion). LIKE meta-character escape covers all three of `%`, `_`, AND `\` (backslash matters because it's Postgres LIKE's default escape char). Single-quote doubling on SQL string literals so injection-style inputs are inert text.
|
||||
- `src/commands/eval.ts` — `gbrain eval` command: single-run table + A/B config comparison. v0.25.0 adds sub-subcommand dispatch on `args[0]` so `gbrain eval export` + `gbrain eval prune` + `gbrain eval replay` route into session-capture handlers; bare `gbrain eval --qrels …` fall-through preserves the legacy IR-metrics flow.
|
||||
- `src/commands/eval-export.ts` (v0.25.0) — streams `eval_candidates` rows as NDJSON to stdout with `schema_version: 1` prefix on every line. EPIPE-safe, progress heartbeats on stderr, stable id-desc tiebreaker so `--since` windows never dupe/miss rows.
|
||||
- `src/commands/eval-prune.ts` (v0.25.0) — explicit retention cleanup. Requires `--older-than DUR`. `--dry-run` reports would-delete count.
|
||||
- `src/commands/eval-replay.ts` (v0.25.0) — contributor-facing replay tool. Reads NDJSON from `gbrain eval export`, re-runs each captured `query` / `search` op against the current brain, computes set-Jaccard@k between captured + current `retrieved_slugs`, top-1 stability rate, and latency Δ. Stable JSON shape (`schema_version: 1`) for CI gating; human mode prints a regression table. Pure Bun, zero new deps. The dev-loop half of BrainBench-Real that closes the gap between "data captured" and "data used to gate a PR." See `docs/eval-bench.md` for the workflow.
|
||||
- `docs/eval-bench.md` (v0.25.0) — contributor guide for using captured data to benchmark retrieval changes before merging. Linked from CONTRIBUTING.md under "Running real-world eval benchmarks (touching retrieval code)".
|
||||
- `src/core/eval-capture.ts` (v0.25.0) — op-layer capture wrapper called from `src/core/operations.ts` `query` + `search` handlers. Catches MCP + CLI + subagent tool-bridge from one site. Fire-and-forget; failures route to `engine.logEvalCaptureFailure` so `gbrain doctor` sees drops cross-process. **Capture is off by default** — `isEvalCaptureEnabled` resolution: explicit `config.eval.capture` (true/false) wins, else `process.env.GBRAIN_CONTRIBUTOR_MODE === '1'`, else off. Production users get a quiet brain; contributors set `export GBRAIN_CONTRIBUTOR_MODE=1` in `.zshrc` to enable the dev loop. PII scrubber gate is independent and defaults to true regardless of CONTRIBUTOR_MODE.
|
||||
- `src/core/eval-capture-scrub.ts` (v0.25.0) — zero-deps PII scrubber: emails, phones, SSN, Luhn-verified credit cards, JWT-shaped tokens, bearer tokens.
|
||||
- `src/core/search/hybrid.ts` — Cathedral II `Promise<SearchResult[]>` return shape unchanged in v0.25.0. Adds `onMeta?: (m: HybridSearchMeta) => void` callback so op-layer capture can record what hybridSearch actually did. Existing callers leave it undefined.
|
||||
- `docs/eval-capture.md` (v0.25.0) — stable NDJSON schema reference for gbrain-evals consumers.
|
||||
- `test/public-exports.test.ts` (v0.25.0 / R2) — runtime contract test. Imports each of the 17 public subpaths via package name and pins a canary symbol per module. Paired with `scripts/check-exports-count.sh`.
|
||||
- `src/core/embedding.ts` — OpenAI text-embedding-3-large, batch, retry, backoff
|
||||
- `src/core/check-resolvable.ts` — Resolver validation: reachability, MECE overlap, DRY checks, structured fix objects. v0.14.1: `CROSS_CUTTING_PATTERNS.conventions` is an array (notability gate accepts both `conventions/quality.md` and `_brain-filing-rules.md`). New `extractDelegationTargets()` parses `> **Convention:**`, `> **Filing rule:**`, and inline backtick references. DRY suppression is proximity-based via `DRY_PROXIMITY_LINES = 40`.
|
||||
- `src/core/repo-root.ts` — Shared `findRepoRoot(startDir?)` (v0.16.4): walks up from `startDir` (default `process.cwd()`) looking for `skills/RESOLVER.md`. Zero-dependency module imported by both `doctor.ts` and `check-resolvable.ts`. Parameterized `startDir` makes tests hermetic.
|
||||
- `src/commands/check-resolvable.ts` — Standalone CLI wrapper (v0.16.4) over `checkResolvable()`. Exports `parseFlags`, `resolveSkillsDir`, `DEFERRED`, `runCheckResolvable`. Exit rule: **1 on any issue (warnings OR errors)**, stricter than doctor's `ok` flag — honors README:259. Stable JSON envelope `{ok, skillsDir, report, autoFix, deferred, error, message}` — same shape on success and error paths. `--fix` path runs `autoFixDryViolations` BEFORE `checkResolvable` (same ordering as doctor). `scripts/skillify-check.ts` subprocess-calls `gbrain check-resolvable --json` (cached per process) and fails loud on binary-missing — no silent false-pass. **v0.19:** AGENTS.md workspaces now resolve natively (see `src/core/resolver-filenames.ts`) — gbrain inspects the 107-skill OpenClaw deployment whether the routing file is `RESOLVER.md` or `AGENTS.md`. `DEFERRED[]` is empty — Checks 5 + 6 shipped as real code, not issue URLs.
|
||||
- `src/core/resolver-filenames.ts` (v0.19) — central list of accepted routing filenames (`RESOLVER.md`, `AGENTS.md`). Shared by `findRepoRoot`, `check-resolvable`, and skillpack install so every code path walks the same fallback chain.
|
||||
- `src/commands/skillify.ts` + `src/core/skillify/{generator,templates}.ts` (v0.19) — `gbrain skillify scaffold <name>` creates all stubs for a new skill in one command: SKILL.md, script, tests, routing-eval.jsonl, resolver entry, filing-rules pointer. `gbrain skillify check <script>` runs the 10-step checklist (LLM evals, routing evals, check-resolvable gate, filing audit) against a candidate skill before it lands.
|
||||
- `src/commands/skillify-check.ts` (v0.19) — `gbrain skillpack-check` agent-readable health report. Exit 0/1/2 for CI pipeline gating; JSON for debugging. Wraps `check-resolvable --json`, `doctor --json`, and migration ledger into one payload so agents can decide whether a human action is required.
|
||||
- `src/commands/skillpack.ts` + `src/core/skillpack/{bundle,installer}.ts` (v0.19) — `gbrain skillpack install` drops gbrain's curated 25-skill bundle into a host workspace, managed-block style. Never clobbers local edits; tracks a skill manifest so subsequent `install --update` diffs cleanly. Bundle builder (`skillpack/bundle.ts`) packages the set from `skills/` into a versioned payload. **v0.24.0:** managed block embeds a `<!-- gbrain:skillpack:manifest cumulative-slugs="..." version="..." -->` receipt inside the fence. Per-skill installs accumulate via `union(prior_receipt, this_call)`; `install --all` is the only path that prunes (drops slugs no longer in the bundle). Rows inside the fence whose slug is in neither the new cumulative set nor the bundle survive as user-added with a stderr `[skillpack] unknown row in managed block: "<slug>" — Investigate: ...` warning. Pre-v0.24 fences upgrade silently on first install (extracted slugs become the prior cumulative set).
|
||||
- `src/core/skill-manifest.ts` (v0.19) — parser for `skill-manifest.json` records. Used by skillpack installer to detect drift between the shipped bundle and the user's local edits, so updates merge instead of overwriting.
|
||||
- `src/commands/routing-eval.ts` + `src/core/routing-eval.ts` (v0.19) — `gbrain routing-eval` catches user phrasings that route to the wrong skill. Reads `skills/<name>/routing-eval.jsonl` fixtures (`{intent, expected_skill, ambiguous_with?}`). Structural layer runs in `check-resolvable` by default (zero API cost). The `--llm` flag is accepted as a placeholder for a future LLM tie-break layer; in v0.24.0 it emits a stderr notice and runs structural only. False positives surface before users hit them.
|
||||
- `src/core/filing-audit.ts` + `skills/_brain-filing-rules.json` (v0.19) — Check 6 of `check-resolvable`. Parses new `writes_pages:` / `writes_to:` frontmatter on skills and audits their filing claims against the filing-rules JSON. Warning-only in v0.19, upgrades to error in v0.20.
|
||||
- `src/core/dry-fix.ts` — `gbrain doctor --fix` engine. `autoFixDryViolations(fixes, {dryRun})` rewrites inlined rules to `> **Convention:** see [path](path).` callouts via three shape-aware expanders (bullet / blockquote / paragraph). Five guards: working-tree-dirty (`getWorkingTreeStatus()` returns 3-state `'clean' | 'dirty' | 'not_a_repo'`), no-git-backup, inside-code-fence, already-delegated (40-line proximity, consistent with detector), ambiguous-multi-match, block-is-callout. `execFileSync` array args (no shell — no injection surface). EOF newline preserved.
|
||||
- `src/core/backoff.ts` — Adaptive load-aware throttling: CPU/memory checks, exponential backoff, active hours multiplier
|
||||
- `src/core/fail-improve.ts` — Deterministic-first, LLM-fallback loop with JSONL failure logging and auto-test generation
|
||||
- `src/core/transcription.ts` — Audio transcription: Groq Whisper (default), OpenAI fallback, ffmpeg segmentation for >25MB
|
||||
- `src/core/enrichment-service.ts` — Global enrichment service: entity slug generation, tier auto-escalation, batch throttling
|
||||
- `src/core/data-research.ts` — Recipe validation, field extraction (MRR/ARR regex), dedup, tracker parsing, HTML stripping
|
||||
- `src/commands/embed.ts` — `gbrain embed [--stale|--all] [--slugs ...]`. v0.22.1 (#409, contributed by @atrevino47): `--stale` path now starts with `engine.countStaleChunks()` (single SELECT count(*) WHERE embedding IS NULL, ~50 bytes wire). On a fully-embedded brain that's a 1-line short-circuit — no further reads. When stale chunks exist, `engine.listStaleChunks()` returns just the chunks needing embeddings (slug + chunk_index + chunk_text + metadata, no `vector(1536)` payload). Caller groups by slug, embeds via OpenAI, re-upserts via `upsertChunks`. Replaces the prior page-walk that pulled every chunk's embedding column over the wire and discarded most.
|
||||
- `src/commands/extract.ts` — `gbrain extract links|timeline|all [--source fs|db]`: batch link/timeline extraction. fs walks markdown files, db walks pages from the engine (mutation-immune snapshot iteration; use this for live brains with no local checkout). As of v0.12.1 there is no in-memory dedup pre-load — candidates are buffered 100 at a time and flushed via `addLinksBatch` / `addTimelineEntriesBatch`; `ON CONFLICT DO NOTHING` enforces uniqueness at the DB layer, and the `created` counter returns real rows inserted (truthful on re-runs). v0.22.1 (#417): `ExtractOpts.slugs?: string[]` enables incremental extract — when set, `extractForSlugs()` reads ONLY those slugs' files (single combined links+timeline pass) instead of the full directory walk. CLI `gbrain extract` keeps full-walk behavior; the cycle path threads sync's `pagesAffected` through. `walkMarkdownFiles(brainDir)` still runs at line 455 to build `allSlugs` for link resolution — see `TODOS.md` for replacing it with `engine.getAllSlugs()`.
|
||||
- `src/commands/graph-query.ts` — `gbrain graph-query <slug> [--type T] [--depth N] [--direction in|out|both]`: typed-edge relationship traversal (renders indented tree)
|
||||
- `src/core/link-extraction.ts` — shared library for the v0.12.0 graph layer. extractEntityRefs (canonical, replaces backlinks.ts duplicate) matches both `[Name](people/slug)` markdown links and Obsidian `[[people/slug|Name]]` wikilinks as of v0.12.3. extractPageLinks, inferLinkType heuristics (attended/works_at/invested_in/founded/advises/source/mentions), parseTimelineEntries, isAutoLinkEnabled config helper. `DIR_PATTERN` covers `people`, `companies`, `deals`, `topics`, `concepts`, `projects`, `entities`, `tech`, `finance`, `personal`, `openclaw`. Used by extract.ts, operations.ts auto-link post-hook, and backlinks.ts.
|
||||
- `src/core/minions/` — Minions job queue: BullMQ-inspired, Postgres-native (queue, worker, backoff, types, protected-names, quiet-hours, stagger, handlers/shell).
|
||||
- `src/core/minions/queue.ts` — MinionQueue class (submit, claim, complete, fail, stall detection, parent-child, depth/child-cap, per-job timeouts, cascade-kill, attachments, idempotency keys, child_done inbox, removeOnComplete/Fail). `add()` takes a 4th `trusted` arg (separate from `opts` to prevent spread leakage); protected names in `PROTECTED_JOB_NAMES` require `{allowProtectedSubmit: true}` and the check runs trim-normalized (whitespace-bypass safe). v0.14.1 #219: `add()` plumbs `max_stalled` through with a `[1, 100]` clamp; omitted values let the schema DEFAULT (5) kick in. v0.19.0: `handleWallClockTimeouts(lockDurationMs)` is Layer 3 kill shot for jobs where `FOR UPDATE SKIP LOCKED` stall detection and the timeout sweep both fail to evict (wedged worker holding a row lock via a pending transaction). v0.19.1: `maxWaiting` coalesce path now uses `pg_advisory_xact_lock` keyed on `(name, queue)` to serialize concurrent submits for the same key, and filters on `queue` in addition to `name` so cross-queue same-name jobs don't suppress each other.
|
||||
- `src/core/minions/worker.ts` — MinionWorker class (handler registry, lock renewal, graceful shutdown, timeout safety net). v0.14.0 abort-path fix: aborted jobs now call `failJob` with reason (`timeout`/`cancel`/`lock-lost`/`shutdown`) instead of returning silently. `shutdownAbort` (instance field) fires on process SIGTERM/SIGINT and propagates to `ctx.shutdownSignal` — shell handler listens to it; non-shell handlers don't. v0.22.1 (#403): per-job timeout fires `abort.abort(new Error('timeout'))` then a 30-second grace-then-evict safety net force-evicts the job from `inFlight` and marks it dead in DB if the handler ignores the abort signal — frees the slot even when a handler wedges (the 98-waiting-0-active prod incident driver).
|
||||
- `src/core/minions/supervisor.ts` — MinionSupervisor process manager. Spawns `gbrain jobs work` as a child, restarts on crash with exponential backoff, periodic health check. v0.22.1 (#406): `consecutiveHealthFailures` counter; on 3 consecutive failures emits `health_warn` with `reason: 'db_connection_degraded'` and calls `engine.reconnect()` to swap in a fresh pool, then resets the counter. Worker exit classifier emits `likely_cause` field on `worker_exited` events: `oom_or_external_kill` (SIGKILL), `graceful_shutdown` (SIGTERM), `runtime_error` (code 1), `clean_exit` (code 0), `unknown`.
|
||||
- `src/core/minions/types.ts` — `MinionJobInput` + `MinionJobStatus` + handler context types. `MinionJobInput.max_stalled` (new in v0.14.1) is optional; omitted values let the schema DEFAULT (5) kick in, provided values are clamped to `[1, 100]`.
|
||||
- `src/core/minions/protected-names.ts` — side-effect-free constant module exporting `PROTECTED_JOB_NAMES` + `isProtectedJobName()`. Kept pure so queue core can import without loading handler modules.
|
||||
- `src/core/minions/handlers/shell.ts` — `shell` job handler. Spawns `/bin/sh -c cmd` (absolute path, PATH-override-safe) or `argv[0] argv[1..]` (no shell). Env allowlist: `PATH, HOME, USER, LANG, TZ, NODE_ENV` + caller `env:` overrides. UTF-8-safe stdout/stderr tail via `string_decoder.StringDecoder`. Abort (either `ctx.signal` or `ctx.shutdownSignal`) fires SIGTERM → 5s grace → SIGKILL on child. Requires `GBRAIN_ALLOW_SHELL_JOBS=1` on worker (gated by `registerBuiltinHandlers`).
|
||||
- `src/core/minions/handlers/shell-audit.ts` — per-submission JSONL audit trail at `~/.gbrain/audit/shell-jobs-YYYY-Www.jsonl` (ISO-week rotation; override via `GBRAIN_AUDIT_DIR`). Best-effort: `mkdirSync(recursive)` + `appendFileSync`; failures logged to stderr, submission not blocked. Logs cmd (first 80 chars) or argv (JSON array). Never logs env values.
|
||||
- `src/core/minions/backpressure-audit.ts` (v0.19.1) — sibling of shell-audit.ts for `maxWaiting` coalesce events. JSONL at `~/.gbrain/audit/backpressure-YYYY-Www.jsonl`. Fires one line per coalesce with `(queue, name, waiting_count, max_waiting, returned_job_id, ts)`. Closes the silent-drop vector the v0.19.0 maxWaiting guard introduced.
|
||||
- `src/core/minions/handlers/subagent.ts` (v0.15) — LLM-loop handler. Two-phase tool persistence (pending → complete/failed), replay reconciliation for mid-dispatch crashes, dual-signal abort (`ctx.signal` + `ctx.shutdownSignal`), Anthropic prompt caching on system + tool defs. `makeSubagentHandler({engine, client?, ...})` factory; `MessagesClient` is an injectable interface the real SDK implements structurally. Throws `RateLeaseUnavailableError` (renewable) when rate-lease capacity is full.
|
||||
- `src/core/minions/handlers/subagent-aggregator.ts` (v0.15) — `subagent_aggregator` handler. Claims AFTER all children resolve (queue changes guarantee every terminal child posts a `child_done` inbox message with outcome). Reads inbox via `ctx.readInbox()`, builds deterministic mixed-outcome markdown summary. No LLM call in v0.15.
|
||||
- `src/core/minions/handlers/subagent-audit.ts` (v0.15) — JSONL audit + heartbeat writer at `~/.gbrain/audit/subagent-jobs-YYYY-Www.jsonl`. Events: `submission` (one line per submit) + `heartbeat` (per turn boundary: `llm_call_started | llm_call_completed | tool_called | tool_result | tool_failed`). Never logs prompts or tool inputs. `readSubagentAuditForJob(jobId, {sinceIso})` is the readback path for `gbrain agent logs`.
|
||||
- `src/core/minions/rate-leases.ts` (v0.15) — lease-based concurrency cap for outbound providers (default key `anthropic:messages`, max via `GBRAIN_ANTHROPIC_MAX_INFLIGHT`). Owner-tagged rows with `expires_at` auto-prune on acquire; `pg_advisory_xact_lock` guards check-then-insert; CASCADE on owning job deletion. `renewLeaseWithBackoff` retries 3x (250/500/1000ms).
|
||||
- `src/core/minions/wait-for-completion.ts` (v0.15) — poll-until-terminal helper for CLI callers. `TimeoutError` does NOT cancel the job; `AbortSignal` exits without throwing. Default `pollMs`: 1000 on Postgres, 250 on PGLite inline.
|
||||
- `src/core/minions/transcript.ts` (v0.15) — renders `subagent_messages` + `subagent_tool_executions` to markdown. Tool rows splice under their owning assistant `tool_use` by `tool_use_id`. UTF-8-safe truncation; unknown block types fall through to fenced JSON.
|
||||
- `src/core/minions/plugin-loader.ts` (v0.15) — `GBRAIN_PLUGIN_PATH` discovery. Absolute paths only, left-wins collision, `gbrain.plugin.json` with `plugin_version: "gbrain-plugin-v1"`, plugins ship DEFS only (no new tools), `allowed_tools:` validated at load time against the derived registry.
|
||||
- `src/core/minions/tools/brain-allowlist.ts` (v0.15, extended v0.23) — derives subagent tool registry from `src/core/operations.ts`. 11-name allow-list. By default `put_page` schema is namespace-wrapped per subagent (`^wiki/agents/<subagentId>/.+`). **v0.23 trusted-workspace path:** when `BuildBrainToolsOpts.allowedSlugPrefixes` is set, the put_page schema instead describes the prefix list to the model and the OperationContext is threaded with `allowedSlugPrefixes`. Trust comes from `PROTECTED_JOB_NAMES` gating subagent submission — MCP cannot reach this field. Only cycle.ts (synthesize/patterns) and direct CLI submitters set it.
|
||||
- `src/mcp/tool-defs.ts` (v0.15) — extracted `buildToolDefs(ops)` helper. MCP server + subagent tool registry both call it; byte-for-byte equivalence pinned by `test/mcp-tool-defs.test.ts`.
|
||||
- `src/core/minions/attachments.ts` — Attachment validation (path traversal, null byte, oversize, base64, duplicate detection)
|
||||
- `src/commands/agent.ts` (v0.16) — `gbrain agent run <prompt> [flags]` CLI. Submits `subagent` (or N children + 1 aggregator) under `{allowProtectedSubmit: true}`. Single-entry `--fanout-manifest` short-circuits. Children get `on_child_fail: 'continue'` + `max_stalled: 3`. `--follow` is the default on TTY; streams logs + polls `waitForCompletion` in parallel. Ctrl-C detaches, does not cancel.
|
||||
- `src/commands/agent-logs.ts` (v0.16) — `gbrain agent logs <job> [--follow] [--since]`. Merges JSONL heartbeat audit + `subagent_messages` into a chronological timeline. `parseSince` accepts ISO-8601 or relative (`5m`, `1h`, `2d`). Transcript tail renders only for terminal jobs.
|
||||
- `src/commands/jobs.ts` — `gbrain jobs` CLI subcommands + `gbrain jobs work` daemon. v0.13.1 surfaces the full `MinionJobInput` retry/backoff/timeout/idempotency surface as first-class CLI flags on `jobs submit`: `--max-stalled`, `--backoff-type fixed|exponential`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key`. `jobs smoke --sigkill-rescue` is the opt-in regression guard for #219. v0.16 wires `registerBuiltinHandlers` to always register `subagent` + `subagent_aggregator` (no env flag — `ANTHROPIC_API_KEY` is the natural cost gate, trust is via `PROTECTED_JOB_NAMES`) and loads `GBRAIN_PLUGIN_PATH` plugins at worker startup with a loud startup-line per plugin. `shell` handler still gated by `GBRAIN_ALLOW_SHELL_JOBS=1` (RCE surface, separate concern). v0.22.10 (#521): the `autopilot-cycle` handler now forwards `job.data.phases` to `runCycle` (was previously discarded — caller-supplied phase selection silently became a full cycle). Phases are validated against `ALL_PHASES` from `src/core/cycle.ts`; invalid names are filtered out and an empty/missing array falls back to the default 6-phase cycle. v0.22.13 (PR #490 CODEX-1+CODEX-4): `sync` handler now resolves `sourceId` at entry by looking up `sources.local_path` (mirrors `cycle.ts:480`'s autopilot fix from PR #475) so multi-source brains read the per-source `last_commit` anchor instead of the global config key. Concurrency routed through the shared `autoConcurrency()` policy in `src/core/sync-concurrency.ts` instead of the prior hardcoded `4`; PGLite stays serial. `noEmbed` default is `true` (embed is a separate job — submit `gbrain embed --stale` after sync, or rely on the autopilot cycle's embed phase).
|
||||
- `src/commands/features.ts` — `gbrain features --json --auto-fix`: usage scan + feature adoption salesman
|
||||
- `src/commands/autopilot.ts` — `gbrain autopilot --install`: self-maintaining brain daemon (sync+extract+embed)
|
||||
- `src/mcp/server.ts` — MCP stdio server (generated from operations). v0.22.7: tool-call handler delegates to `dispatchToolCall` from `src/mcp/dispatch.ts` so stdio + HTTP transports share one validation, context-build, and error-format path.
|
||||
- `src/mcp/dispatch.ts` (v0.22.7) — Shared tool-call dispatch consumed by both stdio (`server.ts`) and HTTP (`http-transport.ts`). Exports `dispatchToolCall(engine, name, params, opts)`, `buildOperationContext(engine, params, opts)`, and `validateParams(op, params)`. Single source of truth for `(ctx, params)` handler arg order and the 5-field `OperationContext` shape (engine + config + logger + dryRun + remote). Defaults to `remote: true` (untrusted); local CLI callers pass `remote: false`. Closed F1 (reversed handler args) + F2 (incomplete OperationContext) + F3 (no param validation) drift bugs in the original v0.22.5 HTTP transport.
|
||||
- `src/mcp/rate-limit.ts` (v0.22.7) — Bounded-LRU token-bucket limiter for `gbrain serve --http`. `buildDefaultLimiters()` returns the two-bucket pipeline used by http-transport: pre-auth IP (default 30/60s, fires BEFORE the DB lookup so brute-force load against `access_tokens` is actually capped) + post-auth token-id (default 60/60s). Tracks `lastTouchedMs` separately from `lastRefillMs` so an exhausted key can't be reset by hammering past the TTL. LRU cap (default 10K keys) bounds memory under attacker-controlled key growth; TTL prune at 2× window evicts abandoned buckets.
|
||||
- `src/mcp/http-transport.ts` (v0.22.7, rewrite) — `gbrain serve --http` HTTP transport. Postgres-only — fails fast at startup on PGLite (the `access_tokens` table only exists on Postgres). Bearer auth against SHA-256 hashes in `access_tokens`. CORS default-deny via `GBRAIN_HTTP_CORS_ORIGIN` allowlist. Body cap stream-counted (1 MiB default via `GBRAIN_HTTP_MAX_BODY_BYTES`) so chunked transfers without Content-Length still hit the cap. `last_used_at` SQL-level debounce (one UPDATE per token per 60s). Per-request audit row in `mcp_request_log` with token_name + operation + status + latency. Optional `GBRAIN_HTTP_TRUST_PROXY=1` honors `X-Forwarded-For` — only safe when bound to a private interface AND the proxy strips client-supplied XFF (otherwise enables IP spoofing past the pre-auth rate limit). `/health` does `SELECT 1` against Postgres and returns 503 + `status:unhealthy` when the DB is unreachable so orchestration doesn't see green pods while clients get misleading 401s. Replaces the standalone OAuth wrapper that was vulnerable to unauthenticated client registration.
|
||||
- `src/commands/auth.ts` — Token management for the HTTP transport. `gbrain auth create/list/revoke/test`. As of v0.22.7 wired into the main CLI (`src/cli.ts`); also runs standalone via `bun run src/commands/auth.ts ...` for environments without a compiled binary. Tokens stored as SHA-256 hashes in `access_tokens` (Postgres-only).
|
||||
- `src/commands/upgrade.ts` — Self-update CLI. `runPostUpgrade()` enumerates migrations from the TS registry (src/commands/migrations/index.ts) and tail-calls `runApplyMigrations(['--yes', '--non-interactive'])` so the mechanical side of every outstanding migration runs unconditionally.
|
||||
- `src/commands/migrations/` — TS migration registry (compiled into the binary; no filesystem walk of `skills/migrations/*.md` needed at runtime). `index.ts` lists migrations in semver order. `v0_11_0.ts` = Minions adoption orchestrator (8 phases). `v0_12_0.ts` = Knowledge Graph auto-wire orchestrator (5 phases: schema → config check → backfill links → backfill timeline → verify). `phaseASchema` has a 600s timeout (bumped from 60s in v0.12.1 for duplicate-heavy brains). `v0_12_2.ts` = JSONB double-encode repair orchestrator (4 phases: schema → repair-jsonb → verify → record). `v0_14_0.ts` = shell-jobs + autopilot cooperative (2 phases: schema ALTER minion_jobs.max_stalled SET DEFAULT 3 — superseded by v0.14.3's schema-level DEFAULT 5 + UPDATE backfill; pending-host-work ping for skills/migrations/v0.14.0.md). All orchestrators are idempotent and resumable from `partial` status. As of v0.14.2 (Bug 3), the RUNNER owns all ledger writes — orchestrators return `OrchestratorResult` and `apply-migrations.ts` persists a canonical `{version, status, phases}` shape after return. Orchestrators no longer call `appendCompletedMigration` directly. `statusForVersion` prefers `complete` over `partial` (never regresses). 3 consecutive partials → wedged → `--force-retry <version>` writes a `'retry'` reset marker. v0.14.3 (fix wave) ships schema-only migrations v14 (`pages_updated_at_index`) + v15 (`minion_jobs_max_stalled_default_5` with UPDATE backfill) via the `MIGRATIONS` array in `src/core/migrate.ts` — no orchestrator phases needed.
|
||||
- `src/commands/repair-jsonb.ts` — `gbrain repair-jsonb [--dry-run] [--json]`: rewrites `jsonb_typeof='string'` rows in place across 5 affected columns (pages.frontmatter, raw_data.data, ingest_log.pages_updated, files.metadata, page_versions.frontmatter). Fixes v0.12.0 double-encode bug on Postgres; PGLite no-ops. Idempotent.
|
||||
- `src/commands/orphans.ts` — `gbrain orphans [--json] [--count] [--include-pseudo]`: surfaces pages with zero inbound wikilinks, grouped by domain. Auto-generated/raw/pseudo pages filtered by default. Also exposed as `find_orphans` MCP operation. Shipped in v0.12.3 (contributed by @knee5).
|
||||
- `src/commands/integrity.ts` — `gbrain integrity check|auto|review|extract`: bare-tweet detection, dead-link detection, three-bucket repair (auto-repair / review-queue / skip). `scanIntegrity()` is the shared library function called from `gbrain doctor` (sampled at limit=500) and `cmdCheck` (full scan). v0.22.8: batch-load fast path on Postgres uses `SELECT DISTINCT ON (slug)` in a single SQL query to fix the PgBouncer round-trip timeout (60s → ~6s) while preserving `engine.getAllSlugs()`'s `Set<string>` semantics on multi-source brains. Gated by `engine.kind === 'postgres'` at the call site so PGLite never enters batch; fallback `catch` logs at `GBRAIN_DEBUG=1` so real Postgres errors are diagnosable.
|
||||
- `src/commands/doctor.ts` — `gbrain doctor [--json] [--fast] [--fix] [--dry-run] [--index-audit]`: health checks. v0.12.3 added `jsonb_integrity` + `markdown_body_completeness` reliability checks. v0.14.1: `--fix` delegates inlined cross-cutting rules to `> **Convention:** see [path](path).` callouts (pipes DRY violations into `src/core/dry-fix.ts`); `--fix --dry-run` previews without writing. v0.14.2: `schema_version` check fails loudly when `version=0` (migrations never ran — the #218 `bun install -g` signature) and routes users to `gbrain apply-migrations --yes`; new opt-in `--index-audit` flag (Postgres-only) reports zero-scan indexes from `pg_stat_user_indexes` (informational only, no auto-drop). v0.15.2: every DB check is wrapped in a progress phase; `markdown_body_completeness` runs under a 1s heartbeat timer so 10+ min scans are observable on 50K-page brains. v0.19.1 added `queue_health` (Postgres-only) with two subchecks: stalled-forever active jobs (started_at > 1h) and waiting-depth-per-name > threshold (default 10, override via `GBRAIN_QUEUE_WAITING_THRESHOLD`). Worker-heartbeat subcheck intentionally deferred to follow-up B7 because it needs a `minion_workers` table to produce ground-truth signal. Fix hints point at `gbrain repair-jsonb`, `gbrain sync --force`, `gbrain apply-migrations`, and `gbrain jobs get/cancel <id>`. v0.22.12 (#500): `sync_failures` check shows `[CODE=N, ...]` breakdown for both unacked entries (warn) and acked-historical entries (ok), surfacing systemic failure modes (`SLUG_MISMATCH=2685`) instead of a bare count.
|
||||
- `src/core/migrate.ts` — schema-migration runner. Owns the `MIGRATIONS` array (source of truth for schema DDL). v0.14.2 extended the `Migration` interface with `sqlFor?: { postgres?, pglite? }` (engine-specific SQL overrides `sql`) and `transaction?: boolean` (set to false for `CREATE INDEX CONCURRENTLY`, which Postgres refuses inside a transaction; ignored on PGLite since it has no concurrent writers). Migration v14 (fix wave) uses a handler branching on `engine.kind` to run CONCURRENTLY on Postgres (with a pre-drop of any invalid remnant via `pg_index.indisvalid`) and plain `CREATE INDEX` on PGLite. v15 bumps `minion_jobs.max_stalled` default 1→5 and backfills existing non-terminal rows. v0.22.6.1: migration v24 (`rls_backfill_missing_tables`) uses `sqlFor: { pglite: '' }` to no-op on PGLite — PGLite has no RLS engine and is single-tenant by definition, and the v24 ALTERs target subagent tables that don't exist in pglite-schema.ts. Closes #395 (contributed by @jdcastro2). **v30 (v0.23):** creates `dream_verdicts (file_path TEXT, content_hash TEXT, worth_processing BOOL, reasons JSONB, judged_at TIMESTAMPTZ, PK(file_path, content_hash))`. RLS-enabled when running as a BYPASSRLS role. The synthesize phase reads/writes this table to avoid re-judging on backfill re-runs.
|
||||
- `src/core/progress.ts` — Shared bulk-action progress reporter. Writes to stderr. Modes: `auto` (TTY: `\r`-rewriting; non-TTY: plain lines), `human`, `json` (JSONL), `quiet`. Rate-gated by `minIntervalMs` and `minItems`. `startHeartbeat(reporter, note)` helper for single long queries. `child()` composes phase paths. Singleton SIGINT/SIGTERM coordinator emits `abort` events for every live phase. EPIPE defense on both sync throws and stream `'error'` events. Zero dependencies. Introduced in v0.15.2.
|
||||
- `src/core/cli-options.ts` — Global CLI flag parser. `parseGlobalFlags(argv)` returns `{cliOpts, rest}` with `--quiet` / `--progress-json` / `--progress-interval=<ms>` stripped. `getCliOptions()` / `setCliOptions()` expose a module-level singleton so commands reach the resolved flags without parameter threading. `cliOptsToProgressOptions()` maps to reporter options. `childGlobalFlags()` returns the flag suffix to append to `execSync('gbrain ...')` calls in migration orchestrators. `OperationContext.cliOpts` extends shared-op dispatch for MCP callers.
|
||||
- `src/core/db-lock.ts` (v0.22.13) — generic `tryAcquireDbLock(engine, lockId, ttlMinutes)` over the existing `gbrain_cycle_locks` table. Parameterized lock id so different scopes can nest cleanly: `gbrain-cycle` for the broad cycle (held by `cycle.ts`) and `gbrain-sync` (`SYNC_LOCK_ID` constant) for `performSync`'s narrower writer window. Same UPSERT-with-TTL semantics as the prior cycle-only helper, just generalized. Survives PgBouncer transaction pooling (unlike session-scoped `pg_try_advisory_lock`); crashed holders auto-release once their TTL expires.
|
||||
- `src/core/sync-concurrency.ts` (v0.22.13) — single source of truth for the parallel-sync policy. Exports `autoConcurrency(engine, fileCount, override?)` (PGLite always serial; explicit override clamped to >=1; auto path returns `DEFAULT_PARALLEL_WORKERS=4` when `fileCount > AUTO_CONCURRENCY_FILE_THRESHOLD=100`), `shouldRunParallel(workers, fileCount, explicit)` (Q1: explicit `--workers` bypasses the >50-file floor), and `parseWorkers(s)` (rejects `'0'`, `'-3'`, `'foo'`, `'1.5'`, trailing chars — replaces the prior parseInt-with-no-validation in both `sync.ts` and `import.ts`). Used by `performSync`, `performFullSync`, `runImport`, and the Minion `sync` handler so the three sites can no longer drift.
|
||||
- `src/commands/sync.ts` — `gbrain sync` CLI + the `performSync` / `performFullSync` library entrypoints (consumed by the autopilot cycle and the Minion sync handler). v0.22.13 (PR #490): `performSync` wraps its body in a `gbrain-sync` writer lock so two concurrent syncs (manual + autopilot, two terminals, two Conductor workspaces) cannot both write `last_commit` and let the last writer win. Head-drift gate after the import phase re-checks `git rev-parse HEAD`; if HEAD moved (someone ran `git checkout` / `git pull` mid-sync), the bookmark refuses to advance. Vanished files now record a failedFiles entry instead of silent-skip — the silent-skip-then-advance pathology that survived prior hardening passes is dead. Worker engines wrap in try/finally so disconnect always fires (panic-path leak fix). Both PGLite-detection sites use `engine.kind === 'pglite'`. CLI accepts `--workers N` (alias `--concurrency N`), validated via `parseWorkers`. Explicit `--workers` bypasses the auto-path file-count floor; auto path defers to `autoConcurrency()`. Banner moved to stderr.
|
||||
- `src/core/cycle.ts` — v0.17 brain maintenance cycle primitive (extended to **8 phases in v0.23**). `runCycle(engine: BrainEngine | null, opts: CycleOpts): Promise<CycleReport>` composes phases in semantically-driven order: **lint → backlinks → sync → synthesize → extract → patterns → embed → orphans**. v0.23's `synthesize` phase runs after sync (cross-references see fresh brain) and before extract (auto-link materializes its writes); `patterns` runs after extract so it reads a fresh graph (codex finding #7 — subagent put_page sets `ctx.remote=true` and skips auto-link/timeline by default; extract is the canonical materialization). Three callers: `gbrain dream` CLI, `gbrain autopilot` daemon's inline path, and the Minions `autopilot-cycle` handler. Coordination via `gbrain_cycle_locks` DB table + `~/.gbrain/cycle.lock` file lock with PID-liveness for PGLite. `CycleReport.schema_version: "1"` is stable; totals additively grew in v0.23 (`transcripts_processed`, `synth_pages_written`, `patterns_written`). `yieldBetweenPhases` runs between phases. **v0.23 added `yieldDuringPhase`** for in-phase keepalive — synthesize/patterns call it during long waits to renew the cycle-lock TTL. Engine nullable; lock-skip on read-only phase selections. v0.22.1 (#403): `CycleOpts.signal?: AbortSignal` propagates the worker's abort signal; `checkAborted()` fires between every phase. v0.22.1 (#417): `runPhaseSync` returns `pagesAffected` via `SyncPhaseResult`; `runCycle` captures it and threads to `runPhaseExtract` as the 4th arg. v0.22.1 (Codex F2): `runPhaseSync` takes `willRunExtractPhase: boolean` and sets `noExtract: phases.includes('extract')` so `gbrain dream --phase sync` doesn't silently lose extraction. v0.22.5 (#475): `resolveSourceForDir(engine, brainDir)` threads `sourceId` to `performSync()` so sync reads the per-source `sources.last_commit` anchor instead of the drift-prone global `config.sync.last_commit` key.
|
||||
- `src/core/cycle/synthesize.ts` (v0.23) — Synthesize phase: conversation-transcript-to-brain pipeline. Reads from `dream.synthesize.session_corpus_dir`, runs cheap Haiku verdict (cached in `dream_verdicts`), then fans out one Sonnet subagent per worth-processing transcript with `allowed_slug_prefixes` (sourced from `skills/_brain-filing-rules.json` `dream_synthesize_paths.globs`). Orchestrator collects slugs from `subagent_tool_executions` (NOT `pages.updated_at` — codex finding #2) and reverse-renders DB → markdown via `serializeMarkdown`. Cooldown via `dream.synthesize.last_completion_ts`, written ONLY on success. Idempotency key `dream:synth:<file_path>:<content_hash>`. Auto-commit deferred to v1.1 (codex #5). `--dry-run` runs Haiku, skips Sonnet (codex #8). Subagent never gets fs-write access. **v0.23.2:** `renderPageToMarkdown` (now exported) stamps `dream_generated: true` and `dream_cycle_date` into every reverse-write's frontmatter; `writeSummaryPage` does the same on the dream-cycle summary index. The marker is the explicit identity surface checked by `isDreamOutput` in `transcript-discovery.ts` — replaces the v0.23.1 content-prefix heuristic that could miss real output (`serializeMarkdown` doesn't embed slugs in body) and false-positive on user transcripts citing brain pages. `judgeSignificance` and `JudgeClient` are exported; `judgeSignificance` accepts a `verdictModel` parameter (default `claude-haiku-4-5-20251001`) loaded from `dream.synthesize.verdict_model` via `loadSynthConfig`.
|
||||
- `src/core/cycle/patterns.ts` (v0.23) — Patterns phase: cross-session theme detection over reflections within `dream.patterns.lookback_days` (default 30). Names a pattern only when ≥`dream.patterns.min_evidence` (default 3) reflections support it. Single Sonnet subagent; same allow-list path as synthesize. Runs AFTER `extract` so the graph is fresh.
|
||||
- `src/core/cycle/transcript-discovery.ts` (v0.23) — Pure filesystem walk for synthesize. `discoverTranscripts(opts)` filters `.txt` files by date range, min_chars, and word-boundary regex `excludePatterns` (Q-3: `medical` matches "medical advice" but NOT "comedical"; power users may pass full regex). `readSingleTranscript(path)` is the `gbrain dream --input <file>` ad-hoc path. **v0.23.2 self-consumption guard:** `DREAM_OUTPUT_MARKER_RE` (anchored at frontmatter open `---\n`, optional BOM + CRLF tolerance, scans first 2000 chars for `dream_generated: true` with case-insensitive value and word boundary on `true`) drives `isDreamOutput(content, bypass=false)`. Both `discoverTranscripts` and `readSingleTranscript` skip matching files and emit a `[dream] skipped <basename>: dream_generated marker` stderr log (no more silent skips). `bypassGuard?: boolean` on `DiscoverOpts` and `readSingleTranscript`'s opts disables the guard for the explicit `--unsafe-bypass-dream-guard` escape hatch only — never auto-applied for `--input`. Replaces v0.23.1's `DREAM_OUTPUT_SLUGS` content-prefix list.
|
||||
- `src/commands/dream.ts` — v0.17 `gbrain dream` CLI; ~80-line thin alias over `runCycle`. brainDir resolution requires explicit `--dir` OR `sync.repo_path` config. Flags: `--dry-run`, `--json`, `--phase <name>`, `--pull`, `--dir <path>`. **v0.23 added** `--input <file>` (ad-hoc transcript, implies `--phase synthesize`), `--date YYYY-MM-DD`, `--from <d> --to <d>` (backfill range). Conflict detection: `--input` + `--date` exits 2. ISO date validation. `--dry-run` runs Haiku significance verdict but skips Sonnet synthesis (codex finding #8 — NOT zero LLM calls). Exit code 1 on status=failed. **v0.23.2 added** `--unsafe-bypass-dream-guard` (long-form intentional, plumbed through `runCycle.synthBypassDreamGuard` → `SynthesizePhaseOpts.bypassDreamGuard` → `discoverTranscripts({bypassGuard})` and `readSingleTranscript({bypassGuard})`). Loud stderr warning fires at synthesize-phase entry when set. Never auto-applied for `--input` so any caller can't silently re-trigger the loop bug.
|
||||
- `src/commands/friction.ts` + `src/core/friction.ts` (v0.23) — `gbrain friction {log,render,list,summary}` reporter. Append-only JSONL under `$GBRAIN_HOME/friction/<run-id>.jsonl`. Schema is a flat extension of `StructuredAgentError` (D20). Render groups by severity → phase, defaults to `--redact` for md output (strips `$HOME`/`$CWD` to placeholders so reports paste safely in PRs). Run-id resolves from `--run-id` > `$GBRAIN_FRICTION_RUN_ID` > `standalone.jsonl`. Skills the claw-test exercises gain a `_friction-protocol.md` callout so agents know when to log friction.
|
||||
- `src/commands/claw-test.ts` + `src/core/claw-test/` (v0.23) — `gbrain claw-test [--scenario <name>] [--live --agent openclaw]`. End-to-end "fresh user" friction harness. Two modes: scripted (CI gate, agent-free) and live (real openclaw subprocess, $1–2 in tokens). Sets `GBRAIN_HOME=<tempdir>` for hermeticity and captures gbrain's `--progress-json` events from each child's stderr to verify expected phases ran (`import.files`, `extract.links_fs`, `doctor.db_checks`). Phases for scripted mode: setup → install_brain (`gbrain init --pglite`) → import (`--no-embed`) → query → extract → verify (`gbrain doctor --json`, asserts `status: 'ok'`) → render. Live mode hands `BRIEF.md` from `test/fixtures/claw-test-scenarios/<name>/` to the agent runner. v1 ships with the OpenClaw runner only (`src/core/claw-test/runners/openclaw.ts`, invokes `openclaw agent --local --agent <name> --message <brief>`); hermes runner deferred to v1.1. Transcript capture (`transcript-capture.ts`) uses `fs.createWriteStream` with `'drain'`-event backpressure — D17 fix for the 256KB-burst child-stall scenario. v0.18 upgrade scenario seeded via `seed-pglite.ts` SQL replay.
|
||||
- `skills/_friction-protocol.md` (v0.23) — shared cross-cutting convention skill (like `_brain-filing-rules.md`). Tells agents when to call `gbrain friction log` and how to choose a severity. Routes to friction CLI from any skill the claw-test exercises.
|
||||
- `scripts/check-progress-to-stdout.sh` — CI guard against regressing to `\r`-on-stdout progress. Wired into `bun run test` via `scripts/check-progress-to-stdout.sh && bun test` in package.json.
|
||||
- `docs/progress-events.md` — Canonical JSON event schema reference. Stable from v0.15.2, additive only.
|
||||
- `src/core/markdown.ts` — Frontmatter parsing + body splitter. `splitBody` requires an explicit timeline sentinel (`<!-- timeline -->`, `--- timeline ---`, or `---` immediately before `## Timeline`/`## History`). Plain `---` in body text is a markdown horizontal rule, not a separator. `inferType` auto-types `/wiki/analysis/` → analysis, `/wiki/guides/` → guide, `/wiki/hardware/` → hardware, `/wiki/architecture/` → architecture, `/writing/` → writing (plus the existing people/companies/deals/etc heuristics).
|
||||
- `scripts/check-jsonb-pattern.sh` — CI grep guard. Fails the build if anyone reintroduces (a) the `${JSON.stringify(x)}::jsonb` interpolation pattern (postgres.js v3 double-encodes it), or (b) `max_stalled INTEGER NOT NULL DEFAULT 1` in any schema source file (v0.15.1 #219 regression guard — must be DEFAULT 5 to preserve SIGKILL-rescue). Wired into `bun test`.
|
||||
- `docker-compose.ci.yml` + `scripts/ci-local.sh` (v0.23.1) — Local CI gate. `bun run ci:local` spins up `pgvector/pgvector:pg16` + `oven/bun:1` with named volumes (`gbrain-ci-pg-data`, `gbrain-ci-node-modules`, `gbrain-ci-bun-cache`), runs gitleaks on host, smoke-tests `scripts/run-e2e.sh` argv handling, runs unit tests with `DATABASE_URL` unset (matches GH Actions structure), then runs all 29 E2E files sequentially. `--diff` swaps in the diff-aware selector; `--no-pull` skips upstream pulls; `--clean` nukes named volumes. Postgres host port defaults to 5434 (avoids 5432 manual `gbrain-test-pg` and 5433 sibling-project conflict); override with `GBRAIN_CI_PG_PORT=NNNN`. Stronger gate than current PR CI's 2-file Tier 1 set — closes the "push-and-wait" feedback loop pre-push.
|
||||
- `scripts/select-e2e.ts` + `scripts/e2e-test-map.ts` (v0.23.1) — Diff-aware E2E test selector. Reads three git sources (committed `origin/master...HEAD`, working-tree `HEAD`, and `git ls-files --others --exclude-standard` for untracked, NOT-gitignored files), classifies as EMPTY / DOC_ONLY / SRC. Fail-closed by design: EMPTY → all 29 files (clean branch shouldn't run nothing), DOC_ONLY (every path matches the README/CLAUDE/AGENTS/CHANGELOG/TODOS allowlist) → empty stdout, SRC → escape-hatch paths (schema, package.json, skills/) trigger all; otherwise the hand-tuned `E2E_TEST_MAP` glob → tests narrows; an unmapped src/ change still emits ALL files, never silently nothing. Pure-function exports (`selectTests`, `classify`, `matchGlob`) so it's trivial to test and fork. `bun run ci:select-e2e` prints the current selection on stdout, pipe-friendly. `test/select-e2e.test.ts` covers all 4 branches plus 3 codex regression guards (skills/, untracked files, unmapped src/) — 24 cases.
|
||||
- `scripts/run-e2e.sh` (v0.23.1 update) — Sequential E2E runner. Now accepts an optional argv-driven file list (used by `ci:local:diff` to pipe in selector output) and a `--dry-run-list` flag that prints the resolved file list and exits (used by `ci-local.sh`'s startup smoke-test). Falls back to `test/e2e/*.test.ts` when invoked with no args.
|
||||
- `scripts/llms-config.ts` + `scripts/build-llms.ts` — Generator for `llms.txt` (llmstxt.org-spec web index) + `llms-full.txt` (inlined single-fetch bundle). Curated config drives both. Run `bun run build:llms` after adding a new doc. `LLMS_REPO_BASE` env var lets forks regenerate with their own URL base. `FULL_SIZE_BUDGET` (600KB) caps the inline bundle; generator WARNs if exceeded. Committed output is not analogous to `schema-embedded.ts` (no runtime consumer); we commit for GitHub browsing and fork-safe fetching.
|
||||
- `AGENTS.md` — Local-clone entry point for non-Claude agents (Codex, Cursor, OpenClaw, Aider). Mirrors `CLAUDE.md` intent via relative links. Claude Code keeps using `CLAUDE.md`.
|
||||
- `docs/UPGRADING_DOWNSTREAM_AGENTS.md` — Patches for downstream agent skill forks to apply when upgrading. Each release appends a new section. v0.10.3 includes diffs for brain-ops, meeting-ingestion, signal-detector, enrich.
|
||||
- `src/core/schema-embedded.ts` — AUTO-GENERATED from schema.sql (run `bun run build:schema`)
|
||||
- `src/schema.sql` — Full Postgres + pgvector DDL (source of truth, generates schema-embedded.ts)
|
||||
- `src/commands/integrations.ts` — Standalone integration recipe management (no DB needed). Exports `getRecipeDirs()` (trust-tagged recipe sources), SSRF helpers (`isInternalUrl`, `parseOctet`, `hostnameToOctets`, `isPrivateIpv4`). Only package-bundled recipes are `embedded=true`; `$GBRAIN_RECIPES_DIR` and cwd `./recipes/` are untrusted and cannot run `command`/`http`/string health checks.
|
||||
- `src/core/search/expansion.ts` — Multi-query expansion via Haiku. Exports `sanitizeQueryForPrompt` + `sanitizeExpansionOutput` (prompt-injection defense-in-depth). Sanitized query is only used for the LLM channel; original query still drives search.
|
||||
- `recipes/` — Integration recipe files (YAML frontmatter + markdown setup instructions)
|
||||
- `docs/guides/` — Individual SKILLPACK guides (broken out from monolith)
|
||||
- `docs/integrations/` — "Getting Data In" guides and integration docs
|
||||
- `docs/architecture/infra-layer.md` — Shared infrastructure documentation
|
||||
- `docs/ethos/THIN_HARNESS_FAT_SKILLS.md` — Architecture philosophy essay
|
||||
- `docs/ethos/MARKDOWN_SKILLS_AS_RECIPES.md` — "Homebrew for Personal AI" essay
|
||||
- `docs/guides/repo-architecture.md` — Two-repo pattern (agent vs brain)
|
||||
- `docs/guides/sub-agent-routing.md` — Model routing table for sub-agents
|
||||
- `docs/guides/skill-development.md` — 5-step skill development cycle + MECE
|
||||
- `docs/guides/idea-capture.md` — Originality distribution, depth test, cross-linking
|
||||
- `docs/guides/quiet-hours.md` — Notification hold + timezone-aware delivery
|
||||
- `docs/guides/diligence-ingestion.md` — Data room to brain pages pipeline
|
||||
- `docs/designs/HOMEBREW_FOR_PERSONAL_AI.md` — 10-star vision for integration system
|
||||
- `docs/mcp/` — Per-client setup guides (Claude Desktop, Code, Cowork, Perplexity)
|
||||
- BrainBench (benchmark suite + corpus): lives in the separate [gbrain-evals](https://github.com/garrytan/gbrain-evals) repo. Not installed alongside gbrain.
|
||||
- `skills/_brain-filing-rules.md` — Cross-cutting brain filing rules (referenced by all brain-writing skills)
|
||||
- `skills/RESOLVER.md` — Skill routing table (based on the agent-fork AGENTS.md pattern)
|
||||
- `skills/conventions/` — Cross-cutting rules (quality, brain-first, model-routing, test-before-bulk, cross-modal)
|
||||
- `skills/_output-rules.md` — Output quality standards (deterministic links, no slop, exact phrasing)
|
||||
- `skills/signal-detector/SKILL.md` — Always-on idea+entity capture on every message
|
||||
- `skills/brain-ops/SKILL.md` — Brain-first lookup, read-enrich-write loop, source attribution
|
||||
- `skills/idea-ingest/SKILL.md` — Links/articles/tweets with author people page mandatory
|
||||
- `skills/media-ingest/SKILL.md` — Video/audio/PDF/book with entity extraction
|
||||
- `skills/meeting-ingestion/SKILL.md` — Transcripts with attendee enrichment chaining
|
||||
- `skills/citation-fixer/SKILL.md` — Citation format auditing and fixing
|
||||
- `skills/repo-architecture/SKILL.md` — Filing rules by primary subject
|
||||
- `skills/skill-creator/SKILL.md` — Create conforming skills with MECE check
|
||||
- `skills/daily-task-manager/SKILL.md` — Task lifecycle with priority levels
|
||||
- `skills/daily-task-prep/SKILL.md` — Morning prep with calendar context
|
||||
- `skills/cross-modal-review/SKILL.md` — Quality gate via second model
|
||||
- `skills/cron-scheduler/SKILL.md` — Schedule staggering, quiet hours, idempotency
|
||||
- `skills/reports/SKILL.md` — Timestamped reports with keyword routing
|
||||
- `skills/testing/SKILL.md` — Skill validation framework
|
||||
- `skills/soul-audit/SKILL.md` — 6-phase interview for SOUL.md, USER.md, ACCESS_POLICY.md, HEARTBEAT.md
|
||||
- `skills/webhook-transforms/SKILL.md` — External events to brain signals
|
||||
- `skills/data-research/SKILL.md` — Structured data research: email-to-tracker pipeline with parameterized YAML recipes
|
||||
- `skills/minion-orchestrator/SKILL.md` — Unified background-work skill (v0.20.4 consolidation of the former `minion-orchestrator` + `gbrain-jobs` split). Two lanes: shell jobs via `gbrain jobs submit shell --params '{"cmd":"..."}'` (operator/CLI only; MCP throws `permission_denied` for protected names) and LLM subagents via `gbrain agent run` (user-facing entrypoint). Shared Preconditions block, parent-child DAGs with depth/cap/timeouts, `child_done` inbox for fan-in, PGLite `--follow` inline path for dev. Triggers narrowed from bare `"gbrain jobs"` to `"gbrain jobs submit"` + `"submit a gbrain job"` so `stats`/`prune`/`retry` questions fall through to `gbrain --help`.
|
||||
- `templates/` — SOUL.md, USER.md, ACCESS_POLICY.md, HEARTBEAT.md templates
|
||||
- `skills/migrations/` — Version migration files with feature_pitch YAML frontmatter
|
||||
- `src/commands/publish.ts` — Deterministic brain page publisher (code+skill pair, zero LLM calls)
|
||||
- `src/commands/backlinks.ts` — Back-link checker and fixer (enforces Iron Law)
|
||||
- `src/commands/lint.ts` — Page quality linter (catches LLM artifacts, placeholder dates)
|
||||
- `src/commands/report.ts` — Structured report saver (audit trail for maintenance/enrichment)
|
||||
- `openclaw.plugin.json` — ClawHub bundle plugin manifest
|
||||
|
||||
### BrainBench — in a sibling repo (v0.20+)
|
||||
|
||||
BrainBench — the public benchmark for personal-knowledge agent stacks — lives in
|
||||
[github.com/garrytan/gbrain-evals](https://github.com/garrytan/gbrain-evals). It
|
||||
depends on gbrain as a consumer; gbrain never pulls in the ~5MB eval corpus or
|
||||
the pdf-parse dev dep at install time.
|
||||
|
||||
gbrain's public API surface (the exports map in `package.json`) is what
|
||||
gbrain-evals consumes: `gbrain/engine`, `gbrain/types`, `gbrain/operations`,
|
||||
`gbrain/pglite-engine`, `gbrain/link-extraction`, `gbrain/import-file`,
|
||||
`gbrain/transcription`, `gbrain/embedding`, `gbrain/config`, `gbrain/markdown`,
|
||||
`gbrain/backoff`, `gbrain/search/hybrid`, `gbrain/search/expansion`,
|
||||
`gbrain/extract`. Removing any of these is a breaking change for the
|
||||
gbrain-evals consumer.
|
||||
|
||||
## Commands
|
||||
|
||||
Run `gbrain --help` or `gbrain --tools-json` for full command reference.
|
||||
|
||||
Key commands added in v0.7:
|
||||
- `gbrain init` — defaults to PGLite (no Supabase needed), scans repo size, suggests Supabase for 1000+ files
|
||||
- `gbrain migrate --to supabase` / `gbrain migrate --to pglite` — bidirectional engine migration
|
||||
|
||||
Key commands added for Minions (job queue):
|
||||
- `gbrain jobs submit <name> [--params JSON] [--follow] [--dry-run]` — submit a background job. v0.13.1 adds first-class flags for every `MinionJobInput` tuning knob: `--max-stalled N`, `--backoff-type fixed|exponential`, `--backoff-delay Nms`, `--backoff-jitter 0..1`, `--timeout-ms N`, `--idempotency-key K`.
|
||||
- `gbrain jobs list [--status S] [--queue Q]` — list jobs with filters
|
||||
- `gbrain jobs get <id>` — job details with attempt history
|
||||
- `gbrain jobs cancel/retry/delete <id>` — manage job lifecycle
|
||||
- `gbrain jobs prune [--older-than 30d]` — clean old completed/dead jobs
|
||||
- `gbrain jobs stats` — job health dashboard
|
||||
- `gbrain jobs smoke [--sigkill-rescue]` — health smoke test. `--sigkill-rescue` is the v0.13.1 regression guard for #219: simulates a killed worker and asserts the stalled job is requeued instead of dead-lettered on first stall.
|
||||
- `gbrain jobs work [--queue Q] [--concurrency N]` — start worker daemon (Postgres only)
|
||||
|
||||
Key commands added in v0.25.0:
|
||||
- `gbrain eval export [--since DUR] [--limit N] [--tool query|search]` — stream captured `eval_candidates` rows as NDJSON to stdout. Every line starts with `"schema_version": 1` per the stable contract in `docs/eval-capture.md`. EPIPE-safe, progress heartbeats on stderr, deterministic ordering. Primary consumer is the sibling `gbrain-evals` repo for BrainBench-Real replay.
|
||||
- `gbrain eval prune --older-than DUR [--dry-run]` — explicit retention cleanup for `eval_candidates`. Requires `--older-than` (never deletes without a window). Duration strings: 30d, 7d, 1h, 90m, 3600s.
|
||||
- `gbrain eval replay --against FILE.ndjson [--limit N] [--top-regressions K] [--json] [--verbose]` — contributor-facing dev loop. Reads a captured NDJSON snapshot, re-runs each `query` / `search` op against the current brain, computes mean set-Jaccard@k between captured + current `retrieved_slugs`, top-1 stability rate, and latency Δ. JSON mode (`schema_version: 1`) for CI gating; human mode prints a regression table sorted worst-first. Closes the gap between "data captured" and "data used to gate a PR." See `docs/eval-bench.md` for the workflow.
|
||||
- `gbrain doctor` gains an `eval_capture` check: reads `eval_capture_failures` for the last 24h, groups by reason, warns when non-zero. Cross-process visibility (doctor runs in a separate process from MCP). Pre-v31 brains get `Skipped (table unavailable)` — non-fatal.
|
||||
- Config addition: `eval: { capture?: boolean, scrub_pii?: boolean }` in `~/.gbrain/config.json`. **File-plane only** — `gbrain config set` writes the DB plane and does NOT control capture.
|
||||
- **`GBRAIN_CONTRIBUTOR_MODE=1` env var** is the contributor-facing toggle. Capture is **off by default** as of v0.25.0; production users get a quiet brain. Resolution order: explicit `eval.capture` config wins both directions, then env var, then off. Documented in README.md, CONTRIBUTING.md, and `docs/eval-bench.md`.
|
||||
|
||||
Key commands added in v0.12.2:
|
||||
- `gbrain repair-jsonb [--dry-run] [--json]` — repair double-encoded JSONB rows left over from v0.12.0-and-earlier Postgres writes. Idempotent; PGLite no-ops. The `v0_12_2` migration runs this automatically on `gbrain upgrade`.
|
||||
|
||||
Key commands added in v0.12.3:
|
||||
- `gbrain orphans [--json] [--count] [--include-pseudo]` — surface pages with zero inbound wikilinks, grouped by domain. Auto-generated/raw/pseudo pages filtered by default. Also exposed as `find_orphans` MCP operation. The natural consumer of the v0.12.0 knowledge graph layer: once edges are captured, find the gaps.
|
||||
- `gbrain doctor` gains two new reliability detection checks: `jsonb_integrity` (v0.12.0 Postgres double-encode damage) and `markdown_body_completeness` (pages truncated by the old splitBody bug). Detection only; fix hints point at `gbrain repair-jsonb` and `gbrain sync --force`.
|
||||
|
||||
Key commands added in v0.14.2:
|
||||
- `gbrain sync --skip-failed` — acknowledge the current set of failed-parse files recorded in `~/.gbrain/sync-failures.jsonl` so the sync bookmark advances past them. Doctor's `sync_failures` check shows previously-skipped as "all acknowledged" instead of warning.
|
||||
- `gbrain sync --retry-failed` — re-walk the unacknowledged failures and re-attempt parsing. If the files now succeed, they clear from the set and the bookmark advances naturally.
|
||||
- `gbrain apply-migrations --force-retry <version>` — reset a wedged migration (3 consecutive partials with no completion) by appending a `'retry'` marker. Next `apply-migrations --yes` treats the version as fresh. `complete` status never regresses to `partial` either before or after a retry marker.
|
||||
- `GBRAIN_POOL_SIZE` env var — honored by both the singleton pool (`src/core/db.ts`) and the parallel-import worker pool (`src/commands/import.ts`). Default is 10; lower to 2 for Supabase transaction pooler to avoid MaxClients crashes during `gbrain upgrade` subprocess spawns. Read at call time via `resolvePoolSize()`.
|
||||
- `gbrain doctor` gains two new checks: `sync_failures` (surfaces unacknowledged parse failures with exact paths + fix hints) and `brain_score` (renders the 5-component breakdown when score < 100: embed coverage / 35, link density / 25, timeline coverage / 15, orphans / 15, dead links / 10 — sum equals total).
|
||||
|
||||
Key commands added in v0.14.3 (fix wave):
|
||||
- `gbrain doctor --index-audit` — opt-in Postgres-only check reporting zero-scan indexes from `pg_stat_user_indexes`. Informational only; never auto-drops.
|
||||
- `gbrain doctor` schema_version check fails loudly when `version=0` — catches `bun install -g github:...` postinstall failures (#218) and routes users to `gbrain apply-migrations --yes`.
|
||||
- `gbrain jobs submit` gains `--max-stalled`, `--backoff-type`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key` — exposing existing `MinionJobInput` fields as first-class CLI flags.
|
||||
- `gbrain jobs smoke --sigkill-rescue` — opt-in regression smoke case simulating a killed worker; asserts the v0.14.3 schema default (`max_stalled=5`) actually rescues on first stall.
|
||||
|
||||
Key commands added in v0.22.13 (PR #490):
|
||||
- `gbrain sync --workers N` (alias `--concurrency N`) — parallelize the import phase using per-worker Postgres engines (small pool of 2 each) with an atomic queue index. Auto-concurrency: defaults to 4 workers when the diff exceeds 100 files. Smaller diffs stay serial. Explicit `--workers` always wins (even on a 30-file diff). PGLite forces serial regardless. Validation rejects `0`, negatives, non-integers loud (replaces the prior silent fall-through to auto-concurrency).
|
||||
- `gbrain import --workers N` — same `parseWorkers()` validation as sync; same try/finally worker-engine cleanup. Behavior surface unchanged.
|
||||
|
||||
Key commands added in v0.22.16 (claw-test friction loop):
|
||||
- `gbrain claw-test [--scenario fresh-install|upgrade-from-v0.18] [--keep-tempdir]` — scripted-mode CI gate that runs the full canonical first-day flow against a fresh tempdir. Asserts every expected `--progress-json` phase fired and doctor's `status === 'ok'`. ~30s, no API keys.
|
||||
- `gbrain claw-test --live --agent openclaw` — friction-discovery mode. Spawns real openclaw, hands it `BRIEF.md`, captures stdin/stdout/stderr to `<run>/transcript.jsonl`, lets the agent log friction via the friction CLI. Run on demand; ~5–10 min and ~$1–2 in tokens.
|
||||
- `gbrain claw-test --list-agents` — reports which agent runners are registered + their detection state (binary path or unavailable reason).
|
||||
- `gbrain friction log --severity {confused|error|blocker|nit} --phase <name> --message <text> [--hint ...] [--kind {friction|delight}] [--run-id ...]` — append a friction or delight entry to the active run JSONL.
|
||||
- `gbrain friction render --run-id <id> [--json] [--transcripts] [--no-redact]` — markdown report grouped by severity + phase; `--redact` is the default for md output (strips `$HOME`/`$CWD` placeholders so reports paste safely in PRs/issues).
|
||||
- `gbrain friction list [--json]` — recent run-ids with friction/delight counts; interrupted runs marked `(interrupted)`.
|
||||
- `gbrain friction summary --run-id <id> [--json]` — two-column friction + delight summary.
|
||||
- `GBRAIN_HOME` env override is now honored uniformly across every gbrain write site (config, audit, friction, sync-failures, import checkpoint, integrity log, integrations heartbeat, migration rollback, etc.) — `gbrainPath(...)` from `src/core/config.ts` is the canonical helper. Read-side host-fingerprint detection (`~/.claude`/`~/.openclaw` etc.) intentionally NOT confined in v1; that's a v1.1 follow-up.
|
||||
|
||||
## Testing
|
||||
|
||||
`bun test` runs all tests. After the v0.12.1 release: ~75 unit test files + 8 E2E test files (1412 unit pass, 119 E2E when `DATABASE_URL` is set — skip gracefully otherwise). Unit tests run
|
||||
without a database. E2E tests skip gracefully when `DATABASE_URL` is not set.
|
||||
|
||||
Unit tests: `test/markdown.test.ts` (frontmatter parsing), `test/chunkers/recursive.test.ts`
|
||||
(chunking), `test/parity.test.ts` (operations contract
|
||||
parity), `test/cli.test.ts` (CLI structure), `test/config.test.ts` (config redaction),
|
||||
`test/files.test.ts` (MIME/hash), `test/import-file.test.ts` (import pipeline),
|
||||
`test/upgrade.test.ts` (schema migrations),
|
||||
`test/file-migration.test.ts` (file migration), `test/file-resolver.test.ts` (file resolution),
|
||||
`test/import-resume.test.ts` (import checkpoints), `test/migrate.test.ts` (migration; v8/v9 helper-btree-index SQL structural assertions + 1000-row wall-clock fixtures that guard the O(n²)→O(n log n) fix + v0.13.1 assertions on v12/v13 SQL shape, `sqlFor` + `transaction:false` runner semantics, the `max_stalled DEFAULT 1` regression guard, and v0.22.6.1 v24 `sqlFor.pglite: ''` no-op assertion),
|
||||
`test/bootstrap.test.ts` (v0.22.6.1 — bootstrap contract: no-op on fresh install, idempotent across two `initSchema()` calls, no-op on modern brain that already has every probed column, full bootstrap path on simulated pre-v0.18 brain, fresh-install regression guard, pre-v0.13 `links` shape coverage),
|
||||
`test/schema-bootstrap-coverage.test.ts` (v0.22.6.1 CI guard — `REQUIRED_BOOTSTRAP_COVERAGE` lists every forward reference in PGLITE_SCHEMA_SQL; the test fails loudly if `applyForwardReferenceBootstrap` skips one. When you add a column-with-index to the embedded schema blob, you extend both arrays or this guard fails. The pattern that broke gbrain ten times in two years is now structurally prevented.),
|
||||
`test/setup-branching.test.ts` (setup flow), `test/slug-validation.test.ts` (slug validation),
|
||||
`test/storage.test.ts` (storage backends), `test/supabase-admin.test.ts` (Supabase admin),
|
||||
`test/yaml-lite.test.ts` (YAML parsing), `test/check-update.test.ts` (version check + update CLI),
|
||||
`test/pglite-engine.test.ts` (PGLite engine, all 40 BrainEngine methods including 11 cases for `addLinksBatch` / `addTimelineEntriesBatch`: empty batch, missing optionals, within-batch dedup via ON CONFLICT, missing-slug rows dropped by JOIN, half-existing batch, batch of 100 + v0.13.1 `connect()` error-wrap assertion (original error nested, #223 link in message, lock released)),
|
||||
`test/engine-factory.test.ts` (engine factory + dynamic imports),
|
||||
`test/integrations.test.ts` (recipe parsing, CLI routing, recipe validation),
|
||||
`test/publish.test.ts` (content stripping, encryption, password generation, HTML output),
|
||||
`test/backlinks.test.ts` (entity extraction, back-link detection, timeline entry generation),
|
||||
`test/lint.test.ts` (LLM artifact detection, code fence stripping, frontmatter validation),
|
||||
`test/report.test.ts` (report format, directory structure),
|
||||
`test/skills-conformance.test.ts` (skill frontmatter + required sections validation),
|
||||
`test/resolver.test.ts` (RESOLVER.md coverage, routing validation + v0.20.4 round-trip: every quoted RESOLVER.md trigger must match a frontmatter `triggers:` entry in the target skill, and every `name="<word>"` reference in any SKILL.md must resolve to a declared op in `src/core/operations.ts` or a Minions handler in `PROTECTED_JOB_NAMES`),
|
||||
`test/search.test.ts` (RRF normalization, compiled truth boost, cosine similarity, dedup key),
|
||||
`test/sql-ranking.test.ts` (v0.22.0 source-boost helpers: 39 cases covering longest-prefix-match in SQL CASE, detail=high temporal-bypass, three-meta-char LIKE escape (%, _, \\), single-quote SQL-literal doubling, env override parsing for GBRAIN_SOURCE_BOOST + GBRAIN_SEARCH_EXCLUDE, resolveBoostMap / resolveHardExcludes merge semantics),
|
||||
`test/dedup.test.ts` (source-aware dedup, compiled truth guarantee, layer interactions),
|
||||
`test/intent.test.ts` (query intent classification: entity/temporal/event/general),
|
||||
`test/eval.test.ts` (retrieval metrics: precisionAtK, recallAtK, mrr, ndcgAtK, parseQrels),
|
||||
`test/check-resolvable.test.ts` (resolver reachability, MECE overlap, gap detection, DRY checks + v0.14.1 proximity-based DRY detection + `extractDelegationTargets` coverage — 13 DRY cases),
|
||||
`test/dry-fix.test.ts` (v0.14.1 auto-fix: three shape-aware expander pure-function tests, five guards — working-tree-dirty, no-git-backup, inside-code-fence, already-delegated within 40 lines, ambiguous-multi-match, block-is-callout — 28 cases),
|
||||
`test/doctor-fix.test.ts` (v0.14.1 `gbrain doctor --fix` CLI integration: dry-run preview, apply path, JSON output shape — 3 cases),
|
||||
`test/backoff.test.ts` (load-aware throttling, concurrency limits, active hours),
|
||||
`test/fail-improve.test.ts` (deterministic/LLM cascade, JSONL logging, test generation, rotation),
|
||||
`test/transcription.test.ts` (provider detection, format validation, API key errors),
|
||||
`test/enrichment-service.test.ts` (entity slugification, extraction, tier escalation),
|
||||
`test/data-research.test.ts` (recipe validation, MRR/ARR extraction, dedup, tracker parsing, HTML stripping),
|
||||
`test/minions.test.ts` (Minions job queue v7: CRUD, state machine, backoff, stall detection, dependencies, worker lifecycle, lock management, claim mechanics, depth/child-cap, timeouts, cascade kill, idempotency, child_done inbox, attachments, removeOnComplete/Fail + v0.13.1 `max_stalled` clamp/default/plumbing coverage),
|
||||
`test/extract.test.ts` (link extraction, timeline extraction, frontmatter parsing, directory type inference),
|
||||
`test/extract-db.test.ts` (gbrain extract --source db: typed link inference, idempotency, --type filter, --dry-run JSON output),
|
||||
`test/extract-fs.test.ts` (gbrain extract --source fs: first-run inserts + second-run reports zero, dry-run dedups candidates across files, second-run perf regression guard — the v0.12.1 N+1 dedup bug),
|
||||
`test/link-extraction.test.ts` (canonical extractEntityRefs both formats, extractPageLinks dedup, inferLinkType heuristics, parseTimelineEntries date variants, isAutoLinkEnabled config),
|
||||
`test/graph-query.test.ts` (direction in/out/both, type filter, indented tree output),
|
||||
`test/features.test.ts` (feature scanning, brain_score calculation, CLI routing, persistence),
|
||||
`test/file-upload-security.test.ts` (symlink traversal, cwd confinement, slug + filename allowlists, remote vs local trust),
|
||||
`test/query-sanitization.test.ts` (prompt-injection stripping, output sanitization, structural boundary),
|
||||
`test/search-limit.test.ts` (clampSearchLimit default/cap behavior across list_pages and get_ingest_log),
|
||||
`test/repair-jsonb.test.ts` (v0.12.2 JSONB repair: TARGETS list, idempotency, engine-awareness),
|
||||
`test/migrations-v0_12_2.test.ts` (v0.12.2 orchestrator phases: schema → repair → verify → record),
|
||||
`test/markdown.test.ts` (splitBody sentinel precedence, horizontal-rule preservation, inferType wiki subtypes),
|
||||
`test/orphans.test.ts` (v0.12.3 orphans command: detection, pseudo filtering, text/json/count outputs, MCP op),
|
||||
`test/postgres-engine.test.ts` (v0.12.3 statement_timeout scoping: `sql.begin` + `SET LOCAL` shape, source-level grep guardrail against reintroduced bare `SET statement_timeout`),
|
||||
`test/sync.test.ts` (sync logic + v0.12.3 regression guard asserting top-level `engine.transaction` is not called),
|
||||
`test/sync-concurrency.test.ts` (v0.22.13 PR #490: 17 cases covering `autoConcurrency()` thresholds + PGLite-forces-serial + explicit-override clamping, `shouldRunParallel()` Q1 explicit-bypasses-floor contract, and `parseWorkers()` validation that rejects `'0'`/`'-3'`/`'foo'`/`'1.5'`/trailing chars),
|
||||
`test/sync-parallel.test.ts` (v0.22.13 PR #490: PGLite-routed coverage of the bookmark gate under concurrency request, head-drift gate, vanished-file failure capture, PGLite-stays-serial, and the `gbrain-sync` writer-lock contract — 7 cases),
|
||||
`test/sync-failures.test.ts` (v0.22.12: 28 cases pinning `classifyErrorCode` regex coverage for all 12 codes against literal production message strings from `markdown.ts:159-244` and `import-file.ts:199, 347, 352, 401`; `summarizeFailuresByCode` sort + pre-classified-honor; `recordSyncFailures` code-field persistence; `acknowledgeSyncFailures` AcknowledgeResult shape + backfill on pre-v0.22.12 entries),
|
||||
`test/doctor.test.ts` (doctor command + v0.12.3 assertions that `jsonb_integrity` scans the four v0.12.0 write sites and `markdown_body_completeness` is present),
|
||||
`test/utils.test.ts` (shared SQL utilities + `tryParseEmbedding` null-return and single-warn semantics),
|
||||
`test/build-llms.test.ts` (llms.txt/llms-full.txt generator: path resolution, idempotence, spec shape, regen-drift guard, content contract, AGENTS.md install-path mirror, size-budget enforcement — 7 cases),
|
||||
`test/check-resolvable-cli.test.ts` (v0.19 CLI wrapper: exit codes, JSON envelope shape, AGENTS.md fallback chain),
|
||||
`test/regression-v0_16_4.test.ts` (findRepoRoot regression guard — hermetic startDir parameterization),
|
||||
`test/filing-audit.test.ts` (v0.19 Check 6: `writes_pages` / `writes_to` frontmatter, filing-rules JSON validation),
|
||||
`test/routing-eval.test.ts` (v0.19 Check 5: fixture parsing, structural routing, ambiguous_with, Haiku tie-break layer),
|
||||
`test/skill-manifest.test.ts` (v0.19 skill manifest parser: drift detection, managed-block markers),
|
||||
`test/skillify-scaffold.test.ts` (v0.19 `gbrain skillify scaffold` stubs: SKILL.md, script, tests, routing-eval fixtures),
|
||||
`test/skillpack-install.test.ts` (v0.19 `gbrain skillpack install` managed-block install / update / no-clobber semantics),
|
||||
`test/skillpack-sync-guard.test.ts` (v0.19 sync-guard: bundled skills stay byte-identical to `skills/` source),
|
||||
`test/http-transport.test.ts` (v0.22.7 HTTP transport: 23 unit cases covering bearer auth + missing/no-Bearer/unknown/revoked + `/health` bypass, F1+F2 round-trip via dispatch.ts, F3 invalid_params, application/json response shape (not SSE), CORS default-deny + allowlist, body cap on Content-Length AND chunked, two-bucket rate limit (refill, exhaust+Retry-After, LRU eviction, TTL prune, pre-auth IP fires before DB), and `mcp_request_log` audit on success + auth_failed).
|
||||
|
||||
E2E tests (`test/e2e/`): Run against real Postgres+pgvector. Require `DATABASE_URL`.
|
||||
- `bun run test:e2e` runs Tier 1 (mechanical, all operations, no API keys). Includes 9 dedicated cases for the postgres-engine `addLinksBatch` / `addTimelineEntriesBatch` bind path — postgres-js's `unnest()` binding is structurally different from PGLite's and gets its own coverage.
|
||||
- `test/e2e/search-quality.test.ts` runs search quality E2E against PGLite (no API keys, in-memory)
|
||||
- `test/e2e/graph-quality.test.ts` runs the v0.10.3 knowledge graph pipeline (auto-link via put_page, reconciliation, traversePaths) against PGLite in-memory
|
||||
- `test/e2e/postgres-jsonb.test.ts` — v0.12.2 regression test. Round-trips all 5 JSONB write sites (pages.frontmatter, raw_data.data, ingest_log.pages_updated, files.metadata, page_versions.frontmatter) against real Postgres and asserts `jsonb_typeof='object'` plus `->>'key'` returns the expected scalar. The test that should have caught the original double-encode bug.
|
||||
- `test/e2e/integrity-batch.test.ts` (v0.22.8) — parity tests for `scanIntegrity`'s batch-load fast path vs sequential. Four cases (dedup, hits, validate, topPages) seed a fixture and assert both paths return identical results. Dedup case uses raw SQL via `getConn().unsafe()` to seed a `(test-source-2, people/alice)` row alongside the default-source row, since `engine.putPage` doesn't take a `source_id`. Pins the codex-caught multi-source overcounting regression.
|
||||
- `test/e2e/jsonb-roundtrip.test.ts` — v0.12.3 companion regression against the 4 doctor-scanned JSONB sites. Assertion-level overlap with `postgres-jsonb.test.ts` is intentional defense-in-depth: if doctor's scan surface ever drifts from the actual write surface, one of these tests catches it.
|
||||
- `test/e2e/sync.test.ts` (v0.22.12 — `--skip-failed` failure-loop test, alongside the existing 13 happy-path tests): exercises the full chain — broken file → `performSync` returns `blocked_by_failures` with grouped breakdown → `performSync({skipFailed: true})` advances bookmark and returns `AcknowledgeResult` with code summary → second broken file → second cycle. Saves and restores the user's real `~/.gbrain/sync-failures.jsonl` so the test is hermetic on a developer machine. Asserts bookmark gating, JSONL state, dedup across paths, summary aggregation, and the literal doctor-rendering string format. This is the integration test that proves the v0.22.12 chain holds together — unit tests cover the pure functions in isolation, this covers the integration.
|
||||
- `test/e2e/upgrade.test.ts` runs check-update E2E against real GitHub API (network required)
|
||||
- `test/e2e/minions-shell-pglite.test.ts` (v0.20.4) exercises the PGLite `--follow` inline shell-job path (in-memory, no `DATABASE_URL` required) — the path the consolidated minion-orchestrator skill documents for dev use
|
||||
- `test/e2e/openclaw-reference-compat.test.ts` (v0.19) — exercises `check-resolvable` + `skillpack install` against a minimal AGENTS.md workspace fixture (`test/fixtures/openclaw-reference-minimal/`), regression guard for the 107-skill OpenClaw deployment shape
|
||||
- `test/e2e/search-swamp.test.ts` (v0.22.0) — reproduces the headline source-swamp case. Seeds a curated `originals/talks/article-outline-fat-code` page against two `wintermute/chat/` pages stuffed with the same multi-word phrase. Asserts the article wins keyword AND vector ranking, that `detail=high` lets the chat swamp re-surface (temporal-query workflow preserved), and that `source_id` passes through the two-stage CTE intact. PGLite in-memory.
|
||||
- `test/e2e/search-exclude.test.ts` (v0.22.0) — verifies `test/` + `archive/` pages are hidden by default, that `include_slug_prefixes` opts back in, and that caller-supplied `exclude_slug_prefixes` adds to defaults. Both keyword and vector search paths covered.
|
||||
- `test/e2e/engine-parity.test.ts` (v0.22.0) — Postgres ↔ PGLite top-result and result-set parity for `searchKeyword` + `searchVector`. Codex flagged that Postgres ranks pages then picks best chunk while PGLite returns chunks directly — without parity coverage the source-boost fix could pass on PGLite and fail on Postgres. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/postgres-bootstrap.test.ts` (v0.22.6.1) — exercises `PostgresEngine.initSchema()` directly against a fresh real Postgres database. Asserts the bootstrap path is no-op on fresh installs and that SCHEMA_SQL replays cleanly through the engine path (not via the standalone `db.initSchema` from `src/core/db.ts`, which would have produced false-positive coverage). Codex caught the E2E-shape gap during plan review.
|
||||
- `test/e2e/http-transport.test.ts` (v0.22.7) — 8 cases against real Postgres covering `gbrain serve --http` end-to-end: bearer auth round-trip, `last_used_at` SQL-level debounce semantics, `mcp_request_log` row insertion on success and auth_failed paths, `/health` DB-down → 503 (DB-probing health check), and the F1+F2+F3 dispatch round-trip with a real operation. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/sync-parallel.test.ts` (v0.22.13 PR #490) — DATABASE_URL-gated. T2: 60-file Postgres sync at concurrency=4 imports all + no connection leak (probes `pg_stat_activity` before/after to confirm worker engines disconnected). P4: 120-file serial-vs-parallel benchmark prints `SYNC_PARALLEL_BENCH N files | serial=Xms | parallel(4)=Yms | speedup=Zx` for CHANGELOG quoting. Asserts parallel ≤ serial × 1.5 (CI-noise tolerant; not a strict speedup gate).
|
||||
- Tier 2 (`skills.test.ts`) requires OpenClaw + API keys, runs nightly in CI
|
||||
- If `.env.testing` doesn't exist in this directory, check sibling worktrees for one:
|
||||
`find ../ -maxdepth 2 -name .env.testing -print -quit` and copy it here if found.
|
||||
- Always run E2E tests when they exist. Do not skip them just because DATABASE_URL
|
||||
is not set. Start the test DB, run the tests, then tear it down.
|
||||
|
||||
### API keys and running ALL tests
|
||||
|
||||
ALWAYS source the user's shell profile before running tests:
|
||||
|
||||
```bash
|
||||
source ~/.zshrc 2>/dev/null || true
|
||||
```
|
||||
|
||||
This loads `OPENAI_API_KEY` and `ANTHROPIC_API_KEY`. Without these, Tier 2 tests
|
||||
skip silently. Do NOT skip Tier 2 tests just because they require API keys — load
|
||||
the keys and run them.
|
||||
|
||||
When asked to "run all E2E tests" or "run tests", that means ALL tiers:
|
||||
- Tier 1: `bun run test:e2e` (mechanical, sync, upgrade — no API keys needed)
|
||||
- Tier 2: `test/e2e/skills.test.ts` (requires OpenAI + Anthropic + openclaw CLI)
|
||||
- Always spin up the test DB, source zshrc, run everything, tear down.
|
||||
|
||||
### E2E test DB lifecycle (ALWAYS follow this)
|
||||
|
||||
You are responsible for spinning up and tearing down the test Postgres container.
|
||||
Do not leave containers running after tests. Do not skip E2E tests.
|
||||
|
||||
1. **Check for `.env.testing`** — if missing, copy from sibling worktree.
|
||||
Read it to get the DATABASE_URL (it has the port number).
|
||||
2. **Check if the port is free:**
|
||||
`docker ps --filter "publish=PORT"` — if another container is on that port,
|
||||
pick a different port (try 5435, 5436, 5437) and start on that one instead.
|
||||
3. **Start the test DB:**
|
||||
```bash
|
||||
docker run -d --name gbrain-test-pg \
|
||||
-e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres \
|
||||
-e POSTGRES_DB=gbrain_test \
|
||||
-p PORT:5432 pgvector/pgvector:pg16
|
||||
```
|
||||
Wait for ready: `docker exec gbrain-test-pg pg_isready -U postgres`
|
||||
4. **Run E2E tests:**
|
||||
`DATABASE_URL=postgresql://postgres:postgres@localhost:PORT/gbrain_test bun run test:e2e`
|
||||
5. **Tear down immediately after tests finish (pass or fail):**
|
||||
`docker stop gbrain-test-pg && docker rm gbrain-test-pg`
|
||||
|
||||
Never leave `gbrain-test-pg` running. If you find a stale one from a previous run,
|
||||
stop and remove it before starting a new one.
|
||||
|
||||
## Skills
|
||||
|
||||
Read the skill files in `skills/` before doing brain operations. GBrain ships 29 skills
|
||||
organized by `skills/RESOLVER.md` (`AGENTS.md` is also accepted as of v0.19):
|
||||
|
||||
**Original 8 (conformance-migrated):** ingest (thin router), query, maintain, enrich,
|
||||
briefing, migrate, setup, publish.
|
||||
|
||||
**Brain skills (ported from an upstream agent fork):** signal-detector, brain-ops, idea-ingest, media-ingest,
|
||||
meeting-ingestion, citation-fixer, repo-architecture, skill-creator, daily-task-manager.
|
||||
|
||||
**Operational + identity:** daily-task-prep, cross-modal-review, cron-scheduler, reports,
|
||||
testing, soul-audit, webhook-transforms, data-research, minion-orchestrator. As of
|
||||
v0.20.4, `minion-orchestrator` is the single unified skill for both lanes of background
|
||||
work (shell jobs via `gbrain jobs submit shell`, LLM subagents via `gbrain agent run`) ...
|
||||
the prior `gbrain-jobs` skill was merged in, Preconditions are shared, and trigger
|
||||
routing is narrowed to what the skill actually covers.
|
||||
|
||||
**Skillify loop (v0.19):** skillify (the markdown orchestration), skillpack-check
|
||||
(agent-readable health report).
|
||||
|
||||
**Operational health (v0.19.1):** smoke-test (8 post-restart health checks with auto-fix
|
||||
for Bun, CLI, DB, worker, Zod CJS, gateway, API key, brain repo; user-extensible via
|
||||
`~/.gbrain/smoke-tests.d/*.sh`).
|
||||
|
||||
**Conventions:** `skills/conventions/` has cross-cutting rules (quality, brain-first,
|
||||
model-routing, test-before-bulk, cross-modal). `skills/_brain-filing-rules.md` and
|
||||
`skills/_output-rules.md` are shared references.
|
||||
|
||||
## Bulk-action progress reporting
|
||||
|
||||
All bulk commands (doctor, embed, import, export, sync, extract, migrate,
|
||||
repair-jsonb, orphans, check-backlinks, lint, integrity auto, eval, files
|
||||
sync, and apply-migrations) stream progress through the shared reporter
|
||||
at `src/core/progress.ts`. Agents get heartbeats within 1 second of every
|
||||
iteration regardless of how slow the underlying work is.
|
||||
|
||||
Rules:
|
||||
- Progress always writes to **stderr**. Stdout stays clean for data output
|
||||
(`--json` payloads, final summaries, JSON action events from `extract`).
|
||||
- Non-TTY default: plain one-line-per-event human text. JSON requires the
|
||||
explicit `--progress-json` flag.
|
||||
- Global flags (`--quiet`, `--progress-json`, `--progress-interval=<ms>`)
|
||||
are parsed by `src/core/cli-options.ts` BEFORE command dispatch.
|
||||
- Phase names are machine-stable `snake_case.dot.path` (e.g.
|
||||
`doctor.db_checks`, `sync.imports`). Documented in
|
||||
`docs/progress-events.md`; additive changes only.
|
||||
- `scripts/check-progress-to-stdout.sh` is a CI guard that fails the build
|
||||
if any new code writes `\r` progress to stdout. Wired into `bun run test`.
|
||||
- Minion handlers pass `job.updateProgress` as the `onProgress` callback
|
||||
to core functions (DB-backed primary progress channel); stderr from
|
||||
`jobs work` stays coarse for daemon liveness only.
|
||||
|
||||
When wiring a new bulk command: `import { createProgress } from '../core/progress.ts'`
|
||||
and `import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts'`.
|
||||
Create a reporter with `createProgress(cliOptsToProgressOptions(getCliOptions()))`,
|
||||
`start(phase, total?)` before the loop, `tick()` inside it, `finish()` after.
|
||||
For single long-running queries, use `startHeartbeat(reporter, note)` with a
|
||||
try/finally to guarantee cleanup. Never call `process.stdout.write('\r...')`
|
||||
in bulk paths, the CI guard will fail the build.
|
||||
|
||||
## Build
|
||||
|
||||
`bun build --compile --outfile bin/gbrain src/cli.ts`
|
||||
|
||||
## Version locations (single source of truth: `VERSION` file)
|
||||
|
||||
Every release advances the version in **five files at once**. Keep these in
|
||||
sync. `/ship` enforces this via Step 12's idempotency check (VERSION vs
|
||||
package.json drift), but the canonical list lives here so future runs and
|
||||
the auto-update agent know where to look.
|
||||
|
||||
**Required (every release must update all five):**
|
||||
|
||||
| File | What lives there | Format |
|
||||
|---|---|---|
|
||||
| `VERSION` | The single source of truth. Read first by `/ship`, the binary, and CI version-gate. | Bare 4-digit string `MAJOR.MINOR.PATCH.MICRO` (e.g. `0.22.1`), no leading `v`, no trailing newline-sensitivity issues. |
|
||||
| `package.json` | Bun/npm package version. `gbrain --version` reads it via the compiled binary's bundled package metadata. CI version-gate cross-checks this against `VERSION` and fails if they drift. | `"version": "0.22.1"` |
|
||||
| `CHANGELOG.md` | Top entry header `## [0.22.1] - YYYY-MM-DD` plus the "To take advantage of v0.22.1" block. | Standard Keep-a-Changelog header. |
|
||||
| `TODOS.md` | Any TODO entries that mention "follow-up from vX.Y.Z" use the version of the release that filed them. Update only when filing NEW follow-up TODOs. | Inline `vX.Y.Z` references in TODO bodies. |
|
||||
| `CLAUDE.md` | The Key Files section's per-file annotations carry `vX.Y.Z (#NNN)` tags noting which release introduced a behavior. Update whenever a wave's annotations get folded in. | Inline `vX.Y.Z (#NNN, contributed by @user)` references. |
|
||||
|
||||
**Auto-derived (no manual edit; refreshed by their own commands):**
|
||||
|
||||
- `bun.lock` — root-package version is auto-pinned from `package.json`. After
|
||||
bumping `package.json`, run `bun install` to refresh the lockfile.
|
||||
- `llms-full.txt` / `llms.txt` — auto-generated documentation bundles. After
|
||||
any release ship that touches the Key Files annotations in `CLAUDE.md`,
|
||||
run `bun run build:llms` to regenerate. The bundles do not contain a
|
||||
version pin per se; they reflect the current state of the docs they index.
|
||||
|
||||
**Historical (DO NOT bump on release):**
|
||||
|
||||
- `skills/migrations/v0.21.0.md` — migration files use the version they
|
||||
shipped FROM as their filename. v0.21.0's migration always says v0.21.0.
|
||||
- `src/commands/migrations/v0_21_0.ts` — same: migration code references
|
||||
the schema version it migrates to.
|
||||
- `test/migrations-v0_21_0.test.ts`, `test/migration-orchestrator-v0_21_0.test.ts`,
|
||||
`test/migrate.test.ts` — migration tests reference historical migration
|
||||
versions; these are correct as-is and should not move.
|
||||
- `src/core/db.ts`, `src/core/migrate.ts`, `src/core/import-file.ts`,
|
||||
`src/commands/reindex-code.ts` — code comments cite the release that
|
||||
introduced a feature. Once written, these are historical record.
|
||||
- `README.md` — references the latest published feature names by version
|
||||
(e.g. "v0.21.0 Code Cathedral"); update only when the README's marketing
|
||||
copy is intentionally being refreshed, NOT on every micro/patch bump.
|
||||
|
||||
**The /ship workflow's version idempotency check:** Step 12 reads
|
||||
`VERSION` and `package.json`, classifies as FRESH / ALREADY_BUMPED /
|
||||
DRIFT_STALE_PKG / DRIFT_UNEXPECTED, and refuses to proceed on
|
||||
DRIFT_UNEXPECTED. This is why the two must move together.
|
||||
|
||||
**The CI version-gate** rejects pushes where `VERSION` and
|
||||
`package.json` disagree, OR where `VERSION` is not strictly greater
|
||||
than master's VERSION. If a queue collision claims your version on
|
||||
master before yours lands, /ship's queue-aware allocator (Step 12)
|
||||
will detect drift and re-bump on the next run.
|
||||
|
||||
## Pre-ship requirements
|
||||
|
||||
Before shipping (/ship) or reviewing (/review), always run the full test suite.
|
||||
Two equivalent paths:
|
||||
|
||||
**Path A — local CI gate (recommended, v0.23.1+):**
|
||||
- `bun run ci:local` runs the entire stack inside Docker: gitleaks (host), unit
|
||||
tests with `DATABASE_URL` unset, and all 29 E2E files sequentially against a
|
||||
fresh pgvector container. Stronger than PR CI's 2-file Tier 1 set; closer to
|
||||
what nightly Tier 1 catches. Spins up + tears down postgres automatically via
|
||||
`docker-compose.ci.yml`. Override the host port with
|
||||
`GBRAIN_CI_PG_PORT=5435 bun run ci:local` if 5434 collides.
|
||||
- `bun run ci:local:diff` runs only the E2E files matched by the diff selector
|
||||
(`scripts/select-e2e.ts`), falling back to all 29 on unmapped src/ paths or
|
||||
schema/skills/package.json changes. Fast iteration during a focused branch.
|
||||
|
||||
**Path B — manual lifecycle (still supported):**
|
||||
- `bun test` — unit tests (no database required)
|
||||
- Follow the "E2E test DB lifecycle" steps above to spin up the test DB,
|
||||
run `bun run test:e2e`, then tear it down.
|
||||
|
||||
Both must pass. Do not ship with failing E2E tests. Do not skip E2E tests.
|
||||
|
||||
**Always run typecheck before pushing.** `bun test` (the bun runner)
|
||||
skips TypeScript type checking — it only enforces runtime behavior.
|
||||
Three ways to actually gate on types:
|
||||
|
||||
1. `bun run test` (npm script in `package.json`) — includes `bun run typecheck`
|
||||
plus the four shell pre-checks (`check-jsonb-pattern.sh`,
|
||||
`check-progress-to-stdout.sh`, `check-trailing-newline.sh`,
|
||||
`check-wasm-embedded.sh`) before the runner. Use this mid-branch.
|
||||
2. `bun run typecheck` — `tsc --noEmit` standalone. Fast (~5s on this repo).
|
||||
3. `bun run ci:local` — the full local CI gate from Path A.
|
||||
|
||||
The trap is: writing a new test, running `bun test test/foo.test.ts`,
|
||||
seeing it pass, pushing — and CI's separate typecheck stage rejects an
|
||||
invalid type literal that the runner accepted. Caught one of these
|
||||
shipping the v0.23.2 round-trip E2E (`type: 'reflection'` is not a
|
||||
member of `PageType`). Run `bun run typecheck` once before push, even
|
||||
when only test files changed.
|
||||
|
||||
## Post-ship requirements (MANDATORY)
|
||||
|
||||
After EVERY /ship, you MUST run /document-release. This is NOT optional. Do NOT
|
||||
skip it. Do NOT say "docs look fine" without running it. The skill reads every .md
|
||||
file in the project, cross-references the diff, and updates anything that drifted.
|
||||
|
||||
If /ship's Step 8.5 triggers document-release automatically, that counts. But if
|
||||
it gets skipped for ANY reason (timeout, error, oversight), you MUST run it manually
|
||||
before considering the ship complete.
|
||||
|
||||
Files that MUST be checked on every ship:
|
||||
- README.md — does it reflect new features, commands, or setup steps?
|
||||
- CLAUDE.md — does it reflect new files, test files, or architecture changes?
|
||||
- CHANGELOG.md — does it cover every commit?
|
||||
- TODOS.md — are completed items marked done?
|
||||
- docs/ — do any guides need updating?
|
||||
|
||||
A ship without updated docs is an incomplete ship. Period.
|
||||
|
||||
## CHANGELOG + VERSION are branch-scoped
|
||||
|
||||
**VERSION and CHANGELOG describe what THIS branch adds vs master, not how we got
|
||||
here.** Every feature branch that ships gets its own version bump and CHANGELOG
|
||||
entry. The entry is product release notes for users; it is not a log of internal
|
||||
decisions, review rounds, or codex findings.
|
||||
|
||||
**Write the CHANGELOG entry at /ship time, not during development.** Mid-branch
|
||||
iterations, review rounds (CEO/Eng/Codex/DX), and implementation detours belong
|
||||
in the plan file at `~/.claude/plans/`, not in the CHANGELOG. One unified entry
|
||||
per branch, covering what the branch added vs the base branch.
|
||||
|
||||
**Never edit a CHANGELOG entry that already landed on master.** If master has
|
||||
v0.18.2 and your branch adds features, bump to the next version (v0.19.0, not
|
||||
editing master's v0.18.2). When merging master into your branch, master may
|
||||
bring new CHANGELOG entries above yours — push your entry above master's
|
||||
latest and verify:
|
||||
|
||||
- Does CHANGELOG have your branch's own entry separate from master's entries?
|
||||
- Is VERSION higher than master's VERSION?
|
||||
- Is your entry the topmost `## [X.Y.Z]` entry?
|
||||
- `grep "^## \[" CHANGELOG.md` shows a contiguous version sequence?
|
||||
|
||||
If any answer is no, fix it before continuing.
|
||||
|
||||
**CHANGELOG is for users, not contributors.** Write like product release notes:
|
||||
|
||||
- Lead with what the user can now **do** that they couldn't before. Sell the capability.
|
||||
- Plain language, not implementation details. "You can now..." not "Refactored the..."
|
||||
- **Never mention internal artifacts**: plan file IDs, decision tags (D-CX-#, F-ENG-#),
|
||||
review rounds, codex findings, subcontractor credits. These are invisible to users.
|
||||
- Put contributor-facing changes in a separate `### For contributors` section at the bottom.
|
||||
- Every entry should make someone think "oh nice, I want to try that."
|
||||
|
||||
**What to omit:**
|
||||
- "Codex caught X that the CEO review missed" — private process detail.
|
||||
- "D-CX-3 split errors/warnings" — tag is meaningless to users; name the feature instead.
|
||||
- "Fix-wave PR #N supersedes #M" — supersede chains belong in PR bodies, not release notes.
|
||||
- "215 new cases, 3 decisions applied, 7 reviews cleared" — these are planning-mode metrics.
|
||||
|
||||
**What to keep:**
|
||||
- The user-facing change: what commands exist now, what flag was added, what behavior fixed.
|
||||
- Numbers that mean something to the user: TTHW, commands that timed out before, detection counts.
|
||||
- Upgrade instructions: `gbrain upgrade` + any manual step if needed.
|
||||
- Credit to external contributors when a community PR was incorporated.
|
||||
|
||||
## CHANGELOG voice + release-summary format
|
||||
|
||||
Every version entry in `CHANGELOG.md` MUST start with a release-summary section in
|
||||
the GStack/Garry voice — one viewport's worth of prose + tables that lands like a
|
||||
verdict, not marketing. The itemized changelog (subsections, bullets, files) goes
|
||||
BELOW that summary, separated by a `### Itemized changes` header.
|
||||
|
||||
The release-summary section gets read by humans, by the auto-update agent, and by
|
||||
anyone deciding whether to upgrade. The itemized list is for agents that need to
|
||||
know exactly what changed.
|
||||
|
||||
### Release-summary template
|
||||
|
||||
Use this structure for the top of every `## [X.Y.Z]` entry:
|
||||
|
||||
1. **Two-line bold headline** (10-14 words total) ... should land like a verdict, not
|
||||
marketing. Sound like someone who shipped today and cares whether it works.
|
||||
2. **Lead paragraph** (3-5 sentences) ... what shipped, what changed for the user.
|
||||
Specific, concrete, no AI vocabulary, no em dashes, no hype.
|
||||
3. **A "The X numbers that matter" section** with:
|
||||
- One short setup paragraph naming the source of the numbers (real production
|
||||
deployment OR a reproducible benchmark ... name the file/command to run).
|
||||
- A table of 3-6 key metrics with BEFORE / AFTER / Δ columns.
|
||||
- A second optional table for per-category breakdown if relevant.
|
||||
- 1-2 sentences interpreting the most striking number in concrete user terms.
|
||||
4. **A "What this means for [audience]" closing paragraph** (2-4 sentences) tying
|
||||
the metrics to a real workflow shift. End with what to do.
|
||||
|
||||
Voice rules:
|
||||
- No em dashes (use commas, periods, "...").
|
||||
- No AI vocabulary (delve, robust, comprehensive, nuanced, fundamental, etc.) or
|
||||
banned phrases ("here's the kicker", "the bottom line", etc.).
|
||||
- Real numbers, real file names, real commands. Not "fast" but "~30s on 30K pages."
|
||||
- Short paragraphs, mix one-sentence punches with 2-3 sentence runs.
|
||||
- Connect to user outcomes: "the agent does ~3x less reading" beats "improved
|
||||
precision."
|
||||
- Be direct about quality. "Well-designed" or "this is a mess." No dancing.
|
||||
|
||||
Source material to pull from:
|
||||
- CHANGELOG.md previous entry for prior context
|
||||
- Latest `gbrain-evals/docs/benchmarks/[latest].md` for headline numbers (sibling repo)
|
||||
- Recent commits (`git log <prev-version>..HEAD --oneline`) for what shipped
|
||||
- Don't make up numbers. If a metric isn't in a benchmark or production data, don't
|
||||
include it. Say "no measurement yet" if asked.
|
||||
|
||||
Target length: ~250-350 words for the summary. Should render as one viewport.
|
||||
|
||||
### "To take advantage of v[version]" block (required, v0.13+)
|
||||
|
||||
After the release-summary and BEFORE `### Itemized changes`, every `## [X.Y.Z]`
|
||||
entry MUST include a human-readable self-repair block under the heading
|
||||
`## To take advantage of v[version]`.
|
||||
|
||||
Why: `gbrain upgrade` runs `gbrain post-upgrade` which runs `gbrain apply-migrations`.
|
||||
This chain has a known weak link — `upgrade.ts` catches post-upgrade failures as
|
||||
best-effort (so the binary still works). When that chain silently fails, users end
|
||||
up with half-upgraded brains. The self-repair block gives them a paste-ready
|
||||
recovery path; the v0.13+ `~/.gbrain/upgrade-errors.jsonl` trail + `gbrain doctor`
|
||||
integration close the loop.
|
||||
|
||||
Template (adapt the verify commands per release):
|
||||
|
||||
```markdown
|
||||
## To take advantage of v[version]
|
||||
|
||||
`gbrain upgrade` should do this automatically. If it didn't, or if `gbrain doctor`
|
||||
warns about a partial migration:
|
||||
|
||||
1. **Run the orchestrator manually:**
|
||||
```bash
|
||||
gbrain apply-migrations --yes
|
||||
```
|
||||
2. **Your agent reads `skills/migrations/v[version].md` the next time you interact with it.**
|
||||
[One sentence on whether headless agents need manual action, or whether the
|
||||
orchestrator already handled the mechanical side.]
|
||||
3. **Verify the outcome:**
|
||||
```bash
|
||||
[release-specific verify commands, e.g. `gbrain graph ... --depth 2`]
|
||||
gbrain stats
|
||||
```
|
||||
4. **If any step fails or the numbers look wrong,** please file an issue:
|
||||
https://github.com/garrytan/gbrain/issues with:
|
||||
- output of `gbrain doctor`
|
||||
- contents of `~/.gbrain/upgrade-errors.jsonl` if it exists
|
||||
- which step broke
|
||||
|
||||
This feedback loop is how the gbrain maintainers find fragile upgrade paths. Thank you.
|
||||
```
|
||||
|
||||
**Skip this block** for patches that are pure bug fixes with zero user-facing action
|
||||
(rare). If the release has a schema migration, data backfill, or new feature the
|
||||
user needs to verify, the block is required.
|
||||
|
||||
The v0.13.0 entry in CHANGELOG.md is the canonical example.
|
||||
|
||||
### Itemized changes (the existing rules)
|
||||
|
||||
Below the release summary, write `### Itemized changes` and continue with the
|
||||
detailed subsections (Knowledge Graph Layer, Schema migrations, Security hardening,
|
||||
Tests, etc.). Same rules as before:
|
||||
|
||||
- Lead with what the user can now DO that they couldn't before
|
||||
- Frame as benefits and capabilities, not files changed or code written
|
||||
- Make the user think "hell yeah, I want that"
|
||||
- Bad: "Added GBRAIN_VERIFY.md installation verification runbook"
|
||||
- Good: "Your agent now verifies the entire GBrain installation end-to-end, catching
|
||||
silent sync failures and stale embeddings before they bite you"
|
||||
- Bad: "Setup skill Phase H and Phase I added"
|
||||
- Good: "New installs automatically set up live sync so your brain never falls behind"
|
||||
- **Always credit community contributions.** When a CHANGELOG entry includes work from
|
||||
a community PR, name the contributor with `Contributed by @username`. Contributors
|
||||
did real work. Thank them publicly every time, no exceptions.
|
||||
|
||||
### Reference: v0.12.0 entry as canonical example
|
||||
|
||||
The v0.12.0 entry in CHANGELOG.md is the canonical example of the format. Match its
|
||||
structure for every future version: bold headline, lead paragraph, "numbers that
|
||||
matter" with BrainBench-style before/after table, "what this means" closer, then
|
||||
`### Itemized changes` with the detailed sections below.
|
||||
|
||||
## Version migrations
|
||||
|
||||
Create a migration file at `skills/migrations/v[version].md` when a release
|
||||
includes changes that existing users need to act on. The auto-update agent
|
||||
reads these files post-upgrade (Section 17, Step 4) and executes them.
|
||||
|
||||
**You need a migration file when:**
|
||||
- New setup step that existing installs don't have (e.g., v0.5.0 added live sync,
|
||||
existing users need to set it up, not just new installs)
|
||||
- New SKILLPACK section with a MUST ADD setup requirement
|
||||
- Schema changes that require `gbrain init` or manual SQL
|
||||
- Changed defaults that affect existing behavior
|
||||
- Deprecated commands or flags that need replacement
|
||||
- New verification steps that should run on existing installs
|
||||
- New cron jobs or background processes that should be registered
|
||||
|
||||
**You do NOT need a migration file when:**
|
||||
- Bug fixes with no behavior changes
|
||||
- Documentation-only improvements (the agent re-reads docs automatically)
|
||||
- New optional features that don't affect existing setups
|
||||
- Performance improvements that are transparent
|
||||
|
||||
**The key test:** if an existing user upgrades and does nothing else, will their
|
||||
brain work worse than before? If yes, migration file. If no, skip it.
|
||||
|
||||
Write migration files as agent instructions, not technical notes. Tell the agent
|
||||
what to do, step by step, with exact commands. See `skills/migrations/v0.5.0.md`
|
||||
for the pattern.
|
||||
|
||||
## Migration is canonical, not advisory
|
||||
|
||||
GBrain's job is to deliver a canonical, working setup to every user on upgrade.
|
||||
Anything that looks like a "host-repo change" — AGENTS.md, cron manifests,
|
||||
launchctl units, config files outside `~/.gbrain/` — is a GBrain migration
|
||||
step, not a nudge we leave for the host-repo maintainer. Migrations edit host
|
||||
files (with backups) to make the canonical setup real. Exceptions: changes
|
||||
that require human judgment (content edits, renames that break semantics,
|
||||
host-specific handler registration where shell-exec would be an RCE surface).
|
||||
Everything mechanical ships in the migration.
|
||||
|
||||
**Test:** if shipping a feature requires a sentence that starts with "in
|
||||
your AGENTS.md, add…" or "in your cron/jobs.json, rewrite…", the migration
|
||||
orchestrator should be doing that edit, not the user.
|
||||
|
||||
**The exception is host-specific code.** For custom Minion handlers
|
||||
(host-specific integrations like inbox sweeps or third-party API scanners), shipping them as a
|
||||
data file the worker would exec is an RCE surface. Those get registered in
|
||||
the host's own repo via the plugin contract (`docs/guides/plugin-handlers.md`);
|
||||
the migration orchestrator emits a structured TODO to
|
||||
`~/.gbrain/migrations/pending-host-work.jsonl` + the host agent walks the
|
||||
TODOs using `skills/migrations/v0.11.0.md` — stays host-agnostic, still
|
||||
canonical.
|
||||
|
||||
## Privacy rule: scrub real names from public docs
|
||||
|
||||
**Never reference real people, companies, funds, or private agent names in any
|
||||
public-facing artifact.** Public artifacts include: `CHANGELOG.md`, `README.md`,
|
||||
`docs/`, `skills/`, PR titles + bodies, commit messages, and comments in checked-in
|
||||
code. Query examples, benchmark stories, and migration guides MUST use generic
|
||||
placeholders.
|
||||
|
||||
Why: gbrain runs a personal knowledge brain containing notes on real people and
|
||||
real companies (YC founders, portfolio companies, funds, investors, meeting
|
||||
attendees). When a doc copies a query like `gbrain graph diana-hu --depth 2` or
|
||||
names a specific agent fork like `Wintermute`, that real name gets indexed by
|
||||
search engines, surfaced in cross-references, and distributed with every release.
|
||||
|
||||
**Name mapping** to use in examples:
|
||||
- Agent forks → `your agent fork`, `a downstream agent`, or `agent-fork`
|
||||
- Example person → `alice-example`, `charlie-example`, or `a-founder`
|
||||
- Example company → `acme-example`, `widget-co`, or `a-company`
|
||||
- Example fund → `fund-a`, `fund-b`, `fund-c`
|
||||
- Example deal → `acme-seed`, `widget-series-a`
|
||||
- Example meeting → `meetings/2026-04-03` (generic date is fine)
|
||||
- Example user → `you` or `the user`, never a proper name
|
||||
|
||||
**Specific rule: never say `Wintermute` in any CHANGELOG, README, doc, PR, or
|
||||
commit message.** When the temptation is to illustrate with the real fork name:
|
||||
- Reader-facing copy → `your OpenClaw` (covers Wintermute, Hermes, AlphaClaw,
|
||||
and any other downstream OpenClaw deployment in one term the reader already
|
||||
recognizes).
|
||||
- First-person / origin-story copy → `Garry's OpenClaw` (honest that this is
|
||||
the production deployment driving the feature, without exposing the private
|
||||
agent's name).
|
||||
|
||||
`Wintermute` may appear in private artifacts (scratch plans under
|
||||
`~/.gstack/projects/…`, memory files, conversation transcripts, CEO-review
|
||||
plans) — those aren't distributed. Anything checked into this repo or shipped
|
||||
in a release must use the OpenClaw phrasing above. Sweeping a stale reference
|
||||
is a small clean-up PR, not a debate.
|
||||
|
||||
**When in doubt, ask yourself:** "Would this query reveal private information
|
||||
about the user's contacts, investments, or portfolio if it were read by a
|
||||
stranger?" If yes, replace with generic placeholders.
|
||||
|
||||
**Illustrative API examples with household-brand companies** (Stripe, Brex, OpenAI,
|
||||
GitHub, etc.) are fine — they're public entities, not contacts in anyone's brain.
|
||||
Do not confuse illustrative API examples with queries that reveal real
|
||||
relationships.
|
||||
|
||||
## Responsible-disclosure rule: don't broadcast attack surface in release notes
|
||||
|
||||
**When a release fixes a security gap or a user-impacting bug, describe the fix
|
||||
functionally. Do not enumerate the attack surface, quantify the exposure window,
|
||||
or highlight the most sensitive records by name in public-facing artifacts.**
|
||||
|
||||
Public-facing artifacts include: `CHANGELOG.md`, `README.md`, `docs/`, PR titles
|
||||
and bodies, commit messages, GitHub issue titles and comments, release pages,
|
||||
tweets, blog posts.
|
||||
|
||||
**Don't write:**
|
||||
- "10 tables were publicly readable by the anon key for months, including X, Y, Z"
|
||||
- "X and Y are the most sensitive ones"
|
||||
- "N tables exposed. Fix: enable RLS on these specific tables: ..."
|
||||
|
||||
**Do write:**
|
||||
- "Security hardening pass. Fresh installs secure by default. Existing brains
|
||||
brought to the same bar automatically on upgrade."
|
||||
- "If `gbrain doctor` still flags anything after upgrade, the message names each
|
||||
table and gives the exact fix."
|
||||
|
||||
Why: anyone reading the release page before they've upgraded now has a directed
|
||||
probe list for unpatched installs. The source code ships the specifics anyway
|
||||
(`src/schema.sql`, `src/core/migrate.ts`, test fixtures) — reverse engineers can
|
||||
get them. But the release page is a broadcast channel. Don't hand attackers a
|
||||
curated list with a banner.
|
||||
|
||||
**The test:** if a reader with no prior context could read the release note and
|
||||
walk away knowing "gbrain at version X has table Y readable by anon key until
|
||||
they patch," the note is too specific. Rewrite until that's no longer possible.
|
||||
|
||||
**What IS fine in public artifacts:**
|
||||
- The mechanism of the fix ("the check now scans every public table instead of
|
||||
a hardcoded allowlist").
|
||||
- User-facing operator ergonomics (the escape-hatch SQL template, the upgrade
|
||||
commands, the breaking-change flag).
|
||||
- Credit to contributors.
|
||||
- Generic framing of severity ("security posture tightening pass") without
|
||||
quantification.
|
||||
|
||||
**What stays in private artifacts (plan files, private memories, internal docs):**
|
||||
- Specific table names, record counts, exposure duration.
|
||||
- Which records stand out as highest-risk.
|
||||
- Detailed before/after tables in the "numbers that matter" format.
|
||||
|
||||
If the CEO/Eng review of a plan produces a detailed exposure table, keep it in
|
||||
the plan file under `~/.claude/plans/` or `~/.gstack/projects/`. Don't copy it
|
||||
into the CHANGELOG or PR body.
|
||||
|
||||
Applies retroactively: if you see a prior CHANGELOG entry naming attack-surface
|
||||
specifics, scrub it as a small cleanup commit, the same way a stale Wintermute
|
||||
reference gets swept.
|
||||
|
||||
## Schema state tracking
|
||||
|
||||
`~/.gbrain/update-state.json` tracks which recommended schema directories the user
|
||||
adopted, declined, or added custom. The auto-update agent (SKILLPACK Section 17)
|
||||
reads this during upgrades to suggest new schema additions without re-suggesting
|
||||
things the user already declined. The setup skill writes the initial state during
|
||||
Phase C/E. Never modify a user's custom directories or re-suggest declined ones.
|
||||
|
||||
## GitHub Actions SHA maintenance
|
||||
|
||||
All GitHub Actions in `.github/workflows/` are pinned to commit SHAs. Before shipping
|
||||
(`/ship`) or reviewing (`/review`), check for stale pins and update them:
|
||||
|
||||
```bash
|
||||
for action in actions/checkout oven-sh/setup-bun actions/upload-artifact actions/download-artifact softprops/action-gh-release gitleaks/gitleaks-action; do
|
||||
tag=$(grep -r "$action@" .github/workflows/ | head -1 | grep -o '#.*' | tr -d '# ')
|
||||
[ -n "$tag" ] && echo "$action@$tag: $(gh api repos/$action/git/ref/tags/$tag --jq .object.sha 2>/dev/null)"
|
||||
done
|
||||
```
|
||||
|
||||
If any SHA differs from what's in the workflow files, update the pin and version comment.
|
||||
|
||||
## PR descriptions cover the whole branch
|
||||
|
||||
Pull request titles and bodies must describe **everything in the PR diff against the
|
||||
base branch**, not just the most recent commit you made. When you open or update a
|
||||
PR, walk the full commit range with `git log --oneline <base>..<head>` and write the
|
||||
body to cover all of it. Group by feature area (schema, code, tests, docs) — not
|
||||
chronologically by commit.
|
||||
|
||||
This matters because reviewers read the PR body to understand what's shipping. If
|
||||
the body only covers your last commit, they miss everything else and can't review
|
||||
properly. A 7-commit PR with a body that describes commit 7 is worse than no body
|
||||
at all — it actively misleads.
|
||||
|
||||
When in doubt, run `gh pr view <N> --json commits --jq '[.commits[].messageHeadline]'`
|
||||
to see what's actually in the PR before writing the body.
|
||||
|
||||
## Community PR wave process
|
||||
|
||||
Never merge external PRs directly into master. Instead, use the "fix wave" workflow:
|
||||
|
||||
1. **Categorize** — group PRs by theme (bug fixes, features, infra, docs)
|
||||
2. **Deduplicate** — if two PRs fix the same thing, pick the one that changes fewer
|
||||
lines. Close the other with a note pointing to the winner.
|
||||
3. **Collector branch** — create a feature branch (e.g. `garrytan/fix-wave-N`), cherry-pick
|
||||
or manually re-implement the best fixes from each PR. Do NOT merge PR branches directly —
|
||||
read the diff, understand the fix, and write it yourself if needed.
|
||||
4. **Test the wave** — verify with `bun test && bun run test:e2e` (full E2E lifecycle).
|
||||
Every fix in the wave must have test coverage.
|
||||
5. **Close with context** — every closed PR gets a comment explaining why and what (if
|
||||
anything) supersedes it. Contributors did real work; respect that with clear communication
|
||||
and thank them.
|
||||
6. **Ship as one PR** — single PR to master with all attributions preserved via
|
||||
`Co-Authored-By:` trailers. Include a summary of what merged and what closed.
|
||||
|
||||
**Community PR guardrails:**
|
||||
- Always AskUserQuestion before accepting commits that touch voice, tone, or
|
||||
promotional material (README intro, CHANGELOG voice, skill templates).
|
||||
- Never auto-merge PRs that remove YC references or "neutralize" the founder perspective.
|
||||
- Preserve contributor attribution in commit messages.
|
||||
|
||||
## Skill routing
|
||||
|
||||
When the user's request matches an available skill, ALWAYS invoke it using the Skill
|
||||
tool as your FIRST action. Do NOT answer directly, do NOT use other tools first.
|
||||
The skill has specialized workflows that produce better results than ad-hoc answers.
|
||||
|
||||
**NEVER hand-roll ship operations.** Do not manually run git commit + push + gh pr
|
||||
create when /ship is available. /ship handles VERSION bump, CHANGELOG, document-release,
|
||||
pre-landing review, test coverage audit, and adversarial review. Manually creating a PR
|
||||
skips all of these. If the user says "commit and ship", "push and ship", "bisect and
|
||||
ship", or any combination that ends with shipping — invoke /ship and let it handle
|
||||
everything including the commits. If the branch name contains a version (e.g.
|
||||
`v0.5-live-sync`), /ship should use that version for the bump.
|
||||
|
||||
Key routing rules:
|
||||
- Product ideas, "is this worth building", brainstorming → invoke office-hours
|
||||
- Bugs, errors, "why is this broken", 500 errors → invoke investigate
|
||||
- Ship, deploy, push, create PR, "commit and ship", "push and ship" → invoke ship
|
||||
- QA, test the site, find bugs → invoke qa
|
||||
- Code review, check my diff → invoke review
|
||||
- Update docs after shipping → invoke document-release
|
||||
- Weekly retro → invoke retro
|
||||
- Design system, brand → invoke design-consultation
|
||||
- Visual audit, design polish → invoke design-review
|
||||
- Architecture review → invoke plan-eng-review
|
||||
- Save progress, checkpoint, resume → invoke checkpoint
|
||||
- Code quality, health check → invoke health
|
||||
-211
@@ -1,211 +0,0 @@
|
||||
# Contributing to GBrain
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
git clone https://github.com/garrytan/gbrain.git
|
||||
cd gbrain
|
||||
bun install
|
||||
bun test
|
||||
```
|
||||
|
||||
Requires Bun 1.0+.
|
||||
|
||||
## Project structure
|
||||
|
||||
```
|
||||
src/
|
||||
cli.ts CLI entry point
|
||||
commands/ CLI-only commands (init, upgrade, import, export, etc.)
|
||||
core/
|
||||
operations.ts Contract-first operation definitions (the foundation)
|
||||
engine.ts BrainEngine interface
|
||||
postgres-engine.ts Postgres implementation
|
||||
db.ts Connection management + schema loader
|
||||
import-file.ts Import pipeline (chunk + embed + tags)
|
||||
types.ts TypeScript types
|
||||
markdown.ts Frontmatter parsing
|
||||
config.ts Config file management
|
||||
storage.ts Pluggable storage interface
|
||||
storage/ Storage backends (S3, Supabase, local)
|
||||
supabase-admin.ts Supabase admin API
|
||||
file-resolver.ts MIME detection + content hashing
|
||||
migrate.ts Migration helpers
|
||||
yaml-lite.ts Lightweight YAML parser
|
||||
chunkers/ 3-tier chunking (recursive, semantic, llm)
|
||||
search/ Hybrid search (vector, keyword, hybrid, expansion, dedup)
|
||||
embedding.ts OpenAI embedding service
|
||||
mcp/
|
||||
server.ts MCP stdio server (generated from operations)
|
||||
schema.sql Postgres DDL
|
||||
skills/ Fat markdown skills for AI agents
|
||||
test/ Unit tests (bun test, no DB required)
|
||||
test/e2e/ E2E tests (requires DATABASE_URL, real Postgres+pgvector)
|
||||
fixtures/ Miniature realistic brain corpus (16 files)
|
||||
helpers.ts DB lifecycle, fixture import, timing
|
||||
mechanical.test.ts All operations against real DB
|
||||
mcp.test.ts MCP tool generation verification
|
||||
skills.test.ts Tier 2 skill tests (requires OpenClaw + API keys)
|
||||
docs/ Architecture docs
|
||||
```
|
||||
|
||||
## Running tests
|
||||
|
||||
```bash
|
||||
# Recommended: full CI guard chain + tests (matches what CI runs)
|
||||
bun run test # privacy + jsonb + progress + wasm + typecheck + bun test
|
||||
|
||||
# Just the test runner (skips CI guards)
|
||||
bun test # all tests (unit + E2E skipped without DB)
|
||||
bun test test/markdown.test.ts # specific unit test
|
||||
|
||||
# E2E tests (requires Postgres with pgvector)
|
||||
docker compose -f docker-compose.test.yml up -d
|
||||
DATABASE_URL=postgresql://postgres:postgres@localhost:5434/gbrain_test bun run test:e2e
|
||||
|
||||
# Or use your own Postgres / Supabase
|
||||
DATABASE_URL=postgresql://... bun run test:e2e
|
||||
```
|
||||
|
||||
Use `bun run test` before pushing. The guard chain catches: banned fork-name leaks
|
||||
(`scripts/check-privacy.sh`), `JSON.stringify(x)::jsonb` interpolation patterns
|
||||
(`scripts/check-jsonb-pattern.sh`), `\r` progress bleed to stdout
|
||||
(`scripts/check-progress-to-stdout.sh`), trailing-newline drift across tracked
|
||||
files (`scripts/check-trailing-newline.sh`), and silent fallback to recursive
|
||||
chunking in the compiled binary (`scripts/check-wasm-embedded.sh`).
|
||||
|
||||
### Local CI gate (recommended before pushing, v0.23.1+)
|
||||
|
||||
```bash
|
||||
bun run ci:local # full gate: gitleaks + unit + ALL 29 E2E files (sequential)
|
||||
bun run ci:local:diff # gate with diff-aware E2E selector
|
||||
bun run ci:select-e2e # print which E2E files the selector would run
|
||||
```
|
||||
|
||||
`ci:local` spins up `pgvector/pgvector:pg16` + `oven/bun:1` via
|
||||
`docker-compose.ci.yml`, runs everything PR CI runs plus the full E2E suite, then
|
||||
tears down. Named volumes keep the install warm across runs (~16-20 min sequential
|
||||
E2E after the first cold pull). Requires Docker (Docker Desktop, OrbStack, or
|
||||
Colima) and `gitleaks` on host (`brew install gitleaks`). Override the postgres
|
||||
host port with `GBRAIN_CI_PG_PORT=5435 bun run ci:local` if 5434 collides.
|
||||
|
||||
Fail-closed selector: an unmapped `src/` change runs all 29 E2E files. Hand-tune
|
||||
narrower mappings via `scripts/e2e-test-map.ts`.
|
||||
|
||||
## Building
|
||||
|
||||
```bash
|
||||
bun build --compile --outfile bin/gbrain src/cli.ts
|
||||
```
|
||||
|
||||
## Adding a new operation
|
||||
|
||||
GBrain uses a contract-first architecture. Add your operation to one file and it
|
||||
automatically appears in the CLI, MCP server, and tools-json:
|
||||
|
||||
1. Add your operation to `src/core/operations.ts` (define params, handler, cliHints)
|
||||
2. Add tests
|
||||
3. That's it. The CLI, MCP server, and tools-json are generated from operations.
|
||||
|
||||
For CLI-only commands (init, upgrade, import, export, files, embed, doctor, sync):
|
||||
1. Create `src/commands/mycommand.ts`
|
||||
2. Add the case to `src/cli.ts`
|
||||
|
||||
Parity tests (`test/parity.test.ts`) verify CLI/MCP/tools-json stay in sync.
|
||||
|
||||
## Adding a new engine
|
||||
|
||||
See `docs/ENGINES.md` for the full guide. In short:
|
||||
|
||||
1. Create `src/core/myengine-engine.ts` implementing `BrainEngine`
|
||||
2. Add to engine factory in `src/core/engine.ts`
|
||||
3. Run the test suite against your engine
|
||||
4. Document in `docs/`
|
||||
|
||||
The SQLite engine is designed and ready for implementation. See `docs/SQLITE_ENGINE.md`.
|
||||
|
||||
## CONTRIBUTOR_MODE — turn on the dev loop
|
||||
|
||||
gbrain captures retrieval traffic so you can replay real queries against
|
||||
your code changes before merging. **This is off by default** (production
|
||||
users get a quiet brain, no surprise data accumulation). Contributors turn
|
||||
it on with one shell rc line:
|
||||
|
||||
```bash
|
||||
# In ~/.zshrc or ~/.bashrc:
|
||||
export GBRAIN_CONTRIBUTOR_MODE=1
|
||||
```
|
||||
|
||||
That's it. Every `query` / `search` you (or agents pointed at your dev
|
||||
brain) run from that shell now writes a row to `eval_candidates`, and the
|
||||
[replay tool](#running-real-world-eval-benchmarks-touching-retrieval-code)
|
||||
has data to work against.
|
||||
|
||||
What CONTRIBUTOR_MODE actually does:
|
||||
|
||||
- Turns on `query`/`search` capture into the local `eval_candidates` table.
|
||||
Without it the gate is closed and capture is a no-op.
|
||||
- That's all. PII scrubbing, retention, and replay are independent.
|
||||
|
||||
Resolution order (most explicit wins):
|
||||
|
||||
1. `eval.capture: true` in `~/.gbrain/config.json` → on
|
||||
2. `eval.capture: false` in `~/.gbrain/config.json` → off
|
||||
3. `GBRAIN_CONTRIBUTOR_MODE=1` → on
|
||||
4. otherwise → off
|
||||
|
||||
Quick check that capture is actually running:
|
||||
|
||||
```bash
|
||||
gbrain query "anything" >/dev/null
|
||||
psql $DATABASE_URL -c 'SELECT count(*) FROM eval_candidates'
|
||||
# (or `gbrain doctor` — surfaces silent capture failures cross-process)
|
||||
```
|
||||
|
||||
To disable capture even with the env var set, write
|
||||
`{"eval": {"capture": false}}` to `~/.gbrain/config.json` — explicit config
|
||||
beats the env var both directions.
|
||||
|
||||
## Running real-world eval benchmarks (touching retrieval code)
|
||||
|
||||
If your PR touches retrieval — search ranking, RRF fusion, embeddings,
|
||||
intent classification, query expansion, source boost, or the `query` /
|
||||
`search` op handlers — run `gbrain eval replay` against a snapshot of
|
||||
real traffic before merging. Requires `CONTRIBUTOR_MODE` (above) so you
|
||||
have captured rows to replay against.
|
||||
|
||||
Quick loop:
|
||||
|
||||
```bash
|
||||
gbrain eval export --since 7d > baseline.ndjson # snapshot before your change
|
||||
# ... make your change ...
|
||||
gbrain eval replay --against baseline.ndjson # diff retrieval, get Jaccard@k
|
||||
```
|
||||
|
||||
Three numbers come back: mean Jaccard@k between captured and current slug
|
||||
sets, top-1 stability, and mean latency Δ. The replay tool flags the worst
|
||||
regressions so you can eyeball whether the change is hurting real queries.
|
||||
|
||||
Trigger paths (rerun if your diff touches any of these):
|
||||
|
||||
- `src/core/search/hybrid.ts`
|
||||
- `src/core/search/source-boost.ts`, `sql-ranking.ts`
|
||||
- `src/core/search/intent.ts`, `expansion.ts`, `dedup.ts`
|
||||
- `src/core/embedding.ts`
|
||||
- `src/core/operations.ts` (query / search handlers)
|
||||
- `src/core/postgres-engine.ts` / `pglite-engine.ts` (searchKeyword /
|
||||
searchVector SQL)
|
||||
|
||||
See [`docs/eval-bench.md`](./docs/eval-bench.md) for the full guide
|
||||
including CI integration, hand-crafted NDJSON corpora (so a fresh checkout
|
||||
without captured data can still replay), and cost considerations. The
|
||||
NDJSON wire format is documented in
|
||||
[`docs/eval-capture.md`](./docs/eval-capture.md).
|
||||
|
||||
## Welcome PRs
|
||||
|
||||
- SQLite engine implementation
|
||||
- Docker Compose for self-hosted Postgres
|
||||
- Additional migration sources
|
||||
- New enrichment API integrations
|
||||
- Performance optimizations
|
||||
@@ -1,171 +0,0 @@
|
||||
# GBrain Installation Guide for AI Agents
|
||||
|
||||
Read this entire file, then follow the steps. Ask the user for API keys when needed.
|
||||
Target: ~30 minutes to a fully working brain.
|
||||
|
||||
## Step 0: If you are not Claude Code
|
||||
|
||||
Read `AGENTS.md` at the repo root first. It's the non-Claude-agent operating
|
||||
protocol (install, read order, trust boundary, common tasks). Claude Code reads
|
||||
`CLAUDE.md` automatically and can skip ahead.
|
||||
|
||||
If you fetched this file by URL without cloning yet, the companion files live at:
|
||||
- `https://raw.githubusercontent.com/garrytan/gbrain/master/AGENTS.md` — start here
|
||||
- `https://raw.githubusercontent.com/garrytan/gbrain/master/llms.txt` — full doc map
|
||||
- `https://raw.githubusercontent.com/garrytan/gbrain/master/llms-full.txt` — same map, inlined
|
||||
|
||||
## Step 1: Install GBrain
|
||||
|
||||
```bash
|
||||
git clone https://github.com/garrytan/gbrain.git ~/gbrain && cd ~/gbrain
|
||||
curl -fsSL https://bun.sh/install | bash
|
||||
export PATH="$HOME/.bun/bin:$PATH"
|
||||
bun install && bun link
|
||||
```
|
||||
|
||||
Verify: `gbrain --version` should print a version number. If `gbrain` is not found,
|
||||
restart the shell or add the PATH export to the shell profile.
|
||||
|
||||
> **Do NOT use `bun install -g github:garrytan/gbrain`.** Bun blocks the top-level
|
||||
> postinstall hook on global installs, so schema migrations never run and the CLI
|
||||
> aborts with `Aborted()` when it opens PGLite. Use the `git clone + bun link` path
|
||||
> above. Tracking issue: [#218](https://github.com/garrytan/gbrain/issues/218).
|
||||
|
||||
## Step 2: API Keys
|
||||
|
||||
Ask the user for these:
|
||||
|
||||
```bash
|
||||
export OPENAI_API_KEY=sk-... # required for vector search
|
||||
export ANTHROPIC_API_KEY=sk-ant-... # optional, improves search quality
|
||||
```
|
||||
|
||||
Save to shell profile or `.env`. Without OpenAI, keyword search still works.
|
||||
Without Anthropic, search works but skips query expansion.
|
||||
|
||||
## Step 3: Create the Brain
|
||||
|
||||
```bash
|
||||
gbrain init # PGLite, no server needed
|
||||
gbrain doctor --json # verify all checks pass
|
||||
```
|
||||
|
||||
The user's markdown files (notes, docs, brain repo) are SEPARATE from this tool repo.
|
||||
Ask the user where their files are, or create a new brain repo:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/brain && cd ~/brain && git init
|
||||
```
|
||||
|
||||
Read `~/gbrain/docs/GBRAIN_RECOMMENDED_SCHEMA.md` and set up the MECE directory
|
||||
structure (people/, companies/, concepts/, etc.) inside the user's brain repo,
|
||||
NOT inside ~/gbrain.
|
||||
|
||||
## Step 4: Import and Index
|
||||
|
||||
```bash
|
||||
gbrain import ~/brain/ --no-embed # import markdown files
|
||||
gbrain embed --stale # generate vector embeddings
|
||||
gbrain query "key themes across these documents?"
|
||||
```
|
||||
|
||||
## Step 4.5: Wire the Knowledge Graph
|
||||
|
||||
If the user already had a brain repo (Step 3 imported existing markdown), backfill
|
||||
the typed-link graph and structured timeline. This populates the `links` and
|
||||
`timeline_entries` tables that future writes will maintain automatically.
|
||||
|
||||
```bash
|
||||
gbrain extract links --source db --dry-run | head -20 # preview
|
||||
gbrain extract links --source db # commit
|
||||
gbrain extract timeline --source db # dated events
|
||||
gbrain stats # verify links > 0
|
||||
```
|
||||
|
||||
For brand-new empty brains, skip this step — auto-link populates the graph as the
|
||||
agent writes pages going forward. There is nothing to backfill yet.
|
||||
|
||||
After this step:
|
||||
- `gbrain graph-query <slug> --depth 2` works (relationship traversal)
|
||||
- Search ranks well-connected entities higher (backlink boost)
|
||||
- Every future `put_page` auto-creates typed links and reconciles stale ones
|
||||
|
||||
If a user has a very large brain (>10K pages), `extract --source db` is idempotent
|
||||
and supports `--since YYYY-MM-DD` for incremental runs.
|
||||
|
||||
## Step 5: Load Skills
|
||||
|
||||
Read `~/gbrain/skills/RESOLVER.md`. This is the skill dispatcher. It tells you which
|
||||
skill to read for any task. Save this to your memory permanently.
|
||||
|
||||
The three most important skills to adopt immediately:
|
||||
|
||||
1. **Signal detector** (`skills/signal-detector/SKILL.md`) — fire this on EVERY
|
||||
inbound message. It captures ideas and entities in parallel. The brain compounds.
|
||||
|
||||
2. **Brain-ops** (`skills/brain-ops/SKILL.md`) — brain-first lookup on every response.
|
||||
Check the brain before any external API call.
|
||||
|
||||
3. **Conventions** (`skills/conventions/quality.md`) — citation format, back-linking
|
||||
iron law, source attribution. These are non-negotiable quality rules.
|
||||
|
||||
## Step 6: Identity (optional)
|
||||
|
||||
Run the soul-audit skill to customize the agent's identity:
|
||||
|
||||
```
|
||||
Read skills/soul-audit/SKILL.md and follow it.
|
||||
```
|
||||
|
||||
This generates SOUL.md (agent identity), USER.md (user profile), ACCESS_POLICY.md
|
||||
(who sees what), and HEARTBEAT.md (operational cadence) from the user's answers.
|
||||
|
||||
If skipped, minimal defaults are installed automatically.
|
||||
|
||||
## Step 7: Recurring Jobs
|
||||
|
||||
Set up using your platform's scheduler (OpenClaw cron, Railway cron, crontab):
|
||||
|
||||
- **Live sync** (every 15 min): `gbrain sync --repo ~/brain && gbrain embed --stale`
|
||||
- **Auto-update** (daily): `gbrain check-update --json` (tell user, never auto-install)
|
||||
- **Dream cycle** (nightly): read `docs/guides/cron-schedule.md` for the full protocol.
|
||||
Entity sweep, citation fixes, memory consolidation, plus (v0.23+) overnight conversation
|
||||
synthesis and cross-session pattern detection. 8 phases, one cron-friendly command. This
|
||||
is what makes the brain compound. Do not skip it.
|
||||
- **Weekly**: `gbrain doctor --json && gbrain embed --stale`
|
||||
|
||||
## Step 8: Integrations
|
||||
|
||||
Run `gbrain integrations list`. Each recipe in `~/gbrain/recipes/` is a self-contained
|
||||
installer. It tells you what credentials to ask for, how to validate, and what cron
|
||||
to register. Ask the user which integrations they want (email, calendar, voice, Twitter).
|
||||
|
||||
Verify: `gbrain integrations doctor` (after at least one is configured)
|
||||
|
||||
## Step 9: Verify
|
||||
|
||||
Read `docs/GBRAIN_VERIFY.md` and run all 7 verification checks. Check #4 (live sync
|
||||
actually works) is the most important.
|
||||
|
||||
## Upgrade
|
||||
|
||||
```bash
|
||||
cd ~/gbrain && git pull origin master && bun install
|
||||
gbrain init # apply schema migrations (idempotent)
|
||||
gbrain post-upgrade # show migration notes for the version range
|
||||
```
|
||||
|
||||
Then read `~/gbrain/skills/migrations/v<NEW_VERSION>.md` (and any intermediate
|
||||
versions you skipped) and run any backfill or verification steps it lists. Skipping
|
||||
this is how features ship in the binary but stay dormant in the user's brain.
|
||||
|
||||
For v0.12.0+ specifically: if your brain was created before v0.12.0, run
|
||||
`gbrain extract links --source db && gbrain extract timeline --source db` to
|
||||
backfill the new graph layer (see Step 4.5 above).
|
||||
|
||||
For v0.12.2+ specifically: if your brain is Postgres- or Supabase-backed and
|
||||
predates v0.12.2, the `v0_12_2` migration runs `gbrain repair-jsonb`
|
||||
automatically during `gbrain post-upgrade` to fix the double-encoded JSONB
|
||||
columns. PGLite brains no-op. If wiki-style imports were truncated by the old
|
||||
`splitBody` bug, run `gbrain sync --full` after upgrading to rebuild
|
||||
`compiled_truth` from source markdown.
|
||||
@@ -1,755 +0,0 @@
|
||||
# GBrain
|
||||
|
||||
Your AI agent is smart but forgetful. GBrain gives it a brain.
|
||||
|
||||
Built by the President and CEO of Y Combinator to run his actual AI agents. The production brain powering his OpenClaw and Hermes deployments: **17,888 pages, 4,383 people, 723 companies**, 21 cron jobs running autonomously, built in 12 days. The agent ingests meetings, emails, tweets, voice calls, and original ideas while you sleep. It enriches every person and company it encounters. It fixes its own citations and consolidates memory overnight. You wake up and the brain is smarter than when you went to bed.
|
||||
|
||||
The brain wires itself. Every page write extracts entity references and creates typed links (`attended`, `works_at`, `invested_in`, `founded`, `advises`) with zero LLM calls. Hybrid search. Self-wiring knowledge graph. Structured timeline. Backlink-boosted ranking. Ask "who works at Acme AI?" or "what did Bob invest in this quarter?" and get answers vector search alone can't reach. Benchmarked side-by-side against the category: gbrain lands **P@5 49.1%, R@5 97.9%** on a 240-page Opus-generated rich-prose corpus, beating its own graph-disabled variant by **+31.4 points P@5** and ripgrep-BM25 + vector-only RAG by a similar margin. The graph layer plus v0.12 extract quality together carry the gap. Full BrainBench scorecards + corpus live in the sibling [gbrain-evals](https://github.com/garrytan/gbrain-evals) repo.
|
||||
|
||||
GBrain is those patterns, generalized. 29 skills. Install in 30 minutes. Your agent does the work. As Garry's personal agent gets smarter, so does yours.
|
||||
|
||||
**New in v0.25.0 — BrainBench-Real (session capture, contributor opt-in):** with `GBRAIN_CONTRIBUTOR_MODE=1` set in your shell, every real `query` + `search` call through MCP, CLI, or the subagent tool-bridge gets captured (PII-scrubbed) into an `eval_candidates` table. Snapshot with `gbrain eval export`, replay against your code change with `gbrain eval replay`. Three numbers come back: mean Jaccard@k between captured and current retrieved slugs, top-1 stability, and latency Δ. **Off by default** for production users — no surprise data accumulation. Walkthrough: [docs/eval-bench.md](docs/eval-bench.md). NDJSON wire format: [docs/eval-capture.md](docs/eval-capture.md).
|
||||
|
||||
> **~30 minutes to a fully working brain.** Database ready in 2 seconds (PGLite, no server). You just answer questions about API keys.
|
||||
|
||||
> **LLMs:** fetch [`llms.txt`](llms.txt) for the documentation map, or [`llms-full.txt`](llms-full.txt) for the same map with core docs inlined in one fetch. **Agents:** start with [`AGENTS.md`](AGENTS.md) (or [`CLAUDE.md`](CLAUDE.md) if you're Claude Code).
|
||||
|
||||
## Install
|
||||
|
||||
### On an agent platform (recommended)
|
||||
|
||||
GBrain is designed to be installed and operated by an AI agent. If you don't have one running yet:
|
||||
|
||||
- **[OpenClaw](https://openclaw.ai)** ... Deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes Agent](https://github.com/NousResearch/hermes-agent)** ... Deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
|
||||
Paste this into your agent:
|
||||
|
||||
```
|
||||
Retrieve and follow the instructions at:
|
||||
https://raw.githubusercontent.com/garrytan/gbrain/master/INSTALL_FOR_AGENTS.md
|
||||
```
|
||||
|
||||
That's it. The agent clones the repo, installs GBrain, sets up the brain, loads 29 skills, and configures recurring jobs. You answer a few questions about API keys. ~30 minutes.
|
||||
|
||||
If your agent doesn't auto-read `AGENTS.md`, point it at that file first:
|
||||
`https://raw.githubusercontent.com/garrytan/gbrain/master/AGENTS.md` is the non-Claude
|
||||
agent operating protocol (install, read order, trust boundary, common tasks). For
|
||||
the full doc map, use `llms.txt` at the same URL root.
|
||||
|
||||
### Standalone CLI (no agent)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/garrytan/gbrain.git && cd gbrain && bun install && bun link
|
||||
gbrain init # local brain, ready in 2 seconds
|
||||
gbrain import ~/notes/ # index your markdown
|
||||
gbrain query "what themes show up across my notes?"
|
||||
```
|
||||
|
||||
**Do NOT use `bun install -g github:garrytan/gbrain`.** Bun blocks the top-level
|
||||
postinstall hook on global installs, so schema migrations never run and the CLI
|
||||
aborts with `Aborted()` the first time it opens PGLite. Use `git clone + bun install
|
||||
&& bun link` as shown above. See [#218](https://github.com/garrytan/gbrain/issues/218).
|
||||
|
||||
```
|
||||
3 results (hybrid search, 0.12s):
|
||||
|
||||
1. concepts/do-things-that-dont-scale (score: 0.94)
|
||||
PG's argument that unscalable effort teaches you what users want.
|
||||
[Source: paulgraham.com, 2013-07-01]
|
||||
|
||||
2. originals/founder-mode-observation (score: 0.87)
|
||||
Deep involvement isn't micromanagement if it expands the team's thinking.
|
||||
|
||||
3. concepts/build-something-people-want (score: 0.81)
|
||||
The YC motto. Connected to 12 other brain pages.
|
||||
```
|
||||
|
||||
### MCP server (Claude Code, Cursor, Windsurf)
|
||||
|
||||
GBrain exposes 30+ MCP tools via stdio:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"gbrain": { "command": "gbrain", "args": ["serve"] }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Add to `~/.claude/server.json` (Claude Code), Settings > MCP Servers (Cursor), or your client's MCP config.
|
||||
|
||||
### Remote MCP (Claude Desktop, Cowork, Perplexity)
|
||||
|
||||
```bash
|
||||
gbrain auth create "claude-desktop" # tokens via the existing CLI
|
||||
gbrain serve --http --port 8787 # built-in HTTP transport (Postgres-only)
|
||||
ngrok http 8787 --url your-brain.ngrok.app # any tunnel works
|
||||
claude mcp add gbrain -t http https://your-brain.ngrok.app/mcp -H "Authorization: Bearer TOKEN"
|
||||
```
|
||||
|
||||
Per-client guides: [`docs/mcp/`](docs/mcp/DEPLOY.md). Hardening defaults, env vars, and threat model: [SECURITY.md](SECURITY.md). ChatGPT requires OAuth 2.1 (not yet implemented).
|
||||
|
||||
### Using gbrain with GStack
|
||||
|
||||
If your engineering agent runs on [GStack](https://github.com/garrytan/gstack), point it at gbrain for code lookup instead of grep+read. Cathedral II (v0.21.0) ships call-graph edges and two-pass retrieval — `/investigate`, `/review`, `/plan-eng-review`, and `/office-hours` all benefit when the agent walks the symbol graph instead of scanning files line by line.
|
||||
|
||||
The five magical-moment commands:
|
||||
|
||||
```bash
|
||||
gbrain code-callers searchKeyword # who calls this symbol?
|
||||
gbrain code-callees searchKeyword # what does this symbol call?
|
||||
gbrain code-def BrainEngine # where is X defined?
|
||||
gbrain code-refs BrainEngine # all reference sites
|
||||
gbrain query "how does N+1 handling work" --near-symbol BrainEngine.searchKeyword --walk-depth 2
|
||||
```
|
||||
|
||||
All five auto-emit JSON on non-TTY (gh-CLI convention) so a GStack subagent shelling out via bash gets a clean parseable response. Run `gbrain sources add <repo> --strategy code` to index a repo, then your agent's brain-first lookup covers code, not just markdown. ([Cathedral II release notes](CHANGELOG.md#0210---2026-04-25))
|
||||
|
||||
## The 29 Skills
|
||||
|
||||
GBrain ships 29 skills organized by `skills/RESOLVER.md` (or your OpenClaw's `AGENTS.md` — both filenames are supported as of v0.19). The resolver tells your agent which skill to read for any task.
|
||||
|
||||
[Skill files are code.](https://x.com/garrytan/status/2042925773300908103) They're the most powerful way to get knowledge work done. A skill file is a fat markdown document that encodes an entire workflow: when to fire, what to check, how to chain with other skills, what quality bar to enforce. The agent reads the skill and executes it. Skills can also call deterministic TypeScript code bundled in GBrain (search, import, embed, sync) for the parts that shouldn't be left to LLM judgment. [Thin harness, fat skills](docs/ethos/THIN_HARNESS_FAT_SKILLS.md): the intelligence lives in the skills, not the runtime.
|
||||
|
||||
### Always-on
|
||||
|
||||
| Skill | What it does |
|
||||
|-------|-------------|
|
||||
| **signal-detector** | Fires on every message. Spawns a cheap model in parallel to capture original thinking and entity mentions. The brain compounds on autopilot. |
|
||||
| **brain-ops** | Brain-first lookup before any external API. The read-enrich-write loop that makes every response smarter. |
|
||||
|
||||
### Content ingestion
|
||||
|
||||
| Skill | What it does |
|
||||
|-------|-------------|
|
||||
| **ingest** | Thin router. Detects input type and delegates to the right ingestion skill. |
|
||||
| **idea-ingest** | Links, articles, tweets become brain pages with analysis, author people pages, and cross-linking. |
|
||||
| **media-ingest** | Video, audio, PDF, books, screenshots, GitHub repos. Transcripts, entity extraction, backlink propagation. |
|
||||
| **meeting-ingestion** | Transcripts become brain pages. Every attendee gets enriched. Every company gets a timeline entry. |
|
||||
|
||||
### Brain operations
|
||||
|
||||
| Skill | What it does |
|
||||
|-------|-------------|
|
||||
| **enrich** | Tiered enrichment (Tier 1/2/3). Creates and updates person/company pages with compiled truth and timelines. |
|
||||
| **query** | 3-layer search with synthesis and citations. Says "the brain doesn't have info on X" instead of hallucinating. |
|
||||
| **maintain** | Periodic health: stale pages, orphans, dead links, citation audit, back-link enforcement, tag consistency. v0.23 adds the dream cycle's synthesize + patterns phases ... overnight conversation transcripts become reflections, originals, and 25-year patterns. |
|
||||
| **citation-fixer** | Scans pages for missing or malformed citations. Fixes format to match the standard. |
|
||||
| **repo-architecture** | Where new brain files go. Decision protocol: primary subject determines directory, not format. |
|
||||
| **publish** | Share brain pages as password-protected HTML. Zero LLM calls. |
|
||||
| **data-research** | Structured data research with parameterized YAML recipes. Extract investor updates, expenses, company metrics from email. |
|
||||
|
||||
### Operational
|
||||
|
||||
| Skill | What it does |
|
||||
|-------|-------------|
|
||||
| **daily-task-manager** | Task lifecycle with priority levels (P0-P3). Stored as searchable brain pages. |
|
||||
| **daily-task-prep** | Morning prep: calendar lookahead with brain context per attendee, open threads, task review. |
|
||||
| **cron-scheduler** | Schedule staggering (5-min offsets), quiet hours (timezone-aware with wake-up override), idempotency. |
|
||||
| **reports** | Timestamped reports with keyword routing. "What's the latest briefing?" finds it instantly. |
|
||||
| **cross-modal-review** | Quality gate via second model. Refusal routing: if one model refuses, silently switch. |
|
||||
| **webhook-transforms** | External events (SMS, meetings, social mentions) converted into brain pages with entity extraction. |
|
||||
| **testing** | Validates every skill has SKILL.md with frontmatter, manifest coverage, resolver coverage. |
|
||||
| **skill-creator** | Create new skills following the conformance standard. MECE check against existing skills. |
|
||||
| **skillify** | The "skillify it!" meta-skill. Orchestrates the 10-step loop so failures become durable skills: scaffold the stubs via `gbrain skillify scaffold`, write the real logic, gate with `gbrain skillify check` + `gbrain check-resolvable`. |
|
||||
| **skillpack-check** | Agent-readable gbrain health report. Exit code for CI; JSON for debugging. Cron-friendly. |
|
||||
| **smoke-test** | 8 post-restart health checks with auto-fix (Bun, CLI, DB, worker, Zod CJS, gateway, API key, brain repo). Drop-in user tests at `~/.gbrain/smoke-tests.d/*.sh`. |
|
||||
| **minion-orchestrator** | Background work in one skill. Shell jobs via `gbrain jobs submit shell` (operator/CLI, MCP blocks protected names) and LLM subagents via `gbrain agent run`. Parent-child DAGs, `child_done` inbox, durability across worker restarts. |
|
||||
|
||||
### Identity and setup
|
||||
|
||||
| Skill | What it does |
|
||||
|-------|-------------|
|
||||
| **soul-audit** | 6-phase interview generating SOUL.md (agent identity), USER.md (user profile), ACCESS_POLICY.md (4-tier privacy), HEARTBEAT.md (operational cadence). |
|
||||
| **setup** | Auto-provision PGLite or Supabase. First import. GStack detection. |
|
||||
| **migrate** | Universal migration from Obsidian, Notion, Logseq, markdown, CSV, JSON, Roam. |
|
||||
| **briefing** | Daily briefing with meeting context, active deals, and citation tracking. |
|
||||
|
||||
### Conventions
|
||||
|
||||
Cross-cutting rules in `skills/conventions/`:
|
||||
- **quality.md** ... citations, back-links, notability gate, source attribution
|
||||
- **brain-first.md** ... 5-step lookup before any external API call
|
||||
- **model-routing.md** ... which model for which task
|
||||
- **test-before-bulk.md** ... test 3-5 items before any batch operation
|
||||
- **cross-modal.yaml** ... review pairs and refusal routing chain
|
||||
|
||||
## How It Works
|
||||
|
||||
```
|
||||
Signal arrives (meeting, email, tweet, link)
|
||||
-> Signal detector captures ideas + entities (parallel, never blocks)
|
||||
-> Brain-ops: check the brain first (gbrain search, gbrain get)
|
||||
-> Respond with full context
|
||||
-> Write: update brain pages with new information + citations
|
||||
-> Auto-link: typed relationships extracted on every write (zero LLM calls)
|
||||
-> Sync: gbrain indexes changes for next query
|
||||
```
|
||||
|
||||
Every cycle adds knowledge. The agent enriches a person page after a meeting. Next time that person comes up, the agent already has context. The difference compounds daily.
|
||||
|
||||
The system gets smarter on its own. Entity enrichment auto-escalates: a person mentioned once gets a stub page (Tier 3). After 3 mentions across different sources, they get web + social enrichment (Tier 2). After a meeting or 8+ mentions, full pipeline (Tier 1). The brain learns who matters without being told. Deterministic classifiers improve over time via a fail-improve loop that logs every LLM fallback and generates better regex patterns from the failures. `gbrain doctor` shows the trajectory: "intent classifier: 87% deterministic, up from 40% in week 1."
|
||||
|
||||
> "Prep me for my meeting with Jordan in 30 minutes"
|
||||
> ... pulls dossier, shared history, recent activity, open threads
|
||||
|
||||
> "What have I said about the relationship between shame and founder performance?"
|
||||
> ... searches YOUR thinking, not the internet
|
||||
|
||||
## Minions: your sub-agents won't drop work anymore
|
||||
|
||||
A durable, Postgres-native job queue built into the brain. Every long-running agent task is now a job that survives gateway restarts, streams progress, gets paused / resumed / steered mid-flight, and shows up in `gbrain jobs list`. Zero infra beyond your existing brain.
|
||||
|
||||
### The production numbers that matter
|
||||
|
||||
Here's my personal OpenClaw deployment: one Render container. Supabase Postgres holding a 45,000-page brain. 19 cron jobs firing on schedule. Real gateway load from real daily work. The task: pull a month of my social posts from an external API and ingest them end-to-end into the brain as a structured page.
|
||||
|
||||
| | Minions | `sessions_spawn` |
|
||||
|--- |--- |--- |
|
||||
| Wall time | **753ms** | **>10,000ms** (gateway timeout) |
|
||||
| Token cost | **$0.00** | ~$0.03 per run |
|
||||
| Success rate | **100%** | **0%** (couldn't even spawn) |
|
||||
| Memory/job | ~2 MB | ~80 MB |
|
||||
|
||||
Under that 19-cron load, sub-agent spawn couldn't clear the 10-second gateway wall. Minions landed it in under a second for zero tokens. **Scaling:** 19,240 posts across 36 months, single bash loop, ~15 min total, $0.00. Sub-agents: ~9 min best case, ~$1.08 in tokens, ~40% spawn failure. **Lab:** durability ∞ (SIGKILL mid-flight, 10/10 rescued), throughput ~10× faster, fan-out ~21× with no failure wall, memory ~400× less.
|
||||
|
||||
Full benchmarks live in [gbrain-evals](https://github.com/garrytan/gbrain-evals/tree/main/docs/benchmarks).
|
||||
|
||||
### The routing rule
|
||||
|
||||
> **Deterministic** (same input → same steps → same output) → **Minions**
|
||||
> **Judgment** (input requires assessment or decision) → **Sub-agents**
|
||||
|
||||
Pull posts, parse JSON, write a brain page, run a sync — deterministic. $0 tokens, survives restart, millisecond runtime. Triage the inbox, assess meeting priority, decide if a cold email deserves a reply — judgment. What sub-agents are actually good at. `minion_mode: pain_triggered` (the default) automates the routing.
|
||||
|
||||
### What's fixed
|
||||
|
||||
The six daily pains — spawn storms, agents that stop responding, forgotten dispatches, gateway crashes mid-run, runaway grandchildren, debugging soup — all belonged to the "deterministic work through a reasoning model" mistake. Minions fixes them by not making that mistake: `max_children` cap, `timeout_ms` + AbortSignal, `child_done` inbox, full `parent_job_id`/`depth`/transcript per job, Postgres durability with stall detection, cascade cancel via recursive CTE. Plus idempotency keys, attachment validation, `removeOnComplete`, and `gbrain jobs smoke` that proves the install in half a second.
|
||||
|
||||
```bash
|
||||
gbrain jobs smoke # verify install
|
||||
gbrain jobs submit sync --params '{}' # fire a background job
|
||||
gbrain jobs stats # health dashboard
|
||||
gbrain jobs supervisor --concurrency 4 # canonical: auto-restarting worker (Postgres only)
|
||||
gbrain jobs work --concurrency 4 # raw worker (no crash recovery — prefer `supervisor`)
|
||||
```
|
||||
|
||||
`gbrain jobs supervisor` keeps the worker alive across crashes with exponential backoff, atomic PID locking, structured audit events at `~/.gbrain/audit/supervisor-*.jsonl`, and a `start --detach` / `status --json` / `stop` subcommand surface for agents. In containers it runs as PID 1; on systemd hosts it's the child of `gbrain-worker.service`. Full deployment guide: [`docs/guides/minions-deployment.md`](docs/guides/minions-deployment.md).
|
||||
|
||||
Read [`skills/minion-orchestrator/SKILL.md`](skills/minion-orchestrator/SKILL.md) for parent-child DAGs, fan-in collection, steering via inbox.
|
||||
|
||||
**Minions is not incrementally better than sub-agents for background work. It's categorically different.** 753ms vs gateway timeout. $0 vs tokens. 100% vs couldn't-spawn. If your agent does deterministic work on a schedule, it runs on Minions now.
|
||||
|
||||
### Health check and self-heal
|
||||
|
||||
Minions is canonical as of v0.11.1 — every `gbrain upgrade` runs the migration automatically (schema → smoke → prefs → host rewrites → env-aware autopilot install). If you ever want to verify manually or wire a cron into your morning briefing:
|
||||
|
||||
```bash
|
||||
gbrain doctor # half-migrated state? prints loud banner + exits non-zero
|
||||
gbrain skillpack-check --quiet # exit 0/1/2 for pipeline gating
|
||||
gbrain skillpack-check | jq # full JSON: {healthy, summary, actions[], doctor, migrations}
|
||||
```
|
||||
|
||||
If anything's off, `actions[]` tells you the exact command to run. For deeper troubleshooting: [`docs/guides/minions-fix.md`](docs/guides/minions-fix.md).
|
||||
|
||||
Moving gateway crons to Minions (deterministic scripts, zero LLM tokens per fire): [`docs/guides/minions-shell-jobs.md`](docs/guides/minions-shell-jobs.md).
|
||||
|
||||
## Durable agents: `gbrain agent` (v0.15)
|
||||
|
||||
Your subagent runs survive crashes now. OpenClaw died mid-run? The worker re-claims on restart and replays from the last committed turn. Fan-out across 50 shards, one shard crashes — the aggregator still claims after every child reaches a terminal state and writes a mixed-outcome summary. Tool calls persist as a two-phase ledger (`pending` → `complete | failed`) so replay is safe by construction, not by hope.
|
||||
|
||||
```bash
|
||||
# Submit a single-subagent run
|
||||
gbrain agent run "summarize my last 10 journal pages"
|
||||
|
||||
# Fan out N prompts across N subagent children + 1 aggregator
|
||||
gbrain agent run "analyze every page" \
|
||||
--fanout-manifest manifests/pages.json \
|
||||
--subagent-def analyzer
|
||||
|
||||
# Tail a running job (heartbeat per turn + full transcript on completion)
|
||||
gbrain agent logs 1247 --follow --since 5m
|
||||
```
|
||||
|
||||
Durability is the point: every Anthropic turn commits to `subagent_messages`, every tool call to `subagent_tool_executions`. Worker kills, OpenClaw crashes, timeouts — all resumable. Host repos (your OpenClaw, etc.) ship their own subagent definitions via `GBRAIN_PLUGIN_PATH` + a `gbrain.plugin.json` manifest: see [`docs/guides/plugin-authors.md`](docs/guides/plugin-authors.md). Requires `ANTHROPIC_API_KEY` on the worker.
|
||||
|
||||
## Skillify: say "skillify it!" and the bug becomes structurally impossible to repeat
|
||||
|
||||
Your OpenClaw hit a new failure. You fix it once in conversation. You say "skillify it!"
|
||||
And now the fix is permanent: a SKILL.md with triggers, a deterministic script with tests, a
|
||||
routing fixture the agent re-evaluates daily, a filing audit that keeps the output from
|
||||
drifting. Ten items. Every one required. The bug can't recur.
|
||||
|
||||
Hermes and similar agent frameworks auto-create skills as a background behavior. Fine until
|
||||
you don't know what the agent shipped. Checklists decay. Tests drift. Resolver entries get
|
||||
stale. Six months later it's an opaque pile nobody has read, nobody has tested, and nobody
|
||||
is sure still works. GBrain ships the same capability except the human stays in the loop
|
||||
and every step is a command you can run.
|
||||
|
||||
### The four verbs you need (v0.19)
|
||||
|
||||
```bash
|
||||
# 1. Scaffold all 5 stub files for a new skill in one shot.
|
||||
gbrain skillify scaffold webhook-verify \
|
||||
--description "verify ngrok webhooks" \
|
||||
--triggers "verify the webhook,check tunnel" \
|
||||
--writes-pages --writes-to people/,companies/
|
||||
|
||||
# 2. Replace the SKILLIFY_STUB sentinels with real logic + real tests.
|
||||
$EDITOR skills/webhook-verify/scripts/webhook-verify.mjs
|
||||
$EDITOR test/webhook-verify.test.ts
|
||||
|
||||
# 3. Run the 10-item audit: SKILL.md exists, script exists, unit + E2E tests,
|
||||
# LLM evals, resolver entry, trigger eval, check-resolvable gate, brain filing.
|
||||
gbrain skillify check skills/webhook-verify/scripts/webhook-verify.mjs
|
||||
|
||||
# 4. Verify the whole tree: reachability, MECE overlap, DRY, routing gaps,
|
||||
# filing audit, SKILLIFY_STUB sentinels (fails if any skill still has one).
|
||||
gbrain check-resolvable # warnings advisory, errors block
|
||||
gbrain check-resolvable --strict # warnings block too (CI opt-in)
|
||||
```
|
||||
|
||||
Idempotent re-runs. `--force` regenerates stub files but NEVER duplicates a resolver row.
|
||||
Scaffold completes in under 2 seconds. The real work (your rule, your script, your tests)
|
||||
is what you spend time on. Everything else is boilerplate the CLI writes for you.
|
||||
|
||||
### `gbrain routing-eval` — catch the routing gaps your users actually hit
|
||||
|
||||
Drop a `routing-eval.jsonl` fixture next to any skill. Each line is `{intent, expected_skill,
|
||||
ambiguous_with?}`. `gbrain check-resolvable` runs the structural layer by default; `gbrain
|
||||
routing-eval` runs the same structural layer as a dedicated CI verb. The `--llm` flag is
|
||||
accepted as a placeholder for a future LLM tie-break layer; in this release it emits a stderr
|
||||
notice and runs structural only. False positives (wrong skill matched), missed routes (no
|
||||
skill matched), and tautological fixtures (intent copies trigger verbatim) all surface as
|
||||
specific advisories with the exact file:line to fix.
|
||||
|
||||
### Works on your OpenClaw, not just gbrain's repo
|
||||
|
||||
v0.19 teaches `gbrain check-resolvable` to accept `AGENTS.md` as a resolver file alongside
|
||||
`RESOLVER.md`, at either the skills directory OR one level up (OpenClaw-native workspace-root
|
||||
layout). The skill manifest auto-derives from walking `skills/*/SKILL.md` when `manifest.json`
|
||||
is missing. Set `OPENCLAW_WORKSPACE=~/your-openclaw/workspace` and everything just works:
|
||||
|
||||
```bash
|
||||
export OPENCLAW_WORKSPACE=~/your-openclaw/workspace
|
||||
gbrain check-resolvable --verbose
|
||||
# Auto-detects: AGENTS.md at workspace root, 107 skills derived from SKILL.md walk,
|
||||
# 15 unreachable errors surfaced, 108 advisory warnings for overlaps and gaps.
|
||||
```
|
||||
|
||||
First run on a real OpenClaw deployment found 15 unreachable skills out of 102 — about 15%
|
||||
of the tree was dark. The essay's "skills the agent can never reach" footgun, now visible.
|
||||
|
||||
### `gbrain skillpack install` — drop 25 curated skills into your OpenClaw
|
||||
|
||||
The skills gbrain ships are a curated bundle. Install them into your workspace with
|
||||
dependency closure (shared conventions come along), per-file diff protection (your local
|
||||
edits are never clobbered without `--overwrite-local`), a file lock that serializes
|
||||
concurrent installers, and an atomic managed-block update to your AGENTS.md so you can
|
||||
see exactly what gbrain wrote.
|
||||
|
||||
```bash
|
||||
gbrain skillpack list # 25 curated skills
|
||||
gbrain skillpack install brain-ops # one skill + its shared conventions
|
||||
gbrain skillpack install --all # the full bundle
|
||||
gbrain skillpack install brain-ops --dry-run # preview; no writes
|
||||
gbrain skillpack diff brain-ops # compare bundle vs your local copy
|
||||
```
|
||||
|
||||
Re-running is safe. The managed-block markers in your AGENTS.md let `skillpack install`
|
||||
accumulate rows across separate single-skill installs instead of overwriting each other.
|
||||
A receipt comment inside the fence (`<!-- gbrain:skillpack:manifest cumulative-slugs="..." -->`)
|
||||
tracks what gbrain has installed across runs. `install --all` is the only path that prunes;
|
||||
per-skill install never deletes what it didn't install. If you hand-add a row inside the fence,
|
||||
gbrain preserves it on reinstall and emits a stderr notice telling your agent to investigate.
|
||||
|
||||
**Skillify is the piece that makes the skills tree survive six months of compounding work.**
|
||||
Read [`skills/skillify/SKILL.md`](skills/skillify/SKILL.md) for the full 10-item checklist
|
||||
and the anti-patterns it catches.
|
||||
|
||||
## Storage tiering: keep bulk content out of git (v0.22.11)
|
||||
|
||||
When your brain crosses 100K files and bulk machine-generated content (tweets, articles, transcripts)
|
||||
becomes the size driver, declare which directories belong in git and which live in the database only.
|
||||
|
||||
```yaml
|
||||
# gbrain.yml at the brain repo root
|
||||
storage:
|
||||
db_tracked:
|
||||
- people/
|
||||
- companies/
|
||||
- deals/
|
||||
db_only:
|
||||
- media/x/
|
||||
- media/articles/
|
||||
- meetings/transcripts/
|
||||
```
|
||||
|
||||
`gbrain sync` auto-manages your `.gitignore` for `db_only` paths. `gbrain export --restore-only --repo .`
|
||||
repopulates missing files from the database (container restart, fresh clone, accidental rm).
|
||||
`gbrain storage status` shows the tier breakdown.
|
||||
|
||||
Full guide: [docs/storage-tiering.md](docs/storage-tiering.md).
|
||||
|
||||
## Getting Data In
|
||||
|
||||
GBrain ships integration recipes that your agent sets up for you. Each recipe tells the agent what credentials to ask for, how to validate, and what cron to register.
|
||||
|
||||
| Recipe | Requires | What It Does |
|
||||
|--------|----------|-------------|
|
||||
| [Public Tunnel](recipes/ngrok-tunnel.md) | — | Fixed URL for MCP + voice (ngrok Hobby $8/mo) |
|
||||
| [Credential Gateway](recipes/credential-gateway.md) | — | Gmail + Calendar access |
|
||||
| [Voice-to-Brain](recipes/twilio-voice-brain.md) | ngrok-tunnel | Phone calls to brain pages (Twilio + OpenAI Realtime) |
|
||||
| [Email-to-Brain](recipes/email-to-brain.md) | credential-gateway | Gmail to entity pages |
|
||||
| [X-to-Brain](recipes/x-to-brain.md) | — | Twitter timeline + mentions + deletions |
|
||||
| [Calendar-to-Brain](recipes/calendar-to-brain.md) | credential-gateway | Google Calendar to searchable daily pages |
|
||||
| [Meeting Sync](recipes/meeting-sync.md) | — | Circleback transcripts to brain pages with attendees |
|
||||
|
||||
**Data research recipes** extract structured data from email into tracked brain pages. Built-in recipes for investor updates (MRR, ARR, runway, headcount), expense tracking, and company metrics. Create your own with `gbrain research init`.
|
||||
|
||||
Run `gbrain integrations` to see status.
|
||||
|
||||
## GBrain + GStack
|
||||
|
||||
[GStack](https://github.com/garrytan/gstack) is the engine. GBrain is the mod.
|
||||
|
||||
- **[GStack](https://github.com/garrytan/gstack)** = coding skills (ship, review, QA, investigate, office-hours, retro). 70,000+ stars, 30,000 developers per day. When your agent codes on itself, it uses GStack.
|
||||
- **GBrain** = everything-else skills (brain ops, signal detection, ingestion, enrichment, cron, reports, identity). When your agent remembers, thinks, and operates, it uses GBrain.
|
||||
- **`hosts/gbrain.ts`** = the bridge. Tells GStack's coding skills to check the brain before coding.
|
||||
|
||||
`gbrain init` detects if GStack is installed and reports mod status. If GStack isn't there, it tells you how to get it.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌──────────────────┐ ┌───────────────┐ ┌──────────────────┐
|
||||
│ Brain Repo │ │ GBrain │ │ AI Agent │
|
||||
│ (git) │ │ (retrieval) │ │ (read/write) │
|
||||
│ │ │ │ │ │
|
||||
│ markdown files │───>│ Postgres + │<──>│ 29 skills │
|
||||
│ = source of │ │ pgvector │ │ define HOW to │
|
||||
│ truth │ │ │ │ use the brain │
|
||||
│ │<───│ hybrid │ │ │
|
||||
│ human can │ │ search │ │ RESOLVER.md │
|
||||
│ always read │ │ (vector + │ │ routes intent │
|
||||
│ & edit │ │ keyword + │ │ to skill │
|
||||
│ │ │ RRF) │ │ │
|
||||
└──────────────────┘ └───────────────┘ └──────────────────┘
|
||||
```
|
||||
|
||||
The repo is the system of record. GBrain is the retrieval layer. The agent reads and writes through both. Human always wins... edit any markdown file and `gbrain sync` picks up the changes.
|
||||
|
||||
## The Knowledge Model
|
||||
|
||||
Every page follows the compiled truth + timeline pattern:
|
||||
|
||||
```markdown
|
||||
---
|
||||
type: concept
|
||||
title: Do Things That Don't Scale
|
||||
tags: [startups, growth, pg-essay]
|
||||
---
|
||||
|
||||
Paul Graham's argument that startups should do unscalable things early on.
|
||||
The key insight: the unscalable effort teaches you what users actually
|
||||
want, which you can't learn any other way.
|
||||
|
||||
---
|
||||
|
||||
- 2013-07-01: Published on paulgraham.com
|
||||
- 2024-11-15: Referenced in batch W25 kickoff talk
|
||||
```
|
||||
|
||||
Above the `---`: **compiled truth**. Your current best understanding. Gets rewritten when new evidence changes the picture. Below: **timeline**. Append-only evidence trail. Never edited, only added to.
|
||||
|
||||
## Knowledge Graph
|
||||
|
||||
Pages aren't just text. Every mention of a person, company, or concept becomes a typed link in a structured graph. The brain wires itself.
|
||||
|
||||
```
|
||||
Write a meeting page mentioning Alice and Acme AI
|
||||
-> Auto-link extracts entity refs from content (zero LLM calls)
|
||||
-> Infers types: meeting page + person ref => `attended`
|
||||
"CEO of X" pattern => `works_at`
|
||||
"invested in" => `invested_in`
|
||||
"advises", "advisor" => `advises`
|
||||
"founded", "co-founded" => `founded`
|
||||
-> Reconciles stale links: edits remove links no longer in content
|
||||
-> Backlinks rank well-connected entities higher in search
|
||||
```
|
||||
|
||||
```bash
|
||||
gbrain graph-query people/alice --type attended --depth 2
|
||||
# returns who Alice met with, transitively
|
||||
```
|
||||
|
||||
The graph powers questions vector search can't: "who works at Acme AI?", "what has Bob invested in?", "find the connection between Alice and Carol". Backfill an existing brain in one command:
|
||||
|
||||
```bash
|
||||
gbrain extract links --source db # wire up the existing 29K pages
|
||||
gbrain extract timeline --source db # extract dated events from markdown timelines
|
||||
```
|
||||
|
||||
Then ask graph questions or watch the search ranking improve. Benchmarked side-by-side against ripgrep-BM25, vector-only RAG (same embedder), and gbrain-with-graph-disabled: gbrain lands **P@5 49.1%, R@5 97.9%** on a 240-page Opus-generated rich-prose corpus, beating hybrid-nograph by **+31.4 points P@5**. Isolate the contribution: v0.11→v0.12 moved the same gbrain codebase from P@5 22.1% → 49.1% on identical inputs, so typed-link extract quality is load-bearing. Full scorecards + reproducible corpus: [gbrain-evals](https://github.com/garrytan/gbrain-evals).
|
||||
|
||||
## Search
|
||||
|
||||
Hybrid search: vector + keyword + RRF fusion + multi-query expansion + 4-layer dedup.
|
||||
|
||||
```
|
||||
Query
|
||||
-> Intent classifier (entity? temporal? event? general?)
|
||||
-> Multi-query expansion (Claude Haiku)
|
||||
-> Vector search (HNSW cosine) + Keyword search (tsvector)
|
||||
-> RRF fusion: score = sum(1/(60 + rank))
|
||||
-> Cosine re-scoring + compiled truth boost
|
||||
-> 4-layer dedup + compiled truth guarantee
|
||||
-> Results
|
||||
```
|
||||
|
||||
Keyword alone misses conceptual matches. Vector alone misses exact phrases. RRF gets both. Search quality is benchmarked and reproducible: `gbrain eval --qrels queries.json` measures P@k, Recall@k, MRR, and nDCG@k. A/B test config changes before deploying them.
|
||||
|
||||
## Why it works: many strategies in concert
|
||||
|
||||
The brain isn't one trick. Every retrieval question goes through ~20 deterministic
|
||||
techniques layered together. No single one is magic; the win comes from stacking
|
||||
them so each layer covers what the others miss.
|
||||
|
||||
```
|
||||
Question
|
||||
│
|
||||
├─ INGESTION (every put_page)
|
||||
│ ├─ Recursive markdown chunking (or semantic / LLM-guided)
|
||||
│ ├─ Embedding cache invalidation on edit
|
||||
│ └─ Idempotent imports (content-hash dedup)
|
||||
│
|
||||
├─ GRAPH EXTRACTION (auto-link post-hook, zero LLM)
|
||||
│ ├─ Entity-ref regex (markdown links + bare slugs)
|
||||
│ ├─ Code-fence stripping (no false-positive slugs in code blocks)
|
||||
│ ├─ Typed inference cascade (FOUNDED → INVESTED → ADVISES → WORKS_AT)
|
||||
│ ├─ Page-role priors (partner-bio language → invested_in)
|
||||
│ ├─ Within-page dedup (same target collapses to one link)
|
||||
│ ├─ Stale-link reconciliation (edits remove dropped refs)
|
||||
│ └─ Multi-type link constraint (same person can works_at AND advises)
|
||||
│
|
||||
├─ SEARCH PIPELINE (every query)
|
||||
│ ├─ Intent classifier (entity / temporal / event / general — auto-routes)
|
||||
│ ├─ Multi-query expansion (Haiku rephrases the question 3 ways)
|
||||
│ ├─ Vector search (HNSW cosine over OpenAI embeddings)
|
||||
│ ├─ Keyword search (Postgres tsvector + websearch_to_tsquery)
|
||||
│ ├─ Source-aware ranking (curated dirs outrank chat/daily swamp at SQL layer)
|
||||
│ ├─ Hard-exclude (test/ archive/ attachments/ .raw/ filtered before retrieval)
|
||||
│ ├─ Reciprocal Rank Fusion (score = sum 1/(60+rank) across both)
|
||||
│ ├─ Cosine re-scoring (re-rank chunks against actual query embedding)
|
||||
│ ├─ Compiled-truth boost (assessments outrank timeline noise)
|
||||
│ ├─ Backlink boost (well-connected entities rank higher)
|
||||
│ └─ Source-aware dedup (one CT chunk per page guaranteed)
|
||||
│
|
||||
├─ GRAPH TRAVERSAL (relational queries)
|
||||
│ ├─ Recursive CTE with cycle prevention (visited-array check)
|
||||
│ ├─ Type-filtered edges (--type works_at, attended, etc.)
|
||||
│ ├─ Direction control (in / out / both)
|
||||
│ └─ Depth-capped (≤10 for remote MCP; DoS prevention)
|
||||
│
|
||||
└─ AGENT WORKFLOW (graph-confident hybrid)
|
||||
├─ Graph-query first (high-precision typed answers)
|
||||
├─ Grep fallback when graph returns nothing
|
||||
└─ Graph hits ranked first in top-K (better P@K and R@K)
|
||||
```
|
||||
|
||||
End-to-end on the BrainBench v1 corpus (240 rich-prose pages, before/after PR #188):
|
||||
|
||||
| Metric | BEFORE PR #188 | AFTER PR #188 | Δ |
|
||||
|-------------------------|----------------|---------------|-------------|
|
||||
| **Precision@5** | 39.2% | **44.7%** | **+5.4 pts**|
|
||||
| **Recall@5** | 83.1% | **94.6%** | **+11.5 pts**|
|
||||
| Correct in top-5 | 217 | 247 | **+30** |
|
||||
| Graph-only F1 (ablation)| 57.8% (grep) | **86.6%** | **+28.8 pts**|
|
||||
|
||||
Plus 5 orthogonal capability checks (identity resolution, temporal queries,
|
||||
performance at 10K-page scale, robustness to malformed input, MCP operation
|
||||
contract). All pass. Full report: [gbrain-evals](https://github.com/garrytan/gbrain-evals).
|
||||
|
||||
The point: each technique handles a class of inputs the others miss. Vector
|
||||
search misses exact slug refs; keyword catches them. Keyword misses conceptual
|
||||
matches; vector catches them. RRF picks the best of both. Compiled-truth boost
|
||||
keeps assessments above timeline noise. Auto-link extraction wires the graph
|
||||
that lets backlink boost rank well-connected entities higher. Graph traversal
|
||||
answers questions search alone can't reach. The agent picks graph-first for
|
||||
precision and falls back to keyword for recall. **All deterministic, all in
|
||||
concert, all measured.**
|
||||
|
||||
## Voice
|
||||
|
||||
Call a phone number. Your AI answers. It knows who's calling, pulls their full context from the brain, and responds like someone who actually knows your world. When the call ends, a brain page appears with the transcript, entity detection, and cross-references.
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/images/voice-client.png" alt="Voice client connected" width="300" />
|
||||
</p>
|
||||
|
||||
> [See it in action](https://x.com/garrytan/status/2043022208512172263)
|
||||
|
||||
The voice recipe ships with GBrain: [Voice-to-Brain](recipes/twilio-voice-brain.md). WebRTC works in a browser tab with zero setup. A real phone number is optional.
|
||||
|
||||
## Engine Architecture
|
||||
|
||||
```
|
||||
CLI / MCP Server
|
||||
(thin wrappers, identical operations)
|
||||
|
|
||||
BrainEngine interface (pluggable)
|
||||
|
|
||||
+--------+--------+
|
||||
| |
|
||||
PGLiteEngine PostgresEngine
|
||||
(default) (Supabase)
|
||||
| |
|
||||
~/.gbrain/ Supabase Pro ($25/mo)
|
||||
brain.pglite Postgres + pgvector
|
||||
embedded PG 17.5
|
||||
|
||||
gbrain migrate --to supabase|pglite
|
||||
(bidirectional migration)
|
||||
```
|
||||
|
||||
PGLite: embedded Postgres, no server, zero config. When your brain outgrows local (1000+ files, multi-device), `gbrain migrate --to supabase` moves everything.
|
||||
|
||||
## File Storage
|
||||
|
||||
Brain repos accumulate binaries. GBrain has a three-stage migration:
|
||||
|
||||
```bash
|
||||
gbrain files mirror <dir> # copy to cloud, local untouched
|
||||
gbrain files redirect <dir> # replace local with .redirect pointers
|
||||
gbrain files clean <dir> # remove pointers, cloud only
|
||||
gbrain files restore <dir> # download everything back (undo)
|
||||
```
|
||||
|
||||
Storage backends: S3-compatible (AWS, R2, MinIO), Supabase Storage, or local.
|
||||
|
||||
## Commands
|
||||
|
||||
```
|
||||
SETUP
|
||||
gbrain init [--supabase|--url] Create brain (PGLite default)
|
||||
gbrain migrate --to supabase|pglite Bidirectional engine migration
|
||||
gbrain upgrade Self-update with feature discovery
|
||||
|
||||
PAGES
|
||||
gbrain get <slug> Read a page (fuzzy slug matching)
|
||||
gbrain put <slug> [< file.md] Write/update (auto-versions)
|
||||
gbrain delete <slug> Delete a page
|
||||
gbrain list [--type T] [--tag T] List with filters
|
||||
|
||||
SEARCH
|
||||
gbrain search <query> Keyword search (tsvector)
|
||||
gbrain query <question> Hybrid search (vector + keyword + RRF)
|
||||
|
||||
IMPORT
|
||||
gbrain import <dir> [--no-embed] [--workers N]
|
||||
Import markdown (idempotent)
|
||||
gbrain sync [--repo <path>] [--workers N]
|
||||
Git-to-brain incremental sync
|
||||
(>100-file diffs auto-parallelize 4 workers on Postgres)
|
||||
gbrain export [--dir ./out/] Export to markdown
|
||||
|
||||
FILES
|
||||
gbrain files list|upload|sync|verify File storage operations
|
||||
|
||||
EMBEDDINGS
|
||||
gbrain embed [<slug>|--all|--stale] Generate/refresh embeddings
|
||||
|
||||
LINKS + GRAPH
|
||||
gbrain link|unlink|backlinks Cross-reference management
|
||||
gbrain extract links|timeline|all Batch backfill from existing pages
|
||||
(--source db|fs, --type, --since, --dry-run)
|
||||
gbrain graph-query <slug> Typed traversal (--type T --depth N
|
||||
--direction in|out|both)
|
||||
|
||||
JOBS (Minions)
|
||||
gbrain jobs submit <name> [--params JSON] [--follow] Submit a background job
|
||||
gbrain jobs list [--status S] [--queue Q] List jobs with filters
|
||||
gbrain jobs get|cancel|retry|delete <id> Manage job lifecycle
|
||||
gbrain jobs prune [--older-than 30d] Clean completed/dead jobs
|
||||
gbrain jobs stats Job health dashboard
|
||||
gbrain jobs smoke One-command health check
|
||||
gbrain jobs work [--queue Q] [--concurrency N] Start worker daemon
|
||||
|
||||
SKILLS (v0.19)
|
||||
gbrain skillify scaffold <name> Create 5 stub files + idempotent resolver row
|
||||
gbrain skillify check [path] 10-item audit of a skill
|
||||
gbrain skillpack list Print the 25 curated skills in the bundle
|
||||
gbrain skillpack install <name> Copy one skill + its shared conventions into target
|
||||
gbrain skillpack install --all Install the full curated bundle
|
||||
gbrain skillpack diff <name> Per-file diff: bundle vs target workspace
|
||||
gbrain check-resolvable [--strict] Resolver audit (reachability, MECE, DRY, routing, filing,
|
||||
SKILLIFY_STUB). Accepts RESOLVER.md OR AGENTS.md.
|
||||
gbrain routing-eval [--llm] [--json] Intent→skill routing accuracy on fixtures
|
||||
|
||||
ADMIN
|
||||
gbrain doctor [--json] [--fast] Health checks (resolver, skills, DB, embeddings)
|
||||
gbrain doctor --fix [--dry-run] Auto-fix DRY violations (delegate inlined rules to conventions)
|
||||
gbrain doctor --locks List idle-in-tx backends (57014 diagnostic, Postgres only)
|
||||
gbrain stats Brain statistics
|
||||
gbrain serve MCP server (stdio)
|
||||
gbrain serve --http --port 8787 MCP server (HTTP, Postgres-only, bearer auth)
|
||||
gbrain auth create|list|revoke|test Token management for the HTTP transport
|
||||
gbrain integrations Integration recipe dashboard
|
||||
gbrain sources list|add|remove|... Multi-source brain management (v0.18)
|
||||
gbrain dream [--dry-run] [--phase N] 8-phase maintenance cycle (lint→backlinks→sync→synthesize
|
||||
→extract→patterns→embed→orphans). v0.23 added synthesize +
|
||||
patterns: transcripts → reflections + cross-session themes.
|
||||
gbrain dream --input <file> Ad-hoc transcript synthesis (implies --phase synthesize)
|
||||
gbrain dream --date YYYY-MM-DD Synthesize a single day; --from/--to for backfill ranges
|
||||
gbrain check-backlinks check|fix Back-link enforcement
|
||||
gbrain lint [--fix] LLM artifact detection
|
||||
gbrain repair-jsonb [--dry-run] Repair v0.12.0 double-encoded JSONB (Postgres)
|
||||
gbrain orphans [--json] [--count] Find pages with zero inbound wikilinks
|
||||
gbrain transcribe <audio> Transcribe audio (Groq Whisper)
|
||||
gbrain research init <name> Scaffold a data-research recipe
|
||||
gbrain research list Show available recipes
|
||||
```
|
||||
|
||||
Run `gbrain --help` for the full reference.
|
||||
|
||||
## Origin Story
|
||||
|
||||
I was setting up my [OpenClaw](https://openclaw.ai) agent and started a markdown brain repo. One page per person, one page per company, compiled truth on top, timeline on the bottom. Within a week: 10,000+ files, 3,000+ people, 13 years of calendar data, 280+ meeting transcripts, 300+ captured ideas.
|
||||
|
||||
The agent runs while I sleep. The dream cycle scans every conversation, enriches missing entities, fixes broken citations, consolidates memory. I wake up and the brain is smarter than when I went to sleep.
|
||||
|
||||
The skills in this repo are those patterns, generalized. What took 11 days to build by hand ships as a mod you install in 30 minutes.
|
||||
|
||||
## Docs
|
||||
|
||||
**For agents:**
|
||||
- **[skills/RESOLVER.md](skills/RESOLVER.md)** ... Start here. The skill dispatcher.
|
||||
- [Individual skill files](skills/) ... 28 standalone instruction sets (25 ship in the curated `gbrain skillpack install` bundle)
|
||||
- [GBRAIN_SKILLPACK.md](docs/GBRAIN_SKILLPACK.md) ... Legacy reference architecture
|
||||
- [Getting Data In](docs/integrations/README.md) ... Integration recipes and data flow
|
||||
- [GBRAIN_VERIFY.md](docs/GBRAIN_VERIFY.md) ... Installation verification
|
||||
|
||||
**For humans:**
|
||||
- [GBRAIN_RECOMMENDED_SCHEMA.md](docs/GBRAIN_RECOMMENDED_SCHEMA.md) ... Brain repo directory structure
|
||||
- [Thin Harness, Fat Skills](docs/ethos/THIN_HARNESS_FAT_SKILLS.md) ... Architecture philosophy
|
||||
- [ENGINES.md](docs/ENGINES.md) ... Pluggable engine interface
|
||||
|
||||
**Reference:**
|
||||
- [GBRAIN_V0.md](docs/GBRAIN_V0.md) ... Full product spec
|
||||
- [CHANGELOG.md](CHANGELOG.md) ... Version history
|
||||
|
||||
**Benchmarks:**
|
||||
- [gbrain-evals](https://github.com/garrytan/gbrain-evals) ... BrainBench, the sibling repo that holds the eval harness, corpus, scorecards, and 4-adapter comparisons. Depends on gbrain; not installed alongside gbrain.
|
||||
|
||||
## Contributing
|
||||
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md). Run `bun test` for unit tests. For the full local CI gate (gitleaks + unit + all 29 E2E files in Docker, the same checks GH Actions runs), use `bun run ci:local` ... or `bun run ci:local:diff` for the diff-aware subset during fast iteration.
|
||||
|
||||
If you're working on retrieval or any of the search/embedding/ranking surface, set `GBRAIN_CONTRIBUTOR_MODE=1` in your shell rc and use `gbrain eval replay` to gate your changes against a snapshot of real captured queries — the dev loop is documented in [`docs/eval-bench.md`](docs/eval-bench.md). Capture is **off by default** for production users (no surprise data accumulation); the env var is the contributor opt-in.
|
||||
|
||||
PRs welcome for: new enrichment APIs, performance optimizations, additional engine backends, new skills following the conformance standard in `skills/skill-creator/SKILL.md`.
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
-168
@@ -1,168 +0,0 @@
|
||||
# Security
|
||||
|
||||
## Reporting Vulnerabilities
|
||||
|
||||
If you discover a security issue in GBrain, please report it privately by opening
|
||||
a [private security advisory](https://github.com/garrytan/gbrain/security/advisories/new)
|
||||
on GitHub.
|
||||
|
||||
Do not open a public issue for security vulnerabilities.
|
||||
|
||||
## Remote MCP Security
|
||||
|
||||
### ⚠️ Do NOT use open OAuth client registration for remote MCP
|
||||
|
||||
If you deploy GBrain's MCP server behind an HTTP wrapper with OAuth 2.1
|
||||
support, **never allow unauthenticated client registration**. An attacker
|
||||
who discovers your server URL can:
|
||||
|
||||
1. Register a new OAuth client via `POST /register`
|
||||
2. Use `client_credentials` grant to obtain a bearer token
|
||||
3. Access all brain data via the MCP tools
|
||||
|
||||
### Recommended: `gbrain serve --http`
|
||||
|
||||
As of v0.22.7, GBrain ships a built-in HTTP transport that uses the
|
||||
existing `access_tokens` table for authentication:
|
||||
|
||||
```bash
|
||||
# Create a token
|
||||
gbrain auth create "my-client"
|
||||
|
||||
# Start the HTTP server
|
||||
gbrain serve --http --port 8787
|
||||
|
||||
# Connect via ngrok, Tailscale, or any tunnel
|
||||
ngrok http 8787 --url your-brain.ngrok.app
|
||||
```
|
||||
|
||||
This is the recommended way to expose GBrain remotely. No OAuth, no
|
||||
registration endpoint, no self-service tokens. Tokens are managed
|
||||
exclusively via `gbrain auth create/list/revoke`.
|
||||
|
||||
### If you must use a custom HTTP wrapper
|
||||
|
||||
1. **Require a secret for client registration** — check a header or body
|
||||
parameter before creating new OAuth clients
|
||||
2. **Disable `client_credentials` grant** — only allow `authorization_code`
|
||||
with browser-based approval
|
||||
3. **Restrict scopes** — never issue tokens with unlimited scope
|
||||
4. **Log all token issuance** — alert on unexpected registrations
|
||||
5. **Rate-limit registration and token endpoints**
|
||||
|
||||
### Token Management
|
||||
|
||||
```bash
|
||||
gbrain auth create "claude-desktop" # Create a new token
|
||||
gbrain auth list # List all tokens
|
||||
gbrain auth revoke "claude-desktop" # Revoke a token
|
||||
gbrain auth test <url> --token <tok> # Smoke-test a remote server
|
||||
```
|
||||
|
||||
Tokens are stored as SHA-256 hashes in the `access_tokens` table. The
|
||||
plaintext token is shown once at creation and never stored.
|
||||
|
||||
## `gbrain serve --http` hardening (v0.22.7+)
|
||||
|
||||
The built-in HTTP transport ships with several layers of hardening on by
|
||||
default. All env vars below are optional; the defaults are intentionally
|
||||
conservative.
|
||||
|
||||
### Postgres-only
|
||||
|
||||
`gbrain serve --http` requires a Postgres engine. PGLite is local-only by
|
||||
design and the `access_tokens` / `mcp_request_log` tables don't exist in
|
||||
the PGLite schema. Local agents continue to use stdio (`gbrain serve`).
|
||||
Running `--http` against a PGLite-backed install fails fast with a clear
|
||||
error message at startup.
|
||||
|
||||
### CORS
|
||||
|
||||
Default-deny: no `Access-Control-Allow-Origin` header is sent unless an
|
||||
allowlist is configured. To allow browser-based MCP clients:
|
||||
|
||||
```bash
|
||||
GBRAIN_HTTP_CORS_ORIGIN=https://claude.ai gbrain serve --http --port 8787
|
||||
# Multiple origins: comma-separated
|
||||
GBRAIN_HTTP_CORS_ORIGIN=https://claude.ai,https://your.app gbrain serve --http
|
||||
```
|
||||
|
||||
When the request `Origin` matches the allowlist, the server echoes it
|
||||
back in `Access-Control-Allow-Origin` (with `Vary: Origin`). Otherwise no
|
||||
CORS header is sent and the browser blocks the request.
|
||||
|
||||
### Rate limiting
|
||||
|
||||
Two buckets, both stored in a bounded LRU map (default 10K keys, evicts
|
||||
least-recently-used on overflow, prunes entries older than 2× the
|
||||
window):
|
||||
|
||||
| Bucket | When it fires | Default | Env var |
|
||||
|---|---|---|---|
|
||||
| Pre-auth IP | Before the DB lookup, on every `/mcp` request | 30 req / 60s | `GBRAIN_HTTP_RATE_LIMIT_IP` |
|
||||
| Post-auth token | After a valid token is resolved | 60 req / 60s | `GBRAIN_HTTP_RATE_LIMIT_TOKEN` |
|
||||
| LRU cap | Maximum distinct keys across both buckets | 10000 | `GBRAIN_HTTP_RATE_LIMIT_LRU` |
|
||||
|
||||
On exhaustion the server returns `429 Too Many Requests` with a
|
||||
`Retry-After` header.
|
||||
|
||||
**Caveat for tunneled deployments (ngrok, Tailscale Funnel, Cloudflare
|
||||
Tunnel):** all requests share one egress IP, so the pre-auth IP bucket
|
||||
becomes effectively shared by all clients on that tunnel. The
|
||||
post-auth token-id bucket is the load-bearing limiter for tunnel-fronted
|
||||
deployments.
|
||||
|
||||
### Reverse-proxy trust
|
||||
|
||||
Disabled by default. To honor `X-Forwarded-For` (or `X-Real-IP`) when
|
||||
gbrain runs behind a trusted reverse proxy:
|
||||
|
||||
```bash
|
||||
GBRAIN_HTTP_TRUST_PROXY=1 gbrain serve --http --port 8787
|
||||
```
|
||||
|
||||
**Critical safety contract:** only set `GBRAIN_HTTP_TRUST_PROXY=1` when
|
||||
**both** of these are true:
|
||||
|
||||
1. gbrain is reachable only via a trusted reverse proxy (not directly
|
||||
exposed to the internet on the configured port). The simplest
|
||||
guarantee is to bind gbrain to `127.0.0.1` or a private interface
|
||||
and have the proxy forward to it.
|
||||
2. The proxy strips any client-supplied `X-Forwarded-For` and `X-Real-IP`
|
||||
headers, then sets them itself. (nginx with `proxy_set_header
|
||||
X-Forwarded-For $remote_addr` does this; Cloudflare and most cloud
|
||||
load balancers handle it automatically.)
|
||||
|
||||
If gbrain is reachable directly AND `GBRAIN_HTTP_TRUST_PROXY=1` is set,
|
||||
clients can spoof their IP by sending arbitrary `X-Forwarded-For`
|
||||
headers, defeating the pre-auth IP rate limit. Without the flag, gbrain
|
||||
ignores all forwarded-for headers and uses the socket peer address,
|
||||
which is the safe default for direct-exposure deployments.
|
||||
|
||||
### Body size cap
|
||||
|
||||
Default 1 MiB, stream-counted (chunked transfers without
|
||||
`Content-Length` are still capped). Override:
|
||||
|
||||
```bash
|
||||
GBRAIN_HTTP_MAX_BODY_BYTES=2097152 gbrain serve --http # 2 MiB
|
||||
```
|
||||
|
||||
Over-cap requests get `413 Payload Too Large` immediately, before any
|
||||
body is materialized in memory.
|
||||
|
||||
### Audit log
|
||||
|
||||
Every `/mcp` request writes one row to `mcp_request_log`:
|
||||
|
||||
```bash
|
||||
psql "$DATABASE_URL" -c \
|
||||
"SELECT created_at, token_name, operation, status, latency_ms
|
||||
FROM mcp_request_log
|
||||
ORDER BY created_at DESC LIMIT 100"
|
||||
```
|
||||
|
||||
`status` is one of: `success`, `error`, `auth_failed`, `rate_limited`,
|
||||
`body_too_large`, `parse_error`, `unknown_method`. Failed-auth rows have
|
||||
`token_name = NULL`. Inserts are fire-and-forget so audit failures
|
||||
never block requests.
|
||||
@@ -1,524 +0,0 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"configVersion": 1,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "gbrain",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "^0.30.0",
|
||||
"@aws-sdk/client-s3": "^3.1028.0",
|
||||
"@dqbd/tiktoken": "^1.0.22",
|
||||
"@electric-sql/pglite": "0.4.3",
|
||||
"@modelcontextprotocol/sdk": "^1.0.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
"marked": "^18.0.0",
|
||||
"openai": "^4.0.0",
|
||||
"pgvector": "^0.2.0",
|
||||
"postgres": "^3.4.0",
|
||||
"tree-sitter-wasms": "0.1.13",
|
||||
"web-tree-sitter": "0.22.6",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"bun-types": "^1.3.13",
|
||||
"typescript": "^5.6.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
"trustedDependencies": [
|
||||
"@electric-sql/pglite",
|
||||
],
|
||||
"packages": {
|
||||
"@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.30.1", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" } }, "sha512-nuKvp7wOIz6BFei8WrTdhmSsx5mwnArYyJgh4+vYu3V4J0Ltb8Xm3odPm51n1aSI0XxNCrDl7O88cxCtUdAkaw=="],
|
||||
|
||||
"@aws-crypto/crc32": ["@aws-crypto/crc32@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg=="],
|
||||
|
||||
"@aws-crypto/crc32c": ["@aws-crypto/crc32c@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag=="],
|
||||
|
||||
"@aws-crypto/sha1-browser": ["@aws-crypto/sha1-browser@5.2.0", "", { "dependencies": { "@aws-crypto/supports-web-crypto": "^5.2.0", "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "@aws-sdk/util-locate-window": "^3.0.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg=="],
|
||||
|
||||
"@aws-crypto/sha256-browser": ["@aws-crypto/sha256-browser@5.2.0", "", { "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", "@aws-crypto/supports-web-crypto": "^5.2.0", "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "@aws-sdk/util-locate-window": "^3.0.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw=="],
|
||||
|
||||
"@aws-crypto/sha256-js": ["@aws-crypto/sha256-js@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA=="],
|
||||
|
||||
"@aws-crypto/supports-web-crypto": ["@aws-crypto/supports-web-crypto@5.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg=="],
|
||||
|
||||
"@aws-crypto/util": ["@aws-crypto/util@5.2.0", "", { "dependencies": { "@aws-sdk/types": "^3.222.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ=="],
|
||||
|
||||
"@aws-sdk/client-s3": ["@aws-sdk/client-s3@3.1028.0", "", { "dependencies": { "@aws-crypto/sha1-browser": "5.2.0", "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.973.27", "@aws-sdk/credential-provider-node": "^3.972.30", "@aws-sdk/middleware-bucket-endpoint": "^3.972.9", "@aws-sdk/middleware-expect-continue": "^3.972.9", "@aws-sdk/middleware-flexible-checksums": "^3.974.7", "@aws-sdk/middleware-host-header": "^3.972.9", "@aws-sdk/middleware-location-constraint": "^3.972.9", "@aws-sdk/middleware-logger": "^3.972.9", "@aws-sdk/middleware-recursion-detection": "^3.972.10", "@aws-sdk/middleware-sdk-s3": "^3.972.28", "@aws-sdk/middleware-ssec": "^3.972.9", "@aws-sdk/middleware-user-agent": "^3.972.29", "@aws-sdk/region-config-resolver": "^3.972.11", "@aws-sdk/signature-v4-multi-region": "^3.996.16", "@aws-sdk/types": "^3.973.7", "@aws-sdk/util-endpoints": "^3.996.6", "@aws-sdk/util-user-agent-browser": "^3.972.9", "@aws-sdk/util-user-agent-node": "^3.973.15", "@smithy/config-resolver": "^4.4.14", "@smithy/core": "^3.23.14", "@smithy/eventstream-serde-browser": "^4.2.13", "@smithy/eventstream-serde-config-resolver": "^4.3.13", "@smithy/eventstream-serde-node": "^4.2.13", "@smithy/fetch-http-handler": "^5.3.16", "@smithy/hash-blob-browser": "^4.2.14", "@smithy/hash-node": "^4.2.13", "@smithy/hash-stream-node": "^4.2.13", "@smithy/invalid-dependency": "^4.2.13", "@smithy/md5-js": "^4.2.13", "@smithy/middleware-content-length": "^4.2.13", "@smithy/middleware-endpoint": "^4.4.29", "@smithy/middleware-retry": "^4.5.0", "@smithy/middleware-serde": "^4.2.17", "@smithy/middleware-stack": "^4.2.13", "@smithy/node-config-provider": "^4.3.13", "@smithy/node-http-handler": "^4.5.2", "@smithy/protocol-http": "^5.3.13", "@smithy/smithy-client": "^4.12.9", "@smithy/types": "^4.14.0", "@smithy/url-parser": "^4.2.13", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", "@smithy/util-defaults-mode-browser": "^4.3.45", "@smithy/util-defaults-mode-node": "^4.2.49", "@smithy/util-endpoints": "^3.3.4", "@smithy/util-middleware": "^4.2.13", "@smithy/util-retry": "^4.3.0", "@smithy/util-stream": "^4.5.22", "@smithy/util-utf8": "^4.2.2", "@smithy/util-waiter": "^4.2.15", "tslib": "^2.6.2" } }, "sha512-KL8PREFJxyWXUjMQR6Krq/OjZ5qbcV1QFjtA7Q7oMW5XaFO9YoSBtBxQeeXO4um6vYSmRVYVDTvEKZDcNbyeXw=="],
|
||||
|
||||
"@aws-sdk/core": ["@aws-sdk/core@3.973.27", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@aws-sdk/xml-builder": "^3.972.17", "@smithy/core": "^3.23.14", "@smithy/node-config-provider": "^4.3.13", "@smithy/property-provider": "^4.2.13", "@smithy/protocol-http": "^5.3.13", "@smithy/signature-v4": "^5.3.13", "@smithy/smithy-client": "^4.12.9", "@smithy/types": "^4.14.0", "@smithy/util-base64": "^4.3.2", "@smithy/util-middleware": "^4.2.13", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-CUZ5m8hwMCH6OYI4Li/WgMfIEx10Q2PLI9Y3XOUTPGZJ53aZ0007jCv+X/ywsaERyKPdw5MRZWk877roQksQ4A=="],
|
||||
|
||||
"@aws-sdk/crc64-nvme": ["@aws-sdk/crc64-nvme@3.972.6", "", { "dependencies": { "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-NMbiqKdruhwwgI6nzBVe2jWMkXjaoQz2YOs3rFX+2F3gGyrJDkDPwMpV/RsTFeq2vAQ055wZNtOXFK4NYSkM8g=="],
|
||||
|
||||
"@aws-sdk/credential-provider-env": ["@aws-sdk/credential-provider-env@3.972.25", "", { "dependencies": { "@aws-sdk/core": "^3.973.27", "@aws-sdk/types": "^3.973.7", "@smithy/property-provider": "^4.2.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-6QfI0wv4jpG5CrdO/AO0JfZ2ux+tKwJPrUwmvxXF50vI5KIypKVGNF6b4vlkYEnKumDTI1NX2zUBi8JoU5QU3A=="],
|
||||
|
||||
"@aws-sdk/credential-provider-http": ["@aws-sdk/credential-provider-http@3.972.27", "", { "dependencies": { "@aws-sdk/core": "^3.973.27", "@aws-sdk/types": "^3.973.7", "@smithy/fetch-http-handler": "^5.3.16", "@smithy/node-http-handler": "^4.5.2", "@smithy/property-provider": "^4.2.13", "@smithy/protocol-http": "^5.3.13", "@smithy/smithy-client": "^4.12.9", "@smithy/types": "^4.14.0", "@smithy/util-stream": "^4.5.22", "tslib": "^2.6.2" } }, "sha512-3V3Usj9Gs93h865DqN4M2NWJhC5kXU9BvZskfN3+69omuYlE3TZxOEcVQtBGLOloJB7BVfJKXVLqeNhOzHqSlQ=="],
|
||||
|
||||
"@aws-sdk/credential-provider-ini": ["@aws-sdk/credential-provider-ini@3.972.29", "", { "dependencies": { "@aws-sdk/core": "^3.973.27", "@aws-sdk/credential-provider-env": "^3.972.25", "@aws-sdk/credential-provider-http": "^3.972.27", "@aws-sdk/credential-provider-login": "^3.972.29", "@aws-sdk/credential-provider-process": "^3.972.25", "@aws-sdk/credential-provider-sso": "^3.972.29", "@aws-sdk/credential-provider-web-identity": "^3.972.29", "@aws-sdk/nested-clients": "^3.996.19", "@aws-sdk/types": "^3.973.7", "@smithy/credential-provider-imds": "^4.2.13", "@smithy/property-provider": "^4.2.13", "@smithy/shared-ini-file-loader": "^4.4.8", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-SiBuAnXecCbT/OpAf3vqyI/AVE3mTaYr9ShXLybxZiPLBiPCCOIWSGAtYYGQWMRvobBTiqOewaB+wcgMMZI2Aw=="],
|
||||
|
||||
"@aws-sdk/credential-provider-login": ["@aws-sdk/credential-provider-login@3.972.29", "", { "dependencies": { "@aws-sdk/core": "^3.973.27", "@aws-sdk/nested-clients": "^3.996.19", "@aws-sdk/types": "^3.973.7", "@smithy/property-provider": "^4.2.13", "@smithy/protocol-http": "^5.3.13", "@smithy/shared-ini-file-loader": "^4.4.8", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-OGOslTbOlxXexKMqhxCEbBQbUIfuhGxU5UXw3Fm56ypXHvrXH4aTt/xb5Y884LOoteP1QST1lVZzHfcTnWhiPQ=="],
|
||||
|
||||
"@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.30", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.25", "@aws-sdk/credential-provider-http": "^3.972.27", "@aws-sdk/credential-provider-ini": "^3.972.29", "@aws-sdk/credential-provider-process": "^3.972.25", "@aws-sdk/credential-provider-sso": "^3.972.29", "@aws-sdk/credential-provider-web-identity": "^3.972.29", "@aws-sdk/types": "^3.973.7", "@smithy/credential-provider-imds": "^4.2.13", "@smithy/property-provider": "^4.2.13", "@smithy/shared-ini-file-loader": "^4.4.8", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-FMnAnWxc8PG+ZrZ2OBKzY4luCUJhe9CG0B9YwYr4pzrYGLXBS2rl+UoUvjGbAwiptxRL6hyA3lFn03Bv1TLqTw=="],
|
||||
|
||||
"@aws-sdk/credential-provider-process": ["@aws-sdk/credential-provider-process@3.972.25", "", { "dependencies": { "@aws-sdk/core": "^3.973.27", "@aws-sdk/types": "^3.973.7", "@smithy/property-provider": "^4.2.13", "@smithy/shared-ini-file-loader": "^4.4.8", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-HR7ynNRdNhNsdVCOCegy1HsfsRzozCOPtD3RzzT1JouuaHobWyRfJzCBue/3jP7gECHt+kQyZUvwg/cYLWurNQ=="],
|
||||
|
||||
"@aws-sdk/credential-provider-sso": ["@aws-sdk/credential-provider-sso@3.972.29", "", { "dependencies": { "@aws-sdk/core": "^3.973.27", "@aws-sdk/nested-clients": "^3.996.19", "@aws-sdk/token-providers": "3.1026.0", "@aws-sdk/types": "^3.973.7", "@smithy/property-provider": "^4.2.13", "@smithy/shared-ini-file-loader": "^4.4.8", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-HWv4SEq3jZDYPlwryZVef97+U8CxxRos5mK8sgGO1dQaFZpV5giZLzqGE5hkDmh2csYcBO2uf5XHjPTpZcJlig=="],
|
||||
|
||||
"@aws-sdk/credential-provider-web-identity": ["@aws-sdk/credential-provider-web-identity@3.972.29", "", { "dependencies": { "@aws-sdk/core": "^3.973.27", "@aws-sdk/nested-clients": "^3.996.19", "@aws-sdk/types": "^3.973.7", "@smithy/property-provider": "^4.2.13", "@smithy/shared-ini-file-loader": "^4.4.8", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-PdMBza1WEKEUPFEmMGCfnU2RYCz9MskU2e8JxjyUOsMKku7j9YaDKvbDi2dzC0ihFoM6ods2SbhfAAro+Gwlew=="],
|
||||
|
||||
"@aws-sdk/middleware-bucket-endpoint": ["@aws-sdk/middleware-bucket-endpoint@3.972.9", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@aws-sdk/util-arn-parser": "^3.972.3", "@smithy/node-config-provider": "^4.3.13", "@smithy/protocol-http": "^5.3.13", "@smithy/types": "^4.14.0", "@smithy/util-config-provider": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-COToYKgquDyligbcAep7ygs48RK+mwe/IYprq4+TSrVFzNOYmzWvHf6werpnKV5VYpRiwdn+Wa5ZXkPqLVwcTg=="],
|
||||
|
||||
"@aws-sdk/middleware-expect-continue": ["@aws-sdk/middleware-expect-continue@3.972.9", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/protocol-http": "^5.3.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-V/FNCjFxnh4VGu+HdSiW4Yg5GELihA1MIDSAdsEPvuayXBVmr0Jaa6jdLAZLH38KYXl/vVjri9DQJWnTAujHEA=="],
|
||||
|
||||
"@aws-sdk/middleware-flexible-checksums": ["@aws-sdk/middleware-flexible-checksums@3.974.7", "", { "dependencies": { "@aws-crypto/crc32": "5.2.0", "@aws-crypto/crc32c": "5.2.0", "@aws-crypto/util": "5.2.0", "@aws-sdk/core": "^3.973.27", "@aws-sdk/crc64-nvme": "^3.972.6", "@aws-sdk/types": "^3.973.7", "@smithy/is-array-buffer": "^4.2.2", "@smithy/node-config-provider": "^4.3.13", "@smithy/protocol-http": "^5.3.13", "@smithy/types": "^4.14.0", "@smithy/util-middleware": "^4.2.13", "@smithy/util-stream": "^4.5.22", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-uU4/ch2CLHB8Phu1oTKnnQ4e8Ujqi49zEnQYBhWYT53zfFvtJCdGsaOoypBr8Fm/pmCBssRmGoIQ4sixgdLP9w=="],
|
||||
|
||||
"@aws-sdk/middleware-host-header": ["@aws-sdk/middleware-host-header@3.972.9", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/protocol-http": "^5.3.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-je5vRdNw4SkuTnmRbFZLdye4sQ0faLt8kwka5wnnSU30q1mHO4X+idGEJOOE+Tn1ME7Oryn05xxkDvIb3UaLaQ=="],
|
||||
|
||||
"@aws-sdk/middleware-location-constraint": ["@aws-sdk/middleware-location-constraint@3.972.9", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-TyfOi2XNdOZpNKeTJwRUsVAGa+14nkyMb2VVGG+eDgcWG/ed6+NUo72N3hT6QJioxym80NSinErD+LBRF0Ir1w=="],
|
||||
|
||||
"@aws-sdk/middleware-logger": ["@aws-sdk/middleware-logger@3.972.9", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-HsVgDrruhqI28RkaXALm8grJ7Agc1wF6Et0xh6pom8NdO2VdO/SD9U/tPwUjewwK/pVoka+EShBxyCvgsPCtog=="],
|
||||
|
||||
"@aws-sdk/middleware-recursion-detection": ["@aws-sdk/middleware-recursion-detection@3.972.10", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@aws/lambda-invoke-store": "^0.2.2", "@smithy/protocol-http": "^5.3.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-RVQQbq5orQ/GHUnXvqEOj2HHPBJm+mM+ySwZKS5UaLBwra5ugRtiH09PLUoOZRl7a1YzaOzXSuGbn9iD5j60WQ=="],
|
||||
|
||||
"@aws-sdk/middleware-sdk-s3": ["@aws-sdk/middleware-sdk-s3@3.972.28", "", { "dependencies": { "@aws-sdk/core": "^3.973.27", "@aws-sdk/types": "^3.973.7", "@aws-sdk/util-arn-parser": "^3.972.3", "@smithy/core": "^3.23.14", "@smithy/node-config-provider": "^4.3.13", "@smithy/protocol-http": "^5.3.13", "@smithy/signature-v4": "^5.3.13", "@smithy/smithy-client": "^4.12.9", "@smithy/types": "^4.14.0", "@smithy/util-config-provider": "^4.2.2", "@smithy/util-middleware": "^4.2.13", "@smithy/util-stream": "^4.5.22", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-qJHcJQH9UNPUrnPlRtCozKjtqAaypQ5IgQxTNoPsVYIQeuwNIA8Rwt3NvGij1vCDYDfCmZaPLpnJEHlZXeFqmg=="],
|
||||
|
||||
"@aws-sdk/middleware-ssec": ["@aws-sdk/middleware-ssec@3.972.9", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-wSA2BR7L0CyBNDJeSrleIIzC+DzL93YNTdfU0KPGLiocK6YsRv1nPAzPF+BFSdcs0Qa5ku5Kcf4KvQcWwKGenQ=="],
|
||||
|
||||
"@aws-sdk/middleware-user-agent": ["@aws-sdk/middleware-user-agent@3.972.29", "", { "dependencies": { "@aws-sdk/core": "^3.973.27", "@aws-sdk/types": "^3.973.7", "@aws-sdk/util-endpoints": "^3.996.6", "@smithy/core": "^3.23.14", "@smithy/protocol-http": "^5.3.13", "@smithy/types": "^4.14.0", "@smithy/util-retry": "^4.3.0", "tslib": "^2.6.2" } }, "sha512-f/sIRzuTfEjg6NsbMYvye2VsmnQoNgntntleQyx5uGacUYzszbfIlO3GcI6G6daWUmTm0IDZc11qMHWwF0o0mQ=="],
|
||||
|
||||
"@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.996.19", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.973.27", "@aws-sdk/middleware-host-header": "^3.972.9", "@aws-sdk/middleware-logger": "^3.972.9", "@aws-sdk/middleware-recursion-detection": "^3.972.10", "@aws-sdk/middleware-user-agent": "^3.972.29", "@aws-sdk/region-config-resolver": "^3.972.11", "@aws-sdk/types": "^3.973.7", "@aws-sdk/util-endpoints": "^3.996.6", "@aws-sdk/util-user-agent-browser": "^3.972.9", "@aws-sdk/util-user-agent-node": "^3.973.15", "@smithy/config-resolver": "^4.4.14", "@smithy/core": "^3.23.14", "@smithy/fetch-http-handler": "^5.3.16", "@smithy/hash-node": "^4.2.13", "@smithy/invalid-dependency": "^4.2.13", "@smithy/middleware-content-length": "^4.2.13", "@smithy/middleware-endpoint": "^4.4.29", "@smithy/middleware-retry": "^4.5.0", "@smithy/middleware-serde": "^4.2.17", "@smithy/middleware-stack": "^4.2.13", "@smithy/node-config-provider": "^4.3.13", "@smithy/node-http-handler": "^4.5.2", "@smithy/protocol-http": "^5.3.13", "@smithy/smithy-client": "^4.12.9", "@smithy/types": "^4.14.0", "@smithy/url-parser": "^4.2.13", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", "@smithy/util-defaults-mode-browser": "^4.3.45", "@smithy/util-defaults-mode-node": "^4.2.49", "@smithy/util-endpoints": "^3.3.4", "@smithy/util-middleware": "^4.2.13", "@smithy/util-retry": "^4.3.0", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-uFkmCDXvmQYLanlYdOFS0+MQWkrj9wPMt/ZCc/0J0fjPim6F5jBVBmEomvGY/j77ILW6GTPwN22Jc174Mhkw6Q=="],
|
||||
|
||||
"@aws-sdk/region-config-resolver": ["@aws-sdk/region-config-resolver@3.972.11", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/config-resolver": "^4.4.14", "@smithy/node-config-provider": "^4.3.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-6Q8B1dcx6BBqUTY1Mc/eROKA0FImEEY5VPSd6AGPEUf0ErjExz4snVqa9kNJSoVDV1rKaNf3qrWojgcKW+SdDg=="],
|
||||
|
||||
"@aws-sdk/signature-v4-multi-region": ["@aws-sdk/signature-v4-multi-region@3.996.16", "", { "dependencies": { "@aws-sdk/middleware-sdk-s3": "^3.972.28", "@aws-sdk/types": "^3.973.7", "@smithy/protocol-http": "^5.3.13", "@smithy/signature-v4": "^5.3.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-EMdXYB4r/k5RWq86fugjRhid5JA+Z6MpS7n4sij4u5/C+STrkvuf9aFu41rJA9MjUzxCLzv8U2XL8cH2GSRYpQ=="],
|
||||
|
||||
"@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1026.0", "", { "dependencies": { "@aws-sdk/core": "^3.973.27", "@aws-sdk/nested-clients": "^3.996.19", "@aws-sdk/types": "^3.973.7", "@smithy/property-provider": "^4.2.13", "@smithy/shared-ini-file-loader": "^4.4.8", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-Ieq/HiRrbEtrYP387Nes0XlR7H1pJiJOZKv+QyQzMYpvTiDs0VKy2ZB3E2Zf+aFovWmeE7lRE4lXyF7dYM6GgA=="],
|
||||
|
||||
"@aws-sdk/types": ["@aws-sdk/types@3.973.7", "", { "dependencies": { "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-reXRwoJ6CfChoqAsBszUYajAF8Z2LRE+CRcKocvFSMpIiLOtYU3aJ9trmn6VVPAzbbY5LXF+FfmUslbXk1SYFg=="],
|
||||
|
||||
"@aws-sdk/util-arn-parser": ["@aws-sdk/util-arn-parser@3.972.3", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-HzSD8PMFrvgi2Kserxuff5VitNq2sgf3w9qxmskKDiDTThWfVteJxuCS9JXiPIPtmCrp+7N9asfIaVhBFORllA=="],
|
||||
|
||||
"@aws-sdk/util-endpoints": ["@aws-sdk/util-endpoints@3.996.6", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/types": "^4.14.0", "@smithy/url-parser": "^4.2.13", "@smithy/util-endpoints": "^3.3.4", "tslib": "^2.6.2" } }, "sha512-2nUQ+2ih7CShuKHpGSIYvvAIOHy52dOZguYG36zptBukhw6iFwcvGfG0tes0oZFWQqEWvgZe9HLWaNlvXGdOrg=="],
|
||||
|
||||
"@aws-sdk/util-locate-window": ["@aws-sdk/util-locate-window@3.965.5", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ=="],
|
||||
|
||||
"@aws-sdk/util-user-agent-browser": ["@aws-sdk/util-user-agent-browser@3.972.9", "", { "dependencies": { "@aws-sdk/types": "^3.973.7", "@smithy/types": "^4.14.0", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-sn/LMzTbGjYqCCF24390WxPd6hkpoSptiUn5DzVp4cD71yqw+yGEGm1YCxyEoPXyc8qciM8UzLJcZBFslxo5Uw=="],
|
||||
|
||||
"@aws-sdk/util-user-agent-node": ["@aws-sdk/util-user-agent-node@3.973.15", "", { "dependencies": { "@aws-sdk/middleware-user-agent": "^3.972.29", "@aws-sdk/types": "^3.973.7", "@smithy/node-config-provider": "^4.3.13", "@smithy/types": "^4.14.0", "@smithy/util-config-provider": "^4.2.2", "tslib": "^2.6.2" }, "peerDependencies": { "aws-crt": ">=1.0.0" }, "optionalPeers": ["aws-crt"] }, "sha512-fYn3s9PtKdgQkczGZCFMgkNEe8aq1JCVbnRqjqN9RSVW43xn2RV9xdcZ3z01a48Jpkuh/xCmBKJxdLOo4Ozg7w=="],
|
||||
|
||||
"@aws-sdk/xml-builder": ["@aws-sdk/xml-builder@3.972.17", "", { "dependencies": { "@smithy/types": "^4.14.0", "fast-xml-parser": "5.5.8", "tslib": "^2.6.2" } }, "sha512-Ra7hjqAZf1OXRRMueB13qex7mFJRDK/pgCvdSFemXBT8KCGnQDPoKzHY1SjN+TjJVmnpSF14W5tJ1vDamFu+Gg=="],
|
||||
|
||||
"@aws/lambda-invoke-store": ["@aws/lambda-invoke-store@0.2.4", "", {}, "sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ=="],
|
||||
|
||||
"@dqbd/tiktoken": ["@dqbd/tiktoken@1.0.22", "", {}, "sha512-RYhO8xeHkMNX5Ixqf4M1Ve3siCYJY/dI0yLnlX4M4oIEDOvjMIQ+E+3OUpAaZcWTaMtQJzGcDAghYfllpx3i/w=="],
|
||||
|
||||
"@electric-sql/pglite": ["@electric-sql/pglite@0.4.3", "", {}, "sha512-ichuWTgtd4mOM1G4SpyGJa5trT03lWbMypDV0fUXUCXg5hiHqVAz/bZyV68NqmkLB7WcYmj1RMJVSp8HV/v/ZQ=="],
|
||||
|
||||
"@hono/node-server": ["@hono/node-server@1.19.12", "", { "peerDependencies": { "hono": "^4" } }, "sha512-txsUW4SQ1iilgE0l9/e9VQWmELXifEFvmdA1j6WFh/aFPj99hIntrSsq/if0UWyGVkmrRPKA1wCeP+UCr1B9Uw=="],
|
||||
|
||||
"@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="],
|
||||
|
||||
"@smithy/chunked-blob-reader": ["@smithy/chunked-blob-reader@5.2.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-St+kVicSyayWQca+I1rGitaOEH6uKgE8IUWoYnnEX26SWdWQcL6LvMSD19Lg+vYHKdT9B2Zuu7rd3i6Wnyb/iw=="],
|
||||
|
||||
"@smithy/chunked-blob-reader-native": ["@smithy/chunked-blob-reader-native@4.2.3", "", { "dependencies": { "@smithy/util-base64": "^4.3.2", "tslib": "^2.6.2" } }, "sha512-jA5k5Udn7Y5717L86h4EIv06wIr3xn8GM1qHRi/Nf31annXcXHJjBKvgztnbn2TxH3xWrPBfgwHsOwZf0UmQWw=="],
|
||||
|
||||
"@smithy/config-resolver": ["@smithy/config-resolver@4.4.14", "", { "dependencies": { "@smithy/node-config-provider": "^4.3.13", "@smithy/types": "^4.14.0", "@smithy/util-config-provider": "^4.2.2", "@smithy/util-endpoints": "^3.3.4", "@smithy/util-middleware": "^4.2.13", "tslib": "^2.6.2" } }, "sha512-N55f8mPEccpzKetUagdvmAy8oohf0J5cuj9jLI1TaSceRlq0pJsIZepY3kmAXAhyxqXPV6hDerDQhqQPKWgAoQ=="],
|
||||
|
||||
"@smithy/core": ["@smithy/core@3.23.14", "", { "dependencies": { "@smithy/protocol-http": "^5.3.13", "@smithy/types": "^4.14.0", "@smithy/url-parser": "^4.2.13", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-middleware": "^4.2.13", "@smithy/util-stream": "^4.5.22", "@smithy/util-utf8": "^4.2.2", "@smithy/uuid": "^1.1.2", "tslib": "^2.6.2" } }, "sha512-vJ0IhpZxZAkFYOegMKSrxw7ujhhT2pass/1UEcZ4kfl5srTAqtPU5I7MdYQoreVas3204ykCiNhY1o7Xlz6Yyg=="],
|
||||
|
||||
"@smithy/credential-provider-imds": ["@smithy/credential-provider-imds@4.2.13", "", { "dependencies": { "@smithy/node-config-provider": "^4.3.13", "@smithy/property-provider": "^4.2.13", "@smithy/types": "^4.14.0", "@smithy/url-parser": "^4.2.13", "tslib": "^2.6.2" } }, "sha512-wboCPijzf6RJKLOvnjDAiBxGSmSnGXj35o5ZAWKDaHa/cvQ5U3ZJ13D4tMCE8JG4dxVAZFy/P0x/V9CwwdfULQ=="],
|
||||
|
||||
"@smithy/eventstream-codec": ["@smithy/eventstream-codec@4.2.13", "", { "dependencies": { "@aws-crypto/crc32": "5.2.0", "@smithy/types": "^4.14.0", "@smithy/util-hex-encoding": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-vYahwBAtRaAcFbOmE9aLr12z7RiHYDSLcnogSdxfm7kKfsNa3wH+NU5r7vTeB5rKvLsWyPjVX8iH94brP7umiQ=="],
|
||||
|
||||
"@smithy/eventstream-serde-browser": ["@smithy/eventstream-serde-browser@4.2.13", "", { "dependencies": { "@smithy/eventstream-serde-universal": "^4.2.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-wwybfcOX0tLqCcBP378TIU9IqrDuZq/tDV48LlZNydMpCnqnYr+hWBAYbRE+rFFf/p7IkDJySM3bgiMKP2ihPg=="],
|
||||
|
||||
"@smithy/eventstream-serde-config-resolver": ["@smithy/eventstream-serde-config-resolver@4.3.13", "", { "dependencies": { "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-ied1lO559PtAsMJzg2TKRlctLnEi1PfkNeMMpdwXDImk1zV9uvS/Oxoy/vcy9uv1GKZAjDAB5xT6ziE9fzm5wA=="],
|
||||
|
||||
"@smithy/eventstream-serde-node": ["@smithy/eventstream-serde-node@4.2.13", "", { "dependencies": { "@smithy/eventstream-serde-universal": "^4.2.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-hFyK+ORJrxAN3RYoaD6+gsGDQjeix8HOEkosoajvXYZ4VeqonM3G4jd9IIRm/sWGXUKmudkY9KdYjzosUqdM8A=="],
|
||||
|
||||
"@smithy/eventstream-serde-universal": ["@smithy/eventstream-serde-universal@4.2.13", "", { "dependencies": { "@smithy/eventstream-codec": "^4.2.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-kRrq4EKLGeOxhC2CBEhRNcu1KSzNJzYY7RK3S7CxMPgB5dRrv55WqQOtRwQxQLC04xqORFLUgnDlc6xrNUULaA=="],
|
||||
|
||||
"@smithy/fetch-http-handler": ["@smithy/fetch-http-handler@5.3.16", "", { "dependencies": { "@smithy/protocol-http": "^5.3.13", "@smithy/querystring-builder": "^4.2.13", "@smithy/types": "^4.14.0", "@smithy/util-base64": "^4.3.2", "tslib": "^2.6.2" } }, "sha512-nYDRUIvNd4mFmuXraRWt6w5UsZTNqtj4hXJA/iiOD4tuseIdLP9Lq38teH/SZTcIFCa2f+27o7hYpIsWktJKEQ=="],
|
||||
|
||||
"@smithy/hash-blob-browser": ["@smithy/hash-blob-browser@4.2.14", "", { "dependencies": { "@smithy/chunked-blob-reader": "^5.2.2", "@smithy/chunked-blob-reader-native": "^4.2.3", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-rtQ5es8r/5v4rav7q5QTsfx9CtCyzrz/g7ZZZBH2xtMmd6G/KQrLOWfSHTvFOUPlVy59RQvxeBYJaLRoybMEyA=="],
|
||||
|
||||
"@smithy/hash-node": ["@smithy/hash-node@4.2.13", "", { "dependencies": { "@smithy/types": "^4.14.0", "@smithy/util-buffer-from": "^4.2.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-4/oy9h0jjmY80a2gOIo75iLl8TOPhmtx4E2Hz+PfMjvx/vLtGY4TMU/35WRyH2JHPfT5CVB38u4JRow7gnmzJA=="],
|
||||
|
||||
"@smithy/hash-stream-node": ["@smithy/hash-stream-node@4.2.13", "", { "dependencies": { "@smithy/types": "^4.14.0", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-WdQ7HwUjINXETeh6dqUeob1UHIYx8kAn9PSp1HhM2WWegiZBYVy2WXIs1lB07SZLan/udys9SBnQGt9MQbDpdg=="],
|
||||
|
||||
"@smithy/invalid-dependency": ["@smithy/invalid-dependency@4.2.13", "", { "dependencies": { "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-jvC0RB/8BLj2SMIkY0Npl425IdnxZJxInpZJbu563zIRnVjpDMXevU3VMCRSabaLB0kf/eFIOusdGstrLJ8IDg=="],
|
||||
|
||||
"@smithy/is-array-buffer": ["@smithy/is-array-buffer@4.2.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-n6rQ4N8Jj4YTQO3YFrlgZuwKodf4zUFs7EJIWH86pSCWBaAtAGBFfCM7Wx6D2bBJ2xqFNxGBSrUWswT3M0VJow=="],
|
||||
|
||||
"@smithy/md5-js": ["@smithy/md5-js@4.2.13", "", { "dependencies": { "@smithy/types": "^4.14.0", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-cNm7I9NXolFxtS20ojROddOEpSAeI1Obq6pd1Kj5HtHws3s9Fkk8DdHDfQSs5KuxCewZuVK6UqrJnfJmiMzDuQ=="],
|
||||
|
||||
"@smithy/middleware-content-length": ["@smithy/middleware-content-length@4.2.13", "", { "dependencies": { "@smithy/protocol-http": "^5.3.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-IPMLm/LE4AZwu6qiE8Rr8vJsWhs9AtOdySRXrOM7xnvclp77Tyh7hMs/FRrMf26kgIe67vFJXXOSmVxS7oKeig=="],
|
||||
|
||||
"@smithy/middleware-endpoint": ["@smithy/middleware-endpoint@4.4.29", "", { "dependencies": { "@smithy/core": "^3.23.14", "@smithy/middleware-serde": "^4.2.17", "@smithy/node-config-provider": "^4.3.13", "@smithy/shared-ini-file-loader": "^4.4.8", "@smithy/types": "^4.14.0", "@smithy/url-parser": "^4.2.13", "@smithy/util-middleware": "^4.2.13", "tslib": "^2.6.2" } }, "sha512-R9Q/58U+qBiSARGWbAbFLczECg/RmysRksX6Q8BaQEpt75I7LI6WGDZnjuC9GXSGKljEbA7N118LhGaMbfrTXw=="],
|
||||
|
||||
"@smithy/middleware-retry": ["@smithy/middleware-retry@4.5.1", "", { "dependencies": { "@smithy/core": "^3.23.14", "@smithy/node-config-provider": "^4.3.13", "@smithy/protocol-http": "^5.3.13", "@smithy/service-error-classification": "^4.2.13", "@smithy/smithy-client": "^4.12.9", "@smithy/types": "^4.14.0", "@smithy/util-middleware": "^4.2.13", "@smithy/util-retry": "^4.3.1", "@smithy/uuid": "^1.1.2", "tslib": "^2.6.2" } }, "sha512-/zY+Gp7Qj2D2hVm3irkCyONER7E9MiX3cUUm/k2ZmhkzZkrPgwVS4aJ5NriZUEN/M0D1hhjrgjUmX04HhRwdWA=="],
|
||||
|
||||
"@smithy/middleware-serde": ["@smithy/middleware-serde@4.2.17", "", { "dependencies": { "@smithy/core": "^3.23.14", "@smithy/protocol-http": "^5.3.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-0T2mcaM6v9W1xku86Dk0bEW7aEseG6KenFkPK98XNw0ZhOqOiD1MrMsdnQw9QsL3/Oa85T53iSMlm0SZdSuIEQ=="],
|
||||
|
||||
"@smithy/middleware-stack": ["@smithy/middleware-stack@4.2.13", "", { "dependencies": { "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-g72jN/sGDLyTanrCLH9fhg3oysO3f7tQa6eWWsMyn2BiYNCgjF24n4/I9wff/5XidFvjj9ilipAoQrurTUrLvw=="],
|
||||
|
||||
"@smithy/node-config-provider": ["@smithy/node-config-provider@4.3.13", "", { "dependencies": { "@smithy/property-provider": "^4.2.13", "@smithy/shared-ini-file-loader": "^4.4.8", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-iGxQ04DsKXLckbgnX4ipElrOTk+IHgTyu0q0WssZfYhDm9CQWHmu6cOeI5wmWRxpXbBDhIIfXMWz5tPEtcVqbw=="],
|
||||
|
||||
"@smithy/node-http-handler": ["@smithy/node-http-handler@4.5.2", "", { "dependencies": { "@smithy/protocol-http": "^5.3.13", "@smithy/querystring-builder": "^4.2.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-/oD7u8M0oj2ZTFw7GkuuHWpIxtWdLlnyNkbrWcyVYhd5RJNDuczdkb0wfnQICyNFrVPlr8YHOhamjNy3zidhmA=="],
|
||||
|
||||
"@smithy/property-provider": ["@smithy/property-provider@4.2.13", "", { "dependencies": { "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-bGzUCthxRmezuxkbu9wD33wWg9KX3hJpCXpQ93vVkPrHn9ZW6KNNdY5xAUWNuRCwQ+VyboFuWirG1lZhhkcyRQ=="],
|
||||
|
||||
"@smithy/protocol-http": ["@smithy/protocol-http@5.3.13", "", { "dependencies": { "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-+HsmuJUF4u8POo6s8/a2Yb/AQ5t/YgLovCuHF9oxbocqv+SZ6gd8lC2duBFiCA/vFHoHQhoq7QjqJqZC6xOxxg=="],
|
||||
|
||||
"@smithy/querystring-builder": ["@smithy/querystring-builder@4.2.13", "", { "dependencies": { "@smithy/types": "^4.14.0", "@smithy/util-uri-escape": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-tG4aOYFCZdPMjbgfhnIQ322H//ojujldp1SrHPHpBSb3NqgUp3dwiUGRJzie87hS1DYwWGqDuPaowoDF+rYCbQ=="],
|
||||
|
||||
"@smithy/querystring-parser": ["@smithy/querystring-parser@4.2.13", "", { "dependencies": { "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-hqW3Q4P+CDzUyQ87GrboGMeD7XYNMOF+CuTwu936UQRB/zeYn3jys8C3w+wMkDfY7CyyyVwZQ5cNFoG0x1pYmA=="],
|
||||
|
||||
"@smithy/service-error-classification": ["@smithy/service-error-classification@4.2.13", "", { "dependencies": { "@smithy/types": "^4.14.0" } }, "sha512-a0s8XZMfOC/qpqq7RCPvJlk93rWFrElH6O++8WJKz0FqnA4Y7fkNi/0mnGgSH1C4x6MFsuBA8VKu4zxFrMe5Vw=="],
|
||||
|
||||
"@smithy/shared-ini-file-loader": ["@smithy/shared-ini-file-loader@4.4.8", "", { "dependencies": { "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-VZCZx2bZasxdqxVgEAhREvDSlkatTPnkdWy1+Kiy8w7kYPBosW0V5IeDwzDUMvWBt56zpK658rx1cOBFOYaPaw=="],
|
||||
|
||||
"@smithy/signature-v4": ["@smithy/signature-v4@5.3.13", "", { "dependencies": { "@smithy/is-array-buffer": "^4.2.2", "@smithy/protocol-http": "^5.3.13", "@smithy/types": "^4.14.0", "@smithy/util-hex-encoding": "^4.2.2", "@smithy/util-middleware": "^4.2.13", "@smithy/util-uri-escape": "^4.2.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-YpYSyM0vMDwKbHD/JA7bVOF6kToVRpa+FM5ateEVRpsTNu564g1muBlkTubXhSKKYXInhpADF46FPyrZcTLpXg=="],
|
||||
|
||||
"@smithy/smithy-client": ["@smithy/smithy-client@4.12.9", "", { "dependencies": { "@smithy/core": "^3.23.14", "@smithy/middleware-endpoint": "^4.4.29", "@smithy/middleware-stack": "^4.2.13", "@smithy/protocol-http": "^5.3.13", "@smithy/types": "^4.14.0", "@smithy/util-stream": "^4.5.22", "tslib": "^2.6.2" } }, "sha512-ovaLEcTU5olSeHcRXcxV6viaKtpkHZumn6Ps0yn7dRf2rRSfy794vpjOtrWDO0d1auDSvAqxO+lyhERSXQ03EQ=="],
|
||||
|
||||
"@smithy/types": ["@smithy/types@4.14.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-OWgntFLW88kx2qvf/c/67Vno1yuXm/f9M7QFAtVkkO29IJXGBIg0ycEaBTH0kvCtwmvZxRujrgP5a86RvsXJAQ=="],
|
||||
|
||||
"@smithy/url-parser": ["@smithy/url-parser@4.2.13", "", { "dependencies": { "@smithy/querystring-parser": "^4.2.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-2G03yoboIRZlZze2+PT4GZEjgwQsJjUgn6iTsvxA02bVceHR6vp4Cuk7TUnPFWKF+ffNUk3kj4COwkENS2K3vw=="],
|
||||
|
||||
"@smithy/util-base64": ["@smithy/util-base64@4.3.2", "", { "dependencies": { "@smithy/util-buffer-from": "^4.2.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-XRH6b0H/5A3SgblmMa5ErXQ2XKhfbQB+Fm/oyLZ2O2kCUrwgg55bU0RekmzAhuwOjA9qdN5VU2BprOvGGUkOOQ=="],
|
||||
|
||||
"@smithy/util-body-length-browser": ["@smithy/util-body-length-browser@4.2.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-JKCrLNOup3OOgmzeaKQwi4ZCTWlYR5H4Gm1r2uTMVBXoemo1UEghk5vtMi1xSu2ymgKVGW631e2fp9/R610ZjQ=="],
|
||||
|
||||
"@smithy/util-body-length-node": ["@smithy/util-body-length-node@4.2.3", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-ZkJGvqBzMHVHE7r/hcuCxlTY8pQr1kMtdsVPs7ex4mMU+EAbcXppfo5NmyxMYi2XU49eqaz56j2gsk4dHHPG/g=="],
|
||||
|
||||
"@smithy/util-buffer-from": ["@smithy/util-buffer-from@4.2.2", "", { "dependencies": { "@smithy/is-array-buffer": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-FDXD7cvUoFWwN6vtQfEta540Y/YBe5JneK3SoZg9bThSoOAC/eGeYEua6RkBgKjGa/sz6Y+DuBZj3+YEY21y4Q=="],
|
||||
|
||||
"@smithy/util-config-provider": ["@smithy/util-config-provider@4.2.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-dWU03V3XUprJwaUIFVv4iOnS1FC9HnMHDfUrlNDSh4315v0cWyaIErP8KiqGVbf5z+JupoVpNM7ZB3jFiTejvQ=="],
|
||||
|
||||
"@smithy/util-defaults-mode-browser": ["@smithy/util-defaults-mode-browser@4.3.45", "", { "dependencies": { "@smithy/property-provider": "^4.2.13", "@smithy/smithy-client": "^4.12.9", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-ag9sWc6/nWZAuK3Wm9KlFJUnRkXLrXn33RFjIAmCTFThqLHY+7wCst10BGq56FxslsDrjhSie46c8OULS+BiIw=="],
|
||||
|
||||
"@smithy/util-defaults-mode-node": ["@smithy/util-defaults-mode-node@4.2.49", "", { "dependencies": { "@smithy/config-resolver": "^4.4.14", "@smithy/credential-provider-imds": "^4.2.13", "@smithy/node-config-provider": "^4.3.13", "@smithy/property-provider": "^4.2.13", "@smithy/smithy-client": "^4.12.9", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-jlN6vHwE8gY5AfiFBavtD3QtCX2f7lM3BKkz7nFKSNfFR5nXLXLg6sqXTJEEyDwtxbztIDBQCfjsGVXlIru2lQ=="],
|
||||
|
||||
"@smithy/util-endpoints": ["@smithy/util-endpoints@3.3.4", "", { "dependencies": { "@smithy/node-config-provider": "^4.3.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-BKoR/ubPp9KNKFxPpg1J28N1+bgu8NGAtJblBP7yHy8yQPBWhIAv9+l92SlQLpolGm71CVO+btB60gTgzT0wog=="],
|
||||
|
||||
"@smithy/util-hex-encoding": ["@smithy/util-hex-encoding@4.2.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-Qcz3W5vuHK4sLQdyT93k/rfrUwdJ8/HZ+nMUOyGdpeGA1Wxt65zYwi3oEl9kOM+RswvYq90fzkNDahPS8K0OIg=="],
|
||||
|
||||
"@smithy/util-middleware": ["@smithy/util-middleware@4.2.13", "", { "dependencies": { "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-GTooyrlmRTqvUen4eK7/K1p6kryF7bnDfq6XsAbIsf2mo51B/utaH+XThY6dKgNCWzMAaH/+OLmqaBuLhLWRow=="],
|
||||
|
||||
"@smithy/util-retry": ["@smithy/util-retry@4.3.1", "", { "dependencies": { "@smithy/service-error-classification": "^4.2.13", "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-FwmicpgWOkP5kZUjN3y+3JIom8NLGqSAJBeoIgK0rIToI817TEBHCrd0A2qGeKQlgDeP+Jzn4i0H/NLAXGy9uQ=="],
|
||||
|
||||
"@smithy/util-stream": ["@smithy/util-stream@4.5.22", "", { "dependencies": { "@smithy/fetch-http-handler": "^5.3.16", "@smithy/node-http-handler": "^4.5.2", "@smithy/types": "^4.14.0", "@smithy/util-base64": "^4.3.2", "@smithy/util-buffer-from": "^4.2.2", "@smithy/util-hex-encoding": "^4.2.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-3H8iq/0BfQjUs2/4fbHZ9aG9yNzcuZs24LPkcX1Q7Z+qpqaGM8+qbGmE8zo9m2nCRgamyvS98cHdcWvR6YUsew=="],
|
||||
|
||||
"@smithy/util-uri-escape": ["@smithy/util-uri-escape@4.2.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-2kAStBlvq+lTXHyAZYfJRb/DfS3rsinLiwb+69SstC9Vb0s9vNWkRwpnj918Pfi85mzi42sOqdV72OLxWAISnw=="],
|
||||
|
||||
"@smithy/util-utf8": ["@smithy/util-utf8@4.2.2", "", { "dependencies": { "@smithy/util-buffer-from": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-75MeYpjdWRe8M5E3AW0O4Cx3UadweS+cwdXjwYGBW5h/gxxnbeZ877sLPX/ZJA9GVTlL/qG0dXP29JWFCD1Ayw=="],
|
||||
|
||||
"@smithy/util-waiter": ["@smithy/util-waiter@4.2.15", "", { "dependencies": { "@smithy/types": "^4.14.0", "tslib": "^2.6.2" } }, "sha512-oUt9o7n8hBv3BL56sLSneL0XeigZSuem0Hr78JaoK33D9oKieyCvVP8eTSe3j7g2mm/S1DvzxKieG7JEWNJUNg=="],
|
||||
|
||||
"@smithy/uuid": ["@smithy/uuid@1.1.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-O/IEdcCUKkubz60tFbGA7ceITTAJsty+lBjNoorP4Z6XRqaFb/OjQjZODophEcuq68nKm6/0r+6/lLQ+XVpk8g=="],
|
||||
|
||||
"@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="],
|
||||
|
||||
"@types/node": ["@types/node@25.5.2", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg=="],
|
||||
|
||||
"@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="],
|
||||
|
||||
"abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="],
|
||||
|
||||
"accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="],
|
||||
|
||||
"agentkeepalive": ["agentkeepalive@4.6.0", "", { "dependencies": { "humanize-ms": "^1.2.1" } }, "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ=="],
|
||||
|
||||
"ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="],
|
||||
|
||||
"ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="],
|
||||
|
||||
"argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="],
|
||||
|
||||
"asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
|
||||
|
||||
"body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="],
|
||||
|
||||
"bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="],
|
||||
|
||||
"bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="],
|
||||
|
||||
"bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="],
|
||||
|
||||
"call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="],
|
||||
|
||||
"call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="],
|
||||
|
||||
"combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="],
|
||||
|
||||
"content-disposition": ["content-disposition@1.0.1", "", {}, "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q=="],
|
||||
|
||||
"content-type": ["content-type@1.0.5", "", {}, "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA=="],
|
||||
|
||||
"cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="],
|
||||
|
||||
"cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="],
|
||||
|
||||
"cors": ["cors@2.8.6", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw=="],
|
||||
|
||||
"cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
|
||||
|
||||
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
||||
|
||||
"delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="],
|
||||
|
||||
"depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="],
|
||||
|
||||
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
|
||||
|
||||
"ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="],
|
||||
|
||||
"encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="],
|
||||
|
||||
"es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="],
|
||||
|
||||
"es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="],
|
||||
|
||||
"es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="],
|
||||
|
||||
"es-set-tostringtag": ["es-set-tostringtag@2.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA=="],
|
||||
|
||||
"escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="],
|
||||
|
||||
"esprima": ["esprima@4.0.1", "", { "bin": { "esparse": "./bin/esparse.js", "esvalidate": "./bin/esvalidate.js" } }, "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="],
|
||||
|
||||
"etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="],
|
||||
|
||||
"event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="],
|
||||
|
||||
"eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="],
|
||||
|
||||
"eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
|
||||
|
||||
"express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="],
|
||||
|
||||
"express-rate-limit": ["express-rate-limit@8.3.2", "", { "dependencies": { "ip-address": "10.1.0" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-77VmFeJkO0/rvimEDuUC5H30oqUC4EyOhyGccfqoLebB0oiEYfM7nwPrsDsBL1gsTpwfzX8SFy2MT3TDyRq+bg=="],
|
||||
|
||||
"extend-shallow": ["extend-shallow@2.0.1", "", { "dependencies": { "is-extendable": "^0.1.0" } }, "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug=="],
|
||||
|
||||
"fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="],
|
||||
|
||||
"fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="],
|
||||
|
||||
"fast-xml-builder": ["fast-xml-builder@1.1.4", "", { "dependencies": { "path-expression-matcher": "^1.1.3" } }, "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg=="],
|
||||
|
||||
"fast-xml-parser": ["fast-xml-parser@5.5.8", "", { "dependencies": { "fast-xml-builder": "^1.1.4", "path-expression-matcher": "^1.2.0", "strnum": "^2.2.0" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-Z7Fh2nVQSb2d+poDViM063ix2ZGt9jmY1nWhPfHBOK2Hgnb/OW3P4Et3P/81SEej0J7QbWtJqxO05h8QYfK7LQ=="],
|
||||
|
||||
"finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="],
|
||||
|
||||
"form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="],
|
||||
|
||||
"form-data-encoder": ["form-data-encoder@1.7.2", "", {}, "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="],
|
||||
|
||||
"formdata-node": ["formdata-node@4.4.1", "", { "dependencies": { "node-domexception": "1.0.0", "web-streams-polyfill": "4.0.0-beta.3" } }, "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ=="],
|
||||
|
||||
"forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="],
|
||||
|
||||
"fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="],
|
||||
|
||||
"function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
|
||||
|
||||
"get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="],
|
||||
|
||||
"get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="],
|
||||
|
||||
"gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="],
|
||||
|
||||
"gray-matter": ["gray-matter@4.0.3", "", { "dependencies": { "js-yaml": "^3.13.1", "kind-of": "^6.0.2", "section-matter": "^1.0.0", "strip-bom-string": "^1.0.0" } }, "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q=="],
|
||||
|
||||
"has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="],
|
||||
|
||||
"has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="],
|
||||
|
||||
"hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
|
||||
|
||||
"hono": ["hono@4.12.10", "", {}, "sha512-mx/p18PLy5og9ufies2GOSUqep98Td9q4i/EF6X7yJgAiIopxqdfIO3jbqsi3jRgTgw88jMDEzVKi+V2EF+27w=="],
|
||||
|
||||
"http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="],
|
||||
|
||||
"humanize-ms": ["humanize-ms@1.2.1", "", { "dependencies": { "ms": "^2.0.0" } }, "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ=="],
|
||||
|
||||
"iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="],
|
||||
|
||||
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
|
||||
|
||||
"ip-address": ["ip-address@10.1.0", "", {}, "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q=="],
|
||||
|
||||
"ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="],
|
||||
|
||||
"is-extendable": ["is-extendable@0.1.1", "", {}, "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw=="],
|
||||
|
||||
"is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="],
|
||||
|
||||
"isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="],
|
||||
|
||||
"jose": ["jose@6.2.2", "", {}, "sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ=="],
|
||||
|
||||
"js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="],
|
||||
|
||||
"json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="],
|
||||
|
||||
"json-schema-typed": ["json-schema-typed@8.0.2", "", {}, "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA=="],
|
||||
|
||||
"kind-of": ["kind-of@6.0.3", "", {}, "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw=="],
|
||||
|
||||
"marked": ["marked@18.0.0", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-2e7Qiv/HJSXj8rDEpgTvGKsP8yYtI9xXHKDnrftrmnrJPaFNM7VRb2YCzWaX4BP1iCJ/XPduzDJZMFoqTCcIMA=="],
|
||||
|
||||
"math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
|
||||
|
||||
"media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="],
|
||||
|
||||
"merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="],
|
||||
|
||||
"mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="],
|
||||
|
||||
"mime-types": ["mime-types@3.0.2", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A=="],
|
||||
|
||||
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
|
||||
|
||||
"negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="],
|
||||
|
||||
"node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="],
|
||||
|
||||
"node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="],
|
||||
|
||||
"object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="],
|
||||
|
||||
"object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="],
|
||||
|
||||
"on-finished": ["on-finished@2.4.1", "", { "dependencies": { "ee-first": "1.1.1" } }, "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg=="],
|
||||
|
||||
"once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="],
|
||||
|
||||
"openai": ["openai@4.104.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" }, "peerDependencies": { "ws": "^8.18.0", "zod": "^3.23.8" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA=="],
|
||||
|
||||
"parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="],
|
||||
|
||||
"path-expression-matcher": ["path-expression-matcher@1.5.0", "", {}, "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ=="],
|
||||
|
||||
"path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="],
|
||||
|
||||
"path-to-regexp": ["path-to-regexp@8.4.2", "", {}, "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA=="],
|
||||
|
||||
"pgvector": ["pgvector@0.2.1", "", {}, "sha512-nKaQY9wtuiidwLMdVIce1O3kL0d+FxrigCVzsShnoqzOSaWWWOvuctb/sYwlai5cTwwzRSNa+a/NtN2kVZGNJw=="],
|
||||
|
||||
"pkce-challenge": ["pkce-challenge@5.0.1", "", {}, "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ=="],
|
||||
|
||||
"postgres": ["postgres@3.4.9", "", {}, "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw=="],
|
||||
|
||||
"proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="],
|
||||
|
||||
"qs": ["qs@6.15.0", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ=="],
|
||||
|
||||
"range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="],
|
||||
|
||||
"raw-body": ["raw-body@3.0.2", "", { "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" } }, "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA=="],
|
||||
|
||||
"require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="],
|
||||
|
||||
"router": ["router@2.2.0", "", { "dependencies": { "debug": "^4.4.0", "depd": "^2.0.0", "is-promise": "^4.0.0", "parseurl": "^1.3.3", "path-to-regexp": "^8.0.0" } }, "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ=="],
|
||||
|
||||
"safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
|
||||
|
||||
"section-matter": ["section-matter@1.0.0", "", { "dependencies": { "extend-shallow": "^2.0.1", "kind-of": "^6.0.0" } }, "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA=="],
|
||||
|
||||
"send": ["send@1.2.1", "", { "dependencies": { "debug": "^4.4.3", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.1", "mime-types": "^3.0.2", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.2" } }, "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ=="],
|
||||
|
||||
"serve-static": ["serve-static@2.2.1", "", { "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "parseurl": "^1.3.3", "send": "^1.2.0" } }, "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw=="],
|
||||
|
||||
"setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="],
|
||||
|
||||
"shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="],
|
||||
|
||||
"shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="],
|
||||
|
||||
"side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="],
|
||||
|
||||
"side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="],
|
||||
|
||||
"side-channel-map": ["side-channel-map@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3" } }, "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA=="],
|
||||
|
||||
"side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="],
|
||||
|
||||
"sprintf-js": ["sprintf-js@1.0.3", "", {}, "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="],
|
||||
|
||||
"statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="],
|
||||
|
||||
"strip-bom-string": ["strip-bom-string@1.0.0", "", {}, "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g=="],
|
||||
|
||||
"strnum": ["strnum@2.2.3", "", {}, "sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg=="],
|
||||
|
||||
"toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="],
|
||||
|
||||
"tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="],
|
||||
|
||||
"tree-sitter-wasms": ["tree-sitter-wasms@0.1.13", "", { "dependencies": { "tree-sitter-wasms": "^0.1.11" } }, "sha512-wT+cR6DwaIz80/vho3AvSF0N4txuNx/5bcRKoXouOfClpxh/qqrF4URNLQXbbt8MaAxeksZcZd1j8gcGjc+QxQ=="],
|
||||
|
||||
"tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
|
||||
|
||||
"type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="],
|
||||
|
||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
"undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
|
||||
|
||||
"unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="],
|
||||
|
||||
"vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="],
|
||||
|
||||
"web-streams-polyfill": ["web-streams-polyfill@4.0.0-beta.3", "", {}, "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug=="],
|
||||
|
||||
"web-tree-sitter": ["web-tree-sitter@0.22.6", "", {}, "sha512-hS87TH71Zd6mGAmYCvlgxeGDjqd9GTeqXNqTT+u0Gs51uIozNIaaq/kUAbV/Zf56jb2ZOyG8BxZs2GG9wbLi6Q=="],
|
||||
|
||||
"webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="],
|
||||
|
||||
"whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="],
|
||||
|
||||
"which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
|
||||
|
||||
"wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
|
||||
|
||||
"zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="],
|
||||
|
||||
"zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="],
|
||||
|
||||
"@anthropic-ai/sdk/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="],
|
||||
|
||||
"@aws-crypto/sha1-browser/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
|
||||
|
||||
"@aws-crypto/sha256-browser/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
|
||||
|
||||
"@aws-crypto/util/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
|
||||
|
||||
"@types/bun/bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
|
||||
|
||||
"form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="],
|
||||
|
||||
"openai/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="],
|
||||
|
||||
"@anthropic-ai/sdk/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="],
|
||||
|
||||
"@aws-crypto/sha1-browser/@smithy/util-utf8/@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="],
|
||||
|
||||
"@aws-crypto/sha256-browser/@smithy/util-utf8/@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="],
|
||||
|
||||
"@aws-crypto/util/@smithy/util-utf8/@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="],
|
||||
|
||||
"form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="],
|
||||
|
||||
"openai/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="],
|
||||
|
||||
"@aws-crypto/sha1-browser/@smithy/util-utf8/@smithy/util-buffer-from/@smithy/is-array-buffer": ["@smithy/is-array-buffer@2.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA=="],
|
||||
|
||||
"@aws-crypto/sha256-browser/@smithy/util-utf8/@smithy/util-buffer-from/@smithy/is-array-buffer": ["@smithy/is-array-buffer@2.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA=="],
|
||||
|
||||
"@aws-crypto/util/@smithy/util-utf8/@smithy/util-buffer-from/@smithy/is-array-buffer": ["@smithy/is-array-buffer@2.2.0", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA=="],
|
||||
}
|
||||
}
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
[test]
|
||||
# PGLite initialization can be slow under parallel test execution.
|
||||
# Default 5s is too short when many test files boot PGLite instances at once.
|
||||
# 60s is the empirical ceiling we observed before the first file's beforeAll
|
||||
# completed on a loaded machine.
|
||||
#
|
||||
# NOTE: this bunfig.toml `timeout` key is read by `bun test` but empirically
|
||||
# does NOT apply to beforeEach/afterEach hook timeouts under `bun run test`
|
||||
# chained behind `bun run typecheck`. The test script in package.json passes
|
||||
# `--timeout=60000` explicitly to cover both per-test and per-hook timeouts.
|
||||
# Leaving both in place as belt-and-suspenders.
|
||||
timeout = 60_000
|
||||
@@ -1,117 +0,0 @@
|
||||
# docker-compose.ci.yml
|
||||
#
|
||||
# Local CI gate with 4-way E2E sharding. Spins up 4 pgvector services + a bun
|
||||
# runner that bind-mounts the repo. Used by `bun run ci:local` and
|
||||
# `bun run ci:local:diff` (see scripts/ci-local.sh).
|
||||
#
|
||||
# All services are pulled as `image:` (no build) so `docker compose pull`
|
||||
# refreshes everything. The bun version floats with `oven/bun:1` to track CI's
|
||||
# `bun-version: latest`. Named volumes isolate the Linux container's deps from
|
||||
# the host's darwin-arm64 deps and keep bun + postgres data warm across runs.
|
||||
#
|
||||
# Why 4 postgres services: bun's E2E suite shares one DB across 36 files and
|
||||
# uses TRUNCATE CASCADE in setupDB(). Running files in parallel against ONE DB
|
||||
# races (file A's TRUNCATE clobbers file B's fixture import). 4 separate DBs
|
||||
# remove the race; we shard the file list 1/4..4/4 and run shards in parallel.
|
||||
# Within a shard, files still run sequentially. Total wall-time on a 16-core
|
||||
# host: ~6 min sequential -> ~1.5-2 min sharded.
|
||||
#
|
||||
# Postgres host ports default to 5434-5437 (avoid 5432 manual `gbrain-test-pg`
|
||||
# and 5433 sibling-project conflicts). Override BASE port with GBRAIN_CI_PG_PORT;
|
||||
# shards take BASE..BASE+3.
|
||||
|
||||
services:
|
||||
postgres-1:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- "${GBRAIN_CI_PG_PORT:-5434}:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d gbrain_test"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
volumes:
|
||||
- gbrain-ci-pg-data-1:/var/lib/postgresql/data
|
||||
|
||||
postgres-2:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- "${GBRAIN_CI_PG_PORT_2:-5435}:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d gbrain_test"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
volumes:
|
||||
- gbrain-ci-pg-data-2:/var/lib/postgresql/data
|
||||
|
||||
postgres-3:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- "${GBRAIN_CI_PG_PORT_3:-5436}:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d gbrain_test"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
volumes:
|
||||
- gbrain-ci-pg-data-3:/var/lib/postgresql/data
|
||||
|
||||
postgres-4:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- "${GBRAIN_CI_PG_PORT_4:-5437}:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d gbrain_test"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
volumes:
|
||||
- gbrain-ci-pg-data-4:/var/lib/postgresql/data
|
||||
|
||||
runner:
|
||||
image: oven/bun:1
|
||||
working_dir: /app
|
||||
depends_on:
|
||||
postgres-1:
|
||||
condition: service_healthy
|
||||
postgres-2:
|
||||
condition: service_healthy
|
||||
postgres-3:
|
||||
condition: service_healthy
|
||||
postgres-4:
|
||||
condition: service_healthy
|
||||
# No global DATABASE_URL — scripts/ci-local.sh sets per-shard URL via -e.
|
||||
# Unit phase explicitly unsets DATABASE_URL so test/e2e/* gracefully skip.
|
||||
volumes:
|
||||
- .:/app
|
||||
# Linux container's node_modules MUST be isolated from host darwin-arm64.
|
||||
# Without this, container `bun install` stomps host node_modules and
|
||||
# subsequent `bun test` on host fails with binary-incompat errors.
|
||||
- gbrain-ci-node-modules:/app/node_modules
|
||||
# Warm install cache across runs.
|
||||
- gbrain-ci-bun-cache:/root/.bun/install/cache
|
||||
|
||||
volumes:
|
||||
gbrain-ci-pg-data-1:
|
||||
gbrain-ci-pg-data-2:
|
||||
gbrain-ci-pg-data-3:
|
||||
gbrain-ci-pg-data-4:
|
||||
gbrain-ci-node-modules:
|
||||
gbrain-ci-bun-cache:
|
||||
@@ -1,14 +0,0 @@
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: gbrain_test
|
||||
ports:
|
||||
- "5434:5432"
|
||||
healthcheck:
|
||||
test: pg_isready -U postgres
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
-234
@@ -1,234 +0,0 @@
|
||||
# Pluggable Engine Architecture
|
||||
|
||||
## The idea
|
||||
|
||||
Every GBrain operation goes through `BrainEngine`. The engine is the contract between "what the brain can do" and "how it's stored." Swap the engine, keep everything else.
|
||||
|
||||
v0 shipped `PostgresEngine` backed by Supabase. v0.7 adds `PGLiteEngine` -- embedded Postgres 17.5 via WASM (@electric-sql/pglite), zero-config default. The interface is designed so a `DuckDBEngine`, `TursoEngine`, or any custom backend could slot in without touching the CLI, MCP server, skills, or any consumer code.
|
||||
|
||||
## Why this matters
|
||||
|
||||
Different users have different constraints:
|
||||
|
||||
| User | Needs | Best engine |
|
||||
|------|-------|-------------|
|
||||
| Getting started | Zero-config, no accounts, no server | PGLiteEngine (default since v0.7) |
|
||||
| Power user (you) | World-class search, 7K+ pages, zero-ops | PostgresEngine + Supabase |
|
||||
| Open source hacker | Single file, no server, git-friendly | PGLiteEngine |
|
||||
| Team/enterprise | Multi-user, RLS, audit trail | PostgresEngine + self-hosted |
|
||||
| Researcher | Analytics, bulk exports, embeddings | DuckDBEngine (someday) |
|
||||
| Edge/mobile | Offline-first, sync later | PGLiteEngine + sync (someday) |
|
||||
|
||||
The engine interface means we don't have to choose. PGLite is the zero-friction default. Supabase is the production scale path. `gbrain migrate --to supabase/pglite` moves between them.
|
||||
|
||||
## The interface
|
||||
|
||||
```typescript
|
||||
// src/core/engine.ts
|
||||
|
||||
export interface BrainEngine {
|
||||
// Lifecycle
|
||||
connect(config: EngineConfig): Promise<void>;
|
||||
disconnect(): Promise<void>;
|
||||
initSchema(): Promise<void>;
|
||||
transaction<T>(fn: (engine: BrainEngine) => Promise<T>): Promise<T>;
|
||||
|
||||
// Pages CRUD
|
||||
getPage(slug: string): Promise<Page | null>;
|
||||
putPage(slug: string, page: PageInput): Promise<Page>;
|
||||
deletePage(slug: string): Promise<void>;
|
||||
listPages(filters: PageFilters): Promise<Page[]>;
|
||||
|
||||
// Search
|
||||
searchKeyword(query: string, opts?: SearchOpts): Promise<SearchResult[]>;
|
||||
searchVector(embedding: Float32Array, opts?: SearchOpts): Promise<SearchResult[]>;
|
||||
|
||||
// Chunks
|
||||
upsertChunks(slug: string, chunks: ChunkInput[]): Promise<void>;
|
||||
getChunks(slug: string): Promise<Chunk[]>;
|
||||
|
||||
// Links
|
||||
addLink(from: string, to: string, context?: string, linkType?: string): Promise<void>;
|
||||
removeLink(from: string, to: string): Promise<void>;
|
||||
getLinks(slug: string): Promise<Link[]>;
|
||||
getBacklinks(slug: string): Promise<Link[]>;
|
||||
traverseGraph(slug: string, depth?: number): Promise<GraphNode[]>;
|
||||
|
||||
// Tags
|
||||
addTag(slug: string, tag: string): Promise<void>;
|
||||
removeTag(slug: string, tag: string): Promise<void>;
|
||||
getTags(slug: string): Promise<string[]>;
|
||||
|
||||
// Timeline
|
||||
addTimelineEntry(slug: string, entry: TimelineInput): Promise<void>;
|
||||
getTimeline(slug: string, opts?: TimelineOpts): Promise<TimelineEntry[]>;
|
||||
|
||||
// Raw data
|
||||
putRawData(slug: string, source: string, data: object): Promise<void>;
|
||||
getRawData(slug: string, source?: string): Promise<RawData[]>;
|
||||
|
||||
// Versions
|
||||
createVersion(slug: string): Promise<PageVersion>;
|
||||
getVersions(slug: string): Promise<PageVersion[]>;
|
||||
revertToVersion(slug: string, versionId: number): Promise<void>;
|
||||
|
||||
// Stats + health
|
||||
getStats(): Promise<BrainStats>;
|
||||
getHealth(): Promise<BrainHealth>;
|
||||
|
||||
// Ingest log
|
||||
logIngest(entry: IngestLogInput): Promise<void>;
|
||||
getIngestLog(opts?: IngestLogOpts): Promise<IngestLogEntry[]>;
|
||||
|
||||
// Config
|
||||
getConfig(key: string): Promise<string | null>;
|
||||
setConfig(key: string, value: string): Promise<void>;
|
||||
|
||||
// Migration + advanced (added v0.7)
|
||||
runMigration(sql: string): Promise<void>;
|
||||
getChunksWithEmbeddings(slug: string): Promise<ChunkWithEmbedding[]>;
|
||||
}
|
||||
```
|
||||
|
||||
### Key design choices
|
||||
|
||||
**Slug-based API, not ID-based.** Every method takes slugs, not numeric IDs. The engine resolves slugs to IDs internally. This keeps the interface portable... slugs are strings, IDs are database-specific.
|
||||
|
||||
**Embedding is NOT in the engine.** The engine stores embeddings and searches by vector, but it doesn't generate embeddings. `src/core/embedding.ts` handles that. This is intentional: embedding is an external API call (OpenAI), not a storage concern. All engines share the same embedding service.
|
||||
|
||||
**Chunking is NOT in the engine.** Same logic. `src/core/chunkers/` handles chunking. The engine stores and retrieves chunks. All engines share the same chunkers.
|
||||
|
||||
**Search returns `SearchResult[]`, not raw rows.** The engine is responsible for its own search implementation (tsvector vs FTS5, pgvector vs sqlite-vss) but must return a uniform result type. RRF fusion and dedup happen above the engine, in `src/core/search/hybrid.ts`.
|
||||
|
||||
**`traverseGraph` exists but is engine-specific.** Postgres uses recursive CTEs. SQLite would use a loop with depth tracking. The interface is the same: give me a slug and max depth, return the graph.
|
||||
|
||||
## How search works across engines
|
||||
|
||||
```
|
||||
+-------------------+
|
||||
| hybrid.ts |
|
||||
| (RRF fusion + |
|
||||
| dedup, shared) |
|
||||
+--------+----------+
|
||||
|
|
||||
+------------+------------+
|
||||
| |
|
||||
+--------v--------+ +--------v--------+
|
||||
| engine.search | | engine.search |
|
||||
| Keyword() | | Vector() |
|
||||
+-----------------+ +-----------------+
|
||||
| |
|
||||
+-----------+-----------+ +---------+---------+
|
||||
| | | |
|
||||
+-------v-------+ +-------v---+ +-------v---+ +----v--------+
|
||||
| Postgres: | | PGLite: | | Postgres: | | PGLite: |
|
||||
| tsvector + | | tsvector +| | pgvector | | pgvector |
|
||||
| ts_rank + | | ts_rank | | HNSW | | HNSW |
|
||||
| websearch_to_ | | (same SQL)| | cosine | | cosine |
|
||||
| tsquery | | | | | | (same SQL) |
|
||||
+---------------+ +-----------+ +-----------+ +-------------+
|
||||
```
|
||||
|
||||
RRF fusion, multi-query expansion, and 4-layer dedup are engine-agnostic. They operate on `SearchResult[]` arrays. Only the raw keyword and vector searches are engine-specific.
|
||||
|
||||
## PostgresEngine (v0, ships)
|
||||
|
||||
**Dependencies:** `postgres` (porsager/postgres), `pgvector`
|
||||
|
||||
**Postgres-specific features used:**
|
||||
- `tsvector` + `GIN` index for full-text search with `ts_rank` weighting
|
||||
- `pgvector` HNSW index for cosine similarity vector search
|
||||
- `pg_trgm` + `GIN` for fuzzy slug resolution
|
||||
- Recursive CTEs for graph traversal
|
||||
- Trigger-based search_vector (spans pages + timeline_entries)
|
||||
- JSONB for frontmatter with GIN index
|
||||
- Connection pooling via Supabase Supavisor (port 6543)
|
||||
|
||||
**Hosting:** Supabase Pro ($25/mo). Zero-ops. Managed Postgres with pgvector built in.
|
||||
|
||||
**Why not self-hosted for v0:** The brain should be infrastructure agents use, not something you maintain. Self-hosted Postgres with Docker is a welcome community PR, but v0 optimizes for zero ops.
|
||||
|
||||
## PGLiteEngine (v0.7, ships)
|
||||
|
||||
**Dependencies:** `@electric-sql/pglite` (v0.4.4+)
|
||||
|
||||
**What it is:** Embedded Postgres 17.5 compiled to WASM via ElectricSQL's PGLite. Runs in-process, no server, no Docker, no accounts. Same SQL as PostgresEngine -- not a separate dialect. All 37 BrainEngine methods implemented.
|
||||
|
||||
**PGLite-specific details:**
|
||||
- Uses `pglite-schema.ts` for DDL (pgvector extension, pg_trgm, triggers, indexes)
|
||||
- Parameterized queries throughout (shared utilities in `src/core/utils.ts`)
|
||||
- `hybridSearch` keyword-only fallback when `OPENAI_API_KEY` is not set
|
||||
- Data stored at `~/.gbrain/brain.db` (configurable)
|
||||
- pgvector HNSW index for cosine similarity vector search (same as Postgres)
|
||||
- tsvector + ts_rank for full-text search (same as Postgres)
|
||||
- pg_trgm for fuzzy slug resolution (same as Postgres)
|
||||
|
||||
**When to use PGLite vs Postgres:**
|
||||
|
||||
| Factor | PGLite | PostgresEngine + Supabase |
|
||||
|--------|--------|--------------------------|
|
||||
| Setup | `gbrain init` (zero-config) | Account + connection string |
|
||||
| Scale | Good for < 1,000 files | Production-proven at 10K+ |
|
||||
| Multi-device | Single machine only | Any device via remote MCP |
|
||||
| Cost | Free | Supabase Pro ($25/mo) |
|
||||
| Concurrency | Single process | Connection pooling |
|
||||
| Backups | Manual (file copy) | Managed by Supabase |
|
||||
|
||||
**Migration:** `gbrain migrate --to supabase` exports everything (pages, chunks, embeddings, links, tags, timeline) and imports into Supabase. `gbrain migrate --to pglite` goes the other direction. Bidirectional, lossless.
|
||||
|
||||
## Adding a new engine
|
||||
|
||||
1. Create `src/core/<name>-engine.ts` implementing `BrainEngine`
|
||||
2. Add to engine factory in `src/core/engine-factory.ts`:
|
||||
```typescript
|
||||
export function createEngine(type: string): BrainEngine {
|
||||
switch (type) {
|
||||
case 'pglite': return new PGLiteEngine();
|
||||
case 'postgres': return new PostgresEngine();
|
||||
case 'myengine': return new MyEngine();
|
||||
default: throw new Error(`Unknown engine: ${type}`);
|
||||
}
|
||||
}
|
||||
```
|
||||
The factory uses dynamic imports so engines are only loaded when selected.
|
||||
3. Store engine type in `~/.gbrain/config.json`: `{ "engine": "myengine", ... }`
|
||||
4. Add tests. The test suite should be engine-agnostic where possible... same test cases, different engine constructor.
|
||||
5. Document in this file + add a design doc in `docs/`
|
||||
|
||||
### What you DON'T need to touch
|
||||
|
||||
- `src/cli.ts` (dispatches to engine, doesn't know which one)
|
||||
- `src/mcp/server.ts` (same)
|
||||
- `src/core/chunkers/*` (shared across engines)
|
||||
- `src/core/embedding.ts` (shared across engines)
|
||||
- `src/core/search/hybrid.ts`, `expansion.ts`, `dedup.ts` (shared, operate on SearchResult[])
|
||||
- `skills/*` (fat markdown, engine-agnostic)
|
||||
|
||||
### What you DO need to implement
|
||||
|
||||
Every method in `BrainEngine`. The full interface. No optional methods, no feature flags. If your engine can't do vector search (e.g., a pure-text engine), implement `searchVector` to return `[]` and document the limitation.
|
||||
|
||||
## Capability matrix
|
||||
|
||||
| Capability | PostgresEngine | PGLiteEngine | Notes |
|
||||
|-----------|---------------|-------------|-------|
|
||||
| CRUD | Full | Full | Same SQL |
|
||||
| Keyword search | tsvector + ts_rank | tsvector + ts_rank | Identical (real Postgres) |
|
||||
| Vector search | pgvector HNSW | pgvector HNSW | Identical (real Postgres) |
|
||||
| Fuzzy slug | pg_trgm | pg_trgm | Identical (real Postgres) |
|
||||
| Graph traversal | Recursive CTE | Recursive CTE | Same SQL |
|
||||
| Transactions | Full ACID | Full ACID | Both support this |
|
||||
| JSONB queries | GIN index | GIN index | Identical |
|
||||
| Concurrent access | Connection pooling | Single process | PGLite limitation |
|
||||
| Hosting | Supabase, self-hosted, Docker | Local file | |
|
||||
| Migration methods | runMigration, getChunksWithEmbeddings | Same | Added v0.7 |
|
||||
|
||||
## Future engine ideas
|
||||
|
||||
**TursoEngine.** libSQL (SQLite fork) with embedded replicas and HTTP edge access. Would give SQLite's simplicity with cloud sync. Interesting for mobile/edge use cases.
|
||||
|
||||
**DuckDBEngine.** Analytical workloads. Bulk exports, embedding analysis, brain-wide statistics. Not for OLTP. Could be a secondary engine for analytics alongside Postgres for operations.
|
||||
|
||||
**Custom/Remote.** The interface is clean enough that someone could build an engine backed by any storage: Firestore, DynamoDB, a REST API, even a flat file system. The interface doesn't assume SQL.
|
||||
|
||||
Note: The original SQLite engine plan (`docs/SQLITE_ENGINE.md`) was superseded by PGLite. PGLite uses the same SQL as Postgres, eliminating the need for a separate SQLite dialect with FTS5/sqlite-vss translation.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,133 +0,0 @@
|
||||
<!-- skillpack-version: 0.7.0 -->
|
||||
<!-- source: https://raw.githubusercontent.com/garrytan/gbrain/master/docs/GBRAIN_SKILLPACK.md -->
|
||||
# GBrain Skillpack: Reference Architecture for AI Agents
|
||||
|
||||
This is a reference architecture for how a production AI agent uses gbrain as its
|
||||
knowledge backbone. Based on patterns from a real deployment with 14,700+ brain
|
||||
files, 40+ skills, and 20+ cron jobs running continuously.
|
||||
|
||||
**The memex vision, realized.** Vannevar Bush imagined a device where an individual
|
||||
stores everything, mechanized so it may be consulted with exceeding speed. GBrain is
|
||||
that device, except the memex builds itself. The agent detects entities, enriches
|
||||
pages, creates cross-references, and maintains compiled truth automatically.
|
||||
|
||||
Each section below is a standalone guide. Click through to the full content.
|
||||
|
||||
---
|
||||
|
||||
## Core Patterns
|
||||
|
||||
The foundational read-write loop and data model.
|
||||
|
||||
| Guide | What It Covers |
|
||||
|-------|---------------|
|
||||
| [The Brain-Agent Loop](guides/brain-agent-loop.md) | The read-write cycle that makes the brain compound over time |
|
||||
| [Entity Detection](guides/entity-detection.md) | Run it on every message. Capture original thinking + entity mentions |
|
||||
| [The Originals Folder](guides/originals-folder.md) | Capturing WHAT YOU THINK, not just what you found |
|
||||
| [Brain-First Lookup](guides/brain-first-lookup.md) | Check the brain before calling any external API |
|
||||
| [Compiled Truth + Timeline](guides/compiled-truth.md) | Above the line: current synthesis. Below: append-only evidence |
|
||||
| [Source Attribution](guides/source-attribution.md) | Every fact needs a citation. Format and hierarchy |
|
||||
|
||||
## Data Pipelines
|
||||
|
||||
Getting data in and keeping it current.
|
||||
|
||||
| Guide | What It Covers |
|
||||
|-------|---------------|
|
||||
| [Enrichment Pipeline](guides/enrichment-pipeline.md) | 7-step protocol, tier system (Tier 1/2/3 by importance) |
|
||||
| [Meeting Ingestion](guides/meeting-ingestion.md) | Always pull complete transcript, propagate to all entity pages |
|
||||
| [Content & Media Ingestion](guides/content-media.md) | YouTube, social media bundles, PDFs/documents |
|
||||
| [Diligence Ingestion](guides/diligence-ingestion.md) | Data room materials: pitch decks, financial models, cap tables |
|
||||
| [Deterministic Collectors](guides/deterministic-collectors.md) | Code for data, LLMs for judgment. The collector pattern |
|
||||
| [Idea Capture & Originals](guides/idea-capture.md) | Depth test, originality distribution, deep cross-linking |
|
||||
| [Getting Data In](integrations/README.md) | Integration recipes: voice, email, X, calendar |
|
||||
|
||||
## Operations
|
||||
|
||||
Running a production brain.
|
||||
|
||||
| Guide | What It Covers |
|
||||
|-------|---------------|
|
||||
| [Reference Cron Schedule](guides/cron-schedule.md) | 20+ recurring jobs, quiet hours, dream cycle |
|
||||
| [Cron via Minions](../skills/conventions/cron-via-minions.md) | Why scheduled work runs as Minion jobs, not `agentTurn`. Auto-applied by v0.11.0 migration for built-in handlers; host-specific handlers use the plugin contract below. |
|
||||
| [Plugin Handlers](guides/plugin-handlers.md) | Registering host-specific Minion handlers via code (no data-file exec surface). |
|
||||
| [Minions fix](guides/minions-fix.md) | Repairing a half-migrated v0.11.0 install. |
|
||||
| [Shell jobs (v0.14.0+)](guides/minions-shell-jobs.md) | Move deterministic crons (API fetch, token refresh, scrape+write) off the LLM gateway. Zero tokens per fire, ~60% gateway headroom. Follow `skills/migrations/v0.14.0.md` for the adoption playbook. |
|
||||
| [Quiet Hours & Timezone](guides/quiet-hours.md) | Hold notifications during sleep, timezone-aware delivery |
|
||||
| [Executive Assistant Pattern](guides/executive-assistant.md) | Email triage, meeting prep, scheduling |
|
||||
| [Operational Disciplines](guides/operational-disciplines.md) | Signal detection, brain-first, sync-after-write, heartbeat, dream cycle |
|
||||
| [Skill Development Cycle](guides/skill-development.md) | 5-step cycle: concept, prototype, evaluate, codify, cron |
|
||||
|
||||
**Subagent routing (v0.11.0+):** agents that dispatch background work should route through
|
||||
`skills/conventions/subagent-routing.md` — it reads `~/.gbrain/preferences.json#minion_mode`
|
||||
and branches between native subagents and Minion jobs. The v0.11.0 migration auto-injects
|
||||
a marker into AGENTS.md pointing at this convention.
|
||||
|
||||
**Cron routing (v0.11.0+):** scheduled work goes through Minions, not OpenClaw's `agentTurn`.
|
||||
See `skills/conventions/cron-via-minions.md` for the rewrite pattern. The v0.11.0 migration
|
||||
auto-rewrites entries whose handler is a gbrain builtin; host-specific handlers (e.g.
|
||||
`ea-inbox-sweep`) need a code-level registration per `docs/guides/plugin-handlers.md`.
|
||||
|
||||
## Architecture
|
||||
|
||||
How to structure your system.
|
||||
|
||||
| Guide | What It Covers |
|
||||
|-------|---------------|
|
||||
| [Two-Repo Architecture](guides/repo-architecture.md) | Agent repo vs brain repo, boundary rules, decision tree |
|
||||
| [Sub-Agent Model Routing](guides/sub-agent-routing.md) | Which model for which task, signal detector pattern, cost optimization |
|
||||
| [The Three Search Modes](guides/search-modes.md) | Keyword, hybrid, direct. When to use each |
|
||||
| [Brain vs Agent Memory](guides/brain-vs-memory.md) | 3 layers: GBrain (world knowledge), agent memory, session |
|
||||
|
||||
## Integrations
|
||||
|
||||
Wiring up your life.
|
||||
|
||||
| Guide | What It Covers |
|
||||
|-------|---------------|
|
||||
| [Credential Gateway](integrations/credential-gateway.md) | ClawVisor / Hermes for Gmail, Calendar, Contacts |
|
||||
| [Meeting & Call Webhooks](integrations/meeting-webhooks.md) | Circleback transcripts + Quo/OpenPhone SMS/calls |
|
||||
| [Voice-to-Brain](../recipes/twilio-voice-brain.md) | Phone calls + WebRTC browser calls create brain pages. 25 production patterns: identity separation, bid system, conversation timing, proactive advisor, prompt compression, caller routing, dynamic VAD, real-time logging, belt-and-suspenders post-call |
|
||||
| [Email-to-Brain](../recipes/email-to-brain.md) | Gmail messages flow into entity pages via deterministic collector |
|
||||
| [X-to-Brain](../recipes/x-to-brain.md) | Twitter monitoring with deletion detection + engagement velocity |
|
||||
| [Calendar-to-Brain](../recipes/calendar-to-brain.md) | Google Calendar events become searchable daily brain pages |
|
||||
| [Meeting Sync](../recipes/meeting-sync.md) | Circleback transcripts auto-import with attendee propagation |
|
||||
|
||||
## Administration
|
||||
|
||||
Keeping it running and up to date.
|
||||
|
||||
| Guide | What It Covers |
|
||||
|-------|---------------|
|
||||
| [Upgrades & Auto-Update](guides/upgrades-auto-update.md) | check-update, agent notifications, migration files |
|
||||
| [Live Sync](guides/live-sync.md) | Keep the index current: cron, --watch, webhook approaches |
|
||||
|
||||
---
|
||||
|
||||
## Appendix: GBrain CLI Quick Reference
|
||||
|
||||
| Command | Purpose |
|
||||
|---------|---------|
|
||||
| `gbrain search "term"` | Keyword search across all brain pages |
|
||||
| `gbrain query "question"` | Hybrid search (vector + keyword + RRF) |
|
||||
| `gbrain get <slug>` | Read a specific brain page by slug |
|
||||
| `gbrain sync` | Sync local markdown repo to gbrain index |
|
||||
| `gbrain import <path>` | Import files into the brain |
|
||||
| `gbrain embed --stale` | Re-embed pages with stale or missing embeddings |
|
||||
| `gbrain integrations` | Manage integration recipes (senses + reflexes) |
|
||||
| `gbrain stats` | Show brain statistics (page count, last sync, etc.) |
|
||||
| `gbrain doctor` | Diagnose brain health issues |
|
||||
| `gbrain check-update` | Check for new versions and integration recipes |
|
||||
|
||||
Run `gbrain --help` for the full command reference.
|
||||
|
||||
---
|
||||
|
||||
## Architecture & Philosophy
|
||||
|
||||
- [Infrastructure Layer](architecture/infra-layer.md) — Import pipeline, chunking, embedding, search
|
||||
- [Thin Harness, Fat Skills](ethos/THIN_HARNESS_FAT_SKILLS.md) — Architecture philosophy
|
||||
- [Markdown Skills as Recipes](ethos/MARKDOWN_SKILLS_AS_RECIPES.md) — Why markdown is code and your agent is a package manager
|
||||
- [Homebrew for Personal AI](designs/HOMEBREW_FOR_PERSONAL_AI.md) — The 10-star vision
|
||||
- [Recommended Schema](GBRAIN_RECOMMENDED_SCHEMA.md) — Directory structure for your brain repo
|
||||
- [Verification Runbook](GBRAIN_VERIFY.md) — End-to-end installation verification
|
||||
@@ -1,544 +0,0 @@
|
||||
# GBrain v0: Postgres-Native Personal Knowledge Brain
|
||||
|
||||
## What this is
|
||||
|
||||
GBrain is a compiled intelligence system. Not a note-taking app. Not "chat with your notes."
|
||||
|
||||
Every page is an intelligence assessment. Above the line: compiled truth (your current best understanding, rewritten when evidence changes). Below the line: timeline (append-only evidence trail). AI agents maintain the brain. MCP clients query it. The intelligence lives in fat markdown skills, not application code.
|
||||
|
||||
The core insight: personal knowledge at scale is an intelligence problem, not a storage problem.
|
||||
|
||||
## Why it exists
|
||||
|
||||
A 7,471-file / 2.3GB markdown wiki is choking git. Git doesn't scale past ~5K files for wiki-style use. The compiled truth + timeline model (Karpathy-style knowledge pages) is right, but it needs a real database underneath.
|
||||
|
||||
There's already a production-grade RAG system (Ruby on Rails, Postgres + pgvector) with 3-tier chunking, hybrid search with RRF, multi-query expansion, and 4-layer dedup. GBrain ports these proven patterns to a standalone Bun + TypeScript tool.
|
||||
|
||||
## The knowledge model
|
||||
|
||||
```
|
||||
+--------------------------------------------------+
|
||||
| Page: concepts/do-things-that-dont-scale |
|
||||
| |
|
||||
| --- frontmatter (YAML) --- |
|
||||
| type: concept |
|
||||
| tags: [startups, growth, pg-essay] |
|
||||
| |
|
||||
| === COMPILED TRUTH === |
|
||||
| Current best understanding. |
|
||||
| Rewritten on new evidence. |
|
||||
| This is the "what we know now" section. |
|
||||
| |
|
||||
| --- |
|
||||
| |
|
||||
| === TIMELINE === |
|
||||
| Append-only evidence trail. |
|
||||
| - 2013-07-01: Published on paulgraham.com |
|
||||
| - 2024-11-15: Referenced in batch kickoff talk |
|
||||
| Never edited, only appended. |
|
||||
+--------------------------------------------------+
|
||||
| |
|
||||
v v
|
||||
[Semantic chunks] [Recursive chunks]
|
||||
(best quality for (predictable format
|
||||
compiled truth) for timeline)
|
||||
| |
|
||||
v v
|
||||
[Embeddings: text-embedding-3-large, 1536 dims]
|
||||
|
|
||||
v
|
||||
[HNSW index + tsvector + pg_trgm]
|
||||
|
|
||||
v
|
||||
[Hybrid search: vector + keyword + RRF fusion]
|
||||
```
|
||||
|
||||
## Architecture decisions
|
||||
|
||||
### v0 stack
|
||||
|
||||
| Layer | Choice | Why |
|
||||
|-------|--------|-----|
|
||||
| Database | Postgres + pgvector | Proven RAG patterns, production-tested. World-class hybrid search. |
|
||||
| Hosting | Supabase Pro ($25/mo) | Zero-ops. Managed Postgres, pgvector, connection pooling. 8GB storage. |
|
||||
| Runtime | Bun + TypeScript | Consistent with GStack ecosystem. Fast. Compiles to single binary. |
|
||||
| Embeddings | OpenAI text-embedding-3-large | 1536 dims (reduced from 3072 via dimensions API). ~$0.13/1M tokens. |
|
||||
| LLM (chunking/expansion) | Claude Haiku | Cheapest model for topic boundary detection and query expansion. |
|
||||
| Background jobs | Trigger.dev | Serverless. Embed backfill, stale detection, orphan audit, tag consistency. |
|
||||
| Distribution | npm package + compiled binary + MCP server | Library for OpenClaw, CLI for humans, MCP for agents. |
|
||||
|
||||
### What we chose and why
|
||||
|
||||
**Postgres over SQLite.** We have 3+ years of proven RAG patterns running on Postgres. tsvector for full-text search, pgvector HNSW for semantic search, pg_trgm for fuzzy slug matching. Porting these to SQLite would mean reimplementing search from scratch. SQLite is a future pluggable engine for lightweight open source users (see `docs/ENGINES.md`).
|
||||
|
||||
**Supabase over self-hosted.** Zero maintenance. The brain should be infrastructure that AI agents use, not something you administer. Free tier has pgvector but only 500MB (not enough for 7K+ pages with embeddings, which need ~750MB). Pro tier at $25/mo gives 8GB. No Docker, no self-hosted Postgres in v1.
|
||||
|
||||
**Full port over minimal viable.** The patterns are proven. The port is mechanical. Shipping the full 3-tier chunking + hybrid search + 4-layer dedup means world-class RAG from day one. "We'll add that later" means rebuilding everything later.
|
||||
|
||||
**Library-first distribution.** gbrain is an npm package. OpenClaw installs it as a dependency (`bun add gbrain`), imports the engine directly. Zero-overhead function calls, shared connection pool, TypeScript types. The CLI and MCP server are thin wrappers over the same engine.
|
||||
|
||||
**Trigger-based tsvector (not generated column).** To include timeline_entries content in full-text search, the tsvector needs to span multiple tables. Generated columns can't do cross-table references. A trigger on pages + timeline_entries updates the search_vector.
|
||||
|
||||
**Auto-embed during import.** No separate embed step. `gbrain import` chunks and embeds in one pass. Progress bar shows status. `--no-embed` flag for users who want to defer. `embedded_at` column enables `gbrain embed --stale` for backfill.
|
||||
|
||||
## Distribution model
|
||||
|
||||
```
|
||||
+-------------------+ +-------------------+ +-------------------+
|
||||
| npm package | | Compiled binary | | MCP server |
|
||||
| (library) | | (CLI) | | (stdio) |
|
||||
+-------------------+ +-------------------+ +-------------------+
|
||||
| | | | | |
|
||||
| bun add gbrain | | GitHub Releases | | gbrain serve |
|
||||
| import { Postgres | | npx gbrain | | in mcp.json |
|
||||
| Engine } | | | | |
|
||||
| | | | | |
|
||||
| WHO: OpenClaw, | | WHO: Humans | | WHO: Claude Code, |
|
||||
| AlphaClaw | | | | Cursor, etc. |
|
||||
+-------------------+ +-------------------+ +-------------------+
|
||||
| | |
|
||||
+-------------------------+-------------------------+
|
||||
|
|
||||
+--------v--------+
|
||||
| BrainEngine |
|
||||
| (pluggable |
|
||||
| interface) |
|
||||
+-----------------+
|
||||
|
|
||||
+-------------+-------------+
|
||||
| |
|
||||
+------v------+ +-------v-------+
|
||||
| Postgres | | SQLite |
|
||||
| Engine | | Engine |
|
||||
| (v0, ships) | | (future, see |
|
||||
+-------------+ | ENGINES.md) |
|
||||
+---------------+
|
||||
```
|
||||
|
||||
package.json exports:
|
||||
- Library: `src/core/index.ts` (BrainEngine interface, PostgresEngine, types)
|
||||
- CLI binary: `src/cli.ts`
|
||||
|
||||
## First-time experience
|
||||
|
||||
### Path 1: OpenClaw user (primary)
|
||||
|
||||
OpenClaw is the AI orchestrator that uses gbrain as its knowledge backend. This is the most common install path.
|
||||
|
||||
```bash
|
||||
# 1. Install gbrain as a ClawHub skill
|
||||
clawhub install gbrain
|
||||
|
||||
# 2. The skill runs guided setup on first use:
|
||||
# - Detects if Supabase CLI is available
|
||||
# - If yes: auto-provisions a new Supabase project
|
||||
# - If no: prompts for connection URL
|
||||
# - Runs schema migration
|
||||
# - Scans for markdown repos and imports user's content
|
||||
# - Shows live entity/edge extraction animation
|
||||
# - Brain is ready
|
||||
|
||||
# 3. From OpenClaw, brain tools are now available:
|
||||
# "Search the brain for [topic from your data]"
|
||||
# "Ingest my meeting notes from today"
|
||||
# "How many pages are in the brain?"
|
||||
```
|
||||
|
||||
Behind the scenes, `clawhub install gbrain`:
|
||||
1. Installs the `gbrain` npm package
|
||||
2. Ships SKILL.md files (ingest, query, maintain, enrich, briefing, migrate)
|
||||
3. Registers brain tools with the orchestrator
|
||||
4. Runs `gbrain init --supabase` on first use (guided wizard)
|
||||
|
||||
### Path 2: CLI user (standalone)
|
||||
|
||||
```bash
|
||||
# 1. Install
|
||||
npm install -g gbrain
|
||||
# or: download binary from GitHub Releases
|
||||
|
||||
# 2. Initialize with Supabase
|
||||
gbrain init --supabase
|
||||
# Guided wizard:
|
||||
# Try 1: Supabase CLI auto-provision (npx supabase)
|
||||
# Try 2: If CLI not installed or not logged in, fallback:
|
||||
# "Enter your Supabase connection URL:"
|
||||
# Then: runs schema migration, verifies pgvector extension
|
||||
# Then: verifies database is ready for import
|
||||
# Output: "Brain ready. Run: gbrain import <your-repo>"
|
||||
|
||||
# 3. Import your data
|
||||
gbrain import /path/to/markdown/wiki/
|
||||
# Progress bar: 7,471 files, auto-chunk, auto-embed
|
||||
# ~30s for text import, ~10-15 min for embedding
|
||||
|
||||
# 4. Query
|
||||
gbrain query "what does PG say about doing things that don't scale?"
|
||||
```
|
||||
|
||||
### Path 3: MCP user (Claude Code, Cursor)
|
||||
|
||||
```json
|
||||
// ~/.config/claude/mcp.json
|
||||
{
|
||||
"mcpServers": {
|
||||
"gbrain": {
|
||||
"command": "gbrain",
|
||||
"args": ["serve"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Then in Claude Code: "Search my brain for people who know about robotics"
|
||||
|
||||
### The init wizard in detail
|
||||
|
||||
`gbrain init --supabase` runs through these steps:
|
||||
|
||||
```
|
||||
Step 1: Database Setup
|
||||
├── Check for Supabase CLI (npx supabase --version)
|
||||
│ ├── Found + logged in → auto-create project
|
||||
│ │ ├── Create project via supabase CLI
|
||||
│ │ ├── Wait for project to be ready
|
||||
│ │ └── Extract connection string
|
||||
│ ├── Found + not logged in →
|
||||
│ │ └── Error: "Supabase CLI found but not logged in."
|
||||
│ │ Cause: "You need to authenticate first."
|
||||
│ │ Fix: "Run: npx supabase login"
|
||||
│ │ Docs: "https://supabase.com/docs/guides/cli"
|
||||
│ └── Not found → fallback to manual
|
||||
│ └── Prompt: "Enter your Supabase connection URL:"
|
||||
│
|
||||
Step 2: Schema Migration
|
||||
├── Connect to database
|
||||
├── CREATE EXTENSION IF NOT EXISTS vector
|
||||
├── CREATE EXTENSION IF NOT EXISTS pg_trgm
|
||||
├── Run src/schema.sql (all tables, indexes, triggers)
|
||||
└── Verify: test insert + vector query
|
||||
|
||||
Step 3: Config
|
||||
├── Write ~/.gbrain/config.json (0600 permissions)
|
||||
│ { "database_url": "...", "service_role_key": "..." }
|
||||
└── Verify connection
|
||||
|
||||
Step 4: Kindling Import
|
||||
├── Import 10 bundled PG essays as demo data
|
||||
├── Chunk + embed each essay
|
||||
├── Show live entity/edge extraction animation:
|
||||
│ "Extracting entities... Paul Graham (person), Y Combinator (company)..."
|
||||
│ "Creating links... Paul Graham → Y Combinator (founded)..."
|
||||
└── Output: "Brain ready. 10 pages imported."
|
||||
|
||||
Step 5: First Query
|
||||
└── "Try: gbrain query 'what does PG say about doing things that don't scale?'"
|
||||
```
|
||||
|
||||
Every error follows the style guide: problem + cause + fix + docs link.
|
||||
|
||||
## CLI commands
|
||||
|
||||
```
|
||||
gbrain init [--supabase|--url <conn>] # create brain
|
||||
gbrain get <slug> # read a page
|
||||
gbrain put <slug> [< file.md] # write/update a page
|
||||
gbrain search <query> # keyword search (tsvector)
|
||||
gbrain query <question> # hybrid search (RRF + expansion)
|
||||
gbrain ingest <file> [--type ...] # ingest a source document
|
||||
gbrain link <from> <to> [--type <type>] # create typed link
|
||||
gbrain unlink <from> <to> # remove link
|
||||
gbrain graph <slug> [--depth 5] # traverse link graph (recursive CTE)
|
||||
gbrain backlinks <slug> # incoming links
|
||||
gbrain tags <slug> # list tags
|
||||
gbrain tag <slug> <tag> # add tag
|
||||
gbrain untag <slug> <tag> # remove tag
|
||||
gbrain timeline [<slug>] # view timeline
|
||||
gbrain timeline-add <slug> <date> <text> # add timeline entry
|
||||
gbrain list [--type] [--tag] [--limit] # list with filters
|
||||
gbrain stats # brain statistics
|
||||
gbrain health # brain health dashboard
|
||||
gbrain import <dir> [--no-embed] # import from markdown directory
|
||||
gbrain export [--dir ./export/] # export to markdown (round-trip)
|
||||
gbrain embed [<slug>|--all|--stale] # generate/refresh embeddings
|
||||
gbrain serve # MCP server (stdio)
|
||||
gbrain call <tool> '<json>' # raw tool invocation
|
||||
gbrain upgrade # self-update (npm, binary, ClawHub)
|
||||
gbrain version # version info
|
||||
gbrain config [get|set] <key> [value] # brain config
|
||||
```
|
||||
|
||||
CLI and MCP expose identical operations. Drift tests assert identical results for all operations across both interfaces.
|
||||
|
||||
## Database schema
|
||||
|
||||
9 tables in Postgres + pgvector:
|
||||
|
||||
```
|
||||
+------------------+ +-------------------+ +------------------+
|
||||
| pages |---->| content_chunks | | links |
|
||||
|------------------| |-------------------| |------------------|
|
||||
| id (PK) | | id (PK) | | id (PK) |
|
||||
| slug (UNIQUE) | | page_id (FK) | | from_page_id(FK) |
|
||||
| type | | chunk_index | | to_page_id (FK) |
|
||||
| title | | chunk_text | | link_type |
|
||||
| compiled_truth | | chunk_source | | context |
|
||||
| timeline | | embedding (1536) | +------------------+
|
||||
| frontmatter(JSONB)| | model |
|
||||
| search_vector | | token_count | +------------------+
|
||||
| created_at | | embedded_at | | tags |
|
||||
| updated_at | +-------------------+ |------------------|
|
||||
+------------------+ | id (PK) |
|
||||
| | page_id (FK) |
|
||||
+-----> +--------------------+ | tag |
|
||||
| | timeline_entries | +------------------+
|
||||
| |--------------------|
|
||||
| | id (PK) | +------------------+
|
||||
| | page_id (FK) | | page_versions |
|
||||
| | date | |------------------|
|
||||
| | source | | id (PK) |
|
||||
| | summary | | page_id (FK) |
|
||||
| | detail (markdown) | | compiled_truth |
|
||||
| +--------------------+ | frontmatter |
|
||||
| | snapshot_at |
|
||||
+-----> +--------------------+ +------------------+
|
||||
| | raw_data |
|
||||
| |--------------------| +------------------+
|
||||
| | id (PK) | | config |
|
||||
| | page_id (FK) | |------------------|
|
||||
| | source | | key (PK) |
|
||||
| | data (JSONB) | | value |
|
||||
| +--------------------+ +------------------+
|
||||
|
|
||||
+-----> +--------------------+
|
||||
| ingest_log |
|
||||
|--------------------|
|
||||
| id (PK) |
|
||||
| source_type |
|
||||
| source_ref |
|
||||
| pages_updated |
|
||||
| summary |
|
||||
+--------------------+
|
||||
```
|
||||
|
||||
Indexes:
|
||||
- `pages.slug`: UNIQUE constraint (implicit B-tree)
|
||||
- `pages.type`: B-tree
|
||||
- `pages.search_vector`: GIN (full-text search)
|
||||
- `pages.frontmatter`: GIN (JSONB queries)
|
||||
- `pages.title`: GIN with pg_trgm (fuzzy slug resolution)
|
||||
- `content_chunks.embedding`: HNSW with cosine ops (vector search)
|
||||
- `content_chunks.page_id`: B-tree
|
||||
- `links.from_page_id`, `links.to_page_id`: B-tree
|
||||
- `tags.tag`, `tags.page_id`: B-tree
|
||||
- `timeline_entries.page_id`, `timeline_entries.date`: B-tree
|
||||
|
||||
## Search architecture
|
||||
|
||||
```
|
||||
Query: "when should you ignore conventional wisdom?"
|
||||
|
|
||||
v
|
||||
+---------------------+
|
||||
| Multi-query expansion|
|
||||
| (Claude Haiku) |
|
||||
| "contrarian thinking"
|
||||
| "going against the crowd"
|
||||
+---------------------+
|
||||
| | |
|
||||
v v v
|
||||
[embed all 3 queries]
|
||||
| | |
|
||||
+---+---+
|
||||
|
|
||||
+----+----+
|
||||
| |
|
||||
v v
|
||||
+--------+ +--------+
|
||||
| Vector | | Keyword|
|
||||
| Search | | Search |
|
||||
| (HNSW | | (tsv + |
|
||||
| cosine)| | ts_rank)|
|
||||
+--------+ +--------+
|
||||
| |
|
||||
+----+----+
|
||||
|
|
||||
v
|
||||
+------------------+
|
||||
| RRF Fusion |
|
||||
| score = sum( |
|
||||
| 1/(60 + rank)) |
|
||||
+------------------+
|
||||
|
|
||||
v
|
||||
+------------------+
|
||||
| 4-Layer Dedup |
|
||||
| 1. By source |
|
||||
| 2. Cosine > 0.85 |
|
||||
| 3. Type cap 60% |
|
||||
| 4. Per-page max |
|
||||
+------------------+
|
||||
|
|
||||
v
|
||||
+------------------+
|
||||
| Stale alerts |
|
||||
| (compiled_truth |
|
||||
| older than |
|
||||
| latest timeline)|
|
||||
+------------------+
|
||||
|
|
||||
v
|
||||
[Results]
|
||||
```
|
||||
|
||||
## Chunking strategies
|
||||
|
||||
| Strategy | Input | Algorithm | When to use |
|
||||
|----------|-------|-----------|-------------|
|
||||
| Recursive | Any text | 5-level delimiter hierarchy (paragraphs > lines > sentences > clauses > whitespace). 300-word chunks, 50-word overlap. | Timeline (predictable format), bulk import |
|
||||
| Semantic | Quality text | Embed each sentence, Savitzky-Golay filter for topic boundaries, cosine similarity minima. Falls back to recursive. | Compiled truth (intelligence assessments) |
|
||||
| LLM-guided | High-value text | Pre-split to 128-word candidates, Claude Haiku finds topic shifts in sliding windows. 3 retries per window. | Explicitly requested via `--chunker llm` |
|
||||
|
||||
Dispatch: compiled_truth gets semantic chunker. Timeline gets recursive chunker. Override with `--chunker` flag or `chunk_strategy` in frontmatter.
|
||||
|
||||
## Skills (fat markdown, no code)
|
||||
|
||||
Each skill is a markdown file that AI agents (Claude Code, OpenClaw) read and follow. The skill contains the workflow, heuristics, and quality rules. No skill logic is in the binary.
|
||||
|
||||
| Skill | What it does |
|
||||
|-------|-------------|
|
||||
| `skills/ingest/SKILL.md` | Ingest meetings, docs, articles. Update compiled truth, append timeline, create links. |
|
||||
| `skills/query/SKILL.md` | 3-layer search (FTS + vector + structured). Synthesize answer with citations. |
|
||||
| `skills/maintain/SKILL.md` | Find contradictions, stale info, orphans, dead links, tag inconsistency. |
|
||||
| `skills/enrich/SKILL.md` | Enrich from external APIs (Crustdata, Happenstance, Exa). Store raw data, distill to compiled truth. |
|
||||
| `skills/briefing/SKILL.md` | Daily briefing: meetings with context, active deals, open threads. |
|
||||
| `skills/migrate/SKILL.md` | Universal migration from Obsidian, Notion, Logseq, plain markdown, CSV, JSON, Roam. |
|
||||
|
||||
## CEO scope expansions (accepted for v0)
|
||||
|
||||
1. **CLI/MCP parity with drift tests.** Both interfaces are thin wrappers over the engine. Tests assert identical output.
|
||||
2. **Smart slug resolution.** Fuzzy matching via pg_trgm for reads. Writes require exact slugs. `gbrain get "dont scale"` resolves to `concepts/do-things-that-dont-scale`.
|
||||
3. **Brain health dashboard.** `gbrain health` shows page count, embed coverage, stale pages, orphans, dead links.
|
||||
4. **Normalized timeline.** `timeline_entries` table only (no TEXT column). `detail` field supports markdown.
|
||||
5. **Page version control.** `page_versions` table stores full snapshots (compiled_truth + frontmatter + links + tags). `gbrain history`, `gbrain diff`, `gbrain revert` commands. Revert re-chunks and re-embeds.
|
||||
6. **Typed links + graph traversal.** `link_type` column (knows, invested_in, works_at, etc.). `gbrain graph` uses recursive CTE with max depth (default 5, configurable via `--depth`).
|
||||
7. **Trigger.dev data cleanup jobs.** Daily embed backfill, weekly stale detection + orphan audit + tag consistency.
|
||||
8. **Stale alert annotations.** Search results flag pages where compiled_truth is older than latest timeline entry.
|
||||
9. **Timeline merge on ingest.** Same event created across all mentioned entities.
|
||||
|
||||
## Security model (v0)
|
||||
|
||||
Single-user, local-only:
|
||||
- Supabase service role key in `~/.gbrain/config.json` (0600 permissions)
|
||||
- MCP stdio transport is inherently local (client spawns `gbrain serve` as subprocess)
|
||||
- No multi-user, no RLS, no OAuth in v0
|
||||
- Multi-user path (future): Supabase RLS + per-user API keys
|
||||
|
||||
## Upgrade mechanism
|
||||
|
||||
`gbrain upgrade` detects the installation method and updates accordingly:
|
||||
|
||||
| Path | How |
|
||||
|------|-----|
|
||||
| npm | `bun update gbrain` (or npm equivalent) |
|
||||
| Compiled binary | Download new binary to temp dir, atomic rename swap, exec new process |
|
||||
| ClawHub | `clawhub update gbrain` |
|
||||
|
||||
Version check: compare local version against latest GitHub release tag.
|
||||
|
||||
## Storage and cost estimates
|
||||
|
||||
### Storage (~750MB for 7,471 pages)
|
||||
|
||||
| Component | Size |
|
||||
|-----------|------|
|
||||
| Page text (compiled_truth + timeline) | ~150MB |
|
||||
| JSONB frontmatter | ~20MB |
|
||||
| tsvector + GIN indexes | ~50MB |
|
||||
| Content chunks (~22K, text) | ~80MB |
|
||||
| Embeddings (22K x 1536 floats x 4 bytes) | ~134MB |
|
||||
| HNSW index overhead (~2x embeddings) | ~270MB |
|
||||
| Links, tags, timeline, raw_data, versions | ~50MB |
|
||||
| **Total** | **~750MB** |
|
||||
|
||||
Supabase free tier (500MB) won't fit. Supabase Pro ($25/mo, 8GB) is the starting point.
|
||||
|
||||
### Embedding cost (~$4-5 for initial import)
|
||||
|
||||
| Step | Cost |
|
||||
|------|------|
|
||||
| Semantic chunker sentence embeddings (~374K sentences) | ~$1 |
|
||||
| Chunk embeddings (~22K chunks) | ~$0.30 |
|
||||
| Query expansion (per query, ~3 embeds) | negligible |
|
||||
| **Total initial import** | **~$4-5** |
|
||||
|
||||
Budget alternative: `gbrain import --chunker recursive` skips sentence-level embeddings, then `gbrain embed --rechunk --chunker semantic` upgrades later.
|
||||
|
||||
## Serverless operations stack
|
||||
|
||||
```
|
||||
+------------------+ +------------------+ +------------------+
|
||||
| Supabase | | Vercel | | Trigger.dev |
|
||||
| (Postgres + | | (web/API, | | (background |
|
||||
| pgvector) | | optional) | | jobs) |
|
||||
+------------------+ +------------------+ +------------------+
|
||||
| Database | | Future web UI | | Embed backfill |
|
||||
| Connection pool | | API endpoints | | Stale detection |
|
||||
| pgvector HNSW | | Edge functions | | Orphan audit |
|
||||
| tsvector FTS | | | | Tag consistency |
|
||||
| pg_trgm fuzzy | | | | Daily briefing |
|
||||
+------------------+ +------------------+ +------------------+
|
||||
```
|
||||
|
||||
The CLI connects directly to Supabase Postgres. Trigger.dev and Vercel are for async/scheduled work. The CLI works without them.
|
||||
|
||||
## Verification checklist
|
||||
|
||||
1. `gbrain import /data/brain/` migrates all 7,471 files losslessly
|
||||
2. `gbrain export` round-trips to semantically identical markdown
|
||||
3. `gbrain query "what does PG say about doing things that don't scale?"` returns relevant hybrid search results
|
||||
4. `gbrain serve` starts MCP server connectable by Claude Code
|
||||
5. All 3 chunkers produce correct output with test fixtures
|
||||
6. `gbrain init --supabase` works end-to-end
|
||||
7. `bun test` passes all tests
|
||||
8. `clawhub install gbrain` installs the skill and runs guided setup
|
||||
9. `bun add gbrain` + `import { PostgresEngine } from 'gbrain'` works in external project
|
||||
10. Drift tests pass: CLI and MCP produce identical results
|
||||
11. `gbrain health` outputs accurate brain health metrics
|
||||
12. Migration skill successfully imports an Obsidian vault
|
||||
|
||||
## Future plans
|
||||
|
||||
See `docs/ENGINES.md` for the pluggable engine architecture and future backend plans.
|
||||
|
||||
### v1 candidates (deferred from v0)
|
||||
|
||||
- **`gbrain ask` natural language CLI alias.** Trivial to add. P1 TODO.
|
||||
- **Intelligence compiler.** Treat every fact as a first-class claim with source span, entity links, validity window, confidence, and contradiction status. "What changed, why, and what evidence would flip it again?" From Codex review. Builds on compiled truth model.
|
||||
- **Active skills via Trigger.dev.** Application-specific briefings, meeting prep. Belongs in OpenClaw, not generic brain infra.
|
||||
- **Multi-user access.** Supabase RLS + per-user API keys. v0 is single-user.
|
||||
- **SQLite engine.** Community PRs welcome. See `docs/SQLITE_ENGINE.md`.
|
||||
- **Docker Compose for self-hosted Postgres.** Community PRs welcome.
|
||||
- **Web UI.** Optional Vercel-hosted dashboard for browsing brain pages.
|
||||
|
||||
### Interface abstraction principle
|
||||
|
||||
All operations go through `BrainEngine`. The engine interface is the contract. Postgres-specific features (tsvector, pgvector HNSW, pg_trgm, recursive CTEs) are implementation details inside `PostgresEngine`. The interface exposes capabilities, not SQL.
|
||||
|
||||
This means:
|
||||
- A SQLite engine can implement `searchKeyword` using FTS5 instead of tsvector
|
||||
- A SQLite engine can implement `searchVector` using sqlite-vss instead of pgvector
|
||||
- A future DuckDB engine could implement analytics-heavy workloads
|
||||
- The CLI, MCP server, and library consumers never know which engine runs underneath
|
||||
|
||||
See `docs/ENGINES.md` for the full interface spec and `docs/SQLITE_ENGINE.md` for the SQLite implementation plan.
|
||||
|
||||
## Review history
|
||||
|
||||
| Review | Runs | Status | Key findings |
|
||||
|--------|------|--------|-------------|
|
||||
| /office-hours | 1 | APPROVED | Builder mode. Full port approach chosen. |
|
||||
| /plan-ceo-review | 1 | CLEAR | 11 proposals, 10 accepted, 1 deferred. SCOPE EXPANSION mode. |
|
||||
| /codex review | 1 | issues_found | 24 points challenged, 3 accepted (fuzzy slug, revert spec, tsvector). |
|
||||
| /plan-eng-review | 2 | CLEAR | 3 issues (upgrade paths, import guardrails, init wizard), 0 critical gaps. |
|
||||
| /plan-devex-review | 1 | CLEAR | DX score 5/10 to 7/10. TTHW 25min to 90s. Champion tier. |
|
||||
@@ -1,293 +0,0 @@
|
||||
# GBrain Installation Verification Runbook
|
||||
|
||||
Run these checks after install to confirm every part of GBrain is working.
|
||||
Each check includes the command, expected output, and what to do if it fails.
|
||||
|
||||
The most important check is #4 (live sync). "Sync ran" is not the same as
|
||||
"sync worked." A sync that silently skips pages because of a pooler bug is
|
||||
worse than no sync at all, because you think it's working.
|
||||
|
||||
---
|
||||
|
||||
## 1. Schema Verification
|
||||
|
||||
**Command:**
|
||||
|
||||
```bash
|
||||
gbrain doctor --json
|
||||
```
|
||||
|
||||
**Expected:** All checks return `"ok"`:
|
||||
- `connection`: connected, N pages
|
||||
- `pgvector`: extension installed
|
||||
- `rls`: enabled on all tables
|
||||
- `schema_version`: current
|
||||
- `embeddings`: coverage percentage
|
||||
|
||||
**If it fails:** The doctor output includes specific fix instructions for each
|
||||
check. See `skills/setup/SKILL.md` Error Recovery table.
|
||||
|
||||
---
|
||||
|
||||
## 2. Skillpack Loaded
|
||||
|
||||
**Check:** Ask the agent: "What is the brain-agent loop?"
|
||||
|
||||
**Expected:** The agent references GBRAIN_SKILLPACK.md Section 2 and describes
|
||||
the read-write cycle: detect entities, read brain, respond with context, write
|
||||
brain, sync.
|
||||
|
||||
**If it fails:** The agent hasn't loaded the skillpack. Run step 6 from the
|
||||
install paste (read `docs/GBRAIN_SKILLPACK.md`).
|
||||
|
||||
---
|
||||
|
||||
## 3. Auto-Update Configured
|
||||
|
||||
**Command:**
|
||||
|
||||
```bash
|
||||
gbrain check-update --json
|
||||
```
|
||||
|
||||
**Expected:** Returns JSON with `current_version`, `latest_version`,
|
||||
`update_available` (boolean). The cron `gbrain-update-check` is registered.
|
||||
|
||||
**If it fails:** Run step 7 from the install paste. See GBRAIN_SKILLPACK.md
|
||||
Section 17.
|
||||
|
||||
---
|
||||
|
||||
## 4. Live Sync Actually Works
|
||||
|
||||
This is the most important check. Three parts.
|
||||
|
||||
### 4a. Coverage Check
|
||||
|
||||
Compare page count in the DB against syncable file count in the repo:
|
||||
|
||||
```bash
|
||||
gbrain stats
|
||||
```
|
||||
|
||||
Then count syncable files:
|
||||
|
||||
```bash
|
||||
find /data/brain -name '*.md' \
|
||||
-not -path '*/.*' \
|
||||
-not -path '*/.raw/*' \
|
||||
-not -path '*/ops/*' \
|
||||
-not -name 'README.md' \
|
||||
-not -name 'index.md' \
|
||||
-not -name 'schema.md' \
|
||||
-not -name 'log.md' \
|
||||
| wc -l
|
||||
```
|
||||
|
||||
**Expected:** Page count in `gbrain stats` should be close to the file count.
|
||||
Some difference is normal (files added since last sync), but if page count is
|
||||
less than half the file count, sync is silently skipping pages.
|
||||
|
||||
**If page count is way too low:** The #1 cause is the connection pooler bug.
|
||||
Check your `DATABASE_URL`:
|
||||
- If it contains `pooler.supabase.com:6543`, verify it's using **Session mode**,
|
||||
not Transaction mode.
|
||||
- Transaction mode breaks `engine.transaction()` and causes `.begin() is not a
|
||||
function` errors.
|
||||
- Fix: switch to Session mode pooler string, then run `gbrain sync --full`
|
||||
to reimport everything.
|
||||
|
||||
### 4b. Embed Check
|
||||
|
||||
```bash
|
||||
gbrain stats
|
||||
```
|
||||
|
||||
**Expected:** Embedded chunk count should be close to total chunk count.
|
||||
|
||||
**If embedded is much lower than total:**
|
||||
|
||||
```bash
|
||||
gbrain embed --stale
|
||||
```
|
||||
|
||||
If `OPENAI_API_KEY` is not set, embeddings can't be generated. Keyword search
|
||||
still works without embeddings, but hybrid/semantic search won't.
|
||||
|
||||
### 4c. End-to-End Test
|
||||
|
||||
This is the real test. Edit a brain page, push, wait, search.
|
||||
|
||||
1. Edit a page in the brain repo (e.g., correct a fact on a person's page):
|
||||
|
||||
```bash
|
||||
# Example: fix a line in Gustaf's page
|
||||
cd /data/brain
|
||||
# Make a small edit to any .md file
|
||||
git add -A && git commit -m "test: verify live sync" && git push
|
||||
```
|
||||
|
||||
2. Wait for the next sync cycle (cron interval or `--watch` poll).
|
||||
|
||||
3. Search for the corrected text:
|
||||
|
||||
```bash
|
||||
gbrain search "<text from the correction>"
|
||||
```
|
||||
|
||||
**Expected:** The search returns the **corrected** text, not the old version.
|
||||
|
||||
**If it returns old text:** Sync failed silently. Check:
|
||||
- Is the sync cron registered and running?
|
||||
- Is `gbrain sync --watch` still alive (if using watch mode)?
|
||||
- Run `gbrain config get sync.last_run` to see when sync last ran.
|
||||
- Run `gbrain sync --repo /data/brain` manually and check for errors.
|
||||
- If you see `.begin() is not a function`, fix the pooler (see 4a above).
|
||||
|
||||
---
|
||||
|
||||
## 5. Embedding Coverage
|
||||
|
||||
**Command:**
|
||||
|
||||
```bash
|
||||
gbrain stats
|
||||
```
|
||||
|
||||
**Expected:** Embedded chunk count matches (or is close to) total chunk count.
|
||||
|
||||
**If zero or very low:** `OPENAI_API_KEY` may be missing or invalid. Check:
|
||||
|
||||
```bash
|
||||
echo $OPENAI_API_KEY | head -c 10
|
||||
```
|
||||
|
||||
If blank, set the key. Then:
|
||||
|
||||
```bash
|
||||
gbrain embed --stale
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Brain-First Lookup Protocol
|
||||
|
||||
**Check:** Ask the agent about a person or concept that exists in the brain.
|
||||
|
||||
**Expected:** The agent uses `gbrain search` or `gbrain query` FIRST, not grep
|
||||
or external APIs. The response includes brain-sourced context with source
|
||||
attribution.
|
||||
|
||||
**If it fails:** The brain-first lookup protocol isn't injected into the agent's
|
||||
system context. See `skills/setup/SKILL.md` Phase D.
|
||||
|
||||
---
|
||||
|
||||
## 7. Knowledge Graph Wired
|
||||
|
||||
The v0.12.0 graph layer needs to be populated for existing brains. New writes are
|
||||
auto-linked, but historical pages need a one-time backfill.
|
||||
|
||||
**Command:**
|
||||
|
||||
```bash
|
||||
gbrain stats | grep -E 'links|timeline'
|
||||
```
|
||||
|
||||
**Expected:** Both `links` and `timeline_entries` are non-zero (assuming the brain
|
||||
has content with entity references and dated markdown).
|
||||
|
||||
**If it's zero on a brain with imported content:** Run the backfill.
|
||||
|
||||
```bash
|
||||
gbrain extract links --source db --dry-run | head -5 # preview
|
||||
gbrain extract links --source db # commit
|
||||
gbrain extract timeline --source db
|
||||
gbrain stats # confirm > 0
|
||||
```
|
||||
|
||||
**Bonus check** — graph traversal works:
|
||||
|
||||
```bash
|
||||
# Pick any well-connected slug from your brain
|
||||
gbrain graph-query people/<some-person-slug> --depth 2
|
||||
```
|
||||
|
||||
**Expected:** Indented tree of typed edges (`--attended-->`, `--works_at-->`, etc.).
|
||||
If the slug has no inbound or outbound links, try a different one or run extract
|
||||
again.
|
||||
|
||||
**If extract finds nothing:** Your pages may not use entity-reference syntax. The
|
||||
extractor matches `[Name](people/slug)`, `[Name](../people/slug.md)`, and bare
|
||||
`people/slug` references. If your brain uses a different format, the auto-link
|
||||
heuristics won't find them — file an issue with a sample page.
|
||||
|
||||
---
|
||||
|
||||
## 8. JSONB Frontmatter Integrity (v0.12.2)
|
||||
|
||||
Postgres-backed brains created before v0.12.2 had double-encoded JSONB columns
|
||||
(`frontmatter->>'key'` returned NULL, GIN indexes were inert). `gbrain upgrade`
|
||||
runs `gbrain repair-jsonb` automatically via the `v0_12_2` orchestrator.
|
||||
Verify the repair succeeded.
|
||||
|
||||
**Command:**
|
||||
|
||||
```bash
|
||||
gbrain repair-jsonb --dry-run --json
|
||||
```
|
||||
|
||||
**Expected:** `totalRepaired: 0` across all 5 columns (`pages.frontmatter`,
|
||||
`raw_data.data`, `ingest_log.pages_updated`, `files.metadata`,
|
||||
`page_versions.frontmatter`). A zero count means every row is properly-typed
|
||||
JSON objects, not string-encoded JSON.
|
||||
|
||||
**If the count is > 0:** The repair didn't run or was interrupted. Re-run
|
||||
without `--dry-run`:
|
||||
|
||||
```bash
|
||||
gbrain repair-jsonb
|
||||
```
|
||||
|
||||
Idempotent. PGLite brains always report 0 (unaffected by the original bug).
|
||||
|
||||
**Bonus check** — frontmatter-keyed queries actually resolve:
|
||||
|
||||
```bash
|
||||
gbrain call list_pages '{"frontmatterKey": "type", "frontmatterValue": "person"}'
|
||||
```
|
||||
|
||||
If this returns rows on a brain with person pages, the JSONB path is healthy.
|
||||
|
||||
---
|
||||
|
||||
## Quick Verification (all checks in one pass)
|
||||
|
||||
```bash
|
||||
# 1. Schema
|
||||
gbrain doctor --json
|
||||
|
||||
# 2. Sync recency
|
||||
gbrain config get sync.last_run
|
||||
|
||||
# 3. Page count + embed coverage
|
||||
gbrain stats
|
||||
|
||||
# 4. Search works
|
||||
gbrain search "test query from your brain content"
|
||||
|
||||
# 5. Catch any unembedded chunks
|
||||
gbrain embed --stale
|
||||
|
||||
# 6. Auto-update
|
||||
gbrain check-update --json
|
||||
|
||||
# 7. Knowledge graph populated (links + timeline > 0)
|
||||
gbrain stats | grep -E 'links|timeline'
|
||||
|
||||
# 8. JSONB integrity (v0.12.2 — Postgres only, PGLite always 0)
|
||||
gbrain repair-jsonb --dry-run --json
|
||||
```
|
||||
|
||||
If all eight return successfully, the installation is healthy. For the full
|
||||
end-to-end sync test (4c), push a real change and verify it appears in search.
|
||||
@@ -1,540 +0,0 @@
|
||||
# Upgrading Downstream Agents
|
||||
|
||||
GBrain ships skills in `skills/`. Downstream agents (custom OpenClaw deployments,
|
||||
agent forks of any kind) often **copy** these skill files into their own workspace and
|
||||
diverge over time — adding agent-specific phases, removing irrelevant ones, tightening
|
||||
language. Once that happens, gbrain can't push updates to those forks. The agent has
|
||||
to apply the diffs by hand.
|
||||
|
||||
This doc lists the exact diffs each downstream agent needs to apply when upgrading.
|
||||
Cross-reference against your fork's local skill files.
|
||||
|
||||
## Why this exists
|
||||
|
||||
`gbrain upgrade` ships the new binary. `gbrain post-upgrade [--execute --yes]` runs
|
||||
the schema migrations and backfills the data. But the **skill files themselves**
|
||||
that tell the agent how to behave — those are user-owned. If your `~/git/<your-agent>/workspace/skills/brain-ops/SKILL.md`
|
||||
says `# Based on gbrain v0.10.0` at the top, it doesn't know about v0.12.0 features.
|
||||
|
||||
The agent will keep manually calling `gbrain link` after every `put_page` (now redundant —
|
||||
auto-link does it), miss out on `gbrain graph-query` for relationship questions, and
|
||||
not know to backfill the structured timeline.
|
||||
|
||||
## How to apply
|
||||
|
||||
1. Identify your forked skill files. Typically at `~/git/<your-agent>/workspace/skills/` or wherever your agent's skill directory lives.
|
||||
2. For each skill listed below, find the matching phase/section in your fork.
|
||||
3. Apply the diff (paste the new block in the indicated location).
|
||||
4. Update the version banner at the top of your fork (`# Based on gbrain v0.12.0`).
|
||||
5. Verify: ask the agent to write a test page and confirm the response includes
|
||||
`auto_links: { created, removed, errors }`.
|
||||
|
||||
Total time: ~10 minutes for all four skills.
|
||||
|
||||
---
|
||||
|
||||
## 1. brain-ops/SKILL.md
|
||||
|
||||
**Where:** Insert a new `### Phase 2.5` section immediately after `### Phase 2: On Every Inbound Signal`.
|
||||
|
||||
**Why:** Phase 2.5 declares that auto-link runs automatically. Without this, the
|
||||
agent's mental model says it must call `gbrain link` after every `put_page`, which
|
||||
is now redundant and can cause double-add warnings.
|
||||
|
||||
```markdown
|
||||
### Phase 2.5: Structured Graph Updates (automatic)
|
||||
|
||||
Every `put_page` call automatically extracts entity references and writes them
|
||||
to the graph (`links` table) with inferred relationship types. Stale links
|
||||
(refs no longer in the page text) are removed in the same call. This is
|
||||
"auto-link" reconciliation.
|
||||
|
||||
- No manual `add_link` calls needed for ordinary page writes.
|
||||
- Inferred link types: `attended` (meeting -> person), `works_at`, `invested_in`,
|
||||
`founded`, `advises`, `source` (frontmatter), `mentions` (default).
|
||||
- The `put_page` MCP response includes `auto_links: { created, removed, errors }`
|
||||
so the agent can verify outcomes.
|
||||
- To disable: `gbrain config set auto_link false`. Default is on.
|
||||
- Timeline entries with specific dates still need explicit `gbrain timeline-add`
|
||||
(or batch via `gbrain extract timeline --source db`).
|
||||
```
|
||||
|
||||
**Also update the Iron Law section.** If your fork still says "Back-links maintained
|
||||
on every brain write (Iron Law)" without qualification, append:
|
||||
|
||||
```markdown
|
||||
**v0.12.0 update:** Auto-link satisfies the Iron Law for entity-reference links
|
||||
on every `put_page`. The agent's Iron Law obligation is now: include the
|
||||
entity reference in the page content (e.g., `[Alice](people/alice)`); auto-link
|
||||
handles the structured row. Manual `add_link` calls are reserved for
|
||||
relationships you can't express in markdown content.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. meeting-ingestion/SKILL.md
|
||||
|
||||
**Where:** Append to the end of `### Phase 3: Attendee enrichment`.
|
||||
|
||||
**Why:** Eliminates redundant `gbrain link` calls per attendee (auto-link handles them
|
||||
when the meeting page references attendees as `[Name](people/slug)`).
|
||||
|
||||
```markdown
|
||||
**Note (v0.12.0):** Once the meeting page is written via `gbrain put`, the
|
||||
auto-link post-hook automatically creates `attended` links from the meeting
|
||||
to each attendee whose page is referenced as `[Name](people/slug)`. You don't
|
||||
need to call `gbrain link` for attendees. You DO still need `gbrain timeline-add`
|
||||
for dated events (auto-link only handles links, not timeline entries).
|
||||
```
|
||||
|
||||
**Where:** In `### Phase 4: Entity propagation`, the line "Back-link from entity page
|
||||
to meeting page" can be replaced with:
|
||||
|
||||
```markdown
|
||||
4. Entity references in the meeting page body auto-create the link via auto-link.
|
||||
For incoming references on the entity page (entity page → meeting page), edit
|
||||
the entity page to mention the meeting and `put_page` it — auto-link handles
|
||||
the rest.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. signal-detector/SKILL.md
|
||||
|
||||
**Where:** Append to the end of `### Phase 2: Entity Detection`.
|
||||
|
||||
**Why:** Same logic as brain-ops — eliminates manual `gbrain link` after writing
|
||||
originals/ideas pages that reference people or companies.
|
||||
|
||||
```markdown
|
||||
**Auto-link (v0.12.0):** When you write/update an originals or ideas page that
|
||||
references a person or company, the auto-link post-hook on `put_page`
|
||||
automatically creates the link from the new page to that entity. You don't
|
||||
need to call `gbrain link` manually. Timeline entries still need explicit calls.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. enrich/SKILL.md
|
||||
|
||||
**Where:** Replace `### Step 7: Cross-reference` with the v0.12.0 version.
|
||||
|
||||
**Why:** Step 7 used to be primarily about creating links between related entity
|
||||
pages. With auto-link, that's automatic. Step 7 is now about content updates,
|
||||
not link creation.
|
||||
|
||||
Old (delete):
|
||||
```markdown
|
||||
### Step 7: Cross-reference
|
||||
|
||||
- Update company pages from person enrichment (and vice versa)
|
||||
- Update related project/deal pages if relevant context surfaced
|
||||
- Check index files if the brain uses them
|
||||
- Add back-links manually via `gbrain link` for any new entity references
|
||||
```
|
||||
|
||||
New (paste):
|
||||
```markdown
|
||||
### Step 7: Cross-reference
|
||||
|
||||
- Update company pages from person enrichment (and vice versa)
|
||||
- Update related project/deal pages if relevant context surfaced
|
||||
- Check index files if the brain uses them
|
||||
|
||||
**Note (v0.12.0):** Links between brain pages are auto-created on every
|
||||
`put_page` call (auto-link post-hook). Step 7 focuses on content
|
||||
cross-references (updating related pages' compiled truth with new signal
|
||||
from this enrichment), not on creating links. Verify via the `auto_links`
|
||||
field in the put_page response (`{ created, removed, errors }`).
|
||||
Timeline entries still need explicit `gbrain timeline-add` calls.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## After all four diffs are applied
|
||||
|
||||
1. **Bump the version banner** at the top of each forked file:
|
||||
```
|
||||
# Based on gbrain v0.12.0 skills/<skill-name>, extended with <your-agent>-specific config
|
||||
```
|
||||
|
||||
2. **Run the v0.12.0 backfill** (this populates the graph for your existing brain):
|
||||
```bash
|
||||
gbrain post-upgrade
|
||||
```
|
||||
The v0.12.0 release wires post-upgrade to call `apply-migrations --yes`
|
||||
automatically, which runs the v0_12_0 orchestrator (schema → config check →
|
||||
`extract links --source db` → `extract timeline --source db` → verify).
|
||||
Idempotent; cheap when nothing is pending.
|
||||
|
||||
3. **Verify auto-link works:** ask the agent to write a test page that references
|
||||
`[Some Person](people/some-person)`. Confirm the put_page response includes
|
||||
`auto_links: { created: 1, removed: 0, errors: 0 }`.
|
||||
|
||||
4. **Verify graph traversal works:**
|
||||
```bash
|
||||
gbrain graph-query people/some-well-connected-person --depth 2
|
||||
```
|
||||
Should return an indented tree of typed edges.
|
||||
|
||||
---
|
||||
|
||||
## v0.12.2 hotfix (data-correctness, no skill edits)
|
||||
|
||||
v0.12.2 is a Postgres data-correctness hotfix. No forked skill files need to
|
||||
change — the skill contracts are unchanged. But you DO need to run the migration,
|
||||
and you should know about one behavior change in markdown parsing.
|
||||
|
||||
### 1. Run the migration (Postgres-backed brains)
|
||||
|
||||
```bash
|
||||
gbrain upgrade
|
||||
```
|
||||
|
||||
The `v0_12_2` orchestrator runs `gbrain repair-jsonb` automatically. It rewrites
|
||||
rows where `jsonb_typeof = 'string'` across `pages.frontmatter`, `raw_data.data`,
|
||||
`ingest_log.pages_updated`, `files.metadata`, and `page_versions.frontmatter`.
|
||||
Idempotent, safe to re-run. PGLite brains no-op cleanly.
|
||||
|
||||
Verify after upgrade:
|
||||
|
||||
```bash
|
||||
gbrain repair-jsonb --dry-run --json # expect totalRepaired: 0
|
||||
```
|
||||
|
||||
### 2. Recover any truncated wiki articles
|
||||
|
||||
If your brain imported wiki-style markdown before v0.12.2, some pages were
|
||||
silently truncated (any standalone `---` in body content was treated as a
|
||||
timeline separator). Re-import from source:
|
||||
|
||||
```bash
|
||||
gbrain sync --full
|
||||
```
|
||||
|
||||
The new `splitBody` rebuilds `compiled_truth` correctly.
|
||||
|
||||
### 3. Know the splitBody contract going forward
|
||||
|
||||
`splitBody` now requires an explicit timeline sentinel. Recognized markers
|
||||
(priority order):
|
||||
|
||||
1. `<!-- timeline -->` (preferred — what `serializeMarkdown` emits)
|
||||
2. `--- timeline ---` (decorated separator)
|
||||
3. `---` directly before `## Timeline` or `## History` heading (backward-compat)
|
||||
|
||||
A bare `---` in body text is now a markdown horizontal rule, not a timeline
|
||||
separator. If your agent writes pages with a bare `---` delimiter, migrate to
|
||||
`<!-- timeline -->` — the `serializeMarkdown` helper already does this.
|
||||
|
||||
### 4. Wiki subtypes now auto-typed
|
||||
|
||||
`inferType` now auto-detects five additional directory patterns as their own
|
||||
page types (previously they all defaulted to `concept`):
|
||||
|
||||
| Path pattern | New type |
|
||||
|------------------------|----------------|
|
||||
| `/wiki/analysis/` | `analysis` |
|
||||
| `/wiki/guides/` | `guide` |
|
||||
| `/wiki/hardware/` | `hardware` |
|
||||
| `/wiki/architecture/` | `architecture` |
|
||||
| `/writing/` | `writing` |
|
||||
|
||||
If your skills or queries filter by `type=concept` and expect wiki content in
|
||||
that bucket, update them to include the new types.
|
||||
|
||||
---
|
||||
|
||||
## v0.13.0 — Frontmatter Relationship Indexing
|
||||
|
||||
**Verdict: no action required for most skills.** v0.13 projects YAML frontmatter fields into the graph as typed edges. The ingestion API is unchanged — keep calling `put_page` with frontmatter the way you do today; the graph auto-populates behind the scenes.
|
||||
|
||||
Three skills get an optional new phase if you want to consume the new `auto_links.unresolved` response field. Without this, unresolvable frontmatter names silently skip (same as v0.12 behavior).
|
||||
|
||||
### 1. meeting-ingestion/SKILL.md (optional)
|
||||
|
||||
**Where:** Add a new section after "Phase 3: Write Meeting Page".
|
||||
|
||||
```markdown
|
||||
### Phase 3.5: Check for unresolved attendees (v0.13+)
|
||||
|
||||
After `put_page`, inspect `response.auto_links.unresolved` — an array of frontmatter
|
||||
references that did not resolve to existing pages. For meetings, this usually means
|
||||
attendees you haven't created a person page for yet.
|
||||
|
||||
If `unresolved.length > 0`:
|
||||
- Option 1 (create pages now): trigger an enrichment pass to build the missing people pages.
|
||||
- Option 2 (defer): log the unresolved names to the enrichment queue for later.
|
||||
- Option 3 (accept the gap): the attendee edge will not be created until a page exists.
|
||||
Re-running `gbrain extract links --source db --include-frontmatter` after creating
|
||||
the page fills in the missing edges.
|
||||
```
|
||||
|
||||
### 2. enrich/SKILL.md (optional)
|
||||
|
||||
**Where:** Add to the enrichment trigger list.
|
||||
|
||||
```markdown
|
||||
### Drain unresolved frontmatter names (v0.13+)
|
||||
|
||||
If any `put_page` response includes `auto_links.unresolved` entries, the enrichment
|
||||
tier should pick up those (field, name) pairs and try to create the missing entity
|
||||
pages. Example flow:
|
||||
|
||||
1. signal-detector captures a meeting with `attendees: [Alice Known, Unknown Person]`
|
||||
2. put_page returns `auto_links.unresolved = [{field: 'attendees', name: 'Unknown Person'}]`
|
||||
3. enrichment tier consumes `Unknown Person` → web search → creates `people/unknown-person.md`
|
||||
4. The next put_page (or a backfill run) wires up the `attended` edge automatically
|
||||
```
|
||||
|
||||
### 3. idea-ingest/SKILL.md (optional)
|
||||
|
||||
**Where:** Same pattern as meeting-ingestion — check `auto_links.unresolved` after `put_page`, route names to enrichment.
|
||||
|
||||
### Unchanged skills (no diffs needed)
|
||||
|
||||
- **brain-ops/SKILL.md** — auto-link mechanics are internal; the write path stays the same.
|
||||
- **signal-detector/SKILL.md** — signal capture path unchanged.
|
||||
- **query/SKILL.md** — `traverse_graph` now returns richer results automatically.
|
||||
- **daily-task-manager/SKILL.md**, **briefing/SKILL.md**, **citation-fixer/SKILL.md**, **media-ingest/SKILL.md** — unchanged.
|
||||
|
||||
### New edge types you can filter in graph queries
|
||||
|
||||
v0.13 edges carry new `link_type` values. If your fork has graph-query skills that filter by type, these are now available:
|
||||
|
||||
- `works_at` (person → company) — from `company:`, `companies:`, or `key_people:`
|
||||
- `founded` (person → company) — from `founded:`
|
||||
- `invested_in` (investor → deal/company) — from `investors:` or `lead:`
|
||||
- `led_round` (lead → deal) — from `lead:`
|
||||
- `yc_partner` (partner → company) — from `partner:`
|
||||
- `attended` (person → meeting) — from `attendees:`
|
||||
- `discussed_in` (source → page) — from `sources:`
|
||||
- `source` (page → source) — from `source:`
|
||||
- `related_to` (page → target) — from `related:` or `see_also:`
|
||||
|
||||
### Migration timing
|
||||
|
||||
`gbrain upgrade` takes 2-5 min on a 46K-page brain (one-time). Runs out-of-process via `gbrain post-upgrade`. If your agent holds a DB connection during the upgrade, reconnect after; otherwise keep serving.
|
||||
|
||||
### Type normalization NOT in v0.13
|
||||
|
||||
Legacy rows with `link_type='attendee'` or `link_type='mention'` coexist with new `'attended'` / `'mentions'` rows. Your queries filtering on old type names keep working. A separate opt-in `gbrain normalize-types` command in v0.14 handles the rename.
|
||||
## v0.14.0 shell jobs (optional adoption, no skill edits)
|
||||
|
||||
Adds a `shell` job type to Minions so deterministic cron scripts (API fetch, token
|
||||
refresh, scrape + write) move off the LLM gateway. Zero tokens per fire. ~60%
|
||||
gateway CPU headroom at typical scale. Feature is **off by default**, existing
|
||||
installs keep running exactly as they did before. Nothing breaks.
|
||||
|
||||
To adopt, follow `skills/migrations/v0.14.0.md`. The short version:
|
||||
|
||||
1. Set `GBRAIN_ALLOW_SHELL_JOBS=1` on the worker process, then `gbrain jobs work`
|
||||
(Postgres). On PGLite, every crontab invocation uses `--follow` for inline
|
||||
execution; no persistent worker.
|
||||
2. Classify each of your host's cron entries: LLM-requiring (keep on gateway) vs
|
||||
deterministic (candidate for shell). Typical splits:
|
||||
- **Deterministic → shell:** `ycli-token-refresh`, `x-oauth2-refresh`,
|
||||
`x-garrytan-unified`, `calendar-sync-to-brain`, `github-pulse`,
|
||||
`frameio-scan`, `flight-tracker`, `x-raw-json-backfill`.
|
||||
- **LLM-requiring → stay:** `social-radar`, `content-ideas`, `adversary-vacuum`,
|
||||
`ea-inbox-sweep`, `morning-briefing`, `brain-maintenance`.
|
||||
3. For each deterministic cron, rewrite as:
|
||||
```cron
|
||||
3 13,16,19,22,1,4,7,10 * * * \
|
||||
gbrain jobs submit shell \
|
||||
--params '{"cmd":"node scripts/your-script.mjs","cwd":"/data/.openclaw/workspace"}' \
|
||||
--max-attempts 3 --timeout-ms 300000
|
||||
```
|
||||
4. Watch `gbrain jobs get <id>` for exit_code / stdout_tail / stderr_tail on each
|
||||
fire. Compare against pre-migration behavior before approving the next batch.
|
||||
|
||||
**No skill edits required.** The handler runs worker-side; skill files don't
|
||||
change. If your host exposed custom handlers via the plugin contract (v0.11.0),
|
||||
they still work the same way.
|
||||
|
||||
Iron rule: **never auto-rewrite the operator's crontab.** Every rewrite is
|
||||
per-cron, human-approved, with a diff. If you want automation later, the
|
||||
upcoming `gbrain crontab-to-minions <file>` helper is P1 in TODOS.
|
||||
|
||||
---
|
||||
|
||||
## v0.16.0: durable agent runtime
|
||||
|
||||
v0.15 ships `gbrain agent run` / `gbrain agent logs`, a new `subagent` handler
|
||||
type in Minions, and a plugin contract for host-repo subagent defs. None of the
|
||||
existing skills need surgery. The question for downstream agents is *how* to
|
||||
adopt the new runtime, not how to patch around a breaking change.
|
||||
|
||||
### 1. Run a worker with an Anthropic key
|
||||
|
||||
The subagent handlers (`subagent` and `subagent_aggregator`) are always
|
||||
registered on the worker. No separate opt-in flag — `ANTHROPIC_API_KEY` is
|
||||
the natural cost gate (no key, the SDK call fails on the first turn), and
|
||||
who-can-submit is already protected (`PROTECTED_JOB_NAMES` + trusted-submit:
|
||||
MCP callers get `permission_denied`; only `gbrain agent run` can insert
|
||||
these rows).
|
||||
|
||||
```bash
|
||||
ANTHROPIC_API_KEY=sk-ant-... gbrain jobs work
|
||||
```
|
||||
|
||||
Worker startup prints:
|
||||
|
||||
```
|
||||
[minion worker] subagent handlers enabled
|
||||
```
|
||||
|
||||
### 2. Ship your subagents as a plugin (OpenClaw + similar)
|
||||
|
||||
Move your custom subagent definitions out of your gbrain fork and into your own
|
||||
repo as a plugin. Concretely:
|
||||
|
||||
```
|
||||
~/<your-agent>/gbrain-plugin/
|
||||
├── gbrain.plugin.json
|
||||
└── subagents/
|
||||
├── meeting-ingestion.md
|
||||
├── signal-detector.md
|
||||
└── daily-task-prep.md
|
||||
```
|
||||
|
||||
`gbrain.plugin.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "your-openclaw",
|
||||
"version": "2026.4.20",
|
||||
"plugin_version": "gbrain-plugin-v1"
|
||||
}
|
||||
```
|
||||
|
||||
Each `subagents/*.md` is a plain-text agent definition — YAML frontmatter +
|
||||
body-as-system-prompt. Recognized frontmatter fields: `name`, `model`,
|
||||
`max_turns`, `allowed_tools` (must subset the derived brain-tool registry).
|
||||
|
||||
Turn it on:
|
||||
|
||||
```bash
|
||||
export GBRAIN_PLUGIN_PATH="$HOME/<your-agent>/gbrain-plugin"
|
||||
```
|
||||
|
||||
Worker startup prints `[plugin-loader] loaded '<name>' v<ver> (N subagents)`
|
||||
per plugin; any rejection (bad manifest, unknown tool in `allowed_tools`,
|
||||
version mismatch) shows up as a loud warning at startup, not a silent dispatch-
|
||||
time failure. See `docs/guides/plugin-authors.md` for the full contract.
|
||||
|
||||
### 3. Replace ephemeral subagent runs with durable ones
|
||||
|
||||
If your agent currently spawns ephemeral subagents (OpenClaw `Agent()`, ad-hoc
|
||||
Anthropic API calls, etc.) for work that should survive crashes, sleeps, or
|
||||
worker restarts, migrate those to `gbrain agent run`. The durability is free:
|
||||
|
||||
```bash
|
||||
gbrain agent run "analyze my last 50 journal pages for recurring themes" \
|
||||
--subagent-def analyzer --fanout-manifest manifests/journal-pages.json
|
||||
```
|
||||
|
||||
Every turn persists to `subagent_messages`, every tool call is a two-phase
|
||||
ledger, and `gbrain agent logs <job>` shows where it died + what the last
|
||||
successful call returned. No more "re-run from scratch because the session
|
||||
context evaporated."
|
||||
|
||||
### 4. `put_page` from subagents writes under an agent namespace
|
||||
|
||||
If you adopted the v0.15 subagent runtime, note that `put_page` calls
|
||||
originating from a subagent's tool dispatch MUST target
|
||||
`wiki/agents/<subagent_id>/...`. The schema shown to the model enforces this
|
||||
on first try; a server-side fail-closed check rejects anything else. This
|
||||
does NOT affect your skill files, CLI put_page calls, or MCP put_page —
|
||||
only tool-dispatched writes from inside an LLM loop.
|
||||
|
||||
Aggregation output (the final "here's what all N children found" brain page)
|
||||
goes via a separate trusted CLI path, not through a subagent tool call, so
|
||||
it can write anywhere you want.
|
||||
|
||||
Iron rule: **never grant an agent write access beyond its namespace**. The
|
||||
server-side check exists because dispatcher bugs happen; treat it as defense
|
||||
in depth, not the primary boundary.
|
||||
|
||||
---
|
||||
|
||||
## v0.22.4 — frontmatter-guard adoption
|
||||
|
||||
### 1. Stop hand-rolling frontmatter validators
|
||||
|
||||
If your fork has scripts that call `js-yaml` directly to validate brain page
|
||||
frontmatter, replace them with `gbrain frontmatter validate` calls. The CLI
|
||||
covers the seven canonical error classes and ships a `--json` envelope that's
|
||||
stable across releases.
|
||||
|
||||
```diff
|
||||
- # Custom validator script
|
||||
- node scripts/validate-frontmatter.mjs <path>
|
||||
+ gbrain frontmatter validate <path> --json
|
||||
```
|
||||
|
||||
For consumers that need the validator inside another script, import from
|
||||
gbrain's `markdown` export instead of duplicating logic:
|
||||
|
||||
```ts
|
||||
import { parseMarkdown } from 'gbrain/markdown';
|
||||
|
||||
const parsed = parseMarkdown(content, filePath, { validate: true, expectedSlug });
|
||||
for (const err of parsed.errors ?? []) {
|
||||
// err.code: MISSING_OPEN | MISSING_CLOSE | YAML_PARSE | SLUG_MISMATCH |
|
||||
// NULL_BYTES | NESTED_QUOTES | EMPTY_FRONTMATTER
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Drop any references to `lib/brain-writer.mjs`
|
||||
|
||||
If your fork's skills or scripts referenced an aspirational
|
||||
`lib/brain-writer.mjs` (it never shipped — the spec was in PR #392 and never
|
||||
landed), replace those references with the gbrain CLI. The `frontmatter-guard`
|
||||
skill lives at `skills/frontmatter-guard/SKILL.md` and points at
|
||||
`gbrain frontmatter validate` / `audit` / `install-hook`.
|
||||
|
||||
### 3. Wire the doctor subcheck into your health pipeline
|
||||
|
||||
`gbrain doctor` now reports `frontmatter_integrity` automatically. If your
|
||||
fork has a custom health pipeline (e.g. a daily Slack post about brain
|
||||
health), pull from `gbrain doctor --json` and surface the
|
||||
`frontmatter_integrity` row counts.
|
||||
|
||||
### 4. (Optional) Install the pre-commit hook on brain repos
|
||||
|
||||
For sources backed by git, the v0.22.4 install-hook helper drops a
|
||||
pre-commit script that blocks commits with malformed frontmatter:
|
||||
|
||||
```bash
|
||||
gbrain frontmatter install-hook
|
||||
```
|
||||
|
||||
Skip this if your brain isn't a git repo or if your downstream agent already
|
||||
enforces validation at write time. See `docs/integrations/pre-commit.md` for
|
||||
the full recipe.
|
||||
|
||||
### 5. Migration ergonomics — read pending-host-work.jsonl
|
||||
|
||||
After `gbrain apply-migrations --yes` runs the v0.22.4 audit, your agent
|
||||
should read `~/.gbrain/migrations/pending-host-work.jsonl` (filter to
|
||||
`migration === "0.22.4"`) and walk each entry's `command` field. Each entry
|
||||
points to a per-source `gbrain frontmatter validate <source_path> --fix`
|
||||
command — surface counts to the user, get explicit consent, then run.
|
||||
|
||||
The migration is **audit-only**. It never mutates brain content during
|
||||
`apply-migrations`. Your agent runs the fix command with user consent.
|
||||
|
||||
---
|
||||
|
||||
## Future versions
|
||||
|
||||
When gbrain ships a new version, this doc will be updated with the diffs for that
|
||||
version. Each new version appends a section; old sections stay so you can catch up
|
||||
multiple versions at once.
|
||||
|
||||
To check what your fork is missing:
|
||||
```bash
|
||||
diff <(grep -A3 "Based on gbrain" ~/<your-fork>/skills/brain-ops/SKILL.md) \
|
||||
<(grep "v[0-9]" ~/gbrain/skills/migrations/ | tail -3)
|
||||
```
|
||||
@@ -1,105 +0,0 @@
|
||||
# GBrain Infrastructure Layer
|
||||
|
||||
The shared foundation that all skills, recipes, and integrations build on.
|
||||
|
||||
## Data Pipeline
|
||||
|
||||
```
|
||||
INPUT (markdown files, git repo)
|
||||
↓
|
||||
FILE RESOLUTION (local → .redirect → .supabase → error)
|
||||
↓
|
||||
MARKDOWN PARSER (gray-matter frontmatter + body)
|
||||
→ compiled_truth + timeline separation
|
||||
↓
|
||||
CONTENT HASH (SHA-256 idempotency check — skip if unchanged)
|
||||
↓
|
||||
CHUNKING (3 strategies, configurable)
|
||||
├── Recursive: 300-word chunks, 50-word overlap, 5-level delimiter hierarchy
|
||||
├── Semantic: embed sentences, cosine similarity, Savitzky-Golay smoothing
|
||||
└── LLM-guided: Claude Haiku identifies topic shifts in 128-word candidates
|
||||
↓
|
||||
EMBEDDING (OpenAI text-embedding-3-large, 1536 dimensions)
|
||||
→ batch 100, exponential backoff, non-fatal if fails
|
||||
↓
|
||||
DATABASE TRANSACTION (atomic: page + chunks + tags + version)
|
||||
↓
|
||||
SEARCH (hybrid, available immediately)
|
||||
```
|
||||
|
||||
## Search Architecture
|
||||
|
||||
GBrain uses Reciprocal Rank Fusion (RRF) to merge vector and keyword search:
|
||||
|
||||
```
|
||||
User Query
|
||||
↓
|
||||
EXPANSION (optional: Claude Haiku generates 2 alternative phrasings)
|
||||
↓
|
||||
├── VECTOR SEARCH (pgvector HNSW, cosine distance)
|
||||
│ → 2x limit results per query variant
|
||||
│
|
||||
└── KEYWORD SEARCH (PostgreSQL tsvector, ts_rank)
|
||||
→ 2x limit results
|
||||
↓
|
||||
RRF MERGE (score = Σ(1/(60 + rank)), balances both fairly)
|
||||
↓
|
||||
4-LAYER DEDUP
|
||||
├── Best 3 chunks per page (source dedup)
|
||||
├── Jaccard similarity > 0.85 (text dedup)
|
||||
├── No type exceeds 60% (diversity)
|
||||
└── Max 2 chunks per page (page cap)
|
||||
↓
|
||||
TOP N RESULTS (default 20)
|
||||
```
|
||||
|
||||
## Key Components
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `src/core/engine.ts` | Pluggable engine interface (BrainEngine) |
|
||||
| `src/core/postgres-engine.ts` | Postgres + pgvector implementation |
|
||||
| `src/core/import-file.ts` | importFromFile + importFromContent pipeline |
|
||||
| `src/core/sync.ts` | Git-based incremental change detection |
|
||||
| `src/core/markdown.ts` | YAML frontmatter + compiled_truth/timeline parsing |
|
||||
| `src/core/embedding.ts` | OpenAI embedding with batch, retry, backoff |
|
||||
| `src/core/chunkers/recursive.ts` | Base chunker (300w, 5-level delimiters) |
|
||||
| `src/core/chunkers/semantic.ts` | Embedding-based topic boundary detection |
|
||||
| `src/core/chunkers/llm.ts` | Claude Haiku guided chunking |
|
||||
| `src/core/search/hybrid.ts` | RRF merge of vector + keyword |
|
||||
| `src/core/search/dedup.ts` | 4-layer result deduplication |
|
||||
| `src/core/search/expansion.ts` | Multi-query expansion via Claude Haiku |
|
||||
| `src/core/storage.ts` | Pluggable storage (S3, Supabase, local) |
|
||||
| `src/core/operations.ts` | Contract-first operation definitions (31 ops) |
|
||||
| `src/schema.sql` | Full DDL (10 tables, RLS, tsvector, HNSW) |
|
||||
|
||||
## Schema Overview
|
||||
|
||||
10 tables in Postgres:
|
||||
|
||||
- **pages** — slug (unique), type, title, compiled_truth, timeline, frontmatter (JSONB)
|
||||
- **content_chunks** — pgvector 1536-dim embedding, chunk_source (compiled_truth|timeline)
|
||||
- **links** — typed edges (knows, works_at, invested_in, founded, etc.)
|
||||
- **tags** — many-to-many page tagging
|
||||
- **timeline_entries** — structured events (date, source, summary, detail)
|
||||
- **page_versions** — snapshot history for diff/revert
|
||||
- **raw_data** — sidecar JSON from external APIs (preserves provenance)
|
||||
- **files** — binary attachments in storage backend
|
||||
- **ingest_log** — audit trail of import operations
|
||||
- **config** — brain-level settings (version, embedding model, chunk strategy)
|
||||
|
||||
Full-text search uses weighted tsvector: title (A), compiled_truth (B), timeline (C).
|
||||
Vector search uses HNSW index with cosine distance on content_chunks.embedding.
|
||||
|
||||
## The Thin Harness Principle
|
||||
|
||||
GBrain is the deterministic layer. Skills and recipes are the latent space layer.
|
||||
|
||||
See [Thin Harness, Fat Skills](../ethos/THIN_HARNESS_FAT_SKILLS.md) for the full
|
||||
architecture philosophy.
|
||||
|
||||
- **GBrain CLI** = thin harness (same input → same output)
|
||||
- **Skills** (ingest, query, maintain, enrich, briefing, migrate, setup) = fat skills
|
||||
- **Recipes** (voice-to-brain, email-to-brain) = fat skills that install infrastructure
|
||||
|
||||
The agent reads the skill/recipe and uses GBrain's deterministic tools to do the work.
|
||||
@@ -1,162 +0,0 @@
|
||||
# Code Cathedral II — v0.20.0 Design
|
||||
|
||||
**Status:** Accepted. CEO + Eng + 2 codex passes CLEARED (2026-04-24). 16 cross-model findings absorbed total: 7 codex pass 1 (structural prereqs) + 6 codex pass 2 (absorption errors including the CHUNKER_VERSION silent-no-op gate and inbound-edge invalidation) + 3 eng-review architectural decisions. DX review recommended post-Layer 8 (new CLI surfaces) before ship.
|
||||
**Supersedes:** Cathedral I (planned v0.18.0–v0.19.0 code indexing, shipped v0.19.0).
|
||||
**Mode:** SCOPE EXPANSION (user explicit: "I want the best code search in the world").
|
||||
**Scale:** 14 bisectable layers, ~20–25 CC hours, 3–5 human-weeks. One schema migration with split edge tables (`code_edges_chunk` + `code_edges_symbol`). Backfill via `CHUNKER_VERSION` bump (automatic on next sync) + explicit `gbrain reindex-code` command.
|
||||
|
||||
## Why v0.20.0
|
||||
|
||||
v0.19.0 shipped code indexing: tree-sitter chunker, 29 active languages, symbol columns, forward doc↔impl linking, incremental embed cache, BrainBench code category. Four cathedral-I items got deferred during shipping: `query --lang` filter, `sync --all` cost preview, markdown fence extraction, reverse-scan doc↔impl backfill.
|
||||
|
||||
Cathedral II is a promise-keeping release for those four, bundled with the leap that makes gbrain *the* code search: structural edges (call graph + references + imports + inheritance), parent-scope capture, doc-comment FTS binding, and two-pass retrieval. No more grep-class retrieval on code.
|
||||
|
||||
## The 10x leap
|
||||
|
||||
Today: agent asks "how does hybrid search handle N+1?" → gets 3 prose chunks of `hybrid.ts`.
|
||||
|
||||
Cathedral II: same query returns the anchor function + its 3 callers + its 2 callees + its JSDoc + the guide in `/docs` that cites it + the test file exercising it + parent scope chain. One walk. Code-aware brain.
|
||||
|
||||
## Scope (5 tiers + Layer 0 prerequisites, 14 bisectable layer commits)
|
||||
|
||||
### Tier 0 — Prerequisites (surfaced by codex outside voice)
|
||||
|
||||
**0a. File-classification widening.** `sync.ts:35` currently classifies only 9 extensions as code (TS, JS, Python, Go, Rust, Ruby, Java, C, C++). Cathedral II's B1 ships 165 lazy-loadable grammars, so the classifier needs to accept any extension the chunker can handle. Also reorders `detectCodeLanguage` so Magika (B2) runs as a fallback for extension-less files, not after a null-return gate.
|
||||
|
||||
**0b. Chunk-grain FTS.** Current keyword search lives on `pages.search_vector`. Adding doc-comments or two-pass anchoring at the chunk level has zero ranking effect against a page-grain primitive. Layer 0b adds `content_chunks.search_vector` with a trigger building from qualified symbol name + doc-comment (weight A) and chunk_text (weight B), plus rewrites `searchKeyword` to rank chunks directly. Page-level search_vector stays for title-heavy searches.
|
||||
|
||||
Both Layer 0 items are prerequisites for the 10x leap to actually move retrieval metrics.
|
||||
|
||||
### Tier A — Structural edges (the 10x leap)
|
||||
|
||||
**A1. Call-graph + reference extraction with qualified symbol identity.** Per-language tree-sitter queries at `importCodeFile` time capture:
|
||||
|
||||
- `calls` — function call-sites
|
||||
- `imports` — module deps
|
||||
- `extends` / `implements` — type hierarchies
|
||||
- `mixes_in` — Ruby `include`/`extend`/`prepend`
|
||||
- `type_refs` — parameter + return type usage
|
||||
- `declares` — chunk owns a symbol definition
|
||||
|
||||
**Qualified symbol identity across all 8 langs.** `parent_symbol_path` (A3) is the source of truth for scope; edges use qualified names built from it. Examples: `Admin::UsersController#render` (Ruby instance), `Admin::UsersController.find_all` (Ruby singleton), `admin.users_controller.UsersController.render` (Python), `(*UsersController).Render` (Go), `users::UsersController::render` (Rust), `com.acme.admin.UsersController.render` (Java). Per-lang delimiter + method/class-method distinction. Ruby ships fully in ranker (CLI + A2 two-pass) — no deferral.
|
||||
|
||||
**Split schema (two tables, not one polymorphic):**
|
||||
```sql
|
||||
CREATE TABLE code_edges_chunk (
|
||||
from_chunk_id INTEGER NOT NULL REFERENCES content_chunks(id) ON DELETE CASCADE,
|
||||
to_chunk_id INTEGER NOT NULL REFERENCES content_chunks(id) ON DELETE CASCADE,
|
||||
from_symbol_qualified TEXT NOT NULL,
|
||||
to_symbol_qualified TEXT NOT NULL,
|
||||
edge_type TEXT NOT NULL,
|
||||
source_id TEXT REFERENCES sources(id) ON DELETE CASCADE,
|
||||
UNIQUE (from_chunk_id, to_chunk_id, edge_type)
|
||||
);
|
||||
CREATE TABLE code_edges_symbol (
|
||||
from_chunk_id INTEGER NOT NULL REFERENCES content_chunks(id) ON DELETE CASCADE,
|
||||
from_symbol_qualified TEXT NOT NULL,
|
||||
to_symbol_qualified TEXT NOT NULL,
|
||||
edge_type TEXT NOT NULL,
|
||||
source_id TEXT REFERENCES sources(id) ON DELETE CASCADE,
|
||||
UNIQUE (from_chunk_id, to_symbol_qualified, edge_type)
|
||||
);
|
||||
```
|
||||
`code_edges_chunk` = resolved (both endpoints known). `code_edges_symbol` = unresolved (target symbol exists by qualified name, definition chunk not yet seen). Promotion from symbol→chunk table happens on later import. `source_id` is TEXT matching actual `sources.id` type.
|
||||
|
||||
**Shipped languages:** TypeScript, TSX, JavaScript, Ruby, Python, Go, Rust, Java (8 langs, ~85% of real brain code). Other languages chunk normally (via B1 lazy-load) but don't emit edges in v0.20.0 — extension is one query file + delimiter config per language, shippable as small follow-up PRs.
|
||||
|
||||
**A2. Two-pass retrieval.** Current: keyword + vector → RRF → dedup. New: keyword + vector → anchor set → expand 1–2 hops on `code_edges_chunk` with structural-distance decay → blend into RRF.
|
||||
|
||||
**Default OFF in all cases.** Opt-in only via `--walk-depth N` or `--near-symbol <name>`. Exact-symbol-match auto-on was unsafe (symbol names collide across files). Neighbor cap 50 per hop, depth cap 2. Dedup's per-page cap (currently 2) lifts to `min(10, walkDepth × 5)` when walking so structural neighbors from one file aren't clipped. Distance decay: `1/(1 + hop)` on expanded-neighbor RRF contributions.
|
||||
|
||||
**A3. Parent-scope capture + nested-chunk emission.** Two parts:
|
||||
|
||||
*Part 1:* Nested symbols get `parent_symbol_path text[]` on `content_chunks`. Embedded into chunk header: `[TypeScript] src/foo.ts:42-58 function formatResult (in BrainEngine.searchKeyword)`. Scope flows into embedding. Dual-use: drives A1's qualified symbol identity.
|
||||
|
||||
*Part 2:* Extend `splitLargeNode` to emit nested functions/methods/inner-classes as their own chunks. The current chunker is top-level-node oriented — a `class Foo { method1() {} method2() {} }` emits one chunk. Parent_symbol_path on top-level nodes is empty (no parent above top level), so A3 contributes nothing without sub-top-level chunks. Part 2 makes the scope annotation load-bearing.
|
||||
|
||||
**A4. Doc-comment → symbol binding.** Leading AST comment extracted to `doc_comment text`. Lands on **chunk-grain** search_vector (Layer 0b prerequisite) with FTS weight `'A'`. Natural-language queries rank docstring matches above body text and below title. `'A' > 'B' > 'C' > 'D'` per Postgres FTS weight convention.
|
||||
|
||||
### Tier B — Coverage (honest Chonkie parity)
|
||||
|
||||
**B1.** Lazy-load tree-sitter-language-pack (~165 languages). Replace 36 committed WASMs with a manifest + per-process parser cache. Cathedral I promised this and didn't deliver — Cathedral II does.
|
||||
|
||||
**B2.** Magika auto-detect for extension-less files (Dockerfile, Makefile, `.envrc`). ~1MB bundled asset. Falls back to null → recursive chunker if classifier fails to load.
|
||||
|
||||
### Tier C — Agent CLI surfaces
|
||||
|
||||
- `query --lang <lang>` — filter by `content_chunks.language`
|
||||
- `query --symbol-kind function|class|method|type|interface|enum` — filter by `symbol_type`
|
||||
- `query --near-symbol <name> --depth 1..2` — two-pass retrieval anchored at a known symbol
|
||||
- `code-callers <symbol>` — uses A1 `calls` edges, reversed
|
||||
- `code-callees <symbol>` — uses A1 `calls` edges, forward
|
||||
|
||||
All auto-JSON on non-TTY. `StructuredAgentError` envelopes on failure. `code-signature` deferred to v0.20.1 (needs per-language type captures).
|
||||
|
||||
### Tier D — Bridge items (cathedral I promises)
|
||||
|
||||
**D1.** `sync --all` cost preview. `estimateTokens` extracted from `chunkers/code.ts` to new `tokens.ts` module. Before per-source loop: walk sync-diff set, sum tokens, compute $ estimate. TTY + !json + !yes → interactive `[y/N]`. Non-TTY or `--json` or piped → emit `ConfirmationRequired` envelope, exit 2. `--yes` skips. `--dry-run` previews + exit 0. Preview on `--all` only, not single-source (DX review pain is first-time large-sync surprise bills).
|
||||
|
||||
**D2.** Markdown fence extraction in `importFromContent`. After `parseMarkdown`, iterate marked lexer tokens for `{type:'code', lang, text}`. Map fence tag → language. Chunk each fence through `chunkCodeText`. Persist as `chunk_source='fenced_code'`. Cap 100 fences per markdown page (DOS defense). Per-fence try/catch — one bad fence doesn't break the page import.
|
||||
|
||||
**D3.** `reconcile-links` batch command. Walks markdown pages, calls existing v0.19.0 `extractCodeRefs` per page, emits `addLink(md, code, ..., 'documents')` + reverse. `ON CONFLICT DO NOTHING` handles idempotency. Statement-timeout scoped via `sql.begin` + `SET LOCAL`. Progress reporter + final summary (edges added / existed / missing-target). Respects `auto_link` config.
|
||||
|
||||
### Tier E — Eval, backfill, honesty
|
||||
|
||||
**E1.** BrainBench code sub-categories: `call_graph_recall` (callers of X → expected set), `parent_scope_coverage` (nested-symbol queries return correct scope), `doc_comment_matching` (NL queries rank doc-comments above prose). Regression gates against A1/A3/A4 drift.
|
||||
|
||||
**E2.** Backfill: schema migrates automatically (zero cost). **`CHUNKER_VERSION` bumps 3 → 4** — that constant is folded into each code page's `content_hash`, so every code page's hash changes on upgrade. Next `gbrain sync` won't short-circuit on "git HEAD unchanged"; it re-chunks every code file. New `gbrain reindex-code [--source <id>] [--dry-run] [--yes] [--force]` provides explicit full backfill with cost preview (reuses D1 infra) and `--force` bypasses content_hash skip entirely. Users control when to pay; silent no-op path closed.
|
||||
|
||||
**E3.** Honest CHANGELOG. Retire "Chonkie superset" framing. Run BrainBench before/after for real numbers: 150+ languages loaded (after B1), MRR on NL→code queries, P@1 call-graph precision, P@k on symbol_name queries, sync cost preview on 5K-file repo. Back every claim with a runnable command.
|
||||
|
||||
## Implementation ordering (14 layers, post-codex)
|
||||
|
||||
1. **0a** — File-classification widening (sync.ts:35) + Magika reordered as fallback
|
||||
2. **0b** — Chunk-grain FTS (content_chunks.search_vector + trigger + searchKeyword chunk-level rewrite)
|
||||
3. **Foundation** — schema migration (split edge tables, qualified name columns on content_chunks) + engine method stubs + types
|
||||
4. **B1** — lazy-load grammar manifest + bun --compile guard
|
||||
5. **A1** — edge-extractor + 8 per-lang query files + qualified symbol identity + tests
|
||||
6. **A3** — parent-scope column + doc-comment column + splitLargeNode nested-chunk emission
|
||||
7. **A4** — doc-comment FTS weight A on chunk-grain search_vector
|
||||
8. **A2** — two-pass retrieval, default OFF, opt-in only; dedup cap lifts when walking
|
||||
9. **D tier bundled** — cost preview + fence extraction + reconcile-links
|
||||
10. **B2** — Magika auto-detect
|
||||
11. **C tier** — 5 CLI surfaces
|
||||
12. **E1** — BrainBench sub-categories + CHUNKER_VERSION 3→4 bump
|
||||
13. **E2** — `reindex-code` with `--force` + migration orchestrator with backfill-prompt phase
|
||||
14. **E3 + release** — honest CHANGELOG + docs + migration skill + `/ship`
|
||||
|
||||
## Size and cost
|
||||
|
||||
- Diff: ~5500–6500 lines (~2.5x v0.19.0 post-codex expansion)
|
||||
- Tests: ~2000 lines (8 langs × qualified-name + edge-extraction fixtures + Layer 0b FTS migration tests)
|
||||
- Files: ~36 new, ~25 modified
|
||||
- CC time: ~20–25 hours focused (was 14–18 pre-codex; +6h for Layer 0a/0b + qualified identity across 8 langs + nested-chunk emission + CHUNKER_VERSION bump layer)
|
||||
- Human-equivalent: 3–5 weeks
|
||||
- First-sync cost bump for upgraded v0.19.0 users: every code page re-chunks on first sync after upgrade (CHUNKER_VERSION bump forces invalidation). Users run `gbrain reindex-code --dry-run` for cost preview, then `--yes` or accept gradual backfill over time as files change.
|
||||
- Daily autopilot cost post-backfill: unchanged (edges extracted at chunk time, no per-query LLM)
|
||||
|
||||
## Risks and mitigations
|
||||
|
||||
1. **Schema migration on live Postgres.** Test against production-shape DB before ship. v0.12.0 JSONB incident is the canary.
|
||||
2. **Per-language tree-sitter queries are fiddly.** Hand-verified edge-set fixtures per language. Ruby gets extra coverage for dynamic-dispatch false negatives.
|
||||
3. **Two-pass retrieval regression.** Default off for prose. BrainBench Cat 1 MUST show no regression before shipping.
|
||||
4. **Backfill shape (G1 resolved).** Three composable layers: schema-auto migrates columns empty (zero cost). Lazy on-touch catches 80% over time (zero cost). Explicit `reindex-code` with cost preview for users wanting immediate full benefit. No surprise bills.
|
||||
5. **Magika bundle (G2 resolved).** +1MB asset, `bun --compile` guard extension. If bundling surfaces bugs late in implementation, B2 is the only tier that can fall back to v0.20.1 without blocking the cathedral — it's self-contained at Layer 8.
|
||||
6. **High-fan-out symbols.** `console.log`-style symbols have 100K callers. Neighbor cap 50, depth cap 2. Chaos test fixture required.
|
||||
|
||||
## Review gates
|
||||
|
||||
- CEO review (cathedral II) — CLEARED 2026-04-24
|
||||
- Outside voice (codex) — run during cathedral II CEO review
|
||||
- `/plan-devex-review` — up next (per user request, 5 new CLI surfaces + reindex-code need DX polish review before eng)
|
||||
- `/plan-eng-review` — required before implementation begins
|
||||
- `/review` + `/codex review` — required before `/ship`
|
||||
|
||||
## What's deferred to later cathedrals
|
||||
|
||||
- **C6** `code-signature "(A, B) => C"` — per-language type captures. v0.20.1.
|
||||
- **Call-graph langs beyond 8 shipped** — PHP, Swift, Kotlin, Scala, C#, C++, Elixir, etc. One small PR per language.
|
||||
- **LSP integration** for live precision. v0.22+ cathedral.
|
||||
- **Code-tour generator** (cathedral I T1).
|
||||
- **Private-code redaction pre-embed** (cathedral I T3).
|
||||
- **`gbrain doctor --chunker-debug`** AST dump.
|
||||
@@ -1,154 +0,0 @@
|
||||
# Homebrew for Personal AI Infrastructure
|
||||
|
||||
The 10-star vision for GBrain's integration system. Ship Approach B (v0.7.0),
|
||||
build toward this over subsequent releases.
|
||||
|
||||
## The Vision
|
||||
|
||||
GBrain becomes a personal infrastructure operating system where every signal in
|
||||
your life flows through the brain automatically. Integrations are **senses**
|
||||
(data inputs) and **reflexes** (automated responses to patterns). Users subscribe
|
||||
to the creator's actual operating system, then customize it.
|
||||
|
||||
```
|
||||
$ gbrain integrations
|
||||
|
||||
SENSES (data inputs) STATUS
|
||||
-------------------------------------------------------
|
||||
voice-to-brain Phone calls -> brain pages ACTIVE last call: 2h ago
|
||||
email-to-brain Gmail -> entity updates ACTIVE 47 emails today
|
||||
x-to-brain Twitter -> media pages ACTIVE 312 tweets tracked
|
||||
calendar-to-brain Google Cal -> meeting prep ACTIVE 3 meetings tomorrow
|
||||
photos-to-brain Camera roll -> visual mem AVAILABLE
|
||||
slack-to-brain Slack -> conversation index AVAILABLE
|
||||
rss-to-brain RSS feeds -> media pages AVAILABLE
|
||||
|
||||
REFLEXES (automated responses) STATUS
|
||||
-------------------------------------------------------
|
||||
meeting-prep Brief me before meetings ACTIVE next: 9am tomorrow
|
||||
entity-enrich Auto-enrich new contacts ACTIVE 12 enriched today
|
||||
dream-cycle Overnight brain maintenance ACTIVE last run: 3am
|
||||
deal-tracker Alert on deal changes AVAILABLE
|
||||
follow-up-nudge Remind on stale threads AVAILABLE
|
||||
|
||||
This week: 1,247 signals ingested. Top: email (47%), voice (23%), X (18%).
|
||||
34 new entity pages created. 7 calls transcribed.
|
||||
|
||||
Run 'gbrain integrations show <id>' for setup details.
|
||||
```
|
||||
|
||||
The user feels: "My brain is alive. It's watching everything I care about, and
|
||||
it's getting smarter every day. I didn't have to write any code. I just said yes
|
||||
when the agent asked."
|
||||
|
||||
## Architecture: Senses & Reflexes
|
||||
|
||||
### Recipe Format (YAML frontmatter + markdown body)
|
||||
|
||||
```yaml
|
||||
---
|
||||
id: voice-to-brain
|
||||
name: Voice-to-Brain
|
||||
version: 0.7.0
|
||||
description: Phone calls create brain pages via Twilio + OpenAI Realtime + GBrain MCP
|
||||
category: sense
|
||||
requires: [credential-gateway]
|
||||
secrets:
|
||||
- name: TWILIO_ACCOUNT_SID
|
||||
description: Twilio account SID
|
||||
where: https://console.twilio.com
|
||||
- name: OPENAI_API_KEY
|
||||
description: OpenAI API key (for Realtime voice)
|
||||
where: https://platform.openai.com/api-keys
|
||||
health_checks:
|
||||
- curl -s https://api.twilio.com/2010-04-01 > /dev/null
|
||||
- curl -s https://api.openai.com/v1/models > /dev/null
|
||||
setup_time: 30 min
|
||||
---
|
||||
|
||||
[Opinionated setup instructions the agent executes...]
|
||||
```
|
||||
|
||||
### Dependency Graph
|
||||
|
||||
Recipes declare `requires` in frontmatter. The CLI resolves dependencies before
|
||||
setup. If voice-to-brain requires credential-gateway, the agent sets up
|
||||
credential-gateway first.
|
||||
|
||||
```
|
||||
credential-gateway
|
||||
├── voice-to-brain (requires credentials for Twilio)
|
||||
├── email-to-brain (requires credentials for Gmail)
|
||||
└── calendar-to-brain (requires credentials for Google Calendar)
|
||||
|
||||
x-to-brain (standalone, uses X API directly)
|
||||
```
|
||||
|
||||
### Health Dashboard
|
||||
|
||||
`gbrain integrations doctor` runs health_checks from every configured recipe:
|
||||
```
|
||||
$ gbrain integrations doctor
|
||||
voice-to-brain: ✓ Twilio reachable ✓ OpenAI key valid ✓ ngrok tunnel up
|
||||
email-to-brain: ✓ Gmail auth valid ✗ No emails in 48h (check cron)
|
||||
OVERALL: 1 warning
|
||||
```
|
||||
|
||||
### Sense Analytics
|
||||
|
||||
`gbrain integrations stats` aggregates heartbeat data:
|
||||
```
|
||||
$ gbrain integrations stats
|
||||
This week: 1,247 signals ingested
|
||||
Top sources: email (47%), voice (23%), X (18%), calendar (12%)
|
||||
34 new entity pages created
|
||||
7 calls transcribed
|
||||
Brain growth: 12,400 → 12,834 pages (+434)
|
||||
```
|
||||
|
||||
### Reflex Rules Engine (future)
|
||||
|
||||
Reflexes are recipes that trigger on brain state changes:
|
||||
|
||||
```yaml
|
||||
---
|
||||
id: deal-tracker
|
||||
category: reflex
|
||||
triggers:
|
||||
- type: page_updated
|
||||
filter: {type: deal, field: status}
|
||||
- type: timeline_entry
|
||||
filter: {source: email, mentions: deal}
|
||||
action: alert
|
||||
---
|
||||
|
||||
When a deal page's status changes or a new email mentions a deal,
|
||||
alert the user with context from the brain.
|
||||
```
|
||||
|
||||
## Roadmap
|
||||
|
||||
| Version | What Ships | Key Recipe |
|
||||
|---------|-----------|------------|
|
||||
| v0.7.0 | Recipe format, CLI, SKILLPACK breakout | voice-to-brain |
|
||||
| v0.8.0 | 3 more senses, reflex format | email, X, calendar |
|
||||
| v0.9.0 | Community recipes, install executor | community submissions |
|
||||
| v1.0.0 | Full senses/reflexes, health dashboard | meeting-prep, dream-cycle |
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
1. **GBrain is deterministic infrastructure.** Cross-sense correlation, pattern
|
||||
detection, and intelligent responses are the agent's job (OpenClaw/Hermes).
|
||||
GBrain provides the plumbing.
|
||||
|
||||
2. **Agents ARE the runtime.** No npm packages, Docker images, or deterministic
|
||||
scripts. The recipe markdown IS the installer. The agent reads it and does
|
||||
the work.
|
||||
|
||||
3. **Very opinionated defaults.** Ship the creator's exact production setup as
|
||||
the default. Users customize from there. Unknown callers get screened. Quiet
|
||||
hours are enforced. Brain-first lookup happens on every call.
|
||||
|
||||
4. **Agent-readable outputs.** All CLI output must be parseable by agents (--json
|
||||
flag). Migration files include agent instructions. The agent is the primary
|
||||
consumer, not the human.
|
||||
@@ -1,717 +0,0 @@
|
||||
# GBrain Knowledge Runtime — Design Doc
|
||||
|
||||
**Status:** DRAFT for CEO review.
|
||||
**Date:** 2026-04-18.
|
||||
**Supersedes:** The earlier "Feynman Ideas Assessment + Phase A/B" plan.
|
||||
|
||||
---
|
||||
|
||||
## 0. Context
|
||||
|
||||
During a CEO review of a narrow two-feature plan (bare-tweet citation repair + completeness score, borrowed from Feynman), the scope was reframed. The narrow plan duplicated work Garry's OpenClaw already does and missed the real leverage point: **the bespoke abstractions hiding inside OpenClaw — resolvers, enrichment orchestration, scheduling, deterministic output — should live in GBrain as first-class primitives.**
|
||||
|
||||
North star: *"When Garry's OpenClaw's Claw upgrades to this version of GBrain, it should immediately recognize brilliance and completeness and say 'It's time to switch to these abstractions.'"*
|
||||
|
||||
That is the test this document is designed against. Everything else is downstream.
|
||||
|
||||
---
|
||||
|
||||
## 1. The Four Layers
|
||||
|
||||
The design is four layered abstractions. Each is independently useful; together they are the Knowledge Runtime.
|
||||
|
||||
```
|
||||
┌───────────────────────────────────────────────────────────────────┐
|
||||
│ KNOWLEDGE RUNTIME (new) │
|
||||
├───────────────────────────────────────────────────────────────────┤
|
||||
│ Layer 4: Deterministic Output Builder │
|
||||
│ BrainWriter · Scaffolds · Back-link enforcer · Slug registry │
|
||||
│ Rule: LLM picks WHAT to write. Code guarantees WHERE and HOW. │
|
||||
├───────────────────────────────────────────────────────────────────┤
|
||||
│ Layer 3: Scheduler │
|
||||
│ ScheduledResolver · TZ-aware quiet hours (enforced) · │
|
||||
│ Auto-stagger · Durable state · Retry/circuit-break │
|
||||
├───────────────────────────────────────────────────────────────────┤
|
||||
│ Layer 2: Enrichment Orchestrator │
|
||||
│ Trigger convergence · Tier routing · Budget · Cascade · │
|
||||
│ Evidence-weighted completeness · Fail-safe transactions │
|
||||
├───────────────────────────────────────────────────────────────────┤
|
||||
│ Layer 1: Resolver SDK │
|
||||
│ Resolver<I,O> interface · Registry · Factory · Plugin recipes │
|
||||
│ Ported reference impls: X-API, Perplexity, Mistral, brain │
|
||||
└───────────────────────────────────────────────────────────────────┘
|
||||
│ │
|
||||
▼ ▼
|
||||
REUSES (polished primitives already in GBrain) REPLACES (ad-hoc code)
|
||||
FailImproveLoop · backoff · storage factory · enrichment-service ·
|
||||
check-resolvable · operations validators · embedding · transcription ·
|
||||
engine interface · publish · backlinks 2 recipe formats
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. Why This Order (L1 → L4)
|
||||
|
||||
Every higher layer depends on the lower one. **L1 must land first or the rest leaks abstractions.**
|
||||
|
||||
- **L1 (Resolvers)** is the substrate. Without a uniform lookup interface, every orchestrator + writer has bespoke callers.
|
||||
- **L2 (Orchestrator)** uses L1 to fetch; without L1 it's still ad-hoc.
|
||||
- **L3 (Scheduler)** runs L2 periodically; without L2 it's scheduling nothing structured.
|
||||
- **L4 (Output Builder)** is what every layer ultimately writes through; without it we have 14 call sites doing `fs.writeFile` with hand-rolled citation discipline.
|
||||
|
||||
An earlier implementation could ship L1 + L4 first (the two "purest" layers) and have the most immediate integrity impact, then add L2 + L3. But the end-state must include all four.
|
||||
|
||||
---
|
||||
|
||||
## 3. Layer 1 — Resolver SDK
|
||||
|
||||
### 3.1 What's broken today
|
||||
|
||||
Garry's OpenClaw has **69 distinct external-lookup patterns** across X API (14 shapes), Perplexity, Mistral OCR, Gmail, Calendar, Slack, GitHub, YouTube, Diarize.io, YC tools, OSINT collectors, and brain-local lookups. Each one is a bespoke script under `scripts/` with its own error handling, retry logic, and output shape. GBrain has 3 ad-hoc wrappers (`embedding.ts`, `transcription.ts`, `enrichment-service.ts`) that don't share an interface.
|
||||
|
||||
Common consequences:
|
||||
- No uniform retry/backoff strategy (some scripts retry, most don't)
|
||||
- No cost tracking (Perplexity bills eaten silently when calls return no-substance results)
|
||||
- No confidence/provenance propagation (callers can't tell if an answer is verified or inferred)
|
||||
- Users can't add a resolver without forking GBrain
|
||||
|
||||
### 3.2 Interface
|
||||
|
||||
```typescript
|
||||
// src/core/resolvers/interface.ts
|
||||
|
||||
export type ResolverCost = 'free' | 'rate-limited' | 'paid';
|
||||
|
||||
export interface ResolverRequest<I> {
|
||||
input: I;
|
||||
context: ResolverContext;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface ResolverResult<O> {
|
||||
value: O;
|
||||
confidence: number; // 0.0–1.0; 1.0 = deterministic from ground-truth API
|
||||
source: string; // e.g. "x-api-v2", "perplexity-sonar", "brain-local"
|
||||
fetchedAt: Date;
|
||||
costEstimate?: number; // dollars; 0 if free
|
||||
raw?: unknown; // for sidecar preservation via put_raw_data
|
||||
}
|
||||
|
||||
export interface Resolver<I, O> {
|
||||
readonly id: string; // stable, slug-like: "x_handle_to_tweet"
|
||||
readonly cost: ResolverCost;
|
||||
readonly backend: string; // "x-api-v2", "perplexity", "brain-local"
|
||||
readonly inputSchema: JSONSchema;
|
||||
readonly outputSchema: JSONSchema;
|
||||
|
||||
available(ctx: ResolverContext): Promise<boolean>;
|
||||
resolve(req: ResolverRequest<I>): Promise<ResolverResult<O>>;
|
||||
}
|
||||
```
|
||||
|
||||
### 3.3 Context
|
||||
|
||||
```typescript
|
||||
export interface ResolverContext {
|
||||
engine: BrainEngine;
|
||||
storage: StorageBackend;
|
||||
config: GBrainConfig;
|
||||
logger: Logger;
|
||||
metrics: MetricsRecorder;
|
||||
budget: BudgetLedger; // hard spend caps, queried pre-resolve
|
||||
requestId: string;
|
||||
remote: boolean; // trust boundary — untrusted callers get stricter validation
|
||||
deadline?: Date;
|
||||
}
|
||||
```
|
||||
|
||||
### 3.4 Registry + Factory (mirrors `src/core/storage.ts`)
|
||||
|
||||
```typescript
|
||||
// src/core/resolvers/registry.ts
|
||||
export class ResolverRegistry {
|
||||
register<I, O>(r: Resolver<I, O>): void;
|
||||
get(id: string): Resolver<unknown, unknown>;
|
||||
list(filter?: { cost?: ResolverCost; backend?: string }): Resolver[];
|
||||
async resolve<I, O>(id: string, input: I, ctx: ResolverContext): Promise<ResolverResult<O>>;
|
||||
}
|
||||
|
||||
// src/core/resolvers/factory.ts (dynamic import like engine-factory)
|
||||
export async function createResolver(
|
||||
type: 'x-api' | 'perplexity' | 'mistral-ocr' | 'brain-local' | 'plugin',
|
||||
config: ResolverConfig,
|
||||
): Promise<Resolver>;
|
||||
```
|
||||
|
||||
### 3.5 Plugin format (unifies `recipes/` + `data-research` formats)
|
||||
|
||||
A plugin is YAML + JS module, discovered via filesystem scan of `~/.gbrain/resolvers/` and `recipes/`.
|
||||
|
||||
```yaml
|
||||
# Example: resolvers/x-api/handle-to-tweet.yaml
|
||||
id: x_handle_to_tweet
|
||||
version: 1
|
||||
category: lookup
|
||||
cost: rate-limited
|
||||
backend: x-api-v2
|
||||
module: ./handle-to-tweet.ts
|
||||
input_schema:
|
||||
type: object
|
||||
properties:
|
||||
handle: { type: string, pattern: "^[A-Za-z0-9_]{1,15}$" }
|
||||
keywords: { type: string }
|
||||
required: [handle]
|
||||
output_schema:
|
||||
type: object
|
||||
properties:
|
||||
url: { type: string, format: uri }
|
||||
tweet_id: { type: string }
|
||||
text: { type: string }
|
||||
created_at: { type: string, format: date-time }
|
||||
requires:
|
||||
env: [X_API_BEARER_TOKEN]
|
||||
health_check:
|
||||
kind: http
|
||||
url: https://api.twitter.com/2/tweets/1
|
||||
expect: { status: [200, 401] } # 401 = auth failure but endpoint reachable
|
||||
tests:
|
||||
- input: { handle: "garrytan" }
|
||||
expect: { url: { pattern: "^https://x\\.com/garrytan/status/\\d+$" } }
|
||||
```
|
||||
|
||||
Trust flagging follows the existing `src/commands/integrations.ts` pattern: only package-bundled resolvers are `embedded=true` and may run arbitrary commands; user-provided resolvers are restricted to `http` and validated schemas.
|
||||
|
||||
### 3.6 Wraps every resolver with `FailImproveLoop`
|
||||
|
||||
Existing `src/core/fail-improve.ts` is the deterministic-first/LLM-fallback pattern. Every resolver automatically gets wrapped: if the deterministic path (e.g. X API) returns a valid result, use it; if it fails, optionally fall back to an LLM-based resolver; log both paths for future pattern analysis and auto-test generation.
|
||||
|
||||
### 3.7 Reference implementations to ship
|
||||
|
||||
The OpenClaw survey inventoried 69 resolver shapes. Shipping all of them is wrong (over-scoped); shipping zero is under-scoped. The dogfood set:
|
||||
|
||||
| # | Resolver | Purpose | Used by |
|
||||
|---|---|---|---|
|
||||
| 1 | `x_handle_to_tweet` | Bare-tweet citation repair (original Phase A) | `gbrain integrity` |
|
||||
| 2 | `url_reachable` | Dead-link detection | `gbrain integrity` |
|
||||
| 3 | `brain_slug_lookup` | Name/email → slug (wraps existing `resolveSlugs`) | Output Builder |
|
||||
| 4 | `openai_embedding` | Refactor of `src/core/embedding.ts` into Resolver | Import pipeline |
|
||||
| 5 | `perplexity_query` | Query → synthesis + citations | Enrichment Orchestrator |
|
||||
| 6 | `text_to_entities` | LLM entity extraction (structured JSON) | Enrichment Orchestrator |
|
||||
|
||||
The remaining 63 OpenClaw patterns port incrementally, driven by user need. Each port is a new YAML + module under `recipes/` or `~/.gbrain/resolvers/` with no framework changes.
|
||||
|
||||
---
|
||||
|
||||
## 4. Layer 2 — Enrichment Orchestrator
|
||||
|
||||
### 4.1 What's broken today
|
||||
|
||||
Garry's OpenClaw's enrichment is **polished at the data layer, hacky at the control layer**:
|
||||
|
||||
- **Completeness = "length > 500 chars + no `needs-enrichment` tag"** (`lib/enrich.mjs:351-355`). Naïve. A rich page of repetitive Perplexity summaries (see `brain/people/0interestrates.md` — 38 repeating blocks) passes this check.
|
||||
- **30-day auto-re-enrichment** runs forever. No "done" state. A person met once in 2023 still gets re-researched monthly.
|
||||
- **Cascade is convention-only.** Person→company stubs are created automatically; company→investors, company→employees traversals are documented but never implemented.
|
||||
- **No hard budget cap.** Cost is estimated per batch, never enforced across batches or per day.
|
||||
- **Failure is silent.** A bad Perplexity response logs and continues; partial writes can leave a page with a timeline entry but no raw-data sidecar.
|
||||
|
||||
### 4.2 The orchestrator
|
||||
|
||||
```typescript
|
||||
// src/core/enrichment/orchestrator.ts
|
||||
|
||||
export interface EnrichmentRequest {
|
||||
entitySlug: string;
|
||||
trigger: 'mention' | 'stub-creation' | 'cron-sweep' | 'manual' | 'cascade';
|
||||
tier?: 1 | 2 | 3; // optional override; auto-computed if absent
|
||||
cascadeDepth?: number; // 0 = no cascade; default 1
|
||||
}
|
||||
|
||||
export interface EnrichmentResult {
|
||||
entitySlug: string;
|
||||
completenessBefore: number;
|
||||
completenessAfter: number;
|
||||
resolversUsed: string[]; // e.g. ["perplexity_query", "x_handle_to_tweet"]
|
||||
costSpent: number;
|
||||
writtenTo: string[]; // page paths touched, for transaction audit
|
||||
cascadedTo: string[]; // related entities enriched
|
||||
status: 'enriched' | 'skipped' | 'failed' | 'budget-exhausted';
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export class EnrichmentOrchestrator {
|
||||
constructor(
|
||||
private registry: ResolverRegistry,
|
||||
private writer: BrainWriter,
|
||||
private budget: BudgetLedger,
|
||||
private scorer: CompletenessScorer,
|
||||
private graph: EntityGraph,
|
||||
) {}
|
||||
|
||||
async enrich(req: EnrichmentRequest): Promise<EnrichmentResult>;
|
||||
async enrichBatch(reqs: EnrichmentRequest[]): Promise<EnrichmentResult[]>;
|
||||
}
|
||||
```
|
||||
|
||||
### 4.3 Evidence-weighted completeness (replaces length heuristic)
|
||||
|
||||
Completeness is a per-entity-type rubric, stored in frontmatter on write and recomputed on demand.
|
||||
|
||||
```typescript
|
||||
// src/core/enrichment/completeness.ts
|
||||
export interface CompletenessRubric<Page> {
|
||||
entityType: PageType;
|
||||
dimensions: {
|
||||
name: string;
|
||||
weight: number; // sum must = 1.0
|
||||
check: (page: Page) => number; // 0.0–1.0
|
||||
}[];
|
||||
}
|
||||
|
||||
// Example rubric for persons:
|
||||
// - has_role_and_company 0.20
|
||||
// - has_source_urls 0.20 (≥1 URL with resolver-verified reachability)
|
||||
// - has_timeline_entries 0.15 (≥1)
|
||||
// - has_citations 0.15 (every claim has [Source: ...])
|
||||
// - has_backlinks 0.10 (every linked page links back)
|
||||
// - recency_score 0.10 (last_verified within 90 days)
|
||||
// - non_redundancy 0.10 (no repeated blocks; distinct-lines/total-lines > 0.8)
|
||||
```
|
||||
|
||||
**Key property:** `non_redundancy` + `recency_score` explicitly kill the two brain pathologies observed in the audit (Wilco-style repeating blocks; stale pages without `last_verified`).
|
||||
|
||||
The `completeness` field goes in frontmatter as `0.0–1.0`. It becomes queryable via `list_pages(where: completeness < 0.5)`.
|
||||
|
||||
### 4.4 Tier routing with hard budget
|
||||
|
||||
Two-dimensional routing: **importance** (tier 1/2/3 from person-score) × **budget state**.
|
||||
|
||||
```typescript
|
||||
// src/core/enrichment/tiers.ts
|
||||
export const TIER_CONFIG = {
|
||||
1: { models: ['opus', 'sonar-deep'], maxCostUsd: 0.10, cascadeDepth: 2 },
|
||||
2: { models: ['sonar'], maxCostUsd: 0.02, cascadeDepth: 1 },
|
||||
3: { models: ['sonar'], maxCostUsd: 0.005, cascadeDepth: 0 },
|
||||
};
|
||||
|
||||
// src/core/enrichment/budget.ts
|
||||
export class BudgetLedger {
|
||||
// Hard caps. Queryable pre-resolve.
|
||||
dailyCapUsd: number;
|
||||
perEntityCapUsd: number;
|
||||
perResolverCapUsd: Map<string, number>;
|
||||
|
||||
async reserve(resolverId: string, estimateUsd: number): Promise<Reservation | 'exhausted'>;
|
||||
async commit(reservation: Reservation, actualUsd: number): Promise<void>;
|
||||
async rollback(reservation: Reservation): Promise<void>;
|
||||
async state(): Promise<{ spent: number; remaining: number; perResolver: Record<string, number> }>;
|
||||
}
|
||||
```
|
||||
|
||||
**Property:** if the daily cap is reached, `orchestrator.enrich()` returns `status: 'budget-exhausted'` immediately. No silent overages. Circuit-breaker resets at midnight in the user's configured TZ.
|
||||
|
||||
### 4.5 Cascade (entity graph traversal)
|
||||
|
||||
```typescript
|
||||
// src/core/enrichment/cascade.ts
|
||||
export class EntityGraph {
|
||||
// Deterministic, no LLM. Uses engine.getLinks() + engine.getBacklinks().
|
||||
async neighbors(slug: string, depth: number): Promise<string[]>;
|
||||
async cascadeFrom(trigger: string, depth: number): Promise<EnrichmentRequest[]>;
|
||||
}
|
||||
```
|
||||
|
||||
If person X is enriched and gains a new `company: Acme` field, cascade checks: does `companies/acme` exist? If not, create stub + enqueue at tier 2. Does `companies/acme` link back to X? If not, write the back-link. **Iron Law is machine-enforced, not skill-enforced.**
|
||||
|
||||
### 4.6 Fail-safe transactions
|
||||
|
||||
Every enrichment is wrapped in a BrainWriter transaction (Layer 4). Partial writes are rolled back. No asymmetric state like timeline-entry-without-raw-sidecar.
|
||||
|
||||
```typescript
|
||||
await writer.transaction(async (tx) => {
|
||||
const research = await registry.resolve('perplexity_query', {...}, ctx);
|
||||
await tx.appendTimeline(slug, {...});
|
||||
await tx.putRawData(slug, 'perplexity', research.raw);
|
||||
await tx.setFrontmatterField(slug, 'completeness', score);
|
||||
// All-or-nothing commit on exit.
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Layer 3 — Scheduler
|
||||
|
||||
### 5.1 What's broken today
|
||||
|
||||
Garry's OpenClaw's cron is **externally-driven JSON** (`cron/jobs.json`) with ~30 jobs manually stagger-offset at different minutes. GBrain has **zero native scheduling** — `src/commands/autopilot.ts` is a single daemon loop, and `docs/guides/cron-schedule.md` is architectural guidance, not code.
|
||||
|
||||
Failures observed in Garry's OpenClaw's actual state:
|
||||
- `X OAuth2 Token Refresh`: 11 consecutive timeouts (critical-path silent failure)
|
||||
- `flight-tracker daily scan`: 5 consecutive timeouts
|
||||
- `morning-briefing`: 4 consecutive timeouts
|
||||
- Quiet hours are checked at runtime in skills, so a skill that forgets to check will DM at 3 a.m.
|
||||
- Staggering is manual convention; no protection against two jobs colliding after a config edit.
|
||||
|
||||
### 5.2 ScheduledResolver interface
|
||||
|
||||
```typescript
|
||||
// src/core/scheduling/scheduler.ts
|
||||
export interface Schedule {
|
||||
kind: 'cron' | 'interval';
|
||||
expr?: string; // cron string
|
||||
intervalMs?: number;
|
||||
tz: string; // IANA: "America/Los_Angeles"
|
||||
quietHours?: {
|
||||
startHour: number; // 22 = 10 PM local
|
||||
endHour: number; // 7 = 7 AM local
|
||||
policy: 'skip' | 'defer' | 'silent-run';
|
||||
};
|
||||
staggerKey?: string; // jobs with same key auto-offset
|
||||
maxConcurrent?: number; // global concurrency cap
|
||||
maxDurationMs?: number; // timeout
|
||||
}
|
||||
|
||||
export interface ScheduledResolver extends Resolver<void, ScheduledResult> {
|
||||
schedule: Schedule;
|
||||
retryPolicy: { maxRetries: number; backoffMs: number };
|
||||
circuitBreaker: { failureThreshold: number; cooldownMs: number };
|
||||
state: DurableState; // watermark, content-hash, idempotency key
|
||||
}
|
||||
```
|
||||
|
||||
### 5.3 Enforcement vs convention (the key delta from Garry's OpenClaw)
|
||||
|
||||
| Concern | Garry's OpenClaw today | Knowledge Runtime |
|
||||
|---|---|---|
|
||||
| Quiet hours | Checked inside each skill (trust-based) | Enforced at scheduler, skill cannot override |
|
||||
| Staggering | Manual minute-offset in `jobs.json` | Scheduler assigns slots via hashed staggerKey |
|
||||
| Concurrency | `MAX_BATCH_PROCESSES=2` in backoff, ignored by cron | Global semaphore in scheduler |
|
||||
| Timeout | Per-job string in JSON, not always respected | Enforced via `AbortController`, timeout raises `TimeoutError` caught by orchestrator |
|
||||
| Retry | None at cron level | `retryPolicy` with exponential backoff |
|
||||
| Silent failure | "11 consecutive timeouts" unnoticed | Circuit breaker opens at threshold → escalation to user |
|
||||
| Idempotency | State files per job, no framework | `DurableState` primitive: watermark/ID/content-hash |
|
||||
|
||||
### 5.4 Native engine + OS cron adapter
|
||||
|
||||
The scheduler runs as either:
|
||||
1. **Embedded** (default for `gbrain autopilot`): native event loop inside the daemon process. One process, many ScheduledResolvers.
|
||||
2. **OS-driven** (for Railway/launchd/systemd): `gbrain schedule run <id>` invoked by OS cron, scheduler state is durable so cross-invocation dedup still works.
|
||||
|
||||
Both modes share the same `Schedule` config + state.
|
||||
|
||||
### 5.5 Observability
|
||||
|
||||
Every scheduled run emits structured events: `started`, `skipped-quiet-hours`, `deferred-to-active-hours`, `failed-retrying`, `circuit-opened`, `completed`. Events go to:
|
||||
- `~/.gbrain/scheduler/events.jsonl` (local, always)
|
||||
- `engine.logIngest` (audit trail in brain DB)
|
||||
- Optional webhook (Slack/Telegram for the user)
|
||||
|
||||
`gbrain doctor` reads the event log and reports: current circuit-breaker state, any resolver with > 3 consecutive failures, any resolver that hasn't fired within 3× its interval (freshness SLA like Garry's OpenClaw's `freshness-check.mjs` but built-in).
|
||||
|
||||
---
|
||||
|
||||
## 6. Layer 4 — Deterministic Output Builder
|
||||
|
||||
### 6.1 The anti-hallucination invariant
|
||||
|
||||
**Iron Law: LLM picks WHAT. Code guarantees WHERE and HOW.**
|
||||
|
||||
Garry's OpenClaw's existing `lib/enrich.mjs:buildTweetEntry` is close to this — tweet URLs are built from `tweet.id` returned by the X API, never from LLM memory. But:
|
||||
|
||||
- A past incident: *"Sub-agent test #2 FAILED — hallucinated 'Philip Leung' entity links across all daily files. LLM rewriting of daily files is too error-prone."* (Garry's OpenClaw memory log, 2026-04-13.)
|
||||
- Back-links depend on `appendTimeline` being called everywhere; skips are silent.
|
||||
- Slug collisions are unchecked (no conflict detection on `slugify`).
|
||||
- Citation format is post-hoc linted weekly, not pre-write enforced.
|
||||
|
||||
### 6.2 BrainWriter
|
||||
|
||||
```typescript
|
||||
// src/core/output/writer.ts
|
||||
export class BrainWriter {
|
||||
constructor(
|
||||
private engine: BrainEngine,
|
||||
private slugRegistry: SlugRegistry,
|
||||
private scaffolder: Scaffolder,
|
||||
) {}
|
||||
|
||||
async transaction<T>(fn: (tx: WriteTx) => Promise<T>): Promise<T>;
|
||||
}
|
||||
|
||||
export interface WriteTx {
|
||||
// High-level typed operations; never raw string writes.
|
||||
createEntity(input: EntityInput): Promise<string>; // returns slug, conflict-checked
|
||||
appendTimeline(slug: string, entry: TimelineInput): Promise<void>;
|
||||
setCompiledTruth(slug: string, body: CompiledTruthInput): Promise<void>;
|
||||
setFrontmatterField(slug: string, key: string, value: unknown): Promise<void>;
|
||||
putRawData(slug: string, source: string, data: object): Promise<void>;
|
||||
addLink(from: string, to: string, context: string): Promise<void>; // auto-creates reverse back-link
|
||||
|
||||
// Validators (called implicitly on commit)
|
||||
validate(): Promise<ValidationReport>;
|
||||
}
|
||||
```
|
||||
|
||||
### 6.3 Scaffolder — deterministic link + citation construction
|
||||
|
||||
Every user-visible URL/link/citation is built by code from resolver outputs, not from LLM text.
|
||||
|
||||
```typescript
|
||||
// src/core/output/scaffold.ts
|
||||
export class Scaffolder {
|
||||
tweetCitation(handle: string, tweetId: string, dateISO: string): string {
|
||||
// "[Source: [X/garrytan, 2026-04-18](https://x.com/garrytan/status/123456)]"
|
||||
}
|
||||
emailCitation(account: string, messageId: string, subject: string): string {
|
||||
// deterministic Gmail URL per OpenClaw pattern
|
||||
}
|
||||
sourceCitation(resolverResult: ResolverResult<unknown>): string {
|
||||
// pulls .source, .fetchedAt, .raw from the result
|
||||
}
|
||||
entityLink(slug: string): string {
|
||||
// slugRegistry checks existence; returns resolvable wikilink
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 6.4 SlugRegistry — conflict detection
|
||||
|
||||
```typescript
|
||||
// src/core/output/slug-registry.ts
|
||||
export class SlugRegistry {
|
||||
async create(desiredSlug: string, displayName: string, type: PageType): Promise<CreatedSlug>;
|
||||
// Throws SlugCollision if another entity already occupies desiredSlug and isn't
|
||||
// confirmed as the same person (via email / x_handle / disambiguator).
|
||||
// Auto-resolves near-collisions by appending disambiguator.
|
||||
|
||||
async confirmSame(slugA: string, slugB: string, confidence: number): Promise<void>;
|
||||
async merge(canonical: string, duplicate: string): Promise<void>;
|
||||
}
|
||||
```
|
||||
|
||||
### 6.5 Pre-write validators (fail-closed for integrity)
|
||||
|
||||
On `WriteTx.validate()` before commit:
|
||||
|
||||
1. **Citation validator.** Every factual sentence in `compiled_truth` must have an inline `[Source: ...]` within N lines. Non-compliant paragraphs are flagged. Configurable: strict-mode rejects the transaction, lint-mode warns.
|
||||
2. **Link validator.** Every `[text](path)` must point to a page that exists OR to a URL the Scaffolder built (so it's guaranteed-valid). No raw LLM-composed URLs.
|
||||
3. **Back-link validator.** Every outbound link must have a reverse link written in the same transaction.
|
||||
4. **Triple-HR validator.** Compiled truth / timeline split enforced at the schema level.
|
||||
|
||||
**Fails closed**: the default is strict-mode. Loosening requires explicit `writer.transaction({ strictMode: false }, ...)` and logs a warning to the ingest log.
|
||||
|
||||
### 6.6 LLM output sanitization
|
||||
|
||||
Any LLM output destined for a brain page passes through a JSON-Schema-validated parser first. No free-form markdown goes to disk.
|
||||
|
||||
- Entity extraction: JSON array of `{ name, type, context }` per existing `extractEntities` pattern — strict validation.
|
||||
- Compiled-truth synthesis: LLM emits structured `{ sections: [{heading, paragraphs: [{text, sources: [...]}]}]}`, scaffolder renders to markdown.
|
||||
- Timeline entries: LLM emits `{ date, summary, detail, sources }`, scaffolder renders.
|
||||
|
||||
LLM never sees file paths, never writes files, never emits finished markdown.
|
||||
|
||||
---
|
||||
|
||||
## 7. Integration with existing GBrain
|
||||
|
||||
### 7.1 Reuse (already polished)
|
||||
|
||||
| Existing | Used by | Change |
|
||||
|---|---|---|
|
||||
| `src/core/fail-improve.ts` (9/10) | Wraps every Resolver in L1 | None; becomes default wrapper |
|
||||
| `src/core/backoff.ts` (9/10) | ResolverContext.backoff | None |
|
||||
| `src/core/storage.ts` (9/10) | Template for Resolver factory pattern | None; serves as pattern reference |
|
||||
| `src/core/check-resolvable.ts` (9/10) | Extend to validate Resolver plugins | Add `checkResolvers()` mode |
|
||||
| `src/commands/publish.ts` (9/10) | Uses BrainWriter under the hood | Minor: route through L4 |
|
||||
| `src/commands/backlinks.ts` (8/10) | Folded into L4 validator | Keep as CLI-facing lint entry point |
|
||||
| `src/core/operations.ts` validators | Reused in ResolverContext trust enforcement | None |
|
||||
| `src/core/engine.ts` BrainEngine (35 methods) | ResolverContext.engine | Extend with `getResolverRegistry()` |
|
||||
|
||||
### 7.2 Replace (ad-hoc today)
|
||||
|
||||
| Existing | Replace with |
|
||||
|---|---|
|
||||
| `src/core/enrichment-service.ts` (5/10) | `src/core/enrichment/orchestrator.ts` (L2) |
|
||||
| `src/core/embedding.ts` (monolithic) | `src/core/resolvers/builtin/embedding/openai.ts` |
|
||||
| `src/core/transcription.ts` (monolithic) | `src/core/resolvers/builtin/transcription/{groq,openai}.ts` |
|
||||
| `src/commands/integrations.ts` recipe format | Unified Resolver plugin format (§3.5) |
|
||||
| `src/core/data-research.ts` recipe format | Same unified format |
|
||||
| `src/commands/autopilot.ts` hard-coded daemon loop | Wraps a set of ScheduledResolvers |
|
||||
|
||||
### 7.3 Extend
|
||||
|
||||
- `src/core/engine.ts`: add `getResolverRegistry()`, `getWriter()`, `getScheduler()`. Engine becomes the runtime's root container.
|
||||
- `src/core/operations.ts`: `OperationContext` inherits from `ResolverContext` (or vice-versa). Trust flags unified.
|
||||
- `src/core/types.ts`: add `completeness: number` to `Page`, `sourcedBy: string[]` for provenance.
|
||||
|
||||
---
|
||||
|
||||
## 8. Migration Path (phased, shippable)
|
||||
|
||||
Each phase ships independently, passes full E2E, is feature-flagged, and is reversible. No big-bang.
|
||||
|
||||
### Phase 0 — Foundation (human: ~1 wk / CC: ~4 h)
|
||||
- Define `Resolver<I,O>`, `ResolverContext`, `ResolverRegistry`, `ResolverResult` (§3.2–3.4).
|
||||
- Add `src/core/resolvers/index.ts` wiring + tests for registry (register/get/list).
|
||||
- No behavioral change; ship as `v0.11.0-alpha` with feature flag.
|
||||
|
||||
### Phase 1 — Three reference resolvers (human: ~1 wk / CC: ~4 h)
|
||||
- Port `src/core/embedding.ts` → `resolvers/builtin/embedding/openai.ts`.
|
||||
- Implement `resolvers/builtin/brain-local/slug-lookup.ts` (wraps `engine.resolveSlugs`).
|
||||
- Implement `resolvers/builtin/url-reachable.ts` (HEAD-check).
|
||||
- Prove the interface: old callers swap to `registry.resolve('openai_embedding', ...)`.
|
||||
|
||||
### Phase 2 — BrainWriter + Slug Registry (human: ~1.5 wk / CC: ~6 h)
|
||||
- L4 core: `BrainWriter.transaction`, `Scaffolder`, `SlugRegistry` with conflict detection.
|
||||
- Pre-write validators: citation, link, back-link, triple-HR.
|
||||
- Migrate `src/commands/publish.ts` + `src/commands/backlinks.ts` to route through BrainWriter.
|
||||
- **Now** Garry's OpenClaw's "Philip Leung" hallucination is structurally impossible — LLM output passes through JSON-Schema validator before reaching Scaffolder.
|
||||
|
||||
### Phase 3 — `gbrain integrity` command (human: ~0.5 wk / CC: ~2 h)
|
||||
- Ship the originally-scoped user-facing feature on top of the new foundation.
|
||||
- Uses Resolver SDK: `x_handle_to_tweet` + `url_reachable`.
|
||||
- Uses BrainWriter: all auto-repairs go through validated writes.
|
||||
- `--auto --confidence 0.8` mode as user approved in cherry-pick #1.
|
||||
- **User-visible value ships in Phase 3, not Phase 7.**
|
||||
|
||||
### Phase 4 — Enrichment Orchestrator (human: ~2 wk / CC: ~8 h)
|
||||
- L2 core: `EnrichmentOrchestrator`, `BudgetLedger`, `CompletenessScorer`, `EntityGraph.cascadeFrom`.
|
||||
- Migrate `src/core/enrichment-service.ts` callers (deprecate the old file after).
|
||||
- Completeness score in frontmatter on every write (dogfooding cascades).
|
||||
|
||||
### Phase 5 — Scheduler (human: ~2 wk / CC: ~8 h)
|
||||
- L3 core: `Scheduler`, `ScheduledResolver`, `DurableState`, circuit breaker, quiet-hours enforcer.
|
||||
- Migrate `src/commands/autopilot.ts` to a ScheduledResolver set.
|
||||
- Ship `gbrain schedule list|run|pause|tail` CLI for observability.
|
||||
|
||||
### Phase 6 — Port 5–8 OpenClaw resolvers (human: ~1.5 wk / CC: ~6 h)
|
||||
- `perplexity_query`, `text_to_entities`, `mistral_ocr_pdf`, `x_search_all`, `x_user_to_tweets`, `gmail_query_to_threads`, `calendar_date_to_events`.
|
||||
- Each ships as YAML + TS module under `resolvers/builtin/` — **proof of the plugin format.**
|
||||
|
||||
### Phase 7 — OpenClaw Adoption Integration (human: ~1 wk / CC: ~4 h)
|
||||
- Write `docs/openclaw/ADOPTION.md` showing your OpenClaw how to replace its 69 bespoke scripts with calls to `gbrain registry.resolve(...)`.
|
||||
- Ship a `gbrain claw-bridge` subcommand that proxies Garry's OpenClaw's current script invocations to the resolver registry — zero-edit adoption path.
|
||||
- **This is the test of the north star.** If your OpenClaw can stand up a 1-line shim and drop `scripts/x-api-client.mjs`, the abstraction succeeded.
|
||||
|
||||
Total: human: ~10 weeks / CC: ~42 hours / calendar with single implementer: ~3–4 weeks.
|
||||
|
||||
---
|
||||
|
||||
## 9. Critical Files
|
||||
|
||||
### New directories / files
|
||||
|
||||
```
|
||||
src/core/
|
||||
runtime/
|
||||
index.ts # RuntimeContext (engine, storage, config, logger, metrics, budget)
|
||||
registry.ts # ResolverRegistry
|
||||
factory.ts # createResolver()
|
||||
resolvers/
|
||||
interface.ts # Resolver<I, O>
|
||||
fail-improve-wrapper.ts # auto-wraps every resolver in FailImproveLoop
|
||||
builtin/
|
||||
x-api/
|
||||
handle-to-tweet.ts
|
||||
handle-to-tweet.yaml
|
||||
perplexity/
|
||||
query.ts
|
||||
query.yaml
|
||||
brain-local/
|
||||
slug-lookup.ts
|
||||
url-reachable.ts
|
||||
embedding/
|
||||
openai.ts # refactored from src/core/embedding.ts
|
||||
transcription/
|
||||
groq.ts
|
||||
openai.ts
|
||||
enrichment/
|
||||
orchestrator.ts # EnrichmentOrchestrator
|
||||
tiers.ts # TIER_CONFIG
|
||||
budget.ts # BudgetLedger
|
||||
completeness.ts # CompletenessScorer + per-type rubrics
|
||||
cascade.ts # EntityGraph
|
||||
scheduling/
|
||||
scheduler.ts # Scheduler + ScheduledResolver
|
||||
schedule.ts # Schedule type, cron expr parser
|
||||
state.ts # DurableState primitives
|
||||
quiet-hours.ts # TZ-aware enforcement
|
||||
stagger.ts # deterministic slot assignment
|
||||
output/
|
||||
writer.ts # BrainWriter
|
||||
scaffold.ts # Scaffolder (typed URL builders)
|
||||
slug-registry.ts # SlugRegistry (conflict detection)
|
||||
validators/
|
||||
citation.ts
|
||||
link.ts
|
||||
back-link.ts
|
||||
triple-hr.ts
|
||||
|
||||
src/commands/
|
||||
integrity.ts # ships in Phase 3, replaces Feynman Phase A/B
|
||||
schedule.ts # gbrain schedule list|run|pause|tail (Phase 5)
|
||||
|
||||
docs/openclaw/
|
||||
ADOPTION.md # written in Phase 7
|
||||
```
|
||||
|
||||
### Replaced / removed
|
||||
- `src/core/enrichment-service.ts` — folded into `enrichment/orchestrator.ts`
|
||||
- `src/core/embedding.ts` — moved into `resolvers/builtin/embedding/openai.ts`
|
||||
- `src/core/transcription.ts` — moved into `resolvers/builtin/transcription/`
|
||||
|
||||
### Extended
|
||||
- `src/core/engine.ts` — add `getResolverRegistry()`, `getWriter()`, `getScheduler()`
|
||||
- `src/core/operations.ts` — unify with ResolverContext; every operation validator reusable by resolvers
|
||||
- `src/core/types.ts` — add `completeness: number`, `sourcedBy: string[]`, `lastVerified: Date`
|
||||
|
||||
---
|
||||
|
||||
## 10. Testing Strategy
|
||||
|
||||
### Contract tests
|
||||
Every Resolver implementation tested against the interface spec. Table-driven: run the same suite against `openai_embedding`, `x_handle_to_tweet`, etc. Ensures plugin authors can't ship broken resolvers.
|
||||
|
||||
### Property tests
|
||||
- **Idempotency:** running a ScheduledResolver twice with the same state produces the same output and doesn't double-write.
|
||||
- **Atomicity:** a BrainWriter transaction that throws mid-flight leaves the brain bit-for-bit identical to pre-transaction.
|
||||
- **Deterministic scaffolds:** given the same resolver outputs, the Scaffolder produces byte-identical citations/links.
|
||||
|
||||
### Integration tests
|
||||
- `EnrichmentOrchestrator` end-to-end against PGLite (in-memory, no API keys) with mocked resolver registry.
|
||||
- `Scheduler` with fake clock + quiet-hours scenarios.
|
||||
- BrainWriter transaction rollback on validator failure.
|
||||
|
||||
### Chaos tests
|
||||
- Kill the process mid-enrichment; next run must resume cleanly.
|
||||
- Simulate API timeout mid-transaction; transaction must roll back completely.
|
||||
- Corrupted state file; scheduler must escalate, not silently skip.
|
||||
|
||||
### Regression tests vs. Garry's OpenClaw behavior
|
||||
For each OpenClaw pattern we port (e.g. X-handle → tweet URL), a regression test proves the new resolver produces the same answer on real-world inputs from the brain audit. This is the "your OpenClaw would adopt" proof.
|
||||
|
||||
---
|
||||
|
||||
## 11. Open Questions (flagged for CEO re-review)
|
||||
|
||||
1. **Scope shape.** Is this the right four-layer decomposition, or are some layers better left to OpenClaw (e.g. Scheduling lives above GBrain, not in it)?
|
||||
2. **Phase 3 user-value break.** Does Phase 3 (user-visible `gbrain integrity`) ship early enough, or do we need an even smaller MVP?
|
||||
3. **LLM-as-resolver.** Should `text_to_entities` be a Resolver, or does that blur the "code vs LLM" line the invariant relies on?
|
||||
4. **Plugin format.** YAML + TS module (§3.5) vs. pure TS module with decorator-style metadata. Latter is more type-safe; former is more discoverable.
|
||||
5. **Cross-resolver transactions.** Do we support "atomic fetch-from-Perplexity + write-to-brain" at the L2 layer? Current design says yes; implementation is tricky (Perplexity call isn't rollbackable).
|
||||
6. **OpenClaw bridge scope.** Phase 7 `gbrain claw-bridge` — is that worth a phase of its own, or should adoption be documentation-only?
|
||||
7. **Completeness rubric coverage.** Do we define rubrics for all 9 PageTypes upfront, or ship people/company/meeting first and extend incrementally?
|
||||
8. **Budget config UX.** Hard daily cap is strict; should we also expose a soft-cap warning mode, and how is the cap set (env var? config file? prompt on first use?)
|
||||
9. **Backwards compat.** `src/commands/publish.ts` and `src/commands/backlinks.ts` have been running cleanly for weeks. Refactoring through BrainWriter carries migration risk. Acceptable?
|
||||
10. **Existing TODOS alignment.** `TODOS.md` has P0 "Runtime MCP access control" and P2 security hardening. The new RuntimeContext.remote flag interacts with both — do we fold MCP access control into Phase 0 or keep separate?
|
||||
|
||||
---
|
||||
|
||||
## 12. Verification (the "your OpenClaw would adopt" test)
|
||||
|
||||
The design succeeds iff:
|
||||
|
||||
- [ ] A user can add a new resolver by dropping a YAML + TS module in `~/.gbrain/resolvers/` without editing GBrain source.
|
||||
- [ ] Your OpenClaw can delete `scripts/x-api-client.mjs` and replace all callers with 1-line `await registry.resolve('x_handle_to_tweet', ...)`.
|
||||
- [ ] No brain page can be written with a bare tweet reference, a missing back-link, or an unverified URL (validators catch it pre-commit).
|
||||
- [ ] Running `gbrain integrity --auto --confidence 0.8` over a real brain fixes ≥1,000 of the 1,424 known bare-tweet citations without human review.
|
||||
- [ ] Full E2E test suite passes on both PGLite + Postgres engines.
|
||||
- [ ] The Knowledge Runtime ships across 7 phases with each phase individually shippable and reversible.
|
||||
@@ -1,448 +0,0 @@
|
||||
---
|
||||
status: ACTIVE
|
||||
---
|
||||
# CEO Plan: Minions as Universal Agent Orchestration Protocol
|
||||
Generated by /plan-ceo-review on 2026-04-15
|
||||
Branch: garrytan/minions-jobs | Mode: SCOPE EXPANSION
|
||||
Repo: garrytan/gbrain
|
||||
|
||||
## Vision
|
||||
|
||||
### 10x Check
|
||||
Instead of "GBrain has a queue, OpenClaw uses it," make Minions a universal agent
|
||||
orchestration protocol. Any platform (OpenClaw, Hermes, Claude Code, Codex, custom
|
||||
scripts) submits, monitors, steers, and composes agents through the same Postgres-native
|
||||
protocol. GBrain IS the agent control plane.
|
||||
|
||||
### Platonic Ideal (aspirational North Star, NOT in v1 scope)
|
||||
Open a terminal, type `gbrain jobs dashboard`. See every agent across every platform.
|
||||
Their progress, tool calls, token spend. Click any agent for full execution trace.
|
||||
Type a message to redirect a running agent mid-flight. See the governor's decisions
|
||||
visualized. Run A/B tests between agent configurations. The feeling: complete
|
||||
situational awareness of your AI workforce.
|
||||
|
||||
**Note:** The dashboard, A/B testing, and visual governor are future phases. This plan
|
||||
builds the primitives they would sit on top of: real-time events, structured progress,
|
||||
token accounting, inbox with ack, and session transcripts.
|
||||
|
||||
## Scope Decisions
|
||||
|
||||
| # | Proposal | Effort | Decision | Reasoning |
|
||||
|---|----------|--------|----------|-----------|
|
||||
| 1 | pg LISTEN/NOTIFY real-time events | S | ACCEPTED | Sub-second event delivery vs 5s polling. Every platform benefits. |
|
||||
| 2 | Structured progress protocol | S | ACCEPTED | Standard progress makes unified dashboard possible. |
|
||||
| 3 | Job cost tracking (token accounting) | M | ACCEPTED | Token cost is #1 thing users want to know about agent work. |
|
||||
| 4 | Job replay | S | ACCEPTED | Small surface area, high utility for debugging failures. |
|
||||
| 5 | Job groups / waves | M | DEFERRED | Parent-child already provides grouping. Overlap concern. |
|
||||
| 6 | Inbox acknowledgment (read receipts) | S | ACCEPTED | Without it, inbox is fire-and-forget — same problem we're fixing. |
|
||||
| 7 | Universal agent protocol | S | ACCEPTED | Design framing, not extra code. Platform-agnostic naming/docs. |
|
||||
| 8 | Session transcript capture | M | ACCEPTED | Full audit trail of every agent run. |
|
||||
|
||||
## Accepted Scope — Implementation Detail
|
||||
|
||||
### 0a. Pause/resume (from base plan)
|
||||
|
||||
**Schema:** Add `'paused'` to `MinionJobStatus` (already in migration v6 constraint).
|
||||
|
||||
**New methods:**
|
||||
- `MinionQueue.pauseJob(id): MinionJob | null`
|
||||
Transitions `waiting` or `active` → `paused`. For `active` jobs, clears `lock_token`
|
||||
and `lock_until` (worker will detect lock loss and stop). Returns null if job not
|
||||
in pausable state.
|
||||
- `MinionQueue.resumeJob(id): MinionJob | null`
|
||||
Transitions `paused` → `waiting`. Resets for claiming. Returns null if not paused.
|
||||
|
||||
**Worker integration:** Worker's lock renewal loop checks `isActive()`. When a job
|
||||
is paused, the lock is cleared, so `renewLock()` returns false and the worker stops
|
||||
execution gracefully (same path as stall detection). The job's progress and state
|
||||
are preserved in the DB for when it resumes.
|
||||
|
||||
**MCP operations:** `pause_job`, `resume_job` (added in Step 3 of implementation plan).
|
||||
|
||||
**PGLite compatibility:** Full.
|
||||
|
||||
### 0b. Resource governor (from base plan)
|
||||
|
||||
**New file:** `src/core/minions/governor.ts`
|
||||
|
||||
```typescript
|
||||
interface GovernorConfig {
|
||||
maxConcurrency: number; // ceiling
|
||||
minConcurrency: number; // floor (default 1)
|
||||
checkIntervalMs: number; // default 10000
|
||||
cpuThreshold: number; // default 0.80 (80%)
|
||||
memoryThreshold: number; // default 0.85 (85%)
|
||||
circuitBreakerMemory: number; // default 0.90 (90%)
|
||||
}
|
||||
|
||||
class ResourceGovernor {
|
||||
getEffectiveConcurrency(): number; // current allowed concurrency
|
||||
start(): void; // begin polling system metrics
|
||||
stop(): void; // stop polling
|
||||
onCircuitBreak(cb: (jobId) => void): void; // kill callback
|
||||
}
|
||||
```
|
||||
|
||||
**System metrics:** Reuse `getSystemLoad()` from `src/core/backoff.ts` (already
|
||||
implements CPU and memory checks). Add event loop lag measurement via
|
||||
`perf_hooks.monitorEventLoopDelay()`.
|
||||
|
||||
**Worker integration:** `MinionWorker.start()` consults `governor.getEffectiveConcurrency()`
|
||||
before claiming new jobs. If current in-flight count >= effective concurrency, skip claim.
|
||||
|
||||
**Circuit breaker:** If memory > 90%, governor calls `onCircuitBreak` with the
|
||||
lowest-priority active job ID. Worker cancels that job via `failJob()` with
|
||||
`UnrecoverableError("circuit breaker: memory pressure")`.
|
||||
|
||||
**Prerequisite:** Concurrent job processing must be implemented first (see
|
||||
Concurrency Note below).
|
||||
|
||||
**PGLite compatibility:** Full (governor is app-level, not DB-level).
|
||||
|
||||
### 1. pg LISTEN/NOTIFY (real-time events)
|
||||
|
||||
**Schema:** No new columns. Add NOTIFY triggers to state transitions.
|
||||
|
||||
**SQL trigger:**
|
||||
```sql
|
||||
CREATE OR REPLACE FUNCTION notify_minion_job_change() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
PERFORM pg_notify('minion_jobs', json_build_object(
|
||||
'id', NEW.id, 'status', NEW.status, 'name', NEW.name,
|
||||
'queue', NEW.queue, 'prev_status', COALESCE(OLD.status, 'new')
|
||||
)::text);
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER minion_job_notify AFTER INSERT OR UPDATE OF status ON minion_jobs
|
||||
FOR EACH ROW EXECUTE FUNCTION notify_minion_job_change();
|
||||
```
|
||||
|
||||
**New method:** `MinionQueue.subscribe(callback: (event) => void): () => void`
|
||||
Returns unsubscribe function. Requires direct Postgres connection (NOT pooled).
|
||||
|
||||
**PGLite compatibility:** PGLite does NOT support LISTEN/NOTIFY. Fallback: polling
|
||||
via `getJob()` at configurable interval (default 2s). The `subscribe()` method
|
||||
detects engine type and uses polling fallback automatically.
|
||||
|
||||
**Supabase constraint:** Requires direct connection (port 5432), not pgBouncer
|
||||
pooler (port 6543). Document in skill file and setup guide.
|
||||
|
||||
### 2. Structured progress protocol
|
||||
|
||||
**TypeScript interface (convention, not enforced at DB level):**
|
||||
```typescript
|
||||
interface AgentProgress {
|
||||
step: number; // current step (1-based)
|
||||
total: number; // total expected steps (0 = unknown)
|
||||
message: string; // human-readable status
|
||||
tokens_in: number; // cumulative input tokens
|
||||
tokens_out: number; // cumulative output tokens
|
||||
last_tool: string; // name of last tool called
|
||||
started_at: string; // ISO 8601 when this step started
|
||||
}
|
||||
```
|
||||
|
||||
**Storage:** Existing `progress JSONB` column. No schema change needed.
|
||||
Handlers use `ctx.updateProgress(agentProgress)`. Non-agent jobs can use
|
||||
any JSONB shape (backward compatible).
|
||||
|
||||
**Validation:** `updateProgress()` accepts any JSONB. The `AgentProgress`
|
||||
interface is a convention enforced by the agent handler, not by the queue.
|
||||
|
||||
### 3. Job cost tracking (token accounting)
|
||||
|
||||
**Schema changes (migration v6):**
|
||||
```sql
|
||||
ALTER TABLE minion_jobs ADD COLUMN tokens_input INTEGER DEFAULT 0;
|
||||
ALTER TABLE minion_jobs ADD COLUMN tokens_output INTEGER DEFAULT 0;
|
||||
ALTER TABLE minion_jobs ADD COLUMN tokens_cache_read INTEGER DEFAULT 0;
|
||||
ALTER TABLE minion_jobs ADD COLUMN cost_usd NUMERIC(10,6) DEFAULT 0;
|
||||
```
|
||||
|
||||
**New method:** `MinionQueue.updateTokens(id, lockToken, { input, output, cache_read, cost_usd })`
|
||||
Accumulates (adds to existing values, does not replace).
|
||||
|
||||
**Parent rollup:** When `completeJob()` is called, if `parent_job_id` is set,
|
||||
add this job's token counts to the parent's via:
|
||||
```sql
|
||||
UPDATE minion_jobs SET
|
||||
tokens_input = tokens_input + $child_input,
|
||||
tokens_output = tokens_output + $child_output,
|
||||
tokens_cache_read = tokens_cache_read + $child_cache,
|
||||
cost_usd = cost_usd + $child_cost
|
||||
WHERE id = $parent_id;
|
||||
```
|
||||
|
||||
**PGLite compatibility:** Full support (standard columns).
|
||||
|
||||
### 4. Job replay
|
||||
|
||||
**New method:** `MinionQueue.replayJob(id, dataOverrides?: Record<string, unknown>): MinionJob`
|
||||
|
||||
Implementation: Read the completed/failed/dead job. Create a NEW job with:
|
||||
- Same `name`, `queue`, `priority`, `max_attempts`, `backoff_type`, `backoff_delay`
|
||||
- `data` = deep merge of original data + overrides
|
||||
- Fresh `attempts_made: 0`, `status: 'waiting'`
|
||||
- `parent_job_id` = null (replay is a new top-level job, not a child)
|
||||
- Does NOT clone children (replay is a single job, not a DAG)
|
||||
|
||||
**Constraint:** Only works on terminal statuses (completed/failed/dead).
|
||||
Returns the new job record.
|
||||
|
||||
**Idempotency:** Each replay creates a distinct new job. No deduplication.
|
||||
If the original had side effects, the replay may repeat them. Document this
|
||||
in the skill file as a user responsibility.
|
||||
|
||||
### 5. Inbox (sidechannel messaging)
|
||||
|
||||
**Schema changes (migration v6):**
|
||||
```sql
|
||||
ALTER TABLE minion_jobs ADD COLUMN inbox JSONB DEFAULT '[]';
|
||||
```
|
||||
|
||||
**Inbox message format:**
|
||||
```typescript
|
||||
interface InboxMessage {
|
||||
id: string; // UUIDv4
|
||||
sent_at: string; // ISO 8601
|
||||
read_at: string | null; // null until worker reads it
|
||||
sender: string; // 'parent' | 'user' | job ID
|
||||
payload: unknown; // arbitrary directive
|
||||
}
|
||||
```
|
||||
|
||||
**New methods:**
|
||||
- `MinionQueue.sendMessage(jobId, payload, sender?): InboxMessage`
|
||||
Appends message to inbox array via atomic JSONB append
|
||||
(`inbox = inbox || $1::jsonb`), not read-modify-write. Returns the message with id + sent_at.
|
||||
- `MinionQueue.readInbox(jobId, lockToken): InboxMessage[]`
|
||||
Returns unread messages (read_at = null). Marks them as read (sets read_at).
|
||||
Token-fenced: only the worker holding the lock can read.
|
||||
|
||||
**Worker integration:** Agent handler calls `readInbox()` on each iteration.
|
||||
If messages exist, injects them into the agent's context as system messages.
|
||||
|
||||
**PGLite compatibility:** Full support (standard JSONB column).
|
||||
|
||||
### 6. Inbox acknowledgment (read receipts)
|
||||
|
||||
Built into the inbox design above. The `read_at` field on each `InboxMessage`
|
||||
provides the receipt. `sendMessage()` returns the message ID; the sender can
|
||||
later check `getJob(id)` and inspect `inbox` to see which messages have been
|
||||
read.
|
||||
|
||||
No additional schema or methods needed beyond what's in #5.
|
||||
|
||||
### 7. Universal agent protocol (platform-agnostic framing)
|
||||
|
||||
**This is a design decision, not code.** It means:
|
||||
|
||||
1. The skill file (`skills/minion-orchestrator/SKILL.md`) is written for ANY
|
||||
agent platform, not just OpenClaw. Examples show MCP tool calls, not
|
||||
OpenClaw-specific commands.
|
||||
|
||||
2. The agent handler (`agent-handler.ts`) accepts a generic interface:
|
||||
```typescript
|
||||
interface AgentJobData {
|
||||
prompt: string;
|
||||
tools?: string[]; // MCP tool names
|
||||
model?: string; // e.g., 'claude-opus-4-6', 'gpt-4o'
|
||||
context?: string; // additional context
|
||||
platform?: string; // 'openclaw' | 'hermes' | 'claude-code' | 'custom'
|
||||
max_iterations?: number; // agent loop budget
|
||||
}
|
||||
```
|
||||
|
||||
3. The OpenClaw plugin is ONE consumer. Hermes, Claude Code extensions,
|
||||
or custom scripts can submit `agent` jobs through the same MCP operations.
|
||||
|
||||
4. **NOT in v1 scope:** Multi-tenant auth, cross-network connectivity,
|
||||
protocol versioning, API key isolation. These are Phase 2 concerns when
|
||||
actual multi-platform usage materializes. v1 is single-user, single-brain.
|
||||
|
||||
### Agent Handler Architecture (critical design decision)
|
||||
|
||||
The agent handler does NOT live in GBrain. GBrain provides the queue infrastructure
|
||||
and a clean handler contract. The actual agent execution lives in the platform plugin.
|
||||
|
||||
```
|
||||
GBrain (this repo):
|
||||
MinionQueue — queue/claim/complete/inbox/tokens/NOTIFY
|
||||
MinionWorker — poll/lock/stall/governor framework
|
||||
Handler contract — AgentJobData interface + MinionJobContext
|
||||
|
||||
OpenClaw plugin (separate repo):
|
||||
Registers "agent" handler with MinionWorker
|
||||
Handler calls OpenClaw's PI agent core (the actual LLM loop)
|
||||
Each iteration: readInbox → inject as system message, updateProgress, updateTokens
|
||||
Completion: store result + session transcript in job.result + job.stacktrace
|
||||
|
||||
GBrain ships a test/echo handler for unit testing only.
|
||||
```
|
||||
|
||||
**Handler contract (GBrain side):**
|
||||
```typescript
|
||||
// The handler receives this context (already exists in worker.ts)
|
||||
interface MinionJobContext {
|
||||
id: number;
|
||||
name: string;
|
||||
data: Record<string, unknown>; // AgentJobData when name="agent"
|
||||
attempts_made: number;
|
||||
updateProgress(progress: unknown): Promise<void>;
|
||||
updateTokens(tokens: TokenUpdate): Promise<void>; // NEW
|
||||
log(message: string | TranscriptEntry): Promise<void>;
|
||||
isActive(): Promise<boolean>;
|
||||
readInbox(): Promise<InboxMessage[]>; // NEW
|
||||
}
|
||||
```
|
||||
|
||||
**Why this is right:** GBrain is orchestration, not execution. OpenClaw has the
|
||||
PI agent core. Hermes has AIAgent. Claude Code has its own loop. Each platform
|
||||
brings its own engine and registers a handler. GBrain manages lifecycle, progress,
|
||||
steering, cost tracking, and persistence around it.
|
||||
|
||||
### 8. Session transcript capture
|
||||
|
||||
**Extends existing stacktrace mechanism.** The `stacktrace` field (JSONB array
|
||||
of strings) already captures log messages. Session transcripts use the same
|
||||
field with structured entries:
|
||||
|
||||
```typescript
|
||||
type TranscriptEntry =
|
||||
| { type: 'log'; message: string; ts: string }
|
||||
| { type: 'tool_call'; tool: string; args_size: number; result_size: number; ts: string }
|
||||
| { type: 'llm_turn'; model: string; tokens_in: number; tokens_out: number; ts: string }
|
||||
| { type: 'error'; message: string; stack?: string; ts: string };
|
||||
```
|
||||
|
||||
**Storage:** Existing `stacktrace JSONB` column. No schema change.
|
||||
The agent handler appends `TranscriptEntry` objects instead of plain strings.
|
||||
Backward compatible: non-agent jobs continue appending strings.
|
||||
|
||||
**Size concern:** Long agent runs could generate large transcripts. Add a
|
||||
`max_transcript_entries` option (default 1000) that rotates oldest entries
|
||||
when exceeded (FIFO). The full transcript for forensic analysis can be
|
||||
stored as a brain file via `gbrain files upload-raw`.
|
||||
|
||||
## Schema Migration v6
|
||||
|
||||
All schema changes are additive (ALTER TABLE ADD COLUMN). No backfill needed.
|
||||
Existing jobs continue to work with default values.
|
||||
|
||||
```sql
|
||||
-- Migration v6: Agent orchestration primitives
|
||||
ALTER TABLE minion_jobs ADD COLUMN IF NOT EXISTS tokens_input INTEGER DEFAULT 0;
|
||||
ALTER TABLE minion_jobs ADD COLUMN IF NOT EXISTS tokens_output INTEGER DEFAULT 0;
|
||||
ALTER TABLE minion_jobs ADD COLUMN IF NOT EXISTS tokens_cache_read INTEGER DEFAULT 0;
|
||||
|
||||
-- Separate inbox table (not JSONB on job row)
|
||||
CREATE TABLE IF NOT EXISTS minion_inbox (
|
||||
id SERIAL PRIMARY KEY,
|
||||
job_id INTEGER NOT NULL REFERENCES minion_jobs(id) ON DELETE CASCADE,
|
||||
sender TEXT NOT NULL,
|
||||
payload JSONB NOT NULL,
|
||||
sent_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
read_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_minion_inbox_unread
|
||||
ON minion_inbox (job_id) WHERE read_at IS NULL;
|
||||
|
||||
-- Status constraint update: add 'paused'
|
||||
ALTER TABLE minion_jobs DROP CONSTRAINT IF EXISTS minion_jobs_status_check;
|
||||
ALTER TABLE minion_jobs ADD CONSTRAINT minion_jobs_status_check
|
||||
CHECK (status IN ('waiting','active','completed','failed','delayed','dead','cancelled','waiting-children','paused'));
|
||||
|
||||
-- NOTIFY trigger for real-time events (Postgres only, not PGLite)
|
||||
CREATE OR REPLACE FUNCTION notify_minion_job_change() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
PERFORM pg_notify('minion_jobs', json_build_object(
|
||||
'id', NEW.id, 'status', NEW.status, 'name', NEW.name,
|
||||
'queue', NEW.queue, 'prev_status', COALESCE(OLD.status, 'new')
|
||||
)::text);
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER minion_job_notify AFTER INSERT OR UPDATE OF status ON minion_jobs
|
||||
FOR EACH ROW EXECUTE FUNCTION notify_minion_job_change();
|
||||
```
|
||||
|
||||
## PGLite Compatibility Matrix
|
||||
|
||||
| Feature | Postgres | PGLite | Fallback |
|
||||
|---|---|---|---|
|
||||
| Pause/resume | Full | Full | — |
|
||||
| Inbox + ack | Full | Full | — |
|
||||
| Token accounting | Full | Full | — |
|
||||
| Job replay | Full | Full | — |
|
||||
| LISTEN/NOTIFY | Full | NO | Polling (2s interval) |
|
||||
| NOTIFY trigger | Full | NO | Skipped in PGLite schema |
|
||||
| Structured progress | Full | Full | — |
|
||||
| Session transcripts | Full | Full | — |
|
||||
| Resource governor | Full | Full | — |
|
||||
| Worker daemon | Full | NO (existing limitation) | — |
|
||||
|
||||
## Concurrency Note
|
||||
|
||||
The current `MinionWorker.start()` processes jobs sequentially (one at a time)
|
||||
despite `concurrency` being declared in `MinionWorkerOpts`. Implementing actual
|
||||
concurrent job processing (Promise pool) is a prerequisite for the resource
|
||||
governor to be meaningful. The governor adjusts effective concurrency, which
|
||||
requires actual concurrent processing to exist.
|
||||
|
||||
**Action:** Implement concurrent job processing in `worker.ts` before or as
|
||||
part of the governor step. Use a semaphore pattern: maintain up to N in-flight
|
||||
promises, claim new jobs as slots free up.
|
||||
|
||||
## Outside Voice Decisions (from adversarial review)
|
||||
|
||||
1. **AbortController for pause/resume** — Handler contract gets `signal: AbortSignal`.
|
||||
Pause clears lock AND signals abort. Handler must check `signal.aborted` on each
|
||||
iteration. Without this, pausing active jobs creates duplicate execution.
|
||||
|
||||
2. **Drop cost_usd column** — Token counts (input/output/cache_read) are stable facts.
|
||||
USD pricing is volatile. Compute cost at display/read time from a pricing table,
|
||||
not at write time. Removes `cost_usd NUMERIC(10,6)` from migration v6.
|
||||
|
||||
3. **Separate minion_inbox table** — Instead of JSONB array on job row, use a dedicated
|
||||
table for inbox messages. Avoids row bloat from rewriting entire inbox on every send.
|
||||
Properly concurrent-safe with standard INSERT (no JSONB append concerns).
|
||||
```sql
|
||||
CREATE TABLE minion_inbox (
|
||||
id SERIAL PRIMARY KEY,
|
||||
job_id INTEGER NOT NULL REFERENCES minion_jobs(id) ON DELETE CASCADE,
|
||||
sender TEXT NOT NULL,
|
||||
payload JSONB NOT NULL,
|
||||
sent_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
read_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX idx_minion_inbox_unread ON minion_inbox (job_id) WHERE read_at IS NULL;
|
||||
```
|
||||
|
||||
4. **One release, not two** — Ship all features in one migration (v6). User prefers
|
||||
cohesive release over incremental delivery for this feature set.
|
||||
|
||||
5. **Selective column projection** — Fix SELECT * queries in getJobs(), claim(),
|
||||
handleStalled() to exclude stacktrace column. Include stacktrace only in getJob()
|
||||
detail view. Prevents transcript bloat from affecting query performance.
|
||||
|
||||
## Future Phases (accepted trajectory)
|
||||
|
||||
- **Phase 2: Dashboard CLI** — `gbrain jobs dashboard` live TUI showing all agents.
|
||||
Enabled by: LISTEN/NOTIFY, structured progress, token accounting.
|
||||
- **Phase 3: Multi-tenant auth** — Runtime MCP access control, per-platform API keys.
|
||||
Enabled by: platform-agnostic framing, sender validation on inbox.
|
||||
- **Phase 4: Agent composition patterns** — Map-reduce, pipeline, approval gates as
|
||||
first-class primitives. Enabled by: parent-child DAGs, inbox sidechannel.
|
||||
|
||||
## Deferred to TODOS.md
|
||||
- Job groups / waves (parent-child covers this; revisit if real grouping need emerges)
|
||||
- cost_usd column (compute from pricing table at read time when pricing API exists)
|
||||
|
||||
## Key Premises Confirmed
|
||||
1. GBrain is intentionally evolving from knowledge brain to agent infrastructure (user confirmed)
|
||||
2. Coupling between OpenClaw and GBrain's Postgres is acceptable (OpenClaw already depends on GBrain)
|
||||
3. Full Infrastructure approach (all 8+ steps) selected over Minimal Viable or Sidecar Tracking
|
||||
4. Prior learning [agent-dx-instruction-layer] validates that the teaching layer (skill + evals) is mandatory
|
||||
@@ -1,188 +0,0 @@
|
||||
---
|
||||
type: essay
|
||||
title: "Homebrew for Personal AI"
|
||||
subtitle: "Why Markdown is Code and Your Agent is a Package Manager"
|
||||
author: Garry Tan
|
||||
created: 2026-04-11
|
||||
updated: 2026-04-11
|
||||
tags: [ai, gbrain, gstack, markdown-is-code, open-source, software-distribution, agents, openclaw]
|
||||
status: draft-v2
|
||||
prior: "Thin Harness, Fat Skills"
|
||||
---
|
||||
|
||||
# Homebrew for Personal AI
|
||||
|
||||
`brew install` gives you someone else's binary. `npm install` gives you someone else's source code. Both require you to understand the tool, configure it, integrate it, maintain it.
|
||||
|
||||
What if software distribution worked differently? What if you could describe a capability in plain English, hand that description to an AI agent, and the agent built a native implementation tailored to your setup?
|
||||
|
||||
That's what happens when markdown is code.
|
||||
|
||||
## Markdown is code
|
||||
|
||||
Here's a real skill file. This one teaches an AI agent to screen phone calls:
|
||||
|
||||
```markdown
|
||||
# Voice Agent — Your Phone Number
|
||||
|
||||
Caller → Twilio → <Stream> WebSocket → Voice Server (port 8765)
|
||||
↕ audio
|
||||
OpenAI Realtime API
|
||||
↓ tool calls
|
||||
Brain / Calendar / Telegram
|
||||
|
||||
## Call Routing
|
||||
|
||||
Every inbound call routes based on caller phone number + brain lookup:
|
||||
|
||||
### Owner → Authenticated Mode
|
||||
- Send crypto-random 6-digit code to secure channel
|
||||
- Caller reads it back
|
||||
- Match → full assistant mode (brain, calendar, scheduling)
|
||||
- No match → treated as unknown caller
|
||||
|
||||
### Known Person, Inner Circle (brain score ≥ 4) → Forward
|
||||
- Greet by name with brain context
|
||||
- Transfer to cell
|
||||
- If no answer (30s timeout), take message
|
||||
- Text Telegram with who called and context
|
||||
|
||||
### Unknown Caller → Screen
|
||||
- Get their name, look them up in brain
|
||||
- If inner circle → offer to transfer
|
||||
- Otherwise → take message
|
||||
- Create brain entry with phone number (marked UNVERIFIED)
|
||||
```
|
||||
|
||||
That's not pseudocode. That's not documentation. That's a working specification that a model like Claude Opus 4.6 with a million-token context window can read and implement. The architecture diagram tells it the components. The routing table tells it the logic. The security model tells it the constraints. The agent reads this file, understands it, and builds the Twilio integration, the WebSocket server, the Telegram bot hooks, the brain lookup, all of it, shaped to whatever infrastructure the user already has.
|
||||
|
||||
A skill file is a method call. It takes parameters (your phone number, your brain, your preferred messaging app). Same skill, different arguments, different implementation. The procedure is the package. The model is the runtime.
|
||||
|
||||
## The distribution mechanism
|
||||
|
||||
Traditional package managers distribute artifacts: compiled binaries, source tarballs, container images. The consumer runs someone else's code.
|
||||
|
||||
GBrain distributes recipes: markdown files that describe capabilities with enough specificity that an AI agent can implement them from scratch. The consumer gets a native implementation. No dependency hell. No version conflicts. No transitive vulnerability chains. Because there is no upstream code. There's a description of what to build and why.
|
||||
|
||||
Here's how it works:
|
||||
|
||||
1. **Build a feature.** Implement a voice agent, meeting ingestion pipeline, email triage system, investment diligence workflow, whatever.
|
||||
|
||||
2. **GBrain captures the recipe.** Not just the code. The architecture, the integration points, the failure modes, the judgment calls. A markdown file that encodes the full capability.
|
||||
|
||||
3. **Push to the repo.** Open source. Anyone can read it.
|
||||
|
||||
4. **Someone else's agent pulls the recipe.** Reads the markdown. Says: "New recipe available: AI voice agent with caller screening. Want it?" User says yes. The agent reads the spec and builds it.
|
||||
|
||||
No installation. No configuration wizard. No README. The agent read a document and figured it out.
|
||||
|
||||
## Why this works now
|
||||
|
||||
This didn't work two years ago. Two things changed.
|
||||
|
||||
**Context windows hit a million tokens.** A real skill file for meeting ingestion is 200+ lines. The enrichment skill that calls it references a brain schema, a resolver, a citation standard, five external APIs, and a cross-linking protocol. An agent implementing this recipe needs to hold all of that in working memory simultaneously while also understanding the user's existing setup. At 8K tokens, impossible. At 128K, marginal. At 1M, comfortable.
|
||||
|
||||
**Models crossed the judgment threshold.** Here's a snippet from a real enrichment recipe:
|
||||
|
||||
```markdown
|
||||
## Philosophy
|
||||
|
||||
A brain page should read like an intelligence dossier crossed
|
||||
with a therapist's notes, not a LinkedIn scrape. We want:
|
||||
|
||||
- What they believe — ideology, worldview, first principles
|
||||
- What they're building — current projects, what's next
|
||||
- What motivates them — ambition drivers, career arc
|
||||
- What makes them emotional — angry, excited, defensive, proud
|
||||
- Their trajectory — ascending, plateauing, pivoting, declining?
|
||||
- Hard facts — role, company, funding, location, contact info
|
||||
|
||||
Facts are table stakes. Texture is the value.
|
||||
```
|
||||
|
||||
A model implementing this recipe has to understand the difference between a LinkedIn scrape and an intelligence dossier. That's a judgment call about what information is worth capturing and how to weight it. GPT-3 couldn't do this. GPT-4 could sort of do it. Opus 4.6 does it well. The enabling technology is models that are smart enough to interpret intent, not just follow instructions.
|
||||
|
||||
## What a recipe actually contains
|
||||
|
||||
A good recipe has five sections:
|
||||
|
||||
**Architecture.** The component diagram. What talks to what, over what protocol, with what data flow. This is the skeleton the agent builds first.
|
||||
|
||||
**Routing logic.** The decision tree. When X happens, do Y. When Z fails, fall back to W. This is where domain knowledge lives. A voice agent recipe encodes call routing. A diligence recipe encodes how to process pitch decks vs. financial models vs. cap tables. A meeting ingestion recipe encodes how to turn a raw transcript into actionable intelligence.
|
||||
|
||||
**Integration points.** What external systems does this touch? Twilio, Telegram, Gmail, Circleback, Slack, GitHub, Supabase, whatever. The recipe names the integrations; the agent figures out how to connect them given what the user already has configured.
|
||||
|
||||
**Judgment calls.** The hard part. Not "send an email" but "decide whether this email is worth surfacing to the user based on sender importance, time sensitivity, and whether it requires a decision." Recipes that skip the judgment calls produce shallow implementations. The judgment calls are the actual value.
|
||||
|
||||
**Failure modes.** What goes wrong and what to do about it. "If Circleback token expires, message the user and ask them to reconnect. Don't silently skip." "If caller ID is spoofed, never trust it for authentication. Use a challenge-response code via a separate channel." Recipes without failure modes produce brittle systems.
|
||||
|
||||
Here's a real example. This is the diligence recipe's detection logic:
|
||||
|
||||
```markdown
|
||||
## Detection
|
||||
|
||||
Recognize data room materials by:
|
||||
- PDF filenames: "Data Deck", "Intro Deck", "Cap Table",
|
||||
"Financial Model", "Pitch Deck", "Series [A-D]"
|
||||
- Spreadsheets with tabs: Revenue, Retention, Cohorts,
|
||||
CAC, Gross Margin, Unit Economics, ARR
|
||||
- User saying: "data room", "diligence", "deck", "pitch"
|
||||
- Context: shared in the Diligence topic
|
||||
```
|
||||
|
||||
That's a pattern matcher expressed in English. An agent reads this and knows how to classify incoming documents. No regex. No file type configuration. Just a description of the pattern and the model's judgment about whether a given document matches.
|
||||
|
||||
## Pick and choose
|
||||
|
||||
GBrain is not monolithic. Recipes are independent. Take what you want:
|
||||
|
||||
- **Voice agent** — phone screening, caller ID, brain lookup, message routing
|
||||
- **Meeting ingestion** — transcript processing, entity extraction, action item capture, timeline updates
|
||||
- **Email triage** — inbox sweep, priority classification, draft replies, scheduling extraction
|
||||
- **Enrichment pipeline** — people and company research from multiple data sources, diarized into brain pages
|
||||
- **Diligence processing** — data room ingestion, PDF extraction, financial model analysis
|
||||
- **Social monitoring** — X/Twitter timeline analysis, mention tracking, narrative detection
|
||||
- **Content pipeline** — idea capture, link ingestion, article summarization
|
||||
|
||||
Each recipe is self-contained. Your agent knows what you already have. GBrain pings daily: "Three new recipes since last sync. Want any?" You pick. It builds.
|
||||
|
||||
And because the source code is English, forking is trivial. Don't like how the voice agent handles unknown callers? Edit the markdown. Change "take a message" to "ask three screening questions first." The behavior changes because the spec changed.
|
||||
|
||||
## The thin harness, fat skills connection
|
||||
|
||||
This essay is a sequel. The prequel was "Thin Harness, Fat Skills," which argued that the secret to 100x AI productivity isn't better models but better context management. Keep the harness thin (the program running the model). Make the skills fat (markdown procedures encoding judgment and process).
|
||||
|
||||
"Markdown is code" is the distribution corollary. If the skills are fat markdown files, and if models are smart enough to implement from markdown, then the skills are distributable software. The skill file is simultaneously:
|
||||
|
||||
- **Documentation** for humans reading it
|
||||
- **Specification** for the implementing agent
|
||||
- **Package** for the distribution system
|
||||
- **Source code** for the resulting capability
|
||||
|
||||
Four artifacts collapsed into one. That's why this is different from every previous package manager. `brew install` separates the formula from the binary from the docs from the source. GBrain collapses them. The markdown is all four.
|
||||
|
||||
## The architecture underneath
|
||||
|
||||
Three layers, same as the talk:
|
||||
|
||||
**Fat skills** on top. Markdown recipes encoding judgment, process, failure modes, and domain knowledge. This is where 90% of the value lives. This is what gets distributed.
|
||||
|
||||
**Thin harness** in the middle. The program running the model. File operations, tool dispatch, context management, safety enforcement. About 200 lines. OpenClaw or any equivalent. The less the harness constrains, the more the recipes can express.
|
||||
|
||||
**Deterministic foundation** on the bottom. Databases, APIs, CLIs. Same input, same output, every time. SQL queries, HTTP calls, file reads. The skills describe WHEN to call these; the harness executes them.
|
||||
|
||||
Push intelligence UP into skills. Push execution DOWN into deterministic tooling. Distribute the skills. That's the whole system.
|
||||
|
||||
## What this means
|
||||
|
||||
When implementation cost approaches zero, the bottleneck shifts. It's no longer "can we build this?" It's "should we build this?" and "what exactly should it do?"
|
||||
|
||||
Taste, vision, and domain knowledge become the scarce resources. The person who deeply understands call screening and writes a precise recipe creates more value than the person who can implement a Twilio integration from scratch. The recipe IS the implementation.
|
||||
|
||||
This also means the best AI agent setups will be open source by default. Closed, proprietary agent configurations are competing against a world where someone publishes a recipe and a thousand agents implement it overnight. The recipe propagates at the speed of a git push. The moat is taste, not code.
|
||||
|
||||
Software distribution reimagined: the package is a markdown file, the runtime is a sufficiently smart model, the package manager is your AI agent, and the app store is a git repo.
|
||||
|
||||
`gbrain install voice-agent`
|
||||
|
||||
That's it.
|
||||
@@ -1,209 +0,0 @@
|
||||
---
|
||||
type: essay
|
||||
title: "Thin Harness, Fat Skills"
|
||||
subtitle: "How to Make AI Agents Actually Understand Your Data"
|
||||
author: Garry Tan
|
||||
created: 2026-04-09
|
||||
updated: 2026-04-11
|
||||
tags: [ai, agents, gstack, harness-engineering, skills, architecture]
|
||||
status: draft-v4
|
||||
talk: "YC Spring 2026 -- Thin Harness, Fat Skills"
|
||||
thread: https://x.com/garrytan/status/2042925773300908103
|
||||
---
|
||||
|
||||
# Thin Harness, Fat Skills
|
||||
|
||||
Steve Yegge says people using AI coding agents are "10x to 100x as productive as engineers using Cursor and chat today, and roughly 1000x as productive as Googlers were back in 2005."
|
||||
|
||||
That's a real number. I've seen it. I've lived it. But when people hear 100x, they think: better models. Smarter Claude. More parameters.
|
||||
|
||||
That's the wrong frame entirely. The 2x people and the 100x people are using the same models. The difference is five concepts that fit on an index card.
|
||||
|
||||
## The harness is the secret sauce
|
||||
|
||||
On March 31, 2026, Anthropic accidentally shipped the entire source code for Claude Code to the npm registry. 512,000 lines. When I read it, it confirmed everything I'd been teaching at YC. The secret sauce isn't the model. It's the thing wrapping the model: the harness. Live repo context. Prompt caching. Purpose-built tools. Context bloat minimization. Structured session memory. Parallel sub-agents.
|
||||
|
||||
None of that is about making the model smarter. All of it is about giving the model the right context, at the right time, without drowning it in noise.
|
||||
|
||||
That's the only question that matters. And the answer has a specific shape. I call it **thin harness, fat skills**.
|
||||
|
||||
## Five definitions
|
||||
|
||||
The bottleneck is never the model's intelligence. The bottleneck is whether the model understands your schema. Models already know how to reason, synthesize, and write code. They fail because they don't know your data. Five definitions fix this.
|
||||
|
||||
### Definition 1: Skill File
|
||||
|
||||
A skill file is a reusable markdown procedure that teaches the model HOW to do something. Not WHAT to do. The user supplies the specifics. The skill supplies the process.
|
||||
|
||||
**Markdown is actually code.** A skill file is a more perfect encapsulation of capability than rigid source code, because it describes process, judgment, and context in the language the model already thinks in.
|
||||
|
||||
On the left is a skill called `/investigate`. Seven steps: scope the dataset, build a timeline, diarize every document, synthesize, argue both sides, cite sources. It takes three parameters: TARGET, QUESTION, and DATASET.
|
||||
|
||||
On the right are two completely different invocations of the same skill. One points at Dr. Sarah Chen and 2.1 million discovery emails, asking whether a safety scientist was silenced. The other points at Pacific Corporate Services and FEC filings, asking whether shell companies are coordinating campaign donations.
|
||||
|
||||
Same skill. Same seven steps. Same markdown file. In one case it's a medical research analyst. In the other it's a forensic investigator. The skill describes a process of judgment. The invocation supplies the world.
|
||||
|
||||
**This is the key insight most people miss: a skill file works like a method call.** It takes parameters. You invoke it with different arguments. The same procedure produces radically different capabilities depending on what you pass in. This is not prompt engineering. This is software design, using markdown as the programming language and human judgment as the runtime.
|
||||
|
||||
### Definition 2: Harness
|
||||
|
||||
The harness is the program that runs the LLM. It does four things: runs the model in a loop, reads and writes your files, manages context, and enforces safety. That's the "thin."
|
||||
|
||||
The anti-pattern is a fat harness with thin skills: 40+ tool definitions eating half the context window. God tools with 2 to 5 second MCP round-trips. REST API wrappers that turn every endpoint into a tool. 3x the tokens, 3x the latency, 3x the failure rate.
|
||||
|
||||
What you should build instead: a Playwright CLI that does each browser operation in 100 milliseconds. Compare: Chrome MCP takes 15 seconds for screenshot + find + click + wait + read. Playwright CLI takes 200 milliseconds for screenshot + assert. 75x faster. Software doesn't have to be precious anymore. Build exactly what you need.
|
||||
|
||||
### Definition 3: Resolver
|
||||
|
||||
A resolver is a routing table for context. When task type X appears, load document Y first.
|
||||
|
||||
Skills say HOW. Resolvers say WHAT to load WHEN. A developer changes a prompt. Without the resolver, they ship it. With the resolver, the model reads `docs/EVALS.md` first, which says: run the eval suite, compare scores, if accuracy drops more than 2%, revert and investigate. The developer didn't know the eval suite existed. The resolver loaded the right context at the right moment.
|
||||
|
||||
Claude Code has a built-in resolver. Every skill has a description field, and the model matches user intent to skill descriptions automatically. You never have to remember `/ship` exists. The description IS the resolver. It's like Clippy. Except it actually works.
|
||||
|
||||
A confession: my CLAUDE.md was 20,000 lines. Every single thing I ran across went in there. Every quirk, every pattern, every lesson. Completely ridiculous. The model's attention degraded. Claude Code literally told me to cut it back. The fix: about 200 lines. Just pointers to documents. The resolver loads the right one when it matters.
|
||||
|
||||
### Definition 4: Latent vs. Deterministic
|
||||
|
||||
Every step in your system is one or the other.
|
||||
|
||||
**Latent space** is where intelligence lives. The model reads, interprets, decides. Judgment. Synthesis. Pattern recognition.
|
||||
|
||||
**Deterministic** is where trust lives. Same input, same output. Every time. SQL. Code. Numbers.
|
||||
|
||||
An LLM can seat 8 people at a dinner table. Ask it to seat 800 and it will hallucinate a seating chart that looks plausible but is completely wrong. That's a deterministic problem forced into latent space. The worst systems put the wrong work on the wrong side.
|
||||
|
||||
### Definition 5: Diarization
|
||||
|
||||
The model reads everything about a subject and writes a structured profile. Read 50 documents, produce 1 page of judgment.
|
||||
|
||||
No SQL query produces this. No RAG pipeline produces this. The model has to actually read, hold contradictions in mind, notice what changed and when, and write structured intelligence. This is what makes AI useful for real knowledge work.
|
||||
|
||||
## The architecture
|
||||
|
||||
Three layers:
|
||||
|
||||
**Fat skills** on top. Markdown procedures that encode judgment, process, and domain knowledge. This is where 90% of the value lives.
|
||||
|
||||
**Thin CLI harness** in the middle. About 200 lines. JSON in, text out. Read-only by default. CLI first, add MCP later.
|
||||
|
||||
**Your app** on the bottom. QueryDB. ReadDoc. Search. Timeline. The deterministic foundation.
|
||||
|
||||
Push intelligence UP into skills. Push execution DOWN into deterministic tooling. Keep the harness THIN.
|
||||
|
||||
## The system that learns: YC Startup School
|
||||
|
||||
Let me show you all five definitions working together. Not in theory. In an actual system we're building at YC.
|
||||
|
||||
Chase Center. July 2026. 6,000 founders. Each one has a structured application, questionnaire answers, transcripts from 1:1 advisor chats, and public signals: X posts, GitHub commits, Claude Code transcripts showing how fast they ship.
|
||||
|
||||
The traditional approach: a program team of 15 reads applications, makes gut calls, updates a spreadsheet. It works at 200 founders. It breaks at 6,000.
|
||||
|
||||
No human can hold 6,000 profiles in working memory and notice that the three best candidates for the infrastructure-for-AI-agents cohort are a dev tools founder in Lagos, a compliance founder in Singapore, and a CLI-tooling founder in Brooklyn who all described the same pain point in different words during their 1:1 chats.
|
||||
|
||||
The model can.
|
||||
|
||||
**Step 1: Enrich every founder.**
|
||||
|
||||
The `/enrich-founder` skill: pull all sources, run enrichments, diarize, highlight what they SAY vs what they're ACTUALLY BUILDING. On the right, the deterministic calls: SQL to find stale profiles, GitHub stats, browser test on the demo URL, social signal pulls, CrustData for company intel.
|
||||
|
||||
Cron runs nightly at 2am. 6,000 profiles, every night, always fresh.
|
||||
|
||||
The diarization output catches things no keyword search would find:
|
||||
|
||||
```
|
||||
FOUNDER: Maria Santos
|
||||
COMPANY: Contrail (contrail.dev)
|
||||
SAYS: "Datadog for AI agents"
|
||||
ACTUALLY BUILDING: 80% of commits are in billing module.
|
||||
She's building a FinOps tool disguised as observability.
|
||||
```
|
||||
|
||||
"SAYS" vs "ACTUALLY BUILDING." That requires reading the GitHub commit history, the application, and the advisor transcript and holding all three in mind at once.
|
||||
|
||||
**Step 2: Match 6,000 founders. Make judgment calls.**
|
||||
|
||||
This is where skill-as-method-call really shines. Three invocations:
|
||||
|
||||
`/match-breakout`: 1,200 founders, cluster by sector affinity, 30 per room. Embed + deterministic assign.
|
||||
|
||||
`/match-lunch`: 600 founders, serendipity matching (cross-sector), 8 per table, no repeats. The LLM invents the themes, then assigns.
|
||||
|
||||
`/match-live`: whoever is in the zone, nearest-neighbor embedding, real-time at 200ms, 1:1 pairs, not already met.
|
||||
|
||||
Same skill. Three invocations. Three completely different matching strategies. Different parameters, different strategies, different group sizes. The skill describes the process. The arguments shape the output.
|
||||
|
||||
And the model's judgment calls: "Santos and Oram are both AI infra, but they're not competitors. Santos is cost attribution, Oram is orchestration. Put them in the same group." And: "Kim applied as 'developer tools' but his 1:1 transcript reveals he's building compliance automation for SOC2. Move him to FinTech/RegTech."
|
||||
|
||||
No embedding captures the Kim reclassification. No algorithm can do it. The model has to read the entire profile.
|
||||
|
||||
**Step 3: The self-learning loop.**
|
||||
|
||||
After the event, the `/improve` skill reads NPS surveys, diarizes the "OK" responses (not the bad ones, the mediocre ones), and extracts patterns. Then it proposes new rules and writes them back into the matching skills:
|
||||
|
||||
```
|
||||
When attendee says "AI infrastructure"
|
||||
but startup is 80%+ billing code:
|
||||
-> Classify as FinTech, not AI Infra.
|
||||
|
||||
When two attendees in same group
|
||||
already know each other:
|
||||
-> Penalize proximity.
|
||||
Prioritize novel introductions.
|
||||
```
|
||||
|
||||
These rules get written back into the skill file. Next run uses them automatically. The skill rewrites itself.
|
||||
|
||||
July event: 12% "OK" ratings. Next event: 4%. The skill file learned what "OK" actually meant.
|
||||
|
||||
Same pattern as every other domain: retrieve, read, diarize, count, synthesize. Then: survey, investigate, diarize, rewrite the skill. It transfers everywhere.
|
||||
|
||||
## OpenClaw: where the skills live
|
||||
|
||||
I want to tell you about one more harness. Not for coding. For everything else.
|
||||
|
||||
I run a personal AI agent on OpenClaw. It has a persona, knows who I am, and maintains a knowledge base of thousands of interconnected files. But the thing that makes it work is the exact same principle. Thin harness, fat skills.
|
||||
|
||||
I tweeted about this a few days ago:
|
||||
|
||||
> *You are not allowed to do one-off work. If I ask you to do something and it's the kind of thing that will need to happen again, you must: do it manually the first time on 3 to 10 items. Show me the output. If I approve, codify it into a skill file. If it should run automatically, put it on a cron.*
|
||||
|
||||
> *The test: if I have to ask you for something twice, you failed.*
|
||||
|
||||
That resonated: a thousand likes, twenty-five hundred bookmarks. People thought it was a prompt engineering trick. It's not. It's the same architecture.
|
||||
|
||||
Claude Code is the best harness for coding. OpenClaw is the best harness for everything else: email, calendar, meetings, people, research, alerts. Same principle. Thin harness. Fat skills. The skills are method calls. The parameters change. The process stays.
|
||||
|
||||
Every skill I write is a permanent upgrade. It never degrades. It never forgets. It runs at 3 AM while I sleep. And when the next model drops, every skill instantly gets better, because the judgment in the latent steps improves while the deterministic steps stay perfectly reliable.
|
||||
|
||||
That's how you get Yegge's 10x to 100x. Not a smarter model. Fat skills, thin harness, and the discipline to codify everything.
|
||||
|
||||
The system compounds. Build it once. It runs forever.
|
||||
|
||||
Five definitions. Three layers. One principle. Thin harness, fat skills.
|
||||
|
||||
---
|
||||
|
||||
## Agent Decision Guide: Skill or Code?
|
||||
|
||||
When building GBrain features, use this decision guide:
|
||||
|
||||
| Question | If YES | If NO |
|
||||
|----------|--------|-------|
|
||||
| Does the agent need to think, adapt, or ask questions? | **Skill** (recipe markdown) | Code |
|
||||
| Same input always produces same output? | **Code** (CLI command) | Skill |
|
||||
| Does it require judgment about the user's environment? | **Skill** | Code |
|
||||
| Is it a lookup, list, or status check? | **Code** | Probably skill |
|
||||
| Does it change behavior based on conversation context? | **Skill** | Code |
|
||||
|
||||
**GBrain examples:**
|
||||
- `gbrain integrations list` = **Code** (reads files, checks env vars, deterministic)
|
||||
- `gbrain integrations status` = **Code** (checks env vars + heartbeat, deterministic)
|
||||
- `gbrain integrations doctor` = **Code** (runs health checks, deterministic)
|
||||
- `gbrain integrations stats` = **Code** (aggregates JSONL, deterministic)
|
||||
- Recipe setup flow = **Skill** (asks for API keys, adapts to environment, validates)
|
||||
- Recipe changelog surfacing = **Skill** (agent describes changes conversationally)
|
||||
- Entity detection = **Skill** (reads message, decides what's important, creates pages)
|
||||
- Meeting ingestion = **Skill** (reads transcript, extracts entities, updates pages)
|
||||
|
||||
**The rule:** If it's a lookup table, it's code. If the agent needs to think, it's a skill.
|
||||
@@ -1,224 +0,0 @@
|
||||
# Running real-world eval benchmarks against your gbrain changes
|
||||
|
||||
Audience: gbrain maintainers and contributors. If you're touching retrieval
|
||||
(search, ranking, embeddings, intent classification, query expansion, source
|
||||
boost, hybrid fusion), this is the doc.
|
||||
|
||||
For the **NDJSON wire format** consumed by gbrain-evals, see
|
||||
[`eval-capture.md`](./eval-capture.md). This doc is the human dev loop
|
||||
that lives on top of that format.
|
||||
|
||||
## Prerequisite: turn on contributor mode
|
||||
|
||||
Capture is **off by default** for production users (privacy-positive — no
|
||||
surprise data accumulation). Contributors flip it on with one line:
|
||||
|
||||
```bash
|
||||
# In ~/.zshrc or ~/.bashrc:
|
||||
export GBRAIN_CONTRIBUTOR_MODE=1
|
||||
```
|
||||
|
||||
Verify:
|
||||
|
||||
```bash
|
||||
gbrain query "anything" >/dev/null
|
||||
psql $DATABASE_URL -c 'SELECT count(*) FROM eval_candidates' # should be > 0
|
||||
```
|
||||
|
||||
To override (force on/off regardless of env var), edit `~/.gbrain/config.json`:
|
||||
|
||||
```json
|
||||
{"eval": {"capture": true}} // force on
|
||||
{"eval": {"capture": false}} // force off
|
||||
```
|
||||
|
||||
Explicit config beats the env var both directions.
|
||||
|
||||
## The 4-command loop
|
||||
|
||||
```bash
|
||||
# ① Capture: writes to eval_candidates whenever CONTRIBUTOR_MODE is set.
|
||||
# Inspect what's been collected:
|
||||
gbrain doctor # surfaces capture failures
|
||||
psql $DATABASE_URL -c 'SELECT count(*) FROM eval_candidates'
|
||||
|
||||
# ② Snapshot: freeze a baseline before your code change.
|
||||
gbrain eval export --since 7d > baseline.ndjson
|
||||
|
||||
# ③ Code change: do whatever you want — tune RRF_K, swap embed model, edit
|
||||
# hybrid.ts, add a new boost source, change the intent classifier.
|
||||
|
||||
# ④ Replay: re-run every captured query against the current build.
|
||||
gbrain eval replay --against baseline.ndjson
|
||||
```
|
||||
|
||||
Output:
|
||||
|
||||
```
|
||||
Replaying 247 captured queries…
|
||||
...25/247
|
||||
...50/247
|
||||
...
|
||||
Replayed 247 of 247 captured queries (0 skipped, 0 errored)
|
||||
Mean Jaccard@k: 0.927
|
||||
Top-1 stability: 91.5%
|
||||
Mean latency Δ: +14ms (current vs captured)
|
||||
|
||||
Top 5 regression(s):
|
||||
jaccard=0.20 captured=12 current=3 "find every reference to widget-co"
|
||||
jaccard=0.43 captured=14 current=8 "show me everything tagged for review"
|
||||
jaccard=0.50 captured=8 current=4 "what did alice say about the spec"
|
||||
...
|
||||
```
|
||||
|
||||
Three numbers tell you whether the change is safe to land:
|
||||
|
||||
| Metric | What it means | Healthy range |
|
||||
|---|---|---|
|
||||
| **Mean Jaccard@k** | Average overlap between captured retrieved slugs and current run's slugs. 1.0 = identical sets. | ≥0.85 for "neutral" changes. <0.7 means major retrieval shift. |
|
||||
| **Top-1 stability** | Fraction of queries whose #1 result didn't change. | ≥85% for tuning passes. <70% means top-of-funnel broke. |
|
||||
| **Mean latency Δ** | Current minus captured. Positive = slower now. | Within ±50ms of captured. >2× anywhere = regression alarm. |
|
||||
|
||||
## What it actually does
|
||||
|
||||
`gbrain eval replay` reads your NDJSON snapshot and, for each row:
|
||||
|
||||
1. Re-executes the same op (`searchKeyword` for `tool_name='search'`,
|
||||
`hybridSearch` for `tool_name='query'`) with the captured `detail` and
|
||||
`expand_enabled` values threaded back in.
|
||||
2. Captures the current `retrieved_slugs` (deduped, in result order).
|
||||
3. Computes set-Jaccard between captured and current slug sets.
|
||||
4. Records top-1 match (was the #1 result the same slug?).
|
||||
5. Records latency delta vs captured `latency_ms`.
|
||||
|
||||
It does NOT compute MRR or nDCG — those need ground-truth relevance labels,
|
||||
not a baseline comparison. For metric-against-truth eval, use
|
||||
`gbrain eval --qrels <path>` (the legacy IR-eval path, still supported). The
|
||||
replay tool answers a different question: "did my code change move
|
||||
retrieval, and which queries did it move most?"
|
||||
|
||||
## Best-effort by design
|
||||
|
||||
Replay is not pure. Three things can drift between capture and replay:
|
||||
|
||||
1. **Brain state** — your brain probably has more pages now than when the
|
||||
snapshot was taken. Unless you explicitly seed a fixed corpus, mean
|
||||
Jaccard will drop simply because new pages are eligible.
|
||||
2. **Embedding source** — if you changed `OPENAI_API_KEY` between capture
|
||||
and replay (or the embedding model rotated), vector-path results drift
|
||||
even with identical code.
|
||||
3. **Capture cap** — captured `retrieved_slugs` is a deduped set; it doesn't
|
||||
preserve internal ranking metadata. Two tools can return the same slug
|
||||
set with different scores — Jaccard will say 1.0, but a downstream
|
||||
consumer that orders by score may behave differently.
|
||||
|
||||
The metrics are **regression alarms on real queries**, not a hash check.
|
||||
Pair them with manual inspection of the top regressions.
|
||||
|
||||
## Cost
|
||||
|
||||
Every `query` row in the snapshot embeds the query string via OpenAI to run
|
||||
the vector half of `hybridSearch`. Cost is identical to a normal `gbrain
|
||||
query` invocation — text-embedding-3-large at OpenAI list price, batched
|
||||
inside a single replay row.
|
||||
|
||||
If you're iterating locally and don't want to pay per change, use
|
||||
`--limit 50` to cap rows replayed. The 50 most recent rows are usually
|
||||
enough to catch direction; expand for the final pre-merge run.
|
||||
|
||||
```bash
|
||||
# Iteration mode — 50 most recent queries
|
||||
gbrain eval replay --against baseline.ndjson --limit 50
|
||||
|
||||
# Pre-merge — full snapshot
|
||||
gbrain eval replay --against baseline.ndjson --top-regressions 20
|
||||
```
|
||||
|
||||
## CI integration
|
||||
|
||||
```bash
|
||||
gbrain eval replay --against baseline.ndjson --json > replay.json
|
||||
jq -e '.summary.mean_jaccard >= 0.85' replay.json || exit 1
|
||||
jq -e '.summary.top1_stability_rate >= 0.85' replay.json || exit 1
|
||||
```
|
||||
|
||||
Stable JSON shape (schema_version: 1):
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"summary": {
|
||||
"rows_total": 247,
|
||||
"rows_replayed": 247,
|
||||
"rows_skipped": 0,
|
||||
"rows_errored": 0,
|
||||
"mean_jaccard": 0.927,
|
||||
"top1_stability_rate": 0.915,
|
||||
"mean_latency_delta_ms": 14,
|
||||
"rows_over_2x_latency": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`--verbose` adds a `results: [...]` array with one entry per replayed row
|
||||
(useful for piping into jq or a notebook for deeper analysis).
|
||||
|
||||
## When to run this
|
||||
|
||||
Before merging anything that touches:
|
||||
|
||||
- `src/core/search/hybrid.ts` (RRF, fusion, dedup, two-pass retrieval)
|
||||
- `src/core/search/source-boost.ts` / `sql-ranking.ts` (per-source ranking)
|
||||
- `src/core/search/intent.ts` (auto-detail classification)
|
||||
- `src/core/search/expansion.ts` (Haiku query expansion)
|
||||
- `src/core/search/dedup.ts` (cross-page result collapse)
|
||||
- `src/core/embedding.ts` or any embedding model swap
|
||||
- `src/core/operations.ts` `query` or `search` op handlers (capture surface)
|
||||
- `src/core/postgres-engine.ts` / `pglite-engine.ts` `searchKeyword` /
|
||||
`searchVector` SQL
|
||||
|
||||
Skip for: schema-only migrations, doc changes, tests-only PRs, CLI ergonomics
|
||||
that don't touch retrieval.
|
||||
|
||||
## Building your own corpus
|
||||
|
||||
If you don't have captured traffic yet (fresh install, can't dogfood for a
|
||||
week before merging), you can hand-author an NDJSON file:
|
||||
|
||||
```jsonl
|
||||
{"schema_version":1,"id":1,"tool_name":"query","query":"who is alice","retrieved_slugs":["people/alice","people/alice-bio"],"expand_enabled":false,"detail":null,"latency_ms":0,"remote":false}
|
||||
{"schema_version":1,"id":2,"tool_name":"search","query":"acme deal","retrieved_slugs":["deals/acme-seed","companies/acme"],"latency_ms":0,"remote":false}
|
||||
```
|
||||
|
||||
Then run `gbrain eval replay --against handcrafted.ndjson` to confirm the
|
||||
authoritative slugs come back. This is the seam between the BrainBench-Real
|
||||
pipeline (replay against live captures) and the BrainBench fixed-fixture
|
||||
pipeline (`gbrain eval --qrels` with the sibling
|
||||
[gbrain-evals](https://github.com/garrytan/gbrain-evals) corpus).
|
||||
|
||||
## Off-switch
|
||||
|
||||
Two ways to disable capture:
|
||||
|
||||
```bash
|
||||
unset GBRAIN_CONTRIBUTOR_MODE # easy: just unset the env var
|
||||
```
|
||||
|
||||
Or force off regardless of the env var via `~/.gbrain/config.json`:
|
||||
|
||||
```json
|
||||
{"eval": {"capture": false}}
|
||||
```
|
||||
|
||||
Existing `eval_candidates` rows stay until you `gbrain eval prune
|
||||
--older-than 0d` (or just drop the table).
|
||||
|
||||
## Failure modes
|
||||
|
||||
| What you see | What it means |
|
||||
|---|---|
|
||||
| `Mean Jaccard@k: 0.4`, top regressions all in one source dir | Source boost or hard-exclude regression on that prefix |
|
||||
| `Top-1 stability: 30%`, mean Jaccard still high | RRF tuning shifted the rank order without changing the set — re-tune `rrfK` |
|
||||
| `Mean latency Δ: +500ms`, jaccard high | Vector path got slower; check embedding API or HNSW probes |
|
||||
| `rows_errored > 0` | One or more queries threw. Inspect first 3 in human output, or `--json` to see all `error_message` fields |
|
||||
| Many `skipped: empty query` | Capture ran on rows where someone passed empty `query` — check why those were captured |
|
||||
@@ -1,160 +0,0 @@
|
||||
# Eval capture — NDJSON schema reference
|
||||
|
||||
**Status:** stable from v0.21.0. Schema versioning via `schema_version`
|
||||
on every row; additive changes increment the minor version; removals
|
||||
are breaking-schema-v2.
|
||||
|
||||
**Audience:** downstream consumers (primarily the sibling
|
||||
[gbrain-evals](https://github.com/garrytan/gbrain-evals) repo) that
|
||||
replay captured real-world queries as a BrainBench-Real fixture.
|
||||
|
||||
## The pipeline
|
||||
|
||||
```
|
||||
MCP / CLI / subagent tool-bridge caller
|
||||
│
|
||||
▼
|
||||
src/core/operations.ts — query + search op handlers
|
||||
│
|
||||
│ (hybridSearch or searchKeyword)
|
||||
│
|
||||
▼
|
||||
{results, meta: HybridSearchMeta} ┌── captureEvalCandidate
|
||||
│ │ (fire-and-forget)
|
||||
▼ │
|
||||
return to caller ▼
|
||||
scrubPii(query) ←── src/core/eval-capture-scrub.ts
|
||||
│
|
||||
▼
|
||||
buildEvalCandidateInput
|
||||
│
|
||||
▼
|
||||
engine.logEvalCandidate
|
||||
│
|
||||
┌──────────────┴──────────────┐
|
||||
│ success │ fail
|
||||
▼ ▼
|
||||
INSERT into eval_candidates engine.logEvalCaptureFailure
|
||||
(reason: db_down | rls_reject |
|
||||
check_violation |
|
||||
scrubber_exception | other)
|
||||
```
|
||||
|
||||
## `gbrain eval export` — the consumer contract
|
||||
|
||||
```sh
|
||||
gbrain eval export [--since DUR] [--limit N] [--tool query|search]
|
||||
```
|
||||
|
||||
Emits NDJSON to **stdout**. One JSON object per `\n`-terminated line.
|
||||
stderr receives progress heartbeats. Every line starts with
|
||||
`"schema_version": 1` so a forward-compat parser can fail loudly on
|
||||
schema v2 instead of silently misparsing.
|
||||
|
||||
Typical usage from gbrain-evals:
|
||||
|
||||
```sh
|
||||
# Snapshot the last week of real traffic for replay
|
||||
gbrain eval export --since 7d > brainbench-real.ndjson
|
||||
```
|
||||
|
||||
```sh
|
||||
# Stream through jq for ad-hoc analysis
|
||||
gbrain eval export --tool query | jq -c 'select(.latency_ms > 500)'
|
||||
```
|
||||
|
||||
## Row schema (v1)
|
||||
|
||||
Every exported row has this shape. Field order in JSON output is not
|
||||
guaranteed; consumers MUST key by name, not position.
|
||||
|
||||
| Field | Type | Notes |
|
||||
|---|---|---|
|
||||
| `schema_version` | number | Always `1` on v1 rows. Forward-compat gate. |
|
||||
| `id` | number | Autoincrement primary key. Stable across exports. |
|
||||
| `tool_name` | `"query"` \| `"search"` | Which MCP operation captured this row. |
|
||||
| `query` | string | **Already PII-scrubbed** by `scrubPii` unless `eval.scrub_pii: false`. Emails / phones / SSN / Luhn-verified credit cards / JWTs / bearer tokens replaced with `[REDACTED]`. Max length 50KB (CHECK-enforced). |
|
||||
| `retrieved_slugs` | string[] | Deduplicated slugs that came back in `SearchResult[]`. |
|
||||
| `retrieved_chunk_ids` | number[] | Every chunk id in result order (duplicates preserved — one per hit). |
|
||||
| `source_ids` | string[] | Distinct `sources.id` values across the result set (v0.18 multi-source). Empty for pre-v0.18 rows that lacked the column. |
|
||||
| `expand_enabled` | boolean \| null | Whether the caller **requested** Haiku expansion. `null` for `search` (no expansion concept). |
|
||||
| `detail` | `"low"` \| `"medium"` \| `"high"` \| null | Detail level the caller **requested**. `null` when omitted. |
|
||||
| `detail_resolved` | `"low"` \| `"medium"` \| `"high"` \| null | What `hybridSearch` **actually used** after auto-detect. `null` when neither caller nor heuristic classified. |
|
||||
| `vector_enabled` | boolean | True iff vector search actually ran. `false` when `OPENAI_API_KEY` was missing or the embed call failed. **Replay MUST respect this** — rows with `false` only exercised the keyword path. |
|
||||
| `expansion_applied` | boolean | True iff Haiku expansion actually produced variants (not just "was requested"). |
|
||||
| `latency_ms` | number | Wall-clock duration of the op handler (includes capture itself — negligible since it's fire-and-forget). |
|
||||
| `remote` | boolean | `true` for MCP callers (untrusted), `false` for local CLI. Partitions "real agent traffic" from "operator probing." |
|
||||
| `job_id` | number \| null | `OperationContext.jobId` when the caller was a subagent tool-bridge. Null for MCP + CLI. |
|
||||
| `subagent_id` | number \| null | `OperationContext.subagentId` for subagent-owned runs. |
|
||||
| `created_at` | string (ISO 8601) | UTC timestamp of insert. |
|
||||
|
||||
## Ordering + determinism
|
||||
|
||||
`listEvalCandidates` orders by `created_at DESC, id DESC`. Same-
|
||||
millisecond inserts tie on `created_at`; `id DESC` is the stable
|
||||
tiebreaker. Replay tools can consume rows in order and assume:
|
||||
- no duplicate rows across calls with non-overlapping `--since` windows
|
||||
- no missed rows across calls that chain `--since` windows (window end
|
||||
of run 1 is the strict upper bound, not a soft cursor)
|
||||
|
||||
## Schema versioning promise
|
||||
|
||||
- **v1 (shipped v0.21.0)** — this document. All fields listed above.
|
||||
- **Additive changes** increment gbrain minor version (v0.25.0, v0.23.0
|
||||
…) and ship with new optional fields. Consumers keyed on known fields
|
||||
ignore unknown keys and keep working.
|
||||
- **Breaking changes** (rename, type change, removal) increment
|
||||
`schema_version` to 2. Consumers MUST branch on `schema_version` to
|
||||
stay compatible.
|
||||
|
||||
## `eval_capture_failures` — companion audit table
|
||||
|
||||
Not exported by `gbrain eval export`. Surfaced via `gbrain doctor`:
|
||||
|
||||
```sh
|
||||
gbrain doctor # warns when failures in last 24h > 0
|
||||
```
|
||||
|
||||
Reason enum (stable): `db_down` | `rls_reject` | `check_violation` |
|
||||
`scrubber_exception` | `other`. Cross-process visibility is the whole
|
||||
point — `gbrain doctor` runs in its own process and reads the table
|
||||
directly, so in-process counters wouldn't work.
|
||||
|
||||
## Config + CONTRIBUTOR_MODE
|
||||
|
||||
Capture is **off by default** as of v0.25.0 (was on for everyone in
|
||||
earlier drafts). Two paths to turn it on:
|
||||
|
||||
**Path A — env var (contributor opt-in, the common case):**
|
||||
|
||||
```bash
|
||||
export GBRAIN_CONTRIBUTOR_MODE=1 # in ~/.zshrc or ~/.bashrc
|
||||
```
|
||||
|
||||
**Path B — explicit config (`~/.gbrain/config.json`, file-plane only):**
|
||||
|
||||
```json
|
||||
{
|
||||
"engine": "postgres",
|
||||
"database_url": "...",
|
||||
"eval": {
|
||||
"capture": true,
|
||||
"scrub_pii": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Resolution order (most explicit wins):
|
||||
|
||||
1. `eval.capture: true` in config → on
|
||||
2. `eval.capture: false` in config → off (overrides CONTRIBUTOR_MODE=1)
|
||||
3. `GBRAIN_CONTRIBUTOR_MODE === '1'` → on
|
||||
4. otherwise → off
|
||||
|
||||
`scrub_pii` defaults to `true` independent of capture. Set
|
||||
`eval.scrub_pii: false` to preserve raw query text (only if you control
|
||||
the brain's distribution).
|
||||
|
||||
`gbrain config set eval.capture false` does **not** work — that
|
||||
command writes the DB-plane config, and the MCP server reads the
|
||||
file-plane. Edit the JSON directly or use the env var.
|
||||
@@ -1,129 +0,0 @@
|
||||
# The Brain-Agent Loop
|
||||
|
||||
## Goal
|
||||
|
||||
Every conversation makes the brain smarter. Every brain lookup makes responses
|
||||
better. The loop compounds daily.
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: the agent answers from stale context. You discuss a deal on Monday,
|
||||
and by Friday the agent has forgotten. Every conversation starts from zero.
|
||||
|
||||
With this: six months in, the agent knows more about your world than you can hold
|
||||
in working memory. It never forgets. It never stops indexing.
|
||||
|
||||
## The Loop
|
||||
|
||||
```
|
||||
Signal arrives (message, meeting, email, tweet, link)
|
||||
│
|
||||
▼
|
||||
DETECT entities (people, companies, concepts, original thinking)
|
||||
│ → spawn sub-agent (see entity-detection.md)
|
||||
│
|
||||
▼
|
||||
READ: check brain FIRST (before responding)
|
||||
│ → gbrain search "{entity name}"
|
||||
│ → gbrain get {slug} (if you know it)
|
||||
│ → gbrain query "what do we know about {topic}"
|
||||
│
|
||||
▼
|
||||
RESPOND with brain context (every answer is better with context)
|
||||
│
|
||||
▼
|
||||
WRITE: update brain pages (new info → compiled truth + timeline)
|
||||
│ → gbrain put {slug} (update page)
|
||||
│ → add_timeline_entry (append to timeline)
|
||||
│ → add_link (cross-reference to other entities)
|
||||
│
|
||||
▼
|
||||
SYNC: gbrain indexes changes
|
||||
│ → gbrain sync --no-pull --no-embed
|
||||
│
|
||||
▼
|
||||
(next signal arrives — agent is now smarter)
|
||||
```
|
||||
|
||||
## Implementation
|
||||
|
||||
### On Every Inbound Message
|
||||
|
||||
```
|
||||
on_message(text):
|
||||
// 1. DETECT (async, don't block)
|
||||
spawn_entity_detector(text)
|
||||
|
||||
// 2. READ (before composing response)
|
||||
entities = extract_entity_names(text) // quick regex/NER
|
||||
context = []
|
||||
for name in entities:
|
||||
results = gbrain_search(name)
|
||||
if results:
|
||||
page = gbrain_get(results[0].slug)
|
||||
context.append(page.compiled_truth)
|
||||
|
||||
// 3. RESPOND (with brain context injected)
|
||||
response = compose_response(text, context)
|
||||
|
||||
// 4. WRITE (after responding, if new info emerged)
|
||||
if response_contains_new_info(response):
|
||||
for entity in mentioned_entities:
|
||||
gbrain_add_timeline_entry(entity.slug, {
|
||||
date: today,
|
||||
summary: "Discussed {topic}",
|
||||
source: "[Source: User, conversation, {date}]"
|
||||
})
|
||||
|
||||
// 5. SYNC
|
||||
gbrain_sync()
|
||||
```
|
||||
|
||||
### The Two Invariants
|
||||
|
||||
1. **Every READ improves the response.** If you answered a question about a
|
||||
person without checking their brain page first, you gave a worse answer
|
||||
than you could have. The brain almost always has something. External APIs
|
||||
fill gaps, they don't start from scratch.
|
||||
|
||||
2. **Every WRITE improves future reads.** If a meeting transcript mentioned
|
||||
new information about a company and you didn't update the company page,
|
||||
you created a gap that will bite you later.
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Read BEFORE responding, not after.** The temptation is to respond first
|
||||
and update the brain later. But the brain context makes the response better.
|
||||
Read first.
|
||||
|
||||
2. **Don't skip the write step.** "I'll update the brain later" means never.
|
||||
Write immediately after the conversation, while the context is fresh.
|
||||
|
||||
3. **Sync after every write batch.** Without sync, the brain search index is
|
||||
stale. The next query won't find what you just wrote.
|
||||
|
||||
4. **External APIs are fallback, not primary.** `gbrain search` before
|
||||
Brave Search. `gbrain get` before Crustdata. The brain has relationship
|
||||
history, your own assessments, meeting transcripts, cross-references.
|
||||
No external API can provide that.
|
||||
|
||||
## How to Verify It Works
|
||||
|
||||
1. **Mention a person the brain knows.** Ask "what do we know about {name}?"
|
||||
The agent should search the brain and return compiled truth, not hallucinate
|
||||
or do a web search.
|
||||
|
||||
2. **Discuss something new about a known entity.** Say "I heard Acme Corp
|
||||
just raised Series B." After the conversation, check: does Acme Corp's
|
||||
brain page have a new timeline entry?
|
||||
|
||||
3. **Ask about the same person a day later.** The agent should immediately
|
||||
pull brain context without you asking. If it doesn't reference the brain
|
||||
page, the loop isn't running.
|
||||
|
||||
4. **Check the sync.** After a conversation, run `gbrain search "{topic}"`
|
||||
from the CLI. The new information should be searchable.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md). See also: [Entity Detection](entity-detection.md), [Brain-First Lookup](brain-first-lookup.md)*
|
||||
@@ -1,85 +0,0 @@
|
||||
# Brain-First Lookup Protocol
|
||||
|
||||
## Goal
|
||||
|
||||
Check the brain before calling ANY external API. The brain almost always has
|
||||
something. External APIs fill gaps, they don't start from scratch.
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: the agent calls Brave Search for someone you've had 12 meetings with.
|
||||
You get a LinkedIn summary instead of your relationship history.
|
||||
|
||||
With this: the agent pulls your compiled truth, recent timeline entries, and
|
||||
shared context before doing anything else. External APIs only fill gaps.
|
||||
|
||||
## Implementation
|
||||
|
||||
```
|
||||
lookup(name_or_topic):
|
||||
// STEP 1: Keyword search (fast, works day one, no embeddings needed)
|
||||
results = gbrain search "{name_or_topic}"
|
||||
if results.length > 0:
|
||||
page = gbrain get {results[0].slug}
|
||||
return page // done, brain had it
|
||||
|
||||
// STEP 2: Hybrid search (needs embeddings, finds semantic matches)
|
||||
results = gbrain query "what do we know about {name_or_topic}"
|
||||
if results.length > 0:
|
||||
page = gbrain get {results[0].slug}
|
||||
return page
|
||||
|
||||
// STEP 3: Direct slug (if you know or can guess the slug)
|
||||
page = gbrain get "people/{slugify(name_or_topic)}"
|
||||
if page: return page
|
||||
|
||||
// STEP 4: External API (FALLBACK ONLY)
|
||||
// Only reach here if brain has nothing
|
||||
return external_search(name_or_topic)
|
||||
```
|
||||
|
||||
**This is mandatory.** An agent that calls Brave Search before checking the brain
|
||||
is wasting money and giving worse answers.
|
||||
|
||||
## Why Brain First
|
||||
|
||||
The brain has context no external API can provide:
|
||||
- Relationship history (how you know them, what you discussed)
|
||||
- Your own assessments (what you think of them, not their LinkedIn bio)
|
||||
- Meeting transcripts (what was said, what was decided)
|
||||
- Cross-references (who they know, what companies they're connected to)
|
||||
- Timeline (what changed recently, what's trending)
|
||||
|
||||
A LinkedIn scrape gives you their job title. The brain gives you: "co-founded
|
||||
Brex, you had coffee with him 3 times, last discussed the payments infrastructure
|
||||
thesis, he's interested in your take on AI agents."
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Try keyword first, then hybrid.** Keyword search works without embeddings
|
||||
(day one). Hybrid search needs embeddings but finds semantic matches. Try
|
||||
both in sequence.
|
||||
|
||||
2. **Fuzzy slug matching.** `gbrain get` supports fuzzy matching. If the exact
|
||||
slug doesn't exist, it suggests alternatives. Use this for name variants
|
||||
("Pedro" → "pedro-franceschi").
|
||||
|
||||
3. **Don't skip for "simple" questions.** Even "what's Acme Corp's address?"
|
||||
should check the brain first. The brain might have it, and the lookup adds
|
||||
no latency (< 100ms for keyword search).
|
||||
|
||||
4. **Load compiled truth + recent timeline.** The compiled truth gives you the
|
||||
state of play in 30 seconds. The timeline gives you what changed recently.
|
||||
Both together = full context.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. Ask about someone in the brain. Verify the agent searched the brain FIRST
|
||||
(check tool call order in the response).
|
||||
2. Ask about someone NOT in the brain. Verify the agent searched the brain,
|
||||
found nothing, THEN fell back to external search.
|
||||
3. Ask the same question twice. Second time should be instant (brain has it).
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md). See also: [Brain-Agent Loop](brain-agent-loop.md), [Search Modes](search-modes.md)*
|
||||
@@ -1,75 +0,0 @@
|
||||
# Brain vs Memory vs Session
|
||||
|
||||
## Goal
|
||||
Know what goes in GBrain, what goes in agent memory, and what stays in session context -- so every piece of information lands in the right layer.
|
||||
|
||||
## What the User Gets
|
||||
Without this: people dossiers get stored in agent memory (lost on agent reset), user preferences get stored in GBrain (cluttering knowledge pages), and the agent re-asks questions it already knows the answer to. With this: world knowledge persists in the brain, operational state persists in agent memory, and the agent never puts information in the wrong layer.
|
||||
|
||||
## Implementation
|
||||
|
||||
```
|
||||
on new_information(info):
|
||||
# Three layers, three purposes -- route to the right one
|
||||
|
||||
if info.is_about_the_world:
|
||||
# GBRAIN: people, companies, deals, meetings, concepts, ideas
|
||||
# This is world knowledge -- facts about entities external to the agent
|
||||
gbrain put <slug> --content "..."
|
||||
# Examples:
|
||||
# "Pedro is CEO of Brex" -> gbrain (person page)
|
||||
# "Brex raised Series D at $12B" -> gbrain (company page)
|
||||
# "Tuesday's meeting covered Q2" -> gbrain (meeting page)
|
||||
# "The meatsuit maintenance tax" -> gbrain (originals page)
|
||||
|
||||
elif info.is_about_operations:
|
||||
# AGENT MEMORY: preferences, decisions, tool config, session continuity
|
||||
# This is how the agent operates -- not facts about the world
|
||||
memory_write(info)
|
||||
# Examples:
|
||||
# "User prefers concise formatting" -> agent memory
|
||||
# "Deploy to staging before prod" -> agent memory
|
||||
# "Use dark mode in code blocks" -> agent memory
|
||||
# "API key for Crustdata goes in .env" -> agent memory
|
||||
|
||||
elif info.is_current_conversation:
|
||||
# SESSION CONTEXT: what was just said, current task, immediate state
|
||||
# This is automatic -- already in the conversation window
|
||||
# No storage action needed
|
||||
# Examples:
|
||||
# "We were just discussing the board deck" -> session
|
||||
# "You asked me to review this PR" -> session
|
||||
# "The file I just shared" -> session
|
||||
|
||||
# Lookup routing:
|
||||
on user_asks(question):
|
||||
if question.about_person or question.about_company or question.about_meeting:
|
||||
gbrain search "{entity}" # -> world knowledge
|
||||
gbrain get <slug>
|
||||
|
||||
elif question.about_preference or question.about_how_to_operate:
|
||||
memory_search("{topic}") # -> operational state
|
||||
|
||||
elif question.about_current_context:
|
||||
# Already in session -- just reference conversation history
|
||||
pass
|
||||
```
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Don't store people in agent memory.** "Pedro prefers email over Slack" feels like a preference, but it's a fact about Pedro -- it goes in GBrain on Pedro's page. Agent memory is for the agent's own operational state, not facts about people in the world.
|
||||
2. **Don't store user preferences in GBrain.** "User likes bullet points over paragraphs" is about how the agent should behave, not about the world. It goes in agent memory. GBrain pages are for entities, not for agent configuration.
|
||||
3. **Synthesis of external ideas goes in GBrain.** "User's take on Peter Thiel's zero-to-one framework" is the user's original thinking -- it goes in GBrain under originals/, not in agent memory.
|
||||
4. **Agent memory doesn't survive agent resets on some platforms.** Critical world knowledge MUST be in GBrain, which is durable. If the agent loses memory, the brain still has everything.
|
||||
5. **When in doubt, ask: is this about the world or about how to operate?** World -> GBrain. Operations -> agent memory. Current conversation -> session.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. Ask the agent "Who is Pedro?" -- confirm it runs `gbrain search` or `gbrain get`, not `memory_search`. Person lookup should hit GBrain.
|
||||
2. Ask the agent "How should I format responses?" -- confirm it checks agent memory, not GBrain. Preferences are operational state.
|
||||
3. Check that no person or company pages exist in agent memory storage. Run `memory_search "person"` -- it should return preferences, not dossiers.
|
||||
4. Check that GBrain doesn't contain pages about agent behavior. Run `gbrain search "user prefers"` -- it should return nothing (preferences belong in agent memory).
|
||||
5. After an agent reset, confirm GBrain knowledge is still accessible. Run `gbrain get <any_slug>` -- world knowledge should survive the reset.
|
||||
|
||||
---
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,137 +0,0 @@
|
||||
# Compiled Truth + Timeline Pattern
|
||||
|
||||
## Goal
|
||||
|
||||
Every brain page has two zones: compiled truth (current synthesis, rewritten as
|
||||
evidence changes) and timeline (append-only evidence trail, never edited).
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: brain pages are append-only logs. To understand a person, you read
|
||||
200 timeline entries. The answer is buried in entry #147.
|
||||
|
||||
With this: the compiled truth gives you the state of play in 30 seconds. The
|
||||
timeline is the proof. Six months of entries compress into a one-paragraph
|
||||
assessment that's always current.
|
||||
|
||||
## Implementation
|
||||
|
||||
### Page Structure
|
||||
|
||||
```markdown
|
||||
---
|
||||
type: person
|
||||
title: Sarah Chen
|
||||
tags: [engineering, acme-corp]
|
||||
---
|
||||
|
||||
## Executive Summary
|
||||
One paragraph. How you know them, why they matter.
|
||||
|
||||
## State
|
||||
VP Engineering at Acme Corp. Managing 45-person team. Reports to CEO.
|
||||
|
||||
## What They Believe
|
||||
Strong opinions on test coverage. "Ship it when the tests pass, not before."
|
||||
|
||||
## What They're Building
|
||||
Leading the API migration from REST to GraphQL. Target: Q3 completion.
|
||||
|
||||
## Assessment
|
||||
Sharp technical leader. Under-appreciated internally. Watch for signs of burnout.
|
||||
|
||||
## Trajectory
|
||||
Ascending. Likely CTO track if the migration succeeds.
|
||||
|
||||
## Relationship
|
||||
Met through Pedro. Had coffee 3x. Last: discussed API architecture thesis.
|
||||
|
||||
## Contact
|
||||
sarah@acmecorp.com | @sarahchen | linkedin.com/in/sarahchen
|
||||
|
||||
---
|
||||
|
||||
## Timeline
|
||||
|
||||
- **2026-04-07** | Met at team sync. Discussed API migration timeline.
|
||||
Seemed energized about GraphQL pivot.
|
||||
[Source: Meeting notes, 2026-04-07 2:00 PM PT]
|
||||
- **2026-04-03** | Mentioned in email re Q2 planning. Taking lead on ops.
|
||||
[Source: Gmail, sarah@acmecorp.com, 2026-04-03 10:30 AM PT]
|
||||
- **2026-03-15** | First meeting. Intro from Pedro. Strong technical background.
|
||||
[Source: User, direct conversation, 2026-03-15 3:00 PM PT]
|
||||
```
|
||||
|
||||
### Updating a Page
|
||||
|
||||
```
|
||||
update_brain_page(slug, new_info, source):
|
||||
page = gbrain get {slug}
|
||||
|
||||
// TIMELINE: always APPEND (never edit existing entries)
|
||||
gbrain add_timeline_entry {slug} {
|
||||
date: today,
|
||||
summary: new_info.summary,
|
||||
detail: new_info.detail,
|
||||
source: format_source(source) // [Source: who, channel, date time tz]
|
||||
}
|
||||
|
||||
// COMPILED TRUTH: REWRITE (not append)
|
||||
// Read the existing compiled truth
|
||||
// Integrate new information
|
||||
// Write the updated synthesis
|
||||
updated_truth = rewrite_compiled_truth(page.compiled_truth, new_info)
|
||||
gbrain put {slug} {
|
||||
compiled_truth: updated_truth,
|
||||
// timeline is NOT passed — it's managed by add_timeline_entry
|
||||
}
|
||||
```
|
||||
|
||||
### The Rules
|
||||
|
||||
| Zone | Action | Explanation |
|
||||
|------|--------|-------------|
|
||||
| Compiled truth | **REWRITE** | Current synthesis. Changes when evidence changes. |
|
||||
| Timeline | **APPEND** | Evidence trail. Never edited, only added to. |
|
||||
|
||||
**Every compiled truth claim must trace to timeline entries.** If the Assessment
|
||||
says "under-appreciated internally," there should be timeline entries that
|
||||
support that claim.
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **REWRITE means rewrite, not append.** Don't add a new paragraph to compiled
|
||||
truth. Rewrite the entire section with the new information integrated. Old
|
||||
assessments that are no longer accurate should be updated, not kept alongside
|
||||
contradictory new ones.
|
||||
|
||||
2. **Timeline entries are immutable.** Never edit a timeline entry. If information
|
||||
turns out to be wrong, add a NEW entry correcting it:
|
||||
`- 2026-04-10 | Correction: Sarah is VP Eng, not CTO. Previous entry was wrong.`
|
||||
|
||||
3. **GBrain search weights compiled truth higher.** `gbrain query` returns compiled
|
||||
truth chunks with higher relevance than timeline chunks. This means the freshest
|
||||
synthesis surfaces first in search results.
|
||||
|
||||
4. **The --- separator matters.** GBrain uses the first standalone `---` after
|
||||
frontmatter to split compiled_truth from timeline. Everything above is compiled
|
||||
truth, everything below is timeline.
|
||||
|
||||
5. **Don't skip the Assessment section.** The assessment is the value. "Strong
|
||||
technical leader" is something no API can provide. It's YOUR read on this
|
||||
person. That's what makes the brain page better than LinkedIn.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Update a person page.** Add new meeting info. Check: compiled truth was
|
||||
REWRITTEN (not appended), timeline has new entry at the top.
|
||||
2. **Search for the person.** `gbrain query "Sarah Chen"`. The compiled truth
|
||||
(current synthesis) should appear first, not a random timeline entry.
|
||||
3. **Check traceability.** Every claim in compiled truth should have a
|
||||
corresponding timeline entry. Read both sections and verify.
|
||||
4. **Check immutability.** After update, old timeline entries should be unchanged.
|
||||
Dates, sources, and content should match the originals exactly.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md). See also: [Source Attribution](source-attribution.md), [Entity Detection](entity-detection.md)*
|
||||
@@ -1,136 +0,0 @@
|
||||
# Content and Media Ingestion
|
||||
|
||||
## Goal
|
||||
YouTube videos, social media, PDFs, and documents become searchable brain pages with the agent's own analysis and full cross-references to every entity mentioned.
|
||||
|
||||
## What the User Gets
|
||||
Without this: media links are bookmarks that decay -- you remember watching a video but can't find what was said, who said it, or why it mattered. With this: every piece of media is a permanent brain page with the agent's analysis layered on top, every mentioned entity gets a back-link, and the full content is searchable forever.
|
||||
|
||||
## Implementation
|
||||
|
||||
```
|
||||
on user_shares_media(url_or_file):
|
||||
|
||||
# PATTERN 1: YouTube Video Ingestion
|
||||
if media.type == "youtube":
|
||||
# Step 1: Get FULL transcript with speaker diarization
|
||||
# WHO said WHAT -- not just a wall of text
|
||||
# Use Diarize.io or equivalent service
|
||||
transcript = diarize(video_url) # speaker-attributed transcript
|
||||
# NEVER use YouTube's auto-generated summary or AI summary
|
||||
|
||||
# Step 2: Agent writes OWN analysis (this is the value)
|
||||
# NOT a summary. NOT regurgitation. The agent's TAKE:
|
||||
# - What matters and why (given the user's worldview)
|
||||
# - Key quotes attributed to specific speakers
|
||||
# - Connections to existing brain pages
|
||||
# - Implications and follow-up angles
|
||||
analysis = agent_analyze(transcript, user_context)
|
||||
|
||||
# Step 3: Create brain page
|
||||
slug = f"media/youtube/{video_slug}"
|
||||
gbrain put <slug> --content """
|
||||
# {title}
|
||||
**Channel:** {channel} | **Date:** {date} | **Link:** {url}
|
||||
|
||||
## Analysis
|
||||
{agent_analysis}
|
||||
|
||||
## Key Quotes
|
||||
- **{Speaker}** ({timestamp}): "{quote}" -- {why_it_matters}
|
||||
|
||||
---
|
||||
## Full Transcript
|
||||
{diarized_transcript}
|
||||
"""
|
||||
|
||||
# Step 4: Extract and cross-reference entities
|
||||
for person in transcript.mentioned_people:
|
||||
gbrain add_link <slug> <person_slug>
|
||||
gbrain add_link <person_slug> <slug>
|
||||
gbrain add_timeline_entry <person_slug> \
|
||||
--entry "Discussed in {video_title}: {what_was_said}" \
|
||||
--source "YouTube: {url}"
|
||||
|
||||
# PATTERN 2: Social Media Bundles
|
||||
elif media.type == "tweet" or media.type == "social":
|
||||
# Don't just save a tweet -- reconstruct FULL context
|
||||
bundle = {
|
||||
"original": fetch_tweet(url),
|
||||
"thread": reconstruct_thread(url), # quoted tweets, replies
|
||||
"linked_articles": fetch_linked_urls(), # fetch and summarize
|
||||
"engagement": get_engagement_data(), # what resonated
|
||||
}
|
||||
|
||||
slug = f"media/social/{platform}-{author}-{date}"
|
||||
gbrain put <slug> --content """
|
||||
# {author}: {topic}
|
||||
{agent_analysis_of_full_bundle}
|
||||
|
||||
## Thread
|
||||
{reconstructed_thread}
|
||||
|
||||
## Linked Articles
|
||||
{article_summaries}
|
||||
|
||||
---
|
||||
## Raw
|
||||
{original_tweet_text}
|
||||
"""
|
||||
|
||||
# Extract entities and cross-reference
|
||||
for entity in bundle.mentioned_entities:
|
||||
gbrain add_link <slug> <entity_slug>
|
||||
gbrain add_link <entity_slug> <slug>
|
||||
|
||||
# PATTERN 3: PDFs and Documents
|
||||
elif media.type == "pdf" or media.type == "document":
|
||||
# OCR if needed (scanned PDFs)
|
||||
content = ocr_if_needed(file) or extract_text(file)
|
||||
|
||||
# For books and long-form:
|
||||
slug = f"sources/{document_slug}"
|
||||
gbrain put <slug> --content """
|
||||
# {title}
|
||||
**Author:** {author} | **Date:** {date}
|
||||
|
||||
## Chapter Summaries
|
||||
{per_chapter_summary}
|
||||
|
||||
## Key Quotes
|
||||
- p.{page}: "{quote}" -- {why_it_matters}
|
||||
|
||||
## Cross-References
|
||||
{links_to_brain_pages_for_people_and_concepts}
|
||||
|
||||
---
|
||||
## Source
|
||||
{full_text_or_key_sections}
|
||||
"""
|
||||
|
||||
for entity in document.mentioned_entities:
|
||||
gbrain add_link <slug> <entity_slug>
|
||||
gbrain add_link <entity_slug> <slug>
|
||||
|
||||
# Always sync after ingestion
|
||||
gbrain sync
|
||||
```
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Always FULL transcript, never AI summary.** YouTube's auto-summary and AI-generated summaries lose the texture: who said what, exact phrasing, tone, what was left unsaid. The full diarized transcript is the evidence base. The agent's analysis goes above it.
|
||||
2. **The agent's OWN analysis is the value, not regurgitation.** "The video discussed AI safety" is worthless. "Dario made a specific claim about compute scaling that contradicts what Ilya said in the NeurIPS talk -- see media/youtube/ilya-neurips-2025" is useful. The analysis connects the new media to the existing brain.
|
||||
3. **Social media is a bundle, not a single tweet.** A tweet without its thread, quoted tweets, linked articles, and engagement context is a fragment. Reconstruct the full context before creating the brain page.
|
||||
4. **Cross-references make media pages alive.** A YouTube page without back-links to the people and companies mentioned is a dead archive. Every mentioned entity gets a link and a timeline entry.
|
||||
5. **Over time, `media/` becomes a searchable archive.** Every video, podcast, talk, interview, article, and tweet the user has consumed, with the agent's commentary layered on top. This is the memex at full power.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. Ingest a YouTube video. Run `gbrain get media/youtube/{slug}`. Confirm the page has: the agent's analysis (not just a summary), key quotes with speaker attribution, and the full diarized transcript.
|
||||
2. Run `gbrain get_links media/youtube/{slug}`. Confirm back-links exist to brain pages for every person and company mentioned in the video.
|
||||
3. Pick a person mentioned in the video. Run `gbrain get <person_slug>`. Confirm their timeline has a new entry referencing the video with specific context.
|
||||
4. Ingest a tweet. Confirm the brain page includes the thread context, linked article summaries, and entity cross-references -- not just the tweet text.
|
||||
5. Run `gbrain search "{topic_from_video}"`. Confirm the media page appears in search results (verifies the content is indexed and searchable).
|
||||
|
||||
---
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,193 +0,0 @@
|
||||
# Reference Cron Schedule
|
||||
|
||||
## Goal
|
||||
|
||||
A production brain runs 20+ recurring jobs that keep it alive, current, and
|
||||
compounding. This guide shows the schedule, the patterns, and how to set it up.
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: the brain only updates when you manually ingest data. Pages go
|
||||
stale, entities are thin, citations break, and the agent answers from old context.
|
||||
|
||||
With this: the brain maintains itself. Email, social, calendar, and meetings
|
||||
flow in automatically. Thin pages get enriched overnight. Broken citations get
|
||||
fixed. You wake up and the brain is smarter than when you went to sleep.
|
||||
|
||||
## The Schedule
|
||||
|
||||
| Frequency | Job | Brain Interaction | Recipe |
|
||||
|-----------|-----|-------------------|--------|
|
||||
| Every 30 min | Email monitoring | Search sender, update people pages | [email-to-brain](../../recipes/email-to-brain.md) |
|
||||
| Every 30 min | X/Twitter collection | Create/update media pages, entity extraction | [x-to-brain](../../recipes/x-to-brain.md) |
|
||||
| 3x/day (weekdays) | Meeting sync | Full ingestion + attendee propagation | [meeting-sync](../../recipes/meeting-sync.md) |
|
||||
| Weekly | Calendar sync | Daily files + attendee enrichment | [calendar-to-brain](../../recipes/calendar-to-brain.md) |
|
||||
| Daily AM | Morning briefing | Search calendar attendees, deal status, active threads | [briefing skill](../../skills/briefing/SKILL.md) |
|
||||
| Weekly | Brain maintenance | `gbrain doctor`, embed stale, orphan detection | [maintain skill](../../skills/maintain/SKILL.md) |
|
||||
| Nightly | Dream cycle | Entity sweep, enrich thin spots, fix citations | See below |
|
||||
|
||||
## Implementation: Setting Up Cron Jobs
|
||||
|
||||
```bash
|
||||
# Email collector — every 30 minutes
|
||||
*/30 * * * * cd /path/to/email-collector && node email-collector.mjs collect && node email-collector.mjs digest
|
||||
|
||||
# X/Twitter collector — every 30 minutes
|
||||
*/30 * * * * cd /path/to/x-collector && node x-collector.mjs collect >> /tmp/x-collector.log 2>&1
|
||||
|
||||
# Meeting sync — 10 AM, 4 PM, 9 PM on weekdays
|
||||
0 10,16,21 * * 1-5 cd /path/to/meeting-sync && node meeting-sync.mjs >> /tmp/meeting-sync.log 2>&1
|
||||
|
||||
# Calendar sync — Sundays at 10 AM
|
||||
0 10 * * 0 cd /path/to/calendar-sync && node calendar-sync.mjs --start $(date -v-7d +%Y-%m-%d) --end $(date +%Y-%m-%d)
|
||||
|
||||
# Brain health — weekly Mondays at 6 AM
|
||||
0 6 * * 1 gbrain doctor --json >> /tmp/gbrain-health.log 2>&1 && gbrain embed --stale
|
||||
|
||||
# Dream cycle — nightly at 2 AM
|
||||
0 2 * * * /path/to/dream-cycle.sh
|
||||
```
|
||||
|
||||
### Quiet Hours Gate (MANDATORY)
|
||||
|
||||
Every cron job that sends notifications MUST check quiet hours first.
|
||||
See [Quiet Hours](quiet-hours.md) for the full pattern.
|
||||
|
||||
```bash
|
||||
# In every cron script:
|
||||
if ! bash scripts/quiet-hours-gate.sh; then
|
||||
mkdir -p /tmp/cron-held
|
||||
echo "$OUTPUT" > /tmp/cron-held/$(basename "$0" .sh).md
|
||||
exit 0
|
||||
fi
|
||||
# Not quiet hours — send normally
|
||||
```
|
||||
|
||||
### Travel-Aware Timezone Handling
|
||||
|
||||
The agent reads your calendar for flights, hotels, and out-of-office blocks to
|
||||
infer your current location and timezone. All times shown in YOUR local timezone.
|
||||
|
||||
```
|
||||
// Example: user flew to Tokyo
|
||||
// 2 PM Pacific = 3 AM Tokyo = quiet hours
|
||||
// Hold the notification, fold into morning briefing
|
||||
|
||||
get_user_timezone():
|
||||
calendar = gbrain search "flight" --type calendar --recent 7d
|
||||
if recent_flight:
|
||||
return infer_timezone(flight.destination)
|
||||
return config.default_timezone // fallback: US/Pacific
|
||||
```
|
||||
|
||||
When you travel: cron jobs that would fire during your waking hours at home but
|
||||
hit your sleeping hours at the destination get held and folded into the next
|
||||
morning briefing. Zero config change needed.
|
||||
|
||||
## The Dream Cycle
|
||||
|
||||
The most important cron job. Runs while you sleep.
|
||||
|
||||
### What It Does
|
||||
|
||||
```
|
||||
dream_cycle():
|
||||
// Phase 1: Entity Sweep
|
||||
conversations = get_todays_conversations()
|
||||
for message in conversations:
|
||||
entities = detect_entities(message)
|
||||
for entity in entities:
|
||||
page = gbrain search "{entity.name}"
|
||||
if not page:
|
||||
create_page(entity) // new entity, create + enrich
|
||||
elif page.is_thin():
|
||||
enrich_page(entity) // thin page, fill it out
|
||||
else:
|
||||
update_timeline(entity) // existing page, add today's mentions
|
||||
|
||||
// Phase 2: Fix Broken Citations
|
||||
pages = gbrain list --type person --limit 100
|
||||
for page in pages:
|
||||
for entry in page.timeline:
|
||||
if not entry.has_source_attribution():
|
||||
fix_citation(entry) // add [Source: ...] where missing
|
||||
if entry.has_tweet_url() and not entry.url_is_valid():
|
||||
fix_url(entry) // broken tweet links
|
||||
|
||||
// Phase 3: Consolidate Memory
|
||||
patterns = detect_patterns_across_conversations()
|
||||
for pattern in patterns:
|
||||
promote_to_memory(pattern) // ephemeral → durable knowledge
|
||||
|
||||
// Phase 4: Sync
|
||||
gbrain sync --no-pull --no-embed
|
||||
gbrain embed --stale
|
||||
```
|
||||
|
||||
### Setting Up the Dream Cycle
|
||||
|
||||
**OpenClaw:** Ships with DREAMS.md as a default skill. Three phases (light,
|
||||
deep, REM) run automatically during quiet hours.
|
||||
|
||||
**Hermes Agent:**
|
||||
```bash
|
||||
/cron add "0 2 * * *" "Dream cycle: search today's sessions for
|
||||
entities I mentioned. For each person, company, or idea: check
|
||||
if a brain page exists (gbrain search), create or update it if
|
||||
thin. Fix any broken citations. Then consolidate: read MEMORY.md,
|
||||
promote important signals, remove stale entries."
|
||||
--name "nightly-dream-cycle"
|
||||
```
|
||||
|
||||
**Claude Code / Custom agents:** Create a script:
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# dream-cycle.sh
|
||||
|
||||
# Check quiet hours (should be quiet — that's when we run)
|
||||
echo "Dream cycle starting at $(date)"
|
||||
|
||||
# Phase 1: Entity sweep (spawn sub-agent)
|
||||
# Read today's conversation logs, extract entities, update brain
|
||||
|
||||
# Phase 2: Citation hygiene
|
||||
gbrain doctor --json | jq '.checks[] | select(.status=="warn")'
|
||||
|
||||
# Phase 3: Embed any stale content
|
||||
gbrain embed --stale
|
||||
|
||||
echo "Dream cycle complete at $(date)"
|
||||
```
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **The dream cycle is NOT optional.** Without it, signal leaks out of every
|
||||
conversation. With it, nothing is lost. This is the difference between an
|
||||
agent that forgets and one that remembers.
|
||||
|
||||
2. **Quiet hours gate on EVERY notification job.** If you skip it, the user
|
||||
gets pinged at 3 AM. One 3 AM ping and they'll disable the whole system.
|
||||
|
||||
3. **Don't over-cron.** 20+ jobs sounds like a lot. Start with: email (30 min),
|
||||
dream cycle (nightly), brain health (weekly). Add more as you add
|
||||
integration recipes.
|
||||
|
||||
4. **Timezone changes are automatic.** Don't make the user reconfigure cron
|
||||
when they travel. Read the calendar, infer the timezone, adjust delivery.
|
||||
|
||||
5. **Held messages MUST be picked up.** If quiet hours hold a notification,
|
||||
the morning briefing MUST include it. Otherwise information is lost.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Quiet hours:** Set quiet hours to current hour. Run a notification cron.
|
||||
Verify output went to `/tmp/cron-held/`, not to messaging.
|
||||
2. **Dream cycle:** Run the dream cycle manually. Check that thin entity pages
|
||||
got enriched and broken citations were fixed.
|
||||
3. **Email collector cron:** Wait 30 minutes. Check `data/digests/` for new digest.
|
||||
4. **Morning briefing:** Check that held messages appear in the briefing.
|
||||
5. **Health check:** Run `gbrain doctor --json`. All checks should pass.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md). See also: [Quiet Hours](quiet-hours.md), [Operational Disciplines](operational-disciplines.md)*
|
||||
@@ -1,146 +0,0 @@
|
||||
# Deterministic Collectors: Code for Data, LLMs for Judgment
|
||||
|
||||
## Goal
|
||||
|
||||
Separate mechanical work (100% reliable code) from analytical work (LLM judgment) so that deterministic tasks never fail probabilistically.
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: the LLM generates Gmail links, formats tables, and tracks state.
|
||||
It follows the rule for the first 10 items, then drops a link on item 11. You
|
||||
write "NO EXCEPTIONS" in the prompt. It still fails. 90% reliability over 20
|
||||
items means visible failures twice per day. Trust is destroyed.
|
||||
|
||||
With this: code handles URLs, formatting, and state (100% reliable). The LLM
|
||||
reads pre-formatted data and adds judgment, classification, and enrichment.
|
||||
Links are never wrong because the LLM never generates them.
|
||||
|
||||
## Implementation
|
||||
|
||||
```
|
||||
// The pattern: code collects, LLM analyzes
|
||||
|
||||
// STEP 1: Deterministic collector (script, no LLM calls)
|
||||
collector_run():
|
||||
messages = gmail_api.fetch_unread()
|
||||
for msg in messages:
|
||||
structured = {
|
||||
id: msg.id,
|
||||
from: msg.sender,
|
||||
subject: msg.subject,
|
||||
snippet: msg.snippet,
|
||||
gmail_link: f"https://mail.google.com/mail/u/?authuser={account}#inbox/{msg.id}",
|
||||
gmail_markdown: f"[Open in Gmail]({gmail_link})",
|
||||
is_signature: regex_match(msg, DOCUSIGN_PATTERNS),
|
||||
is_noise: regex_match(msg, NOISE_PATTERNS),
|
||||
is_new: msg.id not in state.seen_ids
|
||||
}
|
||||
store(structured)
|
||||
state.seen_ids.add(msg.id)
|
||||
generate_markdown_digest(structured_messages)
|
||||
|
||||
// STEP 2: LLM reads the pre-formatted digest
|
||||
llm_analyze():
|
||||
digest = read("data/digests/today.md") // links already baked in
|
||||
classify_urgency(digest) // judgment call
|
||||
add_commentary(digest) // contextual analysis
|
||||
run_brain_enrichment(notable_entities) // gbrain search + update
|
||||
draft_replies(urgent_items) // creative work
|
||||
surface_to_user(final_output) // delivery
|
||||
|
||||
// STEP 3: Wire into cron
|
||||
cron_job():
|
||||
collector_run() // fast, cheap, deterministic
|
||||
llm_analyze() // slower, expensive, creative
|
||||
```
|
||||
|
||||
### The Architecture
|
||||
|
||||
```
|
||||
+-----------------------------+ +------------------------------+
|
||||
| Deterministic Collector |---->| LLM Agent |
|
||||
| (Node.js / Python script) | | |
|
||||
| | | - Read the pre-formatted |
|
||||
| - Pull data from API | | digest |
|
||||
| - Store structured JSON | | - Classify items |
|
||||
| - Generate links/URLs | | - Add commentary |
|
||||
| - Detect patterns (regex) | | - Run brain enrichment |
|
||||
| - Track state (seen/new) | | - Draft replies |
|
||||
| - Output markdown digest | | - Surface to user |
|
||||
| | | |
|
||||
| CODE — deterministic, | | AI — judgment, context, |
|
||||
| never forgets | | creativity |
|
||||
+-----------------------------+ +------------------------------+
|
||||
```
|
||||
|
||||
### File Structure
|
||||
|
||||
```
|
||||
scripts/email-collector/
|
||||
├── email-collector.mjs # No LLM calls, no external deps
|
||||
├── data/
|
||||
│ ├── state.json # Last pull timestamp, known IDs, pending signatures
|
||||
│ ├── messages/ # Structured JSON per day
|
||||
│ │ └── 2026-04-09.json
|
||||
│ └── digests/ # Pre-formatted markdown
|
||||
│ └── 2026-04-09.md
|
||||
```
|
||||
|
||||
### Where the Pattern Applies
|
||||
|
||||
| Signal Source | Collector Generates | LLM Adds |
|
||||
|--------------|-------------------|----------|
|
||||
| **Email** | Gmail links, sender metadata, signature detection | Urgency classification, enrichment, reply drafts |
|
||||
| **X/Twitter** | Tweet links, engagement metrics, deletion detection | Sentiment analysis, narrative detection, content ideas |
|
||||
| **Calendar** | Event links, attendee lists, conflict detection | Prep briefings, meeting context from brain |
|
||||
| **Slack** | Channel links, thread links, mention detection | Priority classification, action item extraction |
|
||||
| **GitHub** | PR/issue links, diff stats, CI status | Code review context, priority assessment |
|
||||
|
||||
### The Principle
|
||||
|
||||
If a piece of output MUST be present and MUST be formatted correctly every
|
||||
time, generate it in code. If a piece of output requires judgment, context,
|
||||
or creativity, generate it with the LLM. Don't ask the LLM to do both in
|
||||
the same pass.
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **LLMs forget links -- bake them in code.** The LLM will follow the
|
||||
"include a Gmail link" rule for the first 10 items, then silently drop
|
||||
it on item 11. No amount of prompt engineering fixes probabilistic
|
||||
formatting over long outputs. The fix: generate every link in the
|
||||
collector script. The LLM reads pre-formatted markdown where links are
|
||||
already embedded. It can't forget what it didn't generate.
|
||||
|
||||
2. **Noise filtering must be deterministic.** Regex-based noise detection
|
||||
(newsletters, automated receipts, marketing) belongs in the collector,
|
||||
not the LLM. The LLM might classify a newsletter as "possibly important"
|
||||
on one run and "noise" on the next. Code classifies the same input the
|
||||
same way every time.
|
||||
|
||||
3. **Atomic writes prevent corruption.** The collector writes to a state
|
||||
file (`state.json`) that tracks which messages have been seen. If the
|
||||
script crashes mid-write, the state file can be corrupted. Write to a
|
||||
temp file first, then rename atomically. This also prevents the LLM
|
||||
from reading a partial digest if the cron fires during a collection run.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Run the collector and check every link.** Execute the collector script
|
||||
manually. Open the generated digest. Click every `[Open in Gmail]` link
|
||||
(or equivalent). Every single link must resolve to the correct item. If
|
||||
any link is broken or missing, the collector has a bug.
|
||||
|
||||
2. **Verify noise filtering is consistent.** Run the collector twice on the
|
||||
same input data. The noise classification (is_noise field) must be
|
||||
identical both times. If it varies, a probabilistic element leaked into
|
||||
the deterministic layer.
|
||||
|
||||
3. **Verify the LLM reads structured output.** Run the full pipeline
|
||||
(collector then LLM). Check that the LLM's analysis references data
|
||||
from the structured digest, not from its own generation. The links in
|
||||
the final output should be identical to the links in the digest file.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,151 +0,0 @@
|
||||
# Diligence Ingestion: Data Room to Brain Pages
|
||||
|
||||
## Goal
|
||||
|
||||
Turn pitch decks, financial models, and data room materials into searchable, cross-referenced brain pages with bull/bear analysis.
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: pitch decks sit in email attachments. Financial models in Google
|
||||
Drive. No cross-reference to the company brain page. You can't search "what
|
||||
were the key metrics from Acme Corp's Series A deck?"
|
||||
|
||||
With this: every data room document is extracted, diarized, cross-referenced to
|
||||
the company page, and searchable. Index.md gives you the bull/bear case at a
|
||||
glance. `gbrain query "Acme Corp revenue growth"` finds the exact chart.
|
||||
|
||||
## Implementation
|
||||
|
||||
Recognize data room materials by PDF filenames containing "Data Deck", "Intro
|
||||
Deck", "Data Room", "Cap Table", "Financial Model", "Investor Memo", "Pitch
|
||||
Deck", or series round names. Spreadsheet tabs with Revenue, Retention, Cohorts,
|
||||
CAC, Gross Margin, Unit Economics, ARR. User language like "data room",
|
||||
"diligence", "deck", "pitch", "fundraise materials".
|
||||
|
||||
### The 9-Step Pipeline
|
||||
|
||||
**Step 1: Identify the Company.**
|
||||
From the document content or filename, identify the company name.
|
||||
Check if `brain/companies/{slug}.md` exists.
|
||||
|
||||
**Step 2: Create Diligence Directory.**
|
||||
|
||||
```bash
|
||||
mkdir -p brain/diligence/{company-slug}/.raw
|
||||
```
|
||||
|
||||
**Step 3: Extract Content.**
|
||||
|
||||
- **PDFs:** Use PDF extraction tool. For scanned/image-heavy PDFs,
|
||||
use OCR (e.g., Mistral OCR or similar).
|
||||
- **Spreadsheets:** Export each sheet as CSV. For Google Sheets:
|
||||
```
|
||||
https://docs.google.com/spreadsheets/d/{ID}/gviz/tq?tqx=out:csv&sheet={Sheet Name}
|
||||
```
|
||||
|
||||
**Step 4: Diarize and Save.**
|
||||
Write extracted content to `brain/diligence/{company}/{doc-name}.md`:
|
||||
- Document title and type
|
||||
- Section-by-section breakdown with key metrics
|
||||
- Notable footnotes or caveats
|
||||
- Raw data tables where relevant
|
||||
|
||||
**Step 5: Save Raw Files.**
|
||||
Copy original PDFs/files to `brain/diligence/{company}/.raw/`
|
||||
Preserve originals for reference. The diarized version is for search.
|
||||
|
||||
**Step 6: Create or Update index.md.**
|
||||
Every diligence directory needs an `index.md`:
|
||||
|
||||
```markdown
|
||||
# {Company Name} — Diligence
|
||||
|
||||
## Round Details
|
||||
- Stage: Series A
|
||||
- Amount: $10M
|
||||
- Date: 2026-04
|
||||
|
||||
## Document Inventory
|
||||
- [Pitch Deck](pitch-deck.md) — 25 slides, company overview + traction
|
||||
- [Financial Model](financial-model.md) — 5 tabs, 3-year projections
|
||||
- [Cap Table](cap-table.md) — current ownership + option pool
|
||||
|
||||
## Key Findings
|
||||
- Revenue growing 30% MoM for last 6 months
|
||||
- CAC payback period: 4 months
|
||||
- Net retention: 135%
|
||||
|
||||
## Bull Case
|
||||
- Strong product-market fit signal (NPS 72)
|
||||
- Expanding into adjacent vertical
|
||||
|
||||
## Bear Case
|
||||
- Single customer represents 40% of revenue
|
||||
- Burn rate increased 3x last quarter
|
||||
|
||||
## Open Questions
|
||||
- What's the path to profitability?
|
||||
- How defensible is the moat?
|
||||
```
|
||||
|
||||
**Step 7: Enrich Company Brain Page.**
|
||||
Update `brain/companies/{slug}.md`:
|
||||
- Add document sources to frontmatter
|
||||
- Update compiled truth with key findings
|
||||
- Add "See Also" link to diligence directory
|
||||
- If no company page exists, create one via the enrich skill
|
||||
|
||||
**Step 8: Commit.**
|
||||
|
||||
```bash
|
||||
cd brain/ && git add -A && git commit -m "diligence: {Company} — {doc type} ingestion" && git push
|
||||
```
|
||||
|
||||
**Step 9: Publish (if asked).**
|
||||
When the user wants a shareable brief, create a password-protected
|
||||
published version. Strip internal notes and raw assessment language.
|
||||
|
||||
### Quality Bar
|
||||
|
||||
A good diligence page reads like an intelligence assessment:
|
||||
- **What they say** vs **what the data shows** (the gap is the insight)
|
||||
- Explicit bull/bear case (not just a summary)
|
||||
- Key metrics highlighted, not buried
|
||||
- Open questions that need answers before decision
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **PDF extraction is lossy.** Scanned decks and image-heavy PDFs lose
|
||||
tables and charts during extraction. Always check the diarized output
|
||||
against the original `.raw/` file. If key metrics are missing, re-extract
|
||||
with OCR or transcribe manually.
|
||||
|
||||
2. **Idempotency on re-ingestion.** If the user sends an updated deck for
|
||||
the same company, don't create a duplicate directory. Check for an existing
|
||||
`brain/diligence/{company-slug}/` and update in place. Append a version
|
||||
suffix to the document file if the old version should be preserved.
|
||||
|
||||
3. **index.md completeness.** The index.md is the entry point for the entire
|
||||
diligence package. If it's missing the bull/bear case or open questions,
|
||||
the diligence is incomplete. Always generate all sections even if some
|
||||
require judgment calls -- flag uncertain assessments explicitly.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Search for key metrics.** After ingestion, run
|
||||
`gbrain search "revenue growth"` or `gbrain search "{company name} CAC"`.
|
||||
The diarized content should appear in results. If it doesn't, the sync
|
||||
or embedding step was missed.
|
||||
|
||||
2. **Check the company page cross-reference.** Open
|
||||
`brain/companies/{slug}.md` and verify it links to the diligence directory.
|
||||
The compiled truth section should include key findings from the deck.
|
||||
|
||||
3. **Verify index.md has all sections.** Open
|
||||
`brain/diligence/{company}/index.md` and confirm it has Round Details,
|
||||
Document Inventory, Key Findings, Bull Case, Bear Case, and Open Questions.
|
||||
Missing sections mean the pipeline stopped early.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,103 +0,0 @@
|
||||
# Enrichment Pipeline
|
||||
|
||||
## Goal
|
||||
Enrich brain pages from external APIs with tiered spend -- full pipeline for key people, light touch for passing mentions, raw data preserved for auditability.
|
||||
|
||||
## What the User Gets
|
||||
Without this: brain pages are thin shells with only what the user manually typed, API calls are wasted on nobodies, and enrichment data vanishes after the agent session ends. With this: key people have rich, multi-source portraits; spend scales to importance; raw API responses are preserved for re-processing; and cross-references connect the entire graph.
|
||||
|
||||
## Implementation
|
||||
|
||||
```
|
||||
on enrich(entity, trigger):
|
||||
# trigger: meeting mention, email thread, social interaction, user request
|
||||
|
||||
# Step 1: Identify entities from the incoming signal
|
||||
entities = extract_entities(signal)
|
||||
# people names, company names, associations
|
||||
|
||||
# Step 2: Check brain state -- UPDATE or CREATE path?
|
||||
for entity in entities:
|
||||
existing = gbrain search "{entity.name}"
|
||||
if existing:
|
||||
page = gbrain get <entity_slug>
|
||||
path = "UPDATE"
|
||||
else:
|
||||
path = "CREATE"
|
||||
|
||||
# Step 3: Determine tier -- scale spend to importance
|
||||
tier = classify_tier(entity):
|
||||
# Tier 1 (10-15 API calls): key people, inner circle, business partners,
|
||||
# portfolio companies. Full pipeline, ALL data sources.
|
||||
# Tier 2 (3-5 API calls): notable people, occasional interactions.
|
||||
# Web search + social + brain cross-reference.
|
||||
# Tier 3 (1-2 API calls): minor mentions, everyone else worth tracking.
|
||||
# Brain cross-reference + social lookup if handle known.
|
||||
|
||||
# Step 4: Run external lookups (priority order, stop when enough signal)
|
||||
data = {}
|
||||
data["brain"] = gbrain search "{entity.name}" # Always first (free)
|
||||
if tier <= 2:
|
||||
data["web"] = brave_search("{entity.name}") # Background, press, talks
|
||||
if tier <= 2:
|
||||
data["twitter"] = twitter_lookup(entity.handle) # Beliefs, building, network
|
||||
if tier == 1:
|
||||
data["linkedin"] = crustdata_enrich(entity.name) # Career, connections
|
||||
data["research"] = happenstance_research(entity) # Career arcs, web presence
|
||||
data["funding"] = captain_api(entity.company) # Funding, valuation, team
|
||||
data["meetings"] = circleback_search(entity.name) # Transcript search
|
||||
data["contacts"] = google_contacts(entity.email) # Contact data
|
||||
|
||||
# Step 5: Store raw data (auditable, re-processable)
|
||||
gbrain put_raw_data <entity_slug> \
|
||||
--data '{"sources": {"crustdata": {"fetched_at": "...", "data": {...}}, ...}}'
|
||||
# Overwrite on re-enrichment, don't append
|
||||
|
||||
# Step 6: Write to brain page
|
||||
if path == "CREATE":
|
||||
gbrain put <entity_slug> --content "<compiled_truth_from_all_sources>"
|
||||
gbrain add_timeline_entry <entity_slug> --entry "Page created via enrichment"
|
||||
elif path == "UPDATE":
|
||||
# Append timeline, update compiled truth ONLY if materially new
|
||||
gbrain add_timeline_entry <entity_slug> --entry "Enriched: {new_signal}"
|
||||
# Flag contradictions -- don't silently resolve them
|
||||
|
||||
# Step 7: Cross-reference the graph
|
||||
gbrain add_link <person_slug> <company_slug> # person -> company
|
||||
gbrain add_link <company_slug> <person_slug> # company -> person
|
||||
gbrain add_link <person_slug> <deal_slug> # person -> deal
|
||||
# Every entity page links to every other entity page that references it
|
||||
|
||||
# People page sections (not a LinkedIn profile -- a living portrait):
|
||||
# Executive Summary, State, What They Believe, What They're Building,
|
||||
# What Motivates Them, Assessment, Trajectory, Relationship, Contact, Timeline
|
||||
# Facts are table stakes. TEXTURE is the value.
|
||||
|
||||
# Extract texture, not just facts:
|
||||
# Opinion expressed? -> What They Believe
|
||||
# Building or shipping? -> What They're Building
|
||||
# Emotion expressed? -> What Makes Them Tick
|
||||
# Who did they engage with? -> Network / Relationship
|
||||
# Recurring topic? -> Hobby Horses
|
||||
# Committed to something? -> Open Threads
|
||||
# Energy level? -> Trajectory
|
||||
```
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Don't overwrite human-written assessments.** If the user wrote an Assessment section with their own read on someone, API enrichment NEVER overwrites it. API data goes into State, Contact, Timeline. The user's assessment is sacrosanct.
|
||||
2. **Don't re-enrich the same page more than once per week.** Check `put_raw_data` timestamps before running the pipeline again. Enrichment is expensive and data doesn't change that fast.
|
||||
3. **LinkedIn connection count < 20 means wrong person.** Crustdata sometimes returns a different person with the same name. If the LinkedIn profile has fewer than 20 connections, it's almost certainly a false match. Discard it.
|
||||
4. **X/Twitter is the most underrated data source.** When you have someone's handle, their tweets reveal beliefs, what they're building, hobby horses, network (reply patterns), and trajectory (posting frequency, tone shifts). This is richer than LinkedIn for "What They Believe" and "What Makes Them Tick."
|
||||
5. **Cross-references are not optional.** After enriching a person, update their company page. After enriching a company, update founder pages. An enriched page without cross-links is a dead end in the graph.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. Enrich a Tier 1 person. Run `gbrain get <slug>` and confirm the page has Executive Summary, State, What They Believe, Contact, and Timeline sections populated from multiple sources.
|
||||
2. Run `gbrain get_raw_data <slug>`. Confirm raw API responses are stored with `sources.{provider}.fetched_at` timestamps.
|
||||
3. Run `gbrain get_links <slug>`. Confirm cross-reference links exist to the person's company page, deal pages, and related entities.
|
||||
4. Check a page that was enriched AND has a user-written Assessment. Confirm the Assessment section was preserved, not overwritten by API data.
|
||||
5. Try to re-enrich the same person. Confirm the system checks the `fetched_at` timestamp and skips if less than a week old.
|
||||
|
||||
---
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,190 +0,0 @@
|
||||
# Entity Detection: Run It on Every Message
|
||||
|
||||
## Goal
|
||||
|
||||
Every inbound message gets scanned for original thinking AND entity mentions so the brain grows on every conversation, automatically.
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: the agent answers questions but forgets everything. You mention
|
||||
Pedro in a meeting, and next week the agent doesn't know who Pedro is.
|
||||
|
||||
With this: every person, company, and idea mentioned in conversation gets a
|
||||
brain page. Next time Pedro comes up, the agent already has context. The
|
||||
brain compounds.
|
||||
|
||||
## Implementation
|
||||
|
||||
Spawn a lightweight sub-agent on EVERY inbound message. Do NOT wait for it
|
||||
to finish before responding. It runs in parallel.
|
||||
|
||||
```
|
||||
on_every_message(message_text, source_context):
|
||||
|
||||
// 1. SPAWN ASYNC — don't block the response
|
||||
spawn_subagent({
|
||||
model: "sonnet-class", // cheap + fast, not opus
|
||||
timeout: 120, // seconds
|
||||
task: build_detection_prompt(message_text, source_context)
|
||||
})
|
||||
|
||||
// 2. RESPOND TO USER NORMALLY
|
||||
// The sub-agent runs in the background
|
||||
```
|
||||
|
||||
### The Detection Prompt
|
||||
|
||||
```
|
||||
build_detection_prompt(text, source):
|
||||
return `
|
||||
SIGNAL DETECTION — scan this message for ideas AND entities:
|
||||
|
||||
Message: "${text}"
|
||||
Source: [Source: User, ${source.topic}, ${source.platform}, ${source.timestamp}]
|
||||
|
||||
STEP 1 — IDEAS FIRST (highest priority):
|
||||
Is the user expressing an original thought, observation, thesis, or framework?
|
||||
|
||||
If yes:
|
||||
- Create or update brain/originals/{slug}.md
|
||||
- Use the user's EXACT phrasing (the language IS the insight)
|
||||
- "The ambition-to-lifespan ratio has never been more broken" is better
|
||||
than "tension between ambition and mortality"
|
||||
- Include [Source: ...] citation with full context
|
||||
|
||||
If the idea references a world concept: brain/concepts/{slug}.md
|
||||
If it's a product/business idea: brain/ideas/{slug}.md
|
||||
|
||||
STEP 2 — ENTITIES:
|
||||
Extract all person names, company names, media titles.
|
||||
|
||||
For each entity:
|
||||
a. Run: gbrain search "{name}"
|
||||
b. If page exists AND new info: append timeline entry
|
||||
Format: - YYYY-MM-DD | {what happened} [Source: {who}, {context}, {date}]
|
||||
c. If no page AND entity is notable: create page with web enrichment
|
||||
d. If page is thin (< 5 lines compiled truth): spawn background enrichment
|
||||
|
||||
STEP 3 — BACK-LINKING (mandatory):
|
||||
For every entity mentioned, add a back-link FROM their page TO this source.
|
||||
An unlinked mention is a broken brain.
|
||||
Format: - **YYYY-MM-DD** | Referenced in [{page title}]({path}) — {context}
|
||||
|
||||
STEP 4 — SYNC:
|
||||
Run: gbrain sync --no-pull --no-embed
|
||||
|
||||
If nothing to capture, reply "No signals detected" and exit.
|
||||
`
|
||||
```
|
||||
|
||||
### Notability Filtering
|
||||
|
||||
Before creating a new entity page, check notability:
|
||||
|
||||
```
|
||||
is_notable(entity):
|
||||
// CREATE a page for:
|
||||
- People the user knows or discusses with specificity
|
||||
- Companies the user is evaluating, working with, or investing in
|
||||
- Media the user mentions with personal reaction
|
||||
- Anyone the user has explicitly engaged with
|
||||
|
||||
// DON'T create a page for:
|
||||
- Generic references or passing examples
|
||||
- Low-engagement accounts who mentioned the user once
|
||||
- Pure metaphors ("like the Roman Empire...")
|
||||
- One-off encounters with no follow-up
|
||||
|
||||
// If notable AND no page: create FULL page (not a stub)
|
||||
// If not notable: skip silently
|
||||
```
|
||||
|
||||
### What Counts as Original Thinking
|
||||
|
||||
| Capture | Don't Capture |
|
||||
|---------|---------------|
|
||||
| Original observations about how the world works | "ok", "do it", "sure" |
|
||||
| Novel connections between disparate things | Pure questions without observations |
|
||||
| Frameworks and mental models | Echoing back what the agent said |
|
||||
| Pattern recognition ("I keep seeing X in every Y") | Acknowledgments and reactions |
|
||||
| Hot takes with reasoning | Routine operational messages |
|
||||
| Metaphors that reveal new angles | Requests without embedded insight |
|
||||
|
||||
### Filing Rules
|
||||
|
||||
| Signal | Destination |
|
||||
|--------|-------------|
|
||||
| User generated the idea | `brain/originals/{slug}.md` |
|
||||
| User's synthesis of others' ideas | `brain/originals/` (the synthesis is original) |
|
||||
| World concept someone else coined | `brain/concepts/{slug}.md` |
|
||||
| Product or business idea | `brain/ideas/{slug}.md` |
|
||||
| Person mentioned | `brain/people/{slug}.md` |
|
||||
| Company mentioned | `brain/companies/{slug}.md` |
|
||||
| Media referenced | `brain/media/{type}/{slug}.md` |
|
||||
|
||||
### The Iron Law of Back-Linking
|
||||
|
||||
Every entity mention MUST create a back-link FROM the entity page TO the
|
||||
source. This is not optional.
|
||||
|
||||
```
|
||||
// When message mentions "Pedro" and creates a meeting page:
|
||||
|
||||
// 1. Update the meeting page (normal)
|
||||
brain/meetings/2026-04-10-board-sync.md:
|
||||
- Pedro presented Q1 numbers
|
||||
|
||||
// 2. ALSO update Pedro's page (back-link)
|
||||
brain/people/pedro-franceschi.md:
|
||||
## Timeline
|
||||
- **2026-04-10** | Presented Q1 numbers at board sync
|
||||
[Source: User, board meeting, 2026-04-10]
|
||||
```
|
||||
|
||||
Without back-links, you can't traverse the graph. "Show me everything related
|
||||
to Pedro" only works if Pedro's page links back to every mention.
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Don't block the conversation.** Entity detection runs async. The user
|
||||
should see a response immediately, not wait 2 minutes while the sub-agent
|
||||
enriches 5 entity pages.
|
||||
|
||||
2. **Sonnet, not Opus.** Entity detection is pattern matching, not deep
|
||||
reasoning. Sonnet is 5-10x cheaper and fast enough. Use Opus for the
|
||||
main conversation.
|
||||
|
||||
3. **Exact phrasing matters.** "Markdown is actually code" is an insight.
|
||||
"Markdown can be used as code" is a summary. Capture the first version.
|
||||
|
||||
4. **Don't create stubs.** If you create a page, make it good. Run a web
|
||||
search, build out the compiled truth, add context. A stub page with just
|
||||
a name is worse than no page (it gives false confidence).
|
||||
|
||||
5. **Dedup before creating.** Always `gbrain search` before creating a page.
|
||||
Variant spellings, nicknames, and company abbreviations cause duplicates.
|
||||
"Pedro Franceschi" and "Pedro" might be the same person.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Send a message mentioning a person.** Say "I had coffee with Sarah Chen
|
||||
from Acme Corp today." Verify: brain/people/sarah-chen.md was created or
|
||||
updated, brain/companies/acme-corp.md was created or updated, both have
|
||||
timeline entries with today's date.
|
||||
|
||||
2. **Send a message with an original idea.** Say "What if we could distribute
|
||||
software as markdown files that agents execute?" Verify:
|
||||
brain/originals/{slug}.md was created with your exact phrasing.
|
||||
|
||||
3. **Check back-links.** Open Sarah Chen's page. It should have a timeline
|
||||
entry linking back to today's conversation. Open Acme Corp's page. Same.
|
||||
|
||||
4. **Send a boring message.** Say "ok sounds good." Verify: nothing was
|
||||
created. The detector should report "No signals detected."
|
||||
|
||||
5. **Check for duplicates.** Mention "Pedro" then later "Pedro Franceschi."
|
||||
Verify: one page, not two.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,109 +0,0 @@
|
||||
# Executive Assistant Pattern
|
||||
|
||||
## Goal
|
||||
Email triage, meeting prep, and scheduling powered by brain context -- so every interaction is informed by the full history of the relationship.
|
||||
|
||||
## What the User Gets
|
||||
Without this: the agent triages email mechanically ("you have 12 unread"), preps for meetings with generic LinkedIn bios, and schedules without relationship context. With this: the agent knows who every sender is before reading their email, surfaces shared history before every meeting, and nudges scheduling based on relationship temperature and open threads.
|
||||
|
||||
## Implementation
|
||||
|
||||
```
|
||||
# WORKFLOW 1: Email Triage
|
||||
on email_batch(emails):
|
||||
for email in emails:
|
||||
# Step 1: Search sender BEFORE reading the email body
|
||||
# Brain context makes triage 10x better
|
||||
sender_page = gbrain search "{email.sender_name}"
|
||||
if sender_page:
|
||||
context = gbrain get <sender_slug>
|
||||
# Now you know: who they are, relationship history,
|
||||
# what they care about, open threads
|
||||
|
||||
# Step 2: Read the email WITH brain context loaded
|
||||
# Classification is now informed, not mechanical
|
||||
|
||||
# Step 3: Classify with context
|
||||
if context.relationship == "inner_circle" or context.has_open_threads:
|
||||
priority = "urgent"
|
||||
elif context.is_known_entity:
|
||||
priority = "normal"
|
||||
else:
|
||||
priority = "noise" # unknown sender, no brain page
|
||||
|
||||
# Step 4: Draft reply with relationship context
|
||||
if needs_reply(email):
|
||||
draft = compose_reply(
|
||||
email,
|
||||
context=context, # their brain page
|
||||
open_threads=context.open_threads, # what you're working on together
|
||||
relationship=context.relationship # tone calibration
|
||||
)
|
||||
|
||||
# WORKFLOW 2: Meeting Prep
|
||||
on upcoming_meeting(meeting):
|
||||
briefing = {}
|
||||
for attendee in meeting.attendees:
|
||||
# Search brain for each attendee
|
||||
results = gbrain search "{attendee.name}"
|
||||
if results:
|
||||
page = gbrain get <attendee_slug>
|
||||
briefing[attendee] = {
|
||||
"compiled_truth": page.compiled_truth,
|
||||
"last_interaction": page.timeline[0], # most recent
|
||||
"open_threads": page.open_threads,
|
||||
"relationship_temperature": page.relationship,
|
||||
"relevant_deals": gbrain get_links <attendee_slug>,
|
||||
}
|
||||
else:
|
||||
briefing[attendee] = "No brain page -- consider enriching"
|
||||
|
||||
# Surface: shared history, what to follow up on, what to watch for
|
||||
# "Last time you discussed the Series B timeline. Pedro was concerned
|
||||
# about burn rate. Here's the latest from his company page."
|
||||
|
||||
# WORKFLOW 3: Post-Inbox Brain Updates
|
||||
on inbox_cleared():
|
||||
for email in processed_emails:
|
||||
if email.contained_new_information:
|
||||
# Update the sender's brain page with new signal
|
||||
gbrain add_timeline_entry <sender_slug> \
|
||||
--entry "Email re: {subject}. Key info: {extracted_signal}" \
|
||||
--source "email from {sender} re {subject}, {date}"
|
||||
|
||||
# Update any mentioned entity pages too
|
||||
for entity in email.mentioned_entities:
|
||||
gbrain add_timeline_entry <entity_slug> \
|
||||
--entry "{what_was_said_about_them}" \
|
||||
--source "email from {sender}, {date}"
|
||||
|
||||
# WORKFLOW 4: Scheduling Nudges
|
||||
on schedule_request(meeting):
|
||||
for attendee in meeting.attendees:
|
||||
page = gbrain get <attendee_slug>
|
||||
if page.last_interaction > 6_weeks_ago:
|
||||
nudge("You haven't met with {attendee} in {weeks} weeks")
|
||||
if page.has_open_threads:
|
||||
nudge("{attendee} has an open thread about {topic}")
|
||||
if page.relationship_temperature == "cooling":
|
||||
nudge("Relationship with {attendee} may need attention")
|
||||
```
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Search sender BEFORE reading the email.** This is counterintuitive but critical. Loading brain context first means you know who they are, what you're working on together, and what they care about -- before you even see the subject line. The triage is informed, not mechanical.
|
||||
2. **Unknown senders with no brain page are almost always noise.** If `gbrain search` returns nothing for a sender, they're probably not important. Classify as low priority unless the email content signals otherwise.
|
||||
3. **Meeting prep is the highest-leverage EA workflow.** The user walks into every meeting already briefed on each attendee: last interaction, open threads, relationship history. This is the difference between "you have a meeting at 3" and "you have a meeting at 3 with Pedro -- last time you discussed the Series B, he was concerned about burn rate."
|
||||
4. **Post-inbox brain updates are where the brain compounds.** Every email is signal. If you clear the inbox without updating brain pages, the information is lost. This is the step most agents skip.
|
||||
5. **Scheduling nudges require timeline data.** "You haven't met with Diana in 6 weeks" only works if meeting pages have been ingested with proper entity propagation (see meeting-ingestion guide).
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. Run meeting prep for tomorrow's calendar. For each attendee, confirm the agent ran `gbrain search` and loaded their brain page before generating the briefing.
|
||||
2. Triage 5 emails. Confirm the agent searched for each sender in the brain before classifying the email.
|
||||
3. After clearing an inbox, check 2 sender brain pages with `gbrain get <slug>`. Confirm new timeline entries were added with information from the emails.
|
||||
4. Check a scheduling suggestion. Confirm the agent referenced the attendee's brain page (last interaction date, open threads) in the nudge.
|
||||
5. Send a test email from someone with a brain page. Confirm the triage response references their relationship context, not just the email content.
|
||||
|
||||
---
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,190 +0,0 @@
|
||||
# Idea Capture: Originals, Depth, and Distribution
|
||||
|
||||
## Goal
|
||||
|
||||
Capture the user's original thinking with exact phrasing, deep context, and cross-links so the originals folder becomes the highest-value content in the brain.
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: brilliant ideas said in conversation disappear. The agent heard
|
||||
"the ambition-to-lifespan ratio has never been more broken" and forgot it.
|
||||
|
||||
With this: every original observation is captured verbatim, cross-linked to
|
||||
the people and ideas that shaped it, and rated for publishing potential. Your
|
||||
intellectual archive grows with every conversation.
|
||||
|
||||
## Implementation
|
||||
|
||||
```
|
||||
capture_idea(message_text, source_context):
|
||||
|
||||
// 1. AUTHORSHIP TEST — where does this idea belong?
|
||||
if user_generated_the_idea(message_text):
|
||||
destination = "brain/originals/{slug}.md"
|
||||
elif user_synthesis_of_others(message_text):
|
||||
destination = "brain/originals/{slug}.md" // synthesis IS original
|
||||
elif world_concept(message_text):
|
||||
destination = "brain/concepts/{slug}.md"
|
||||
elif product_or_business_idea(message_text):
|
||||
destination = "brain/ideas/{slug}.md"
|
||||
elif ghostwritten_by_user(message_text):
|
||||
destination = "brain/originals/{slug}.md" // note ghostwriter in metadata
|
||||
elif article_about_user(message_text):
|
||||
destination = "brain/media/writings/{slug}.md"
|
||||
|
||||
// 2. CAPTURE WITH EXACT PHRASING — never paraphrase
|
||||
page = create_or_update(destination, {
|
||||
content: message_text, // verbatim, not summarized
|
||||
source: source_context, // conversation, meeting, moment
|
||||
reasoning_path: influences, // what led to the insight
|
||||
depth_context: emotional_nuance // the WHY behind the WHAT
|
||||
})
|
||||
|
||||
// 3. ORIGINALITY RATING (for notable ideas)
|
||||
if is_notable(message_text):
|
||||
rate_originality(page, populations=[
|
||||
"general_population", "tech_industry",
|
||||
"intellectual_media", "political_establishment"
|
||||
])
|
||||
|
||||
// 4. CROSS-LINK (mandatory — an original without links is dead)
|
||||
link_to_people(page, mentioned_people)
|
||||
link_to_companies(page, mentioned_companies)
|
||||
link_to_meetings(page, source_meeting)
|
||||
link_to_media(page, influences)
|
||||
link_to_other_originals(page, related_ideas)
|
||||
link_to_concepts(page, referenced_concepts)
|
||||
|
||||
// 5. SYNC
|
||||
gbrain sync --no-pull --no-embed
|
||||
```
|
||||
|
||||
### The Authorship Test
|
||||
|
||||
| Signal | Destination |
|
||||
|--------|-------------|
|
||||
| User generated the idea | `brain/originals/{slug}.md` |
|
||||
| User's unique synthesis of others' ideas | `brain/originals/` (the synthesis is original) |
|
||||
| World concept someone else coined | `brain/concepts/{slug}.md` |
|
||||
| Product or business idea | `brain/ideas/{slug}.md` |
|
||||
| User's ghostwritten book/essay | `brain/originals/` (note ghostwriter in metadata) |
|
||||
| Article ABOUT user | `brain/media/writings/` |
|
||||
|
||||
### Capture Standards
|
||||
|
||||
**Use the user's EXACT phrasing.** The language IS the insight.
|
||||
|
||||
"The ambition-to-lifespan ratio has never been more broken" captures something that
|
||||
"tension between ambition and mortality" doesn't. Don't clean it up. Don't paraphrase.
|
||||
The vivid version is the real version.
|
||||
|
||||
**What counts as worth capturing:**
|
||||
- Original observations about how the world works
|
||||
- Novel connections between disparate things
|
||||
- Frameworks and mental models
|
||||
- Pattern recognition moments ("I keep seeing X in every Y")
|
||||
- Hot takes with reasoning behind them
|
||||
- Metaphors that reveal new angles
|
||||
- Emotional/psychological insights about self or others
|
||||
|
||||
**What does NOT count:**
|
||||
- Routine operational messages ("ok", "do it")
|
||||
- Pure questions without embedded observations
|
||||
- Echoing back something the agent said
|
||||
- Acknowledgments and reactions
|
||||
|
||||
### The Depth Test
|
||||
|
||||
**Could someone unfamiliar with the user read this page and understand not
|
||||
just WHAT they think but WHY and HOW they got there?**
|
||||
|
||||
If the answer is no, it needs more depth. Include:
|
||||
- The reasoning path (what led to the insight)
|
||||
- The influences (what they were reading/watching/experiencing)
|
||||
- The context (conversation, meeting, moment)
|
||||
- The emotional or psychological nuance
|
||||
|
||||
### Originality Distribution Rating
|
||||
|
||||
For notable ideas, rate originality 0-100 across different populations:
|
||||
|
||||
```markdown
|
||||
## Originality Distribution
|
||||
|
||||
- **General population:** 72/100 — most people haven't encountered this framework
|
||||
- **Tech industry:** 45/100 — common in startup circles but novel to most
|
||||
- **Intellectual/media class:** 68/100 — would resonate, not yet articulated
|
||||
- **Political establishment:** 82/100 — completely foreign to policy thinking
|
||||
|
||||
**Publish signal:** Strong essay candidate. Best audience: founders, builders.
|
||||
```
|
||||
|
||||
This tells the user which ideas are worth turning into essays, talks, or videos,
|
||||
and which audience would find them most novel.
|
||||
|
||||
### Deep Cross-Linking Mandate
|
||||
|
||||
**An original without cross-links is a dead original.** The connections ARE
|
||||
the intelligence.
|
||||
|
||||
Every original MUST link to:
|
||||
- **People** who shaped the thinking
|
||||
- **Companies** where the idea played out
|
||||
- **Meetings** where it was discussed
|
||||
- **Books and media** that influenced it
|
||||
- **Other originals** it connects to (ideas form clusters)
|
||||
- **Concepts** it builds on or challenges
|
||||
|
||||
### Notability Filtering
|
||||
|
||||
Before creating any entity page, check notability:
|
||||
|
||||
**Create a page for:**
|
||||
- People you know or discuss with specificity
|
||||
- Companies you're evaluating, working with, or investing in
|
||||
- Media you mention with personal reaction
|
||||
- Anyone you've explicitly engaged with
|
||||
|
||||
**Don't create pages for:**
|
||||
- Generic references or passing examples
|
||||
- Low-engagement accounts who mentioned you once
|
||||
- Pure metaphors ("like the Roman Empire...")
|
||||
- One-off encounters with no follow-up
|
||||
|
||||
**Decision:** If notable AND no page exists, create a full page with web
|
||||
search enrichment. No stubs. If you make a page, make it good.
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Synthesis IS original.** When the user connects two existing ideas in a
|
||||
new way, that synthesis belongs in `brain/originals/`, not `brain/concepts/`.
|
||||
The novel combination is the insight, even if the component ideas aren't new.
|
||||
|
||||
2. **Exact phrasing is non-negotiable.** Never paraphrase, summarize, or
|
||||
"clean up" the user's language. "The ambition-to-lifespan ratio has never
|
||||
been more broken" is the insight. "Tension between ambition and mortality"
|
||||
is a corpse. Capture the first version.
|
||||
|
||||
3. **Cross-links are mandatory, not optional.** An original without links to
|
||||
the people, companies, meetings, and concepts that shaped it is a dead
|
||||
original. The connections ARE the intelligence. Check every original for
|
||||
at least 2 cross-links before considering it captured.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Generate an idea and check the page.** Say something original in
|
||||
conversation (e.g., "What if markdown files are actually distributed
|
||||
software?"). Verify that `brain/originals/{slug}.md` was created with
|
||||
your exact phrasing, not a paraphrase.
|
||||
|
||||
2. **Check cross-links exist.** Open the newly created original page. It
|
||||
should link to at least the people or concepts mentioned. Open those
|
||||
linked pages and verify they back-link to the original.
|
||||
|
||||
3. **Verify the depth test passes.** Read the captured page as if you were
|
||||
a stranger. Can you understand not just WHAT the user thinks but WHY?
|
||||
If the reasoning path and context are missing, the capture is incomplete.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,138 +0,0 @@
|
||||
# Live Sync: Keep the Index Current
|
||||
|
||||
## Goal
|
||||
|
||||
Every markdown change in the brain repo is searchable within minutes, automatically, with no manual intervention.
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: you correct a hallucination in a brain page, but the vector DB
|
||||
keeps serving the old text because nobody ran `gbrain sync`. Stale search
|
||||
results erode trust. The brain becomes unreliable.
|
||||
|
||||
With this: edits show up in search within minutes. The vector DB stays current
|
||||
with the brain repo automatically. You never have to remember to run sync.
|
||||
|
||||
## Implementation
|
||||
|
||||
### Prerequisite: Session Mode Pooler
|
||||
|
||||
Sync uses `engine.transaction()` on every import. If `DATABASE_URL` points to
|
||||
Supabase's **Transaction mode** pooler, sync will throw `.begin() is not a
|
||||
function` and **silently skip most pages**. This is the number one cause of
|
||||
"sync ran but nothing happened."
|
||||
|
||||
Fix: use the **Session mode** pooler string (port 6543, Session mode) or the
|
||||
direct connection (port 5432, IPv6-only). Verify by running `gbrain sync` and
|
||||
checking that the page count in `gbrain stats` matches the syncable file count
|
||||
in the repo.
|
||||
|
||||
### The Primitives
|
||||
|
||||
Always chain sync + embed:
|
||||
|
||||
```bash
|
||||
gbrain sync --repo /path/to/brain && gbrain embed --stale
|
||||
```
|
||||
|
||||
- `gbrain sync --repo <path>` -- one-shot incremental sync. Detects changes via
|
||||
`git diff`, imports only what changed. For small changesets (<= 100 files),
|
||||
embeddings are generated inline during import.
|
||||
- `gbrain embed --stale` -- backfill embeddings for any chunks that don't have
|
||||
them. Safety net for large syncs (>100 files) or prior `--no-embed` runs.
|
||||
- `gbrain sync --watch --repo <path>` -- foreground polling loop, every 60s
|
||||
(configurable with `--interval N`). Embeds inline for small changesets. Exits
|
||||
after 5 consecutive failures, so run under a process manager or pair with a
|
||||
cron fallback.
|
||||
|
||||
### Approach 1: Cron Job (recommended)
|
||||
|
||||
Run every 5-30 minutes. Works with any cron scheduler.
|
||||
|
||||
```bash
|
||||
gbrain sync --repo /data/brain && gbrain embed --stale
|
||||
```
|
||||
|
||||
**OpenClaw:**
|
||||
```
|
||||
Name: gbrain-auto-sync
|
||||
Schedule: */15 * * * *
|
||||
Prompt: "Run: gbrain sync --repo /data/brain && gbrain embed --stale
|
||||
Log the result. If sync fails with .begin() is not a function,
|
||||
the DATABASE_URL is using Transaction mode pooler."
|
||||
```
|
||||
|
||||
**Hermes:**
|
||||
```
|
||||
/cron add "*/15 * * * *" "Run gbrain sync --repo /data/brain &&
|
||||
gbrain embed --stale. Log the result." --name "gbrain-auto-sync"
|
||||
```
|
||||
|
||||
### Approach 2: Long-Lived Watcher
|
||||
|
||||
For near-instant sync (60s polling). Run under a process manager that
|
||||
auto-restarts on exit. Pair with a cron fallback since `--watch` exits
|
||||
on repeated failures.
|
||||
|
||||
```bash
|
||||
gbrain sync --watch --repo /data/brain
|
||||
```
|
||||
|
||||
### Approach 3: Git Hook / Webhook
|
||||
|
||||
Triggers sync on push events for instant sync (<5s).
|
||||
|
||||
- **GitHub webhook:** Set up the webhook to call
|
||||
`gbrain sync --repo /data/brain && gbrain embed --stale`.
|
||||
Verify `X-Hub-Signature-256` against a shared secret.
|
||||
- **Git post-receive hook:** If the brain repo is on the same machine.
|
||||
|
||||
### What Gets Synced
|
||||
|
||||
Sync only indexes "syncable" markdown files. These are excluded by design:
|
||||
- Hidden paths (`.git/`, `.raw/`, etc.)
|
||||
- The `ops/` directory
|
||||
- Meta files: `README.md`, `index.md`, `schema.md`, `log.md`
|
||||
|
||||
### Sync is Idempotent
|
||||
|
||||
Concurrent runs are safe. Two syncs on the same commit no-op because content
|
||||
hashes match. If both a cron and `--watch` fire simultaneously, no conflict.
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Always chain sync + embed.** Running `gbrain sync` without
|
||||
`gbrain embed --stale` leaves new chunks without embeddings. They exist
|
||||
in the database but are invisible to vector search. Always run both
|
||||
commands together. The `&&` ensures embed only runs if sync succeeds.
|
||||
|
||||
2. **--watch polls, it doesn't stream.** The `--watch` flag polls every 60s
|
||||
(configurable). It is not a filesystem watcher or git hook. It exits after
|
||||
5 consecutive failures, so it needs a process manager (systemd, pm2) or a
|
||||
cron fallback to stay alive. Don't assume it runs forever.
|
||||
|
||||
3. **Webhook needs the server running.** If you use a GitHub webhook for
|
||||
instant sync, the receiving server must be running and reachable. If the
|
||||
server is down when a push happens, that sync is missed. Pair webhooks
|
||||
with a cron fallback that catches anything the webhook missed.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Edit a file and search for the change.** Edit a brain markdown file,
|
||||
commit, and push. Wait for the next sync cycle (cron interval or `--watch`
|
||||
poll). Run `gbrain search "<text from the edit>"`. The updated content
|
||||
should appear in results. If it returns old content, sync failed.
|
||||
|
||||
2. **Compare page count to file count.** Run `gbrain stats` and count the
|
||||
syncable markdown files in the brain repo. The page count in the database
|
||||
should match. If they diverge, files are being silently skipped (likely
|
||||
a Transaction mode pooler issue).
|
||||
|
||||
3. **Check embedded chunk count.** In `gbrain stats`, the embedded chunk
|
||||
count should be close to the total chunk count. A large gap means
|
||||
`gbrain embed --stale` isn't running after sync, leaving chunks invisible
|
||||
to vector search.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,80 +0,0 @@
|
||||
# Meeting Ingestion
|
||||
|
||||
## Goal
|
||||
Meeting transcripts become brain pages that update every mentioned entity -- attendees, companies, deals, and action items all propagated in one pass.
|
||||
|
||||
## What the User Gets
|
||||
Without this: meetings vanish into memory, action items are forgotten, and the agent has no idea what was discussed last time you met someone. With this: every meeting is a permanent record that enriches every person and company page it touches, and the user walks into every follow-up already briefed.
|
||||
|
||||
## Implementation
|
||||
|
||||
```
|
||||
on new_meeting_transcript(meeting):
|
||||
# Step 1: Pull the COMPLETE transcript -- NOT the AI summary
|
||||
# AI summaries hallucinate framing ("it was agreed that...")
|
||||
# The transcript is ground truth
|
||||
transcript = fetch_full_transcript(meeting.id) # e.g., Circleback API
|
||||
# Must have speaker diarization: WHO said WHAT
|
||||
|
||||
# Step 2: Create the meeting page
|
||||
slug = f"meetings/{meeting.date}-{short_description}"
|
||||
compiled_truth = agent_analysis(transcript):
|
||||
# Above the bar: agent's OWN analysis, not a generic recap
|
||||
# - Reframe through the user's priorities
|
||||
# - Flag surprises, contradictions, implications
|
||||
# - Name real decisions (not performative ones)
|
||||
# - Call out what was left unsaid or unresolved
|
||||
timeline = format_diarized_transcript(transcript)
|
||||
# Below the bar: full transcript, append-only
|
||||
# Format: **Speaker** (HH:MM:SS): Words.
|
||||
|
||||
gbrain put <slug> --content "<compiled_truth>\n---\n<timeline>"
|
||||
|
||||
# Step 3: Propagate to ALL entity pages (MANDATORY -- most agents skip this)
|
||||
for person in meeting.attendees + meeting.mentioned_people:
|
||||
gbrain add_timeline_entry <person_slug> \
|
||||
--entry "Met in '{meeting.title}' on {date}. Key points: ..." \
|
||||
--source "Meeting notes '{meeting.title}', {date}"
|
||||
# Update their State section if new information surfaced
|
||||
# Update company pages for each person's company if relevant
|
||||
|
||||
for company in meeting.mentioned_companies:
|
||||
gbrain add_timeline_entry <company_slug> \
|
||||
--entry "Discussed in '{meeting.title}': {what_was_said}" \
|
||||
--source "Meeting notes '{meeting.title}', {date}"
|
||||
|
||||
# Step 4: Extract action items
|
||||
action_items = extract_action_items(transcript)
|
||||
# Add to task list with owner attribution
|
||||
|
||||
# Step 5: Back-link everything (bidirectional graph)
|
||||
for entity in all_entities_mentioned:
|
||||
gbrain add_link <slug> <entity_slug> # meeting -> entity
|
||||
gbrain add_link <entity_slug> <slug> # entity -> meeting
|
||||
|
||||
# Step 6: Sync so new pages are immediately searchable
|
||||
gbrain sync
|
||||
|
||||
# Schedule: cron 3x/day (10 AM, 4 PM, 9 PM) to catch new meetings
|
||||
# Source: Circleback (https://circleback.ai) or any service with
|
||||
# speaker diarization + API/webhook access
|
||||
```
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Always pull the COMPLETE transcript, never the AI summary.** AI summaries hallucinate framing -- they editorialize what was "agreed" or "decided" when no such agreement happened. The diarized transcript is ground truth.
|
||||
2. **Entity propagation is the step most agents skip.** A meeting is NOT fully ingested until every attendee's page, every mentioned person's page, and every company's page has a new timeline entry. The meeting page alone is useless without propagation.
|
||||
3. **Mentioned people are not just attendees.** If the meeting discussed "Sarah's team at Brex," then Sarah's page AND Brex's page need updates -- even though Sarah wasn't in the room.
|
||||
4. **The agent's analysis is the value, not a summary.** "They discussed Q2 targets" is worthless. "Pedro pushed back on the burn rate, Diana didn't commit to the timeline, and nobody addressed the pricing gap" is useful.
|
||||
5. **Back-links must be bidirectional.** The meeting page links to attendee pages AND attendee pages link back to the meeting. The graph is bidirectional. Always.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. After ingesting a meeting, run `gbrain get meetings/{date}-{slug}`. Confirm the page has the agent's analysis above the bar and the full diarized transcript below it.
|
||||
2. For each attendee, run `gbrain get <attendee_slug>`. Check that their timeline has a new entry referencing the meeting with specific insights (not just "attended meeting").
|
||||
3. Pick a company mentioned in the meeting. Run `gbrain get <company_slug>`. Confirm a timeline entry exists referencing what was discussed about the company.
|
||||
4. Run `gbrain get_links meetings/{date}-{slug}`. Verify back-links exist to all attendee and entity pages.
|
||||
5. Run `gbrain search "{meeting_topic}"`. Confirm the meeting page appears in search results (verifies sync ran).
|
||||
|
||||
---
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,13 +0,0 @@
|
||||
# Procfile — Render / Railway / Heroku.
|
||||
#
|
||||
# Fly.io users: see fly.toml.partial instead.
|
||||
#
|
||||
# Set secrets via the platform's env UI or CLI (e.g. `heroku config:set`,
|
||||
# `render env:set`, `railway variables set`). At minimum:
|
||||
# DATABASE_URL=postgresql://...
|
||||
# GBRAIN_ALLOW_SHELL_JOBS=1 # only if submitting shell jobs
|
||||
|
||||
# Two-layer supervision: the platform restarts the container on host
|
||||
# events (OOM, deploy); `gbrain jobs supervisor` restarts the worker
|
||||
# on in-process crashes with exponential backoff.
|
||||
worker: gbrain jobs supervisor --concurrency 2
|
||||
@@ -1,24 +0,0 @@
|
||||
# fly.toml — partial. Merge into your existing fly.toml.
|
||||
#
|
||||
# Set secrets once (never commit them):
|
||||
# fly secrets set DATABASE_URL='postgresql://user:pass@host:6543/db?prepare=false'
|
||||
# fly secrets set GBRAIN_ALLOW_SHELL_JOBS=1 # only if submitting shell jobs
|
||||
# fly secrets set ANTHROPIC_API_KEY=... # optional
|
||||
#
|
||||
# Two-layer supervision: Fly restarts the VM on host events; the
|
||||
# `gbrain jobs supervisor` process restarts the worker on in-process
|
||||
# crashes with exponential backoff and a structured audit trail.
|
||||
|
||||
[processes]
|
||||
worker = "gbrain jobs supervisor --concurrency 2"
|
||||
|
||||
# Scale the worker process to 1 machine (job queue serializes work; more
|
||||
# machines means higher concurrency but also more Postgres connections).
|
||||
# fly scale count worker=1
|
||||
|
||||
# If you want the worker in its own VM size:
|
||||
# [[vm]]
|
||||
# processes = ["worker"]
|
||||
# memory = "512mb"
|
||||
# cpu_kind = "shared"
|
||||
# cpus = 1
|
||||
@@ -1,35 +0,0 @@
|
||||
# /etc/gbrain.env — secrets + env for the gbrain worker.
|
||||
#
|
||||
# Install:
|
||||
# sudo install -m 600 -o $GBRAIN_WORKER_USER -g $GBRAIN_WORKER_USER \
|
||||
# gbrain.env.example /etc/gbrain.env
|
||||
# sudoedit /etc/gbrain.env # fill in real values
|
||||
#
|
||||
# Referenced from crontab via BASH_ENV=/etc/gbrain.env, or from systemd
|
||||
# via EnvironmentFile=/etc/gbrain.env. Never commit real secrets.
|
||||
|
||||
# --- Required ---------------------------------------------------------------
|
||||
|
||||
# Postgres connection string. For Supabase transaction pooler, include
|
||||
# prepare=false (see CLAUDE.md #284/#286).
|
||||
DATABASE_URL=postgresql://user:pass@host:6543/db?prepare=false
|
||||
|
||||
# --- Required if you submit `shell` jobs ------------------------------------
|
||||
# Only the worker process needs this. Submitters do not.
|
||||
GBRAIN_ALLOW_SHELL_JOBS=1
|
||||
|
||||
# --- Optional ---------------------------------------------------------------
|
||||
|
||||
# LLM provider keys (needed for `subagent` handler, transcription, enrichment).
|
||||
# ANTHROPIC_API_KEY=
|
||||
# OPENAI_API_KEY=
|
||||
|
||||
# Custom handler plugins (see docs/guides/plugin-handlers.md).
|
||||
# GBRAIN_PLUGIN_PATH=/etc/gbrain/plugins
|
||||
|
||||
# Pool size tuning for Supabase transaction pooler (default 10; drop to 2
|
||||
# if you hit MaxClients during upgrade subprocess spawns).
|
||||
# GBRAIN_POOL_SIZE=2
|
||||
|
||||
# Connection-level concurrency cap for Anthropic Messages API.
|
||||
# GBRAIN_ANTHROPIC_MAX_INFLIGHT=4
|
||||
@@ -1,50 +0,0 @@
|
||||
[Unit]
|
||||
Description=gbrain minion worker
|
||||
Documentation=https://github.com/garrytan/gbrain/blob/master/docs/guides/minions-deployment.md
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
# Runs as an unprivileged user that owns the brain repo and any shell-job cwds.
|
||||
# Create with: sudo useradd --system --home /srv/gbrain --shell /usr/sbin/nologin gbrain
|
||||
User=gbrain
|
||||
Group=gbrain
|
||||
WorkingDirectory=/srv/gbrain
|
||||
|
||||
# Env file is mode 600, owned by User=. Do not put secrets in this unit.
|
||||
EnvironmentFile=/etc/gbrain.env
|
||||
|
||||
# Two-layer supervision: systemd restarts `gbrain jobs supervisor` on host
|
||||
# events (reboot, unit crash); the supervisor restarts `gbrain jobs work`
|
||||
# on in-process crashes with exponential backoff + structured audit.
|
||||
ExecStart=/usr/local/bin/gbrain jobs supervisor --concurrency 2
|
||||
|
||||
# systemd restarts the supervisor on any non-zero exit. The supervisor
|
||||
# itself handles worker-level crash recovery.
|
||||
Restart=always
|
||||
RestartSec=10s
|
||||
|
||||
# Graceful shutdown: SIGTERM → wait → SIGKILL. 30s matches worker grace
|
||||
# for in-flight jobs and the shell handler's 5s child SIGTERM window.
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30s
|
||||
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=gbrain-worker
|
||||
|
||||
# Default 1024 is tight for Bun + Postgres pool + concurrent subagent LLM calls.
|
||||
LimitNOFILE=65535
|
||||
|
||||
# Hardening (optional — remove if they break your deployment).
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=read-only
|
||||
# ReadWritePaths must include the brain workspace AND ~/.gbrain (PID file +
|
||||
# audit log written by the supervisor).
|
||||
ReadWritePaths=/srv/gbrain /home/gbrain/.gbrain
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,332 +0,0 @@
|
||||
# Minions Worker Deployment Guide
|
||||
|
||||
Keep `gbrain jobs work` running across crashes, reboots, and Postgres
|
||||
connection blips. Written for agents to execute line-by-line.
|
||||
|
||||
## The problem
|
||||
|
||||
The persistent worker can die silently from:
|
||||
|
||||
- Database connection drops (Supabase/Postgres maintenance or network blips).
|
||||
- Lock-renewal failures → the stall detector eventually dead-letters jobs.
|
||||
- Bun process crashes with no automatic restart.
|
||||
- Internal event-loop death (PID alive, worker loop stopped).
|
||||
|
||||
When the worker dies, submitted jobs sit in `waiting` forever. The
|
||||
canonical answer is `gbrain jobs supervisor` — a first-class CLI that
|
||||
spawns `gbrain jobs work` as a child and auto-restarts it on crash.
|
||||
|
||||
## Worker supervision
|
||||
|
||||
### The canonical pattern
|
||||
|
||||
`gbrain jobs supervisor` is an auto-restarting wrapper around
|
||||
`gbrain jobs work`. It writes a PID file, restarts the worker on crash
|
||||
with exponential backoff (1s → 60s cap), emits lifecycle events to an
|
||||
audit file, and drains gracefully on SIGTERM (35s worker-drain window
|
||||
before SIGKILL). Exit codes are documented so agents can branch on them.
|
||||
|
||||
**Typical commands:**
|
||||
|
||||
```bash
|
||||
# Start in the foreground (blocks; Ctrl-C to stop).
|
||||
gbrain jobs supervisor --concurrency 4
|
||||
|
||||
# Start detached — returns {"event":"started","supervisor_pid":…} on stdout.
|
||||
gbrain jobs supervisor start --detach --json
|
||||
|
||||
# Check liveness without reading log files.
|
||||
gbrain jobs supervisor status --json
|
||||
|
||||
# Graceful stop (SIGTERM + drain wait + SIGKILL fallback).
|
||||
gbrain jobs supervisor stop
|
||||
```
|
||||
|
||||
**Exit codes:**
|
||||
|
||||
| Code | Meaning |
|
||||
|---|---|
|
||||
| 0 | Clean shutdown (SIGTERM/SIGINT received, worker drained) |
|
||||
| 1 | Max crashes exceeded (worker kept dying) |
|
||||
| 2 | Another supervisor holds the PID lock |
|
||||
| 3 | PID file unwritable (permission / path error) |
|
||||
|
||||
An agent seeing exit=2 can safely treat it as "one is already running";
|
||||
exit=1 should page a human.
|
||||
|
||||
### Which supervisor when?
|
||||
|
||||
The supervisor solves in-process crash recovery. Platform-level
|
||||
supervision (systemd, Fly, Render) handles host-level failures. You
|
||||
usually want both.
|
||||
|
||||
| Environment | Recommendation |
|
||||
|---|---|
|
||||
| **Container (Fly / Railway / Render / Heroku)** | `gbrain jobs supervisor` runs as PID 1. The platform restarts the container on OOM / host loss; supervisor restarts the worker on crash. See [Fly.io](#flyio) / [Render / Railway / Heroku](#render--railway--heroku). |
|
||||
| **Linux VM with systemd** | Two-layer recommended: systemd supervises `gbrain jobs supervisor`, which in turn supervises `gbrain jobs work`. Buys you automatic restart on reboot (systemd) plus fast crash recovery (supervisor). See [systemd](#systemd). |
|
||||
| **Dev laptop / macOS** | `gbrain jobs supervisor` in a terminal. Ctrl-C stops it. No system-level setup needed. |
|
||||
|
||||
### Variables used in this guide
|
||||
|
||||
Substitute these once before copy-pasting any snippet.
|
||||
|
||||
| Variable | Meaning | Typical value |
|
||||
|---|---|---|
|
||||
| `$GBRAIN_BIN` | Absolute path to the `gbrain` binary | `$(command -v gbrain)` — often `/usr/local/bin/gbrain` or `~/.bun/bin/gbrain` |
|
||||
| `$GBRAIN_WORKER_USER` | OS user that owns the worker process | the same user that ran `gbrain init`; never `root` |
|
||||
| `$GBRAIN_WORKSPACE` | `cwd` for shell jobs submitted by this deployment | absolute path, e.g. `/srv/my-brain` |
|
||||
| `$GBRAIN_ENV_FILE` | Secrets file sourced by systemd / shell | `/etc/gbrain.env` (mode 600) |
|
||||
|
||||
### Preconditions
|
||||
|
||||
Run these before any deployment step.
|
||||
|
||||
```bash
|
||||
# 1. gbrain is on PATH and resolves to an absolute location.
|
||||
command -v gbrain || { echo "gbrain not on PATH. Install, then retry."; exit 1; }
|
||||
|
||||
# 2. DATABASE_URL points at reachable Postgres.
|
||||
# (Supervisor is Postgres-only. PGLite's exclusive file lock blocks the
|
||||
# separate worker process. If `config.engine === 'pglite'` the CLI rejects
|
||||
# with a clear error.)
|
||||
gbrain doctor --fast --json | jq '.checks[] | select(.name=="db_connectivity")'
|
||||
|
||||
# 3. Schema is up to date. If version=0 or status=="fail":
|
||||
# gbrain apply-migrations --yes
|
||||
gbrain doctor --fast --json | jq '.checks[] | select(.name=="schema_version")'
|
||||
|
||||
# 4. If you plan to submit `shell` jobs, pass --allow-shell-jobs to the
|
||||
# supervisor (or export GBRAIN_ALLOW_SHELL_JOBS=1 before starting).
|
||||
# Without the flag, the shell handler is disabled at worker startup.
|
||||
```
|
||||
|
||||
## Agent usage (OpenClaw / Hermes / Cursor / Codex)
|
||||
|
||||
Three-command pattern an agent can drive without shell archaeology:
|
||||
|
||||
```bash
|
||||
# Start (returns PIDs + pid_file on stdout as JSON, then detaches)
|
||||
gbrain jobs supervisor start --detach --json
|
||||
# → {"event":"started","supervisor_pid":1234,"worker_pid":1235,"pid_file":"/Users/you/.gbrain/supervisor.pid"}
|
||||
|
||||
# Check health (machine-parseable JSON, no log scraping)
|
||||
gbrain jobs supervisor status --json
|
||||
# → {"running":true,"supervisor_pid":1234,"last_start":"2026-04-23T15:30:22Z","crashes_24h":0, ...}
|
||||
|
||||
# Stop cleanly (SIGTERM + 35s drain + SIGKILL fallback)
|
||||
gbrain jobs supervisor stop
|
||||
```
|
||||
|
||||
Every lifecycle event (spawn, crash, backoff, health warning, max-crashes,
|
||||
shutdown) is also written to `${GBRAIN_AUDIT_DIR:-~/.gbrain/audit}/supervisor-YYYY-Www.jsonl`
|
||||
for historical inspection. `gbrain doctor` reads that file and surfaces
|
||||
a `supervisor` check in its health report.
|
||||
|
||||
## Deployment: systemd
|
||||
|
||||
For long-running Linux VMs with shell access.
|
||||
|
||||
```bash
|
||||
# Create the worker user if it doesn't exist.
|
||||
sudo useradd --system --home "$GBRAIN_WORKSPACE" --shell /usr/sbin/nologin gbrain \
|
||||
2>/dev/null || true
|
||||
sudo mkdir -p "$GBRAIN_WORKSPACE" && sudo chown gbrain:gbrain "$GBRAIN_WORKSPACE"
|
||||
|
||||
# Install the env file (secrets stay out of the unit file).
|
||||
sudo install -m 600 -o gbrain -g gbrain \
|
||||
docs/guides/minions-deployment-snippets/gbrain.env.example /etc/gbrain.env
|
||||
sudoedit /etc/gbrain.env
|
||||
# Fill in DATABASE_URL, optional GBRAIN_ALLOW_SHELL_JOBS=1.
|
||||
|
||||
# Install the unit file, substituting /srv/gbrain → your workspace path.
|
||||
sudo install -m 644 docs/guides/minions-deployment-snippets/systemd.service \
|
||||
/etc/systemd/system/gbrain-worker.service
|
||||
sudo sed -i "s|/srv/gbrain|$GBRAIN_WORKSPACE|g" \
|
||||
/etc/systemd/system/gbrain-worker.service
|
||||
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now gbrain-worker
|
||||
sudo systemctl status gbrain-worker
|
||||
journalctl -u gbrain-worker -n 50
|
||||
```
|
||||
|
||||
The shipped unit file invokes `gbrain jobs supervisor` (not `gbrain jobs work`
|
||||
directly) so you get two-layer supervision: systemd restarts the supervisor
|
||||
on host reboot, supervisor restarts the worker on in-process crash.
|
||||
|
||||
`Restart=always` + `RestartSec=10s` handle the supervisor-level recovery.
|
||||
The unit runs as unprivileged `gbrain` with `PrivateTmp`, `ProtectSystem=strict`,
|
||||
and `ReadWritePaths=$GBRAIN_WORKSPACE,$HOME/.gbrain` (for the PID file and
|
||||
audit log). `LimitNOFILE=65535` covers Bun + Postgres pool + concurrent
|
||||
LLM subagent calls without hitting the default 1024 cap.
|
||||
|
||||
## Deployment: Fly.io
|
||||
|
||||
```bash
|
||||
# Merge the [processes] block from fly.toml.partial into your fly.toml.
|
||||
cat docs/guides/minions-deployment-snippets/fly.toml.partial >> fly.toml
|
||||
# Review + edit as needed.
|
||||
|
||||
# Set secrets (Fly handles restart on crash).
|
||||
fly secrets set DATABASE_URL='postgres://…' GBRAIN_ALLOW_SHELL_JOBS=1
|
||||
```
|
||||
|
||||
The `[processes]` block runs `gbrain jobs supervisor` as PID 1. Fly
|
||||
restarts the container on host failure; the supervisor restarts the
|
||||
worker on in-process crash.
|
||||
|
||||
## Deployment: Render / Railway / Heroku
|
||||
|
||||
Drop [`Procfile`](./minions-deployment-snippets/Procfile) at the repo
|
||||
root. The shipped Procfile calls `gbrain jobs supervisor`. Set
|
||||
`DATABASE_URL` + optional `GBRAIN_ALLOW_SHELL_JOBS=1` via the platform's
|
||||
env UI or CLI.
|
||||
|
||||
## Deployment: inline `--follow` (no persistent worker)
|
||||
|
||||
For short deterministic scripts on a fixed schedule where you don't need
|
||||
a persistent worker between runs. Each cron run brings its own temporary
|
||||
worker. `--follow` starts one on the queue and blocks until the
|
||||
just-submitted job reaches a terminal state (`completed` / `failed` /
|
||||
`dead` / `cancelled`). 2-3 s startup overhead per job; negligible vs job
|
||||
duration for scheduled work.
|
||||
|
||||
```bash
|
||||
GBRAIN_ALLOW_SHELL_JOBS=1 gbrain jobs submit shell \
|
||||
--queue nightly-enrich \
|
||||
--params "{\"cmd\":\"$GBRAIN_BIN embed --stale\",\"cwd\":\"$GBRAIN_WORKSPACE\"}" \
|
||||
--follow \
|
||||
--timeout-ms 600000
|
||||
```
|
||||
|
||||
Replace `gbrain embed --stale` with whichever gbrain subcommand you're
|
||||
scheduling (`sync`, `extract`, `orphans`, `doctor`, `check-backlinks`,
|
||||
`lint`, `autopilot`). For strict single-job semantics on shared queues,
|
||||
use a dedicated queue name like `nightly-enrich` above.
|
||||
|
||||
## Upgrading from an older deployment
|
||||
|
||||
### From `minion-watchdog.sh` (pre-v0.20)
|
||||
|
||||
Earlier versions of this guide shipped a 68-line bash watchdog
|
||||
(`minion-watchdog.sh`). It's been replaced by `gbrain jobs supervisor`
|
||||
which handles everything the script did, plus atomic PID locking,
|
||||
structured audit events, queue-scoped health checks, and graceful
|
||||
drain on SIGTERM.
|
||||
|
||||
**Migration:**
|
||||
|
||||
```bash
|
||||
# 1. Stop and remove the old watchdog.
|
||||
sudo kill $(head -n1 /tmp/gbrain-worker.pid) 2>/dev/null
|
||||
sudo rm -f /usr/local/bin/minion-watchdog.sh /tmp/gbrain-worker.pid \
|
||||
/tmp/gbrain-worker.log
|
||||
crontab -e # delete the "*/5 * * * * /usr/local/bin/minion-watchdog.sh" line
|
||||
|
||||
# 2. Start the supervisor (systemd users: reinstall the unit from
|
||||
# docs/guides/minions-deployment-snippets/systemd.service, which
|
||||
# now calls `gbrain jobs supervisor`).
|
||||
gbrain jobs supervisor start --detach --json
|
||||
# Or: sudo systemctl restart gbrain-worker
|
||||
|
||||
# 3. Verify.
|
||||
gbrain jobs supervisor status --json
|
||||
gbrain doctor # 'supervisor' check should report running=true
|
||||
```
|
||||
|
||||
### Schema / migration hygiene
|
||||
|
||||
Regardless of which deployment path you're upgrading from:
|
||||
|
||||
1. **Stop the worker before upgrading.** `gbrain jobs supervisor stop`
|
||||
(or `sudo systemctl stop gbrain-worker`). Skipping this risks an
|
||||
in-flight job landing partial schema.
|
||||
2. **Run `gbrain upgrade`**. Then `gbrain apply-migrations --yes` if
|
||||
`gbrain doctor` reports any migration as `partial` or `pending`.
|
||||
3. **If you run shell jobs:** from v0.14 onward, pass
|
||||
`--allow-shell-jobs` to the supervisor (or keep
|
||||
`GBRAIN_ALLOW_SHELL_JOBS=1` in `/etc/gbrain.env`). Submitters don't
|
||||
need the flag; only the worker does.
|
||||
4. **Verify.** `gbrain doctor` should report zero `pending` or `partial`
|
||||
migrations plus a healthy `supervisor` check. `gbrain jobs stats`
|
||||
should show no unexplained growth in `dead` between pre- and
|
||||
post-upgrade.
|
||||
|
||||
## Known issues
|
||||
|
||||
### Supabase connection drops
|
||||
|
||||
The worker uses a single Postgres connection. If Supabase drops it
|
||||
(maintenance, connection limits, network blip), lock renewal fails
|
||||
silently. The stall detector then dead-letters the job after
|
||||
`max_stalled` misses.
|
||||
|
||||
**Current defaults that make this worse:**
|
||||
|
||||
- `lockDuration: 30000` (30 s) — too short for long jobs during
|
||||
connection blips.
|
||||
- `max_stalled: 5` (schema column default — see `src/schema.sql` and
|
||||
`src/core/pglite-schema.ts`). Five missed heartbeats before dead-letter.
|
||||
- `stalledInterval: 30000` (30 s) — checks too aggressively.
|
||||
|
||||
**Tune per-job today.** `gbrain jobs submit` accepts `--max-stalled N`,
|
||||
`--backoff-type fixed|exponential`, `--backoff-delay <ms>`,
|
||||
`--backoff-jitter 0..1`, and `--timeout-ms N` as first-class flags
|
||||
(since v0.13.1). These write onto the job row at submit time — which is
|
||||
what `handleStalled()` reads — so per-job tuning is the real knob today.
|
||||
|
||||
### DO NOT pass `maxStalledCount` to `MinionWorker`
|
||||
|
||||
It's a no-op. The stall detector reads the row's `max_stalled` column
|
||||
(set at submit time), not the worker opt in `src/core/minions/worker.ts:74`.
|
||||
Use `gbrain jobs submit --max-stalled N` per-job instead.
|
||||
|
||||
### Zombie shell children
|
||||
|
||||
When the Bun worker crashes hard, child processes from shell jobs can
|
||||
become zombies. The supervisor's SIGTERM → 35s drain → SIGKILL window
|
||||
covers the shell handler's 5 s child-kill grace (`KILL_GRACE_MS`). For
|
||||
long-running shell jobs, prefer timeouts via `--timeout-ms` on submit
|
||||
over relying on hard kills.
|
||||
|
||||
## Smoke test
|
||||
|
||||
```bash
|
||||
# Supervisor alive?
|
||||
gbrain jobs supervisor status --json | jq .running
|
||||
|
||||
# Aggregate queue health.
|
||||
gbrain jobs stats
|
||||
|
||||
# Jobs currently stalled (still `active` with expired lock_until, pre-requeue).
|
||||
gbrain jobs list --status active --limit 10
|
||||
|
||||
# Dead-lettered jobs.
|
||||
gbrain jobs list --status dead --limit 10
|
||||
|
||||
# Shell handler registered? (check supervisor audit log or worker stderr.)
|
||||
gbrain jobs supervisor status --json | jq '.worker_config.allow_shell_jobs'
|
||||
```
|
||||
|
||||
## Uninstall
|
||||
|
||||
**`gbrain jobs supervisor`** (foreground or `--detach`):
|
||||
|
||||
```bash
|
||||
gbrain jobs supervisor stop
|
||||
```
|
||||
|
||||
**systemd:**
|
||||
|
||||
```bash
|
||||
sudo systemctl disable --now gbrain-worker
|
||||
sudo rm /etc/systemd/system/gbrain-worker.service /etc/gbrain.env
|
||||
sudo systemctl daemon-reload
|
||||
```
|
||||
|
||||
**Fly / Render / Railway:** delete the `worker` process from `fly.toml`
|
||||
/ `Procfile` and redeploy. Secrets set via `fly secrets` persist until
|
||||
`fly secrets unset`.
|
||||
|
||||
**Inline `--follow`:** remove the cron entry. Nothing else to clean up
|
||||
— temporary workers exit with their jobs.
|
||||
@@ -1,159 +0,0 @@
|
||||
# Minions fix — repairing a half-migrated install
|
||||
|
||||
**tl;dr:** on v0.11.1+ everything should self-heal. If Minions is partially
|
||||
set up (no `~/.gbrain/preferences.json`, autopilot still inline, cron jobs
|
||||
still on `agentTurn`), run:
|
||||
|
||||
```bash
|
||||
gbrain apply-migrations --yes
|
||||
```
|
||||
|
||||
It's idempotent. On v0.11.1 installs that already migrated it's a cheap
|
||||
no-op.
|
||||
|
||||
## Context
|
||||
|
||||
v0.11.0 shipped the Minions schema, queue, worker, and migration skill —
|
||||
but the migration skill itself never fired on upgrade. `runPostUpgrade`
|
||||
printed the feature pitch and stopped. v0.11.0 was never released
|
||||
publicly; v0.11.1 is the first public Minions ship and fixes the
|
||||
mega-bug (migration fires automatically on `gbrain upgrade` and via
|
||||
the `postinstall` hook).
|
||||
|
||||
If you're on a pre-v0.11.1 branch build (e.g. running the
|
||||
`minions-jobs` branch before v0.11.1 tagged), Minions may be installed
|
||||
but not wired: schema is v7, but no `~/.gbrain/preferences.json`,
|
||||
autopilot still runs inline, cron jobs still call `agentTurn`.
|
||||
|
||||
This guide covers both paths: the canonical v0.11.1+ fix, and the
|
||||
stopgap for pre-v0.11.1 binaries that don't have `apply-migrations`.
|
||||
|
||||
## Detecting the half-migrated state
|
||||
|
||||
```bash
|
||||
gbrain doctor
|
||||
```
|
||||
|
||||
If the install is half-migrated, you'll see:
|
||||
|
||||
```
|
||||
[FAIL] minions_migration: MINIONS HALF-INSTALLED (partial migration: 0.11.0). Run: gbrain apply-migrations --yes
|
||||
```
|
||||
|
||||
or
|
||||
|
||||
```
|
||||
[FAIL] minions_config: MINIONS HALF-INSTALLED (schema v7+ but no ~/.gbrain/preferences.json). Run: gbrain apply-migrations --yes
|
||||
```
|
||||
|
||||
For a machine-readable report (cron-friendly):
|
||||
|
||||
```bash
|
||||
gbrain skillpack-check --quiet && echo healthy || echo needs_action
|
||||
gbrain skillpack-check | jq -r '.actions[]' # prints the exact commands to run
|
||||
```
|
||||
|
||||
## The fix (v0.11.1 or later)
|
||||
|
||||
```bash
|
||||
gbrain apply-migrations --yes
|
||||
```
|
||||
|
||||
Reads `~/.gbrain/migrations/completed.jsonl`, diffs against the TS
|
||||
migration registry, runs whatever's pending. Seven phases:
|
||||
|
||||
```
|
||||
A. Schema gbrain init --migrate-only
|
||||
B. Smoke gbrain jobs smoke
|
||||
C. Mode prompt (or --yes default pain_triggered)
|
||||
D. Prefs write ~/.gbrain/preferences.json
|
||||
E. Host AGENTS.md marker injection + cron rewrites for gbrain
|
||||
builtins; JSONL TODOs for host-specific handlers
|
||||
F. Install gbrain autopilot --install (env-aware)
|
||||
G. Record append completed.jsonl status:"complete"
|
||||
```
|
||||
|
||||
If Phase E emits TODOs for host-specific handlers (e.g. your OpenClaw's
|
||||
~29 non-gbrain crons), the migration finishes with `status: "partial"`.
|
||||
Your host agent walks the TODOs using `skills/migrations/v0.11.0.md` +
|
||||
`docs/guides/plugin-handlers.md`, ships handler registrations in the
|
||||
host repo, then re-runs `gbrain apply-migrations --yes`. Newly
|
||||
registerable cron entries get rewritten and the JSONL rows mark
|
||||
`status: "complete"`.
|
||||
|
||||
## The stopgap (pre-v0.11.1 binary, no apply-migrations yet)
|
||||
|
||||
If you're stuck on a branch build that doesn't have `apply-migrations`:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/garrytan/gbrain/v0.11.1/scripts/fix-v0.11.0.sh | bash
|
||||
```
|
||||
|
||||
This bash script does what apply-migrations does from a shell environment:
|
||||
|
||||
1. `gbrain init --migrate-only` — schema v7.
|
||||
2. `gbrain jobs smoke` — verify Minions health.
|
||||
3. Prompt for `minion_mode` (defaults `pain_triggered` on non-TTY).
|
||||
4. Write `~/.gbrain/preferences.json` atomically.
|
||||
5. Append `~/.gbrain/migrations/completed.jsonl` with `status: "partial"`
|
||||
and `apply_migrations_pending: true`. That partial record is the
|
||||
signal to v0.11.1's `apply-migrations` to pick up remaining phases
|
||||
after the user upgrades.
|
||||
6. Detect host agent repos and PRINT rewrite instructions (never
|
||||
auto-edits from a curl-piped script).
|
||||
7. Print the next step: `Run: gbrain autopilot --install`.
|
||||
|
||||
Once v0.11.1 is installed, re-run `gbrain apply-migrations --yes` to
|
||||
finish the remaining phases (host rewrites + autopilot install). The
|
||||
stopgap's `status: "partial"` record is designed to resume cleanly
|
||||
(it doesn't poison the permanent migration path).
|
||||
|
||||
## Verify the fix landed
|
||||
|
||||
```bash
|
||||
# 1. Preferences exist and are readable
|
||||
cat ~/.gbrain/preferences.json
|
||||
|
||||
# 2. Migration recorded
|
||||
cat ~/.gbrain/migrations/completed.jsonl
|
||||
|
||||
# 3. Autopilot is supervising a Minions worker child
|
||||
gbrain autopilot --status
|
||||
ps aux | grep 'jobs work'
|
||||
|
||||
# 4. Jobs show up in the queue
|
||||
gbrain jobs list
|
||||
|
||||
# 5. Any host-specific TODOs still pending
|
||||
cat ~/.gbrain/migrations/pending-host-work.jsonl 2>/dev/null || echo "(none — all host work is done)"
|
||||
|
||||
# 6. Doctor + skillpack-check should both be clean
|
||||
gbrain doctor
|
||||
gbrain skillpack-check --quiet && echo ok
|
||||
```
|
||||
|
||||
## If the fix fails
|
||||
|
||||
Each phase is idempotent. Re-running is safe. Common failure modes:
|
||||
|
||||
- **Phase B smoke fails:** the schema didn't apply. Check
|
||||
`~/.gbrain/config.json` has a valid `database_url` (or `database_path`
|
||||
for PGLite). Run `gbrain init --migrate-only` directly and look at
|
||||
the error.
|
||||
- **Phase F install fails:** your host environment doesn't match any
|
||||
detected target. Pass `--target <macos|linux-systemd|ephemeral-container|linux-cron>`
|
||||
explicitly.
|
||||
- **Pending host work never clears:** your host agent hasn't shipped
|
||||
handler registrations yet. Read
|
||||
`~/.gbrain/migrations/pending-host-work.jsonl`, open
|
||||
`skills/migrations/v0.11.0.md`, and follow the host-agent instruction
|
||||
manual.
|
||||
|
||||
## Related
|
||||
|
||||
- `skills/migrations/v0.11.0.md` — full migration skill for host agents.
|
||||
- `skills/skillpack-check/SKILL.md` — when and how to run the health check.
|
||||
- `docs/guides/plugin-handlers.md` — plugin contract for host-specific
|
||||
handlers.
|
||||
- `skills/conventions/cron-via-minions.md` — the canonical cron rewrite
|
||||
pattern.
|
||||
@@ -1,167 +0,0 @@
|
||||
# Minions shell jobs — move deterministic crons off the gateway
|
||||
|
||||
## 30 seconds
|
||||
|
||||
```bash
|
||||
# Run your first shell job:
|
||||
GBRAIN_ALLOW_SHELL_JOBS=1 gbrain jobs submit shell \
|
||||
--params '{"cmd":"echo hello","cwd":"/tmp"}' --follow
|
||||
# → exit_code: 0, stdout_tail: "hello\n", duration_ms: 43
|
||||
```
|
||||
|
||||
That's it. Your cron scripts now have a home with retry, backoff, DLQ, and
|
||||
`gbrain jobs list` visibility, without each one booting a full LLM session.
|
||||
|
||||
**PGLite users:** `gbrain jobs work` does not run on PGLite (exclusive file
|
||||
lock). Every crontab invocation must use `--follow` for inline execution.
|
||||
Postgres users can run a persistent worker; see recipes below.
|
||||
|
||||
---
|
||||
|
||||
## Why it exists
|
||||
|
||||
If your agent runs deterministic scripts from cron (token refresh, API fetch,
|
||||
scrape + write), each one pays the cost of a full LLM session on the gateway.
|
||||
Fourteen simultaneous fires on a Series A deployment pin CPU at 100% and block
|
||||
live messages. None of those scripts need reasoning. They need a shell.
|
||||
|
||||
Shell jobs move them to the Minions worker: one deterministic-script execution
|
||||
per cron, zero LLM tokens, unified visibility and retry.
|
||||
|
||||
---
|
||||
|
||||
## Security model (read this)
|
||||
|
||||
Shell exec is a large blast radius. We ship two independent gates, both must
|
||||
pass:
|
||||
|
||||
1. **MCP boundary.** `submit_job` with `name: 'shell'` is rejected when
|
||||
`ctx.remote === true` (MCP callers). Independent of the env flag. Remote
|
||||
agents can never submit shell jobs. `MinionQueue.add('shell', ...)` has its
|
||||
own guard too, so an in-process handler can't programmatically bypass this.
|
||||
2. **Env flag.** The worker only registers the shell handler when
|
||||
`GBRAIN_ALLOW_SHELL_JOBS=1` is set on the worker process. Default: off. Your
|
||||
agent opts in per-host.
|
||||
|
||||
**What the env allowlist does AND does not do.** Shell jobs run with a minimal
|
||||
env: `PATH, HOME, USER, LANG, TZ, NODE_ENV`. Your secrets like `OPENAI_API_KEY`
|
||||
and `DATABASE_URL` are NOT passed to the child. You opt-in additional keys per
|
||||
job via `env: { ... }`. This stops accidental `$OPENAI_API_KEY` interpolation in
|
||||
a user-authored script. It does **not** sandbox filesystem reads: a shell
|
||||
script can `cat ~/.env` or any file the worker process can read. The operator
|
||||
picks a safe `cwd`. That is the trust boundary.
|
||||
|
||||
**Audit trail, not forensic insurance.** Every submission writes a JSONL line
|
||||
to `~/.gbrain/audit/shell-jobs-YYYY-Www.jsonl` (ISO-week rotation; override
|
||||
with `GBRAIN_AUDIT_DIR`). Failures log to stderr and don't block submission, so
|
||||
a disk-full adversary could silently disable the trail. Good for "what did
|
||||
this cron submit last Tuesday", not for security-critical forensics.
|
||||
|
||||
**The command text is logged as-is.** If you embed a secret in `cmd`
|
||||
(`curl -H 'Authorization: Bearer ...'`), it shows up in the audit file. Put
|
||||
secrets in `env:` instead.
|
||||
|
||||
---
|
||||
|
||||
## Migrate a cron
|
||||
|
||||
### Postgres worker (recommended)
|
||||
|
||||
On one terminal, start a persistent worker:
|
||||
|
||||
```bash
|
||||
GBRAIN_ALLOW_SHELL_JOBS=1 gbrain jobs work
|
||||
```
|
||||
|
||||
Rewrite crontab to submit shell jobs (no `--follow`):
|
||||
|
||||
```cron
|
||||
# Before (LLM gateway):
|
||||
# OpenClaw cron: x-garrytan-unified
|
||||
# After (Minions worker):
|
||||
3 13,16,19,22,1,4,7,10 * * * \
|
||||
gbrain jobs submit shell \
|
||||
--params '{"cmd":"node scripts/x-garrytan-daily.mjs","cwd":"/data/.openclaw/workspace"}' \
|
||||
--max-attempts 3 --timeout-ms 300000
|
||||
```
|
||||
|
||||
Worker claims the job on next poll, runs it, records `exit_code` +
|
||||
`stdout_tail` + `stderr_tail` in the result. Failures retry per
|
||||
`--max-attempts` with exponential backoff.
|
||||
|
||||
### PGLite (inline execution)
|
||||
|
||||
PGLite doesn't support the persistent worker daemon. Every crontab invocation
|
||||
uses `--follow` to run inline:
|
||||
|
||||
```cron
|
||||
# Each cron tick spawns a short-lived worker that runs the job inline.
|
||||
3 13,16,19,22,1,4,7,10 * * * \
|
||||
GBRAIN_ALLOW_SHELL_JOBS=1 gbrain jobs submit shell \
|
||||
--params '{"cmd":"node scripts/x-garrytan-daily.mjs","cwd":"/data/.openclaw/workspace"}' \
|
||||
--follow --timeout-ms 300000
|
||||
```
|
||||
|
||||
Note: `--follow` blocks the crontab slot until the job finishes. If 14 shell
|
||||
crons land at the same minute and each takes 30s, they serialize through
|
||||
crontab's spawning limits. Postgres + persistent worker scales better.
|
||||
|
||||
### Submitting with `argv` (no shell interpolation)
|
||||
|
||||
For programmatic callers assembling commands from JSON, use `argv` instead of
|
||||
`cmd`. No shell, no injection surface:
|
||||
|
||||
```bash
|
||||
gbrain jobs submit shell \
|
||||
--params '{"argv":["node","scripts/fetch.mjs","--date","2026-04-19"],"cwd":"/data"}' \
|
||||
--follow
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Debug a failed job
|
||||
|
||||
```bash
|
||||
# List dead shell jobs
|
||||
gbrain jobs list --status dead
|
||||
|
||||
# Inspect one
|
||||
gbrain jobs get 42
|
||||
# → error_text, stacktrace, result.stdout_tail, result.stderr_tail
|
||||
|
||||
# Submission audit log (operator trail, not forensic)
|
||||
cat ~/.gbrain/audit/shell-jobs-*.jsonl | jq '.'
|
||||
|
||||
# First-time failure mode: submitted without env flag on the worker
|
||||
gbrain jobs list --status waiting --name shell
|
||||
# If rows pile up here, no worker with GBRAIN_ALLOW_SHELL_JOBS=1 is running.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Limitations
|
||||
|
||||
- **Filesystem reads are not sandboxed.** See "Security model" above. Don't
|
||||
point `cwd` at a directory full of secrets.
|
||||
- **Audit log is advisory.** Disk-full or EACCES silently disables it.
|
||||
- **Cancel latency is lock-renewal-bounded** (~7-15 s by default). A cancelled
|
||||
child keeps running until the next lock-renewal tick fails.
|
||||
- **`--follow` claim order** is by priority/created_at. If another job is
|
||||
waiting in the same queue at the time of `--follow`, that one runs first.
|
||||
- **`cwd` symlink TOCTOU.** The absolute-path check doesn't guard against
|
||||
symlinks pointing elsewhere at execution time. Operator-scope concern.
|
||||
|
||||
---
|
||||
|
||||
## Errors {#errors}
|
||||
|
||||
| Error | What it means | Fix |
|
||||
|---|---|---|
|
||||
| `shell: specify exactly one of cmd or argv` | `cmd` and `argv` are mutually exclusive. Both absent is also invalid. | Choose one. `cmd` for shell-interpolated strings; `argv` for structured args. |
|
||||
| `shell: cwd is required and must be an absolute path` | `cwd` must be a string starting with `/`. | Set `cwd` in `--params` to an absolute path. |
|
||||
| `shell: argv must be an array of strings` | `argv` has a non-string entry or isn't an array. | Pass `argv: ["bin","arg1","arg2"]`. |
|
||||
| `shell: env values must all be strings` | `env` has a number/bool/object value. | Stringify: `"env":{"COUNT":"3"}` not `"env":{"COUNT":3}`. |
|
||||
| `permission_denied: shell jobs cannot be submitted over MCP` | An MCP client tried to submit a shell job. By design CLI-only. | Submit from CLI or via a trusted operation handler (`ctx.remote === false`). |
|
||||
| `protected job name 'shell' requires CLI or operation-local submitter` | A caller invoked `MinionQueue.add('shell', ...)` without the `trusted` opt-in. | Pass `{ allowProtectedSubmit: true }` as the 4th arg. CLI and `submit_job` do this automatically. |
|
||||
| `aborted: timeout` / `aborted: cancel` / `aborted: shutdown` / `aborted: lock-lost` | The worker's abort signal fired mid-execution. Child got SIGTERM, 5s grace, then SIGKILL. | Expected: timeout / user cancel / deploy restart / stall. Inspect `gbrain jobs get` to see which. |
|
||||
| `exit N: <stderr_tail_500>` | Script exited non-zero. | Read `stderr_tail` in `gbrain jobs get`. |
|
||||
@@ -1,182 +0,0 @@
|
||||
# Multi-source brains
|
||||
|
||||
**A single gbrain database can hold multiple knowledge repos.** Each one
|
||||
is a `source`: a logical brain-within-the-brain with its own slug
|
||||
namespace, its own sync state, and its own federation policy. The rest
|
||||
of this guide walks the three canonical scenarios.
|
||||
|
||||
## The three scenarios
|
||||
|
||||
### 1. Unified knowledge recall (wiki + gstack)
|
||||
|
||||
You have a personal wiki and a `gstack` checkout. Both belong to you,
|
||||
both are knowledge you want your agent to recall across. When you ask
|
||||
"what did I learn about X?" you want the best hit whether it lives in
|
||||
the wiki or in a gstack plan.
|
||||
|
||||
```bash
|
||||
# Register the gstack source, federate so it joins cross-source search
|
||||
gbrain sources add gstack --path ~/.gstack --federated
|
||||
|
||||
# Pin the directory so `gbrain sync` knows which source it's walking
|
||||
cd ~/.gstack && gbrain sources attach gstack
|
||||
|
||||
# Initial sync
|
||||
gbrain sync --source gstack
|
||||
|
||||
# Now `gbrain search "retry budgets"` returns hits from BOTH wiki and
|
||||
# gstack. Each result includes source_id so the agent can cite properly.
|
||||
```
|
||||
|
||||
Result: wiki pages and gstack plans are separate (different source_ids,
|
||||
different slug namespaces) but share the search surface.
|
||||
|
||||
### 2. Purpose-separated brains (yc-media + garrys-list)
|
||||
|
||||
You run two completely different content pipelines on the same backend.
|
||||
YC Media covers portfolio news and founder profiles. Garry's List is
|
||||
personal writing. You explicitly DON'T want them mixed in search — YC
|
||||
portfolio content leaking into essay searches is a bug, not a feature.
|
||||
|
||||
```bash
|
||||
# Two sources, both isolated (federated=false)
|
||||
gbrain sources add yc-media --path ~/yc-media --no-federated
|
||||
gbrain sources add garrys-list --path ~/writing --no-federated
|
||||
|
||||
# Pin each checkout directory
|
||||
(cd ~/yc-media && gbrain sources attach yc-media)
|
||||
(cd ~/writing && gbrain sources attach garrys-list)
|
||||
|
||||
# Sync each independently
|
||||
gbrain sync --source yc-media
|
||||
gbrain sync --source garrys-list
|
||||
```
|
||||
|
||||
Result: searching from neither directory returns the `default` source
|
||||
(your main brain). Searching from inside `~/yc-media` returns only yc-
|
||||
media hits. Searching from inside `~/writing` returns only garrys-list.
|
||||
Federation is opt-in, not leaked.
|
||||
|
||||
To search across them explicitly on demand:
|
||||
|
||||
```bash
|
||||
gbrain search "tech layoffs" --source yc-media,garrys-list
|
||||
```
|
||||
|
||||
### 3. Mixed (wiki federated + sessions isolated)
|
||||
|
||||
Your main wiki is federated with a few trusted sources. Your session
|
||||
transcripts (coming in v0.18) land in a separate isolated source so
|
||||
they don't dominate every search result.
|
||||
|
||||
```bash
|
||||
# Federated sources
|
||||
gbrain sources add gstack --path ~/.gstack --federated
|
||||
|
||||
# Isolated source (future v0.18 — sessions use this shape today for ingest)
|
||||
gbrain sources add sessions --path ~/.claude/sessions --no-federated
|
||||
```
|
||||
|
||||
## Resolution priority
|
||||
|
||||
When any command needs to pick a source, gbrain walks this list (highest
|
||||
first):
|
||||
|
||||
1. Explicit `--source <id>` flag.
|
||||
2. `GBRAIN_SOURCE` environment variable.
|
||||
3. `.gbrain-source` dotfile in CWD or any ancestor directory.
|
||||
4. A registered source whose `local_path` contains the CWD (longest
|
||||
prefix wins for nested checkouts).
|
||||
5. The brain-level default set via `gbrain sources default <id>`.
|
||||
6. The seeded `default` source.
|
||||
|
||||
So inside `~/.gstack/plans/` on a brain that pinned `gstack` to
|
||||
`~/.gstack` via `.gbrain-source`, `gbrain put-page` implicitly writes to
|
||||
the `gstack` source. Outside any registered directory with no env/dotfile
|
||||
set, it writes to the default.
|
||||
|
||||
## Federation flag
|
||||
|
||||
Every source row stores `config.federated: boolean` in its JSONB config.
|
||||
|
||||
| Value | Meaning |
|
||||
|-------|---------|
|
||||
| `true` | Source participates in unqualified `gbrain search "X"` results. |
|
||||
| `false` (default for new sources) | Source only searched when explicitly named via `--source <id>` or qualified citation. |
|
||||
|
||||
The seeded `default` source is `federated=true` so pre-v0.17 brains
|
||||
behave exactly as before — every page appears in search.
|
||||
|
||||
Flip later with `gbrain sources federate <id>` / `unfederate <id>`.
|
||||
|
||||
## Commands
|
||||
|
||||
Full subcommand reference:
|
||||
|
||||
```
|
||||
gbrain sources add <id> --path <p> [--name <n>] [--federated|--no-federated]
|
||||
Register a source. id: [a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?
|
||||
gbrain sources list [--json] List all sources with page counts + federation state.
|
||||
gbrain sources remove <id> [--yes] [--dry-run] [--keep-storage]
|
||||
Cascade-delete a source (pages, chunks, timeline).
|
||||
gbrain sources rename <id> <new-name>
|
||||
Change display name only; id is immutable.
|
||||
gbrain sources default <id> Set the brain-level default.
|
||||
gbrain sources attach <id> Write .gbrain-source in CWD (like kubectl context).
|
||||
gbrain sources detach Remove .gbrain-source from CWD.
|
||||
gbrain sources federate <id>
|
||||
gbrain sources unfederate <id>
|
||||
```
|
||||
|
||||
## Citation format for agents
|
||||
|
||||
When agents receive multi-source results they MUST cite pages in
|
||||
`[source-id:slug]` form. Example:
|
||||
|
||||
> You told me about the distillation protocol — see [wiki:topics/ai]
|
||||
> and [gstack:plans/multi-repo] for where this came from.
|
||||
|
||||
The citation key is `sources.id` (immutable). Renaming a source via
|
||||
`gbrain sources rename` changes the display name only; existing
|
||||
citations keep working.
|
||||
|
||||
## Writing to a specific source
|
||||
|
||||
```bash
|
||||
# Pass --source explicitly
|
||||
gbrain put-page topics/ai ... --source wiki
|
||||
|
||||
# Or rely on the dotfile / env / CWD match
|
||||
cd ~/.gstack && gbrain put-page plans/multi-repo ...
|
||||
# → source auto-resolves to gstack
|
||||
```
|
||||
|
||||
Reads span federated sources by default. Writes require a resolved
|
||||
source (explicit, inferred, or default). The resolver never picks a
|
||||
source silently when ambiguous — it errors with a clear fix.
|
||||
|
||||
## Upgrading an existing brain
|
||||
|
||||
`gbrain upgrade` runs the v16 + v17 migrations automatically. Your
|
||||
existing pages all move under `source_id='default'`. Behavior is
|
||||
unchanged until you add a second source.
|
||||
|
||||
To add one:
|
||||
|
||||
```bash
|
||||
gbrain sources add gstack --path ~/.gstack --federated
|
||||
cd ~/.gstack && gbrain sources attach gstack && gbrain sync
|
||||
```
|
||||
|
||||
Two commands. The existing default source is untouched.
|
||||
|
||||
## Not in v0.18.0
|
||||
|
||||
- Session transcript ingest (`.jsonl`, raised size cap, session
|
||||
PageType) — v0.18.
|
||||
- Per-source retention/TTL (`gbrain sources prune`) — v0.18.
|
||||
- ACL enforcement via caller-identity — v0.17.1.
|
||||
- `gbrain sources import-from-github <url>` one-shot bootstrap — patch
|
||||
release after the core plumbing stabilizes.
|
||||
|
||||
All of these build on the `sources` primitive shipped here.
|
||||
@@ -1,120 +0,0 @@
|
||||
# Operational Disciplines
|
||||
|
||||
## Goal
|
||||
Five non-negotiable rules that separate a production brain from a demo -- signal detection, brain-first lookup, sync after every write, daily heartbeat, and nightly dream cycle.
|
||||
|
||||
## What the User Gets
|
||||
Without this: the agent misses signals in conversation, wastes money on external APIs when the brain already has the answer, leaves search results stale after writes, and lets the brain rot quietly. With this: every message is scanned for entities, the brain is always consulted first, search is always current, health is monitored daily, and the brain compounds overnight.
|
||||
|
||||
## Implementation
|
||||
|
||||
```
|
||||
# DISCIPLINE 1: Signal Detection on Every Message (MANDATORY)
|
||||
on every_inbound_message(message):
|
||||
# No exceptions. If the user thinks out loud and the brain doesn't
|
||||
# capture it, the system is broken. This is the #1 discipline.
|
||||
|
||||
entities = detect_entities(message)
|
||||
# people, companies, deals, original ideas
|
||||
|
||||
for entity in entities:
|
||||
existing = gbrain search "{entity.name}"
|
||||
if existing:
|
||||
gbrain add_timeline_entry <entity_slug> \
|
||||
--entry "{what_was_said}" \
|
||||
--source "User, direct message, {timestamp}"
|
||||
# else: flag for enrichment if important enough
|
||||
|
||||
originals = detect_original_thinking(message)
|
||||
for idea in originals:
|
||||
gbrain put originals/{slug} --content "{user's exact phrasing}"
|
||||
|
||||
# DISCIPLINE 2: Brain-First Lookup Before External APIs (MANDATORY)
|
||||
on information_needed(topic):
|
||||
# ALWAYS check the brain before reaching for the web
|
||||
brain_result = gbrain search "{topic}"
|
||||
if brain_result:
|
||||
page = gbrain get <slug>
|
||||
# Use brain data first. External APIs FILL GAPS, not replace.
|
||||
else:
|
||||
# Brain has nothing -- now use external APIs
|
||||
external_result = brave_search("{topic}")
|
||||
|
||||
# An agent that reaches for the web before checking its own brain
|
||||
# is wasting money and giving worse answers.
|
||||
|
||||
# DISCIPLINE 3: Sync After Every Write (MANDATORY)
|
||||
on brain_write_complete():
|
||||
gbrain sync
|
||||
# Without this, search results are stale.
|
||||
# The page you just wrote won't appear in gbrain search or gbrain query
|
||||
# until sync runs. Skipping this means the next lookup misses the
|
||||
# most recent data.
|
||||
|
||||
# DISCIPLINE 4: Daily Heartbeat Check
|
||||
on daily_schedule("09:00"):
|
||||
gbrain doctor
|
||||
# Checks: database connectivity, embedding health, sync status,
|
||||
# page count, stale pages, broken links
|
||||
# If doctor reports issues, fix them before doing anything else.
|
||||
|
||||
# DISCIPLINE 5: Nightly Dream Cycle
|
||||
on nightly_schedule("02:00"):
|
||||
# The dream cycle is the most important discipline.
|
||||
# The brain COMPOUNDS overnight.
|
||||
|
||||
# 5a: Entity sweep -- find unlinked mentions
|
||||
pages = gbrain list_pages
|
||||
for page in pages:
|
||||
mentions = extract_entity_mentions(page.content)
|
||||
existing_links = gbrain get_links <page.slug>
|
||||
for mention in mentions:
|
||||
if mention not in existing_links:
|
||||
gbrain add_link <page.slug> <mention_slug> # fix broken graph
|
||||
|
||||
# 5b: Citation audit -- find facts without sources
|
||||
for page in pages:
|
||||
facts_without_sources = audit_citations(page.content)
|
||||
if facts_without_sources:
|
||||
flag_for_remediation(page, facts_without_sources)
|
||||
|
||||
# 5c: Memory consolidation -- update compiled truth from timeline
|
||||
for page in stale_pages(older_than="7d"):
|
||||
timeline = gbrain get_timeline <page.slug>
|
||||
if timeline.has_new_entries_since_last_consolidation:
|
||||
# Re-synthesize compiled truth from accumulated timeline
|
||||
updated_truth = consolidate(page.compiled_truth, timeline.new_entries)
|
||||
gbrain put <page.slug> --content updated_truth
|
||||
|
||||
# 5d: Sync everything
|
||||
gbrain sync
|
||||
|
||||
# BONUS: Durable Skills Over One-Off Work
|
||||
# If you do something twice, make it a skill + cron.
|
||||
# 1. Concept the process
|
||||
# 2. Run it manually for 3-10 items
|
||||
# 3. Revise -- iterate on quality
|
||||
# 4. Codify into a skill
|
||||
# 5. Add to cron -- automate it
|
||||
# Each entity type and signal source has exactly one owner skill.
|
||||
# Two skills creating the same page = coverage violation.
|
||||
```
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **The dream cycle is the most important discipline.** Brains compound overnight. Entity sweeps fix broken graphs, citation audits catch sourceless facts, and memory consolidation keeps compiled truth current. Skip the dream cycle and the brain slowly rots.
|
||||
2. **Skipping Discipline 3 (sync after write) means stale search results.** You write a page, then immediately search for it -- and get nothing back. The page exists but isn't indexed. Always sync after writes.
|
||||
3. **Signal detection must fire on EVERY message.** Not just messages that look important. The user says "I talked to Pedro yesterday about the board seat" in passing -- that's a timeline entry on Pedro's page, a potential update to his State section, and a signal about the board. If the agent doesn't catch it, the system is broken.
|
||||
4. **Brain-first saves money AND gives better answers.** The brain has context that external APIs don't: relationship history, meeting notes, the user's own assessment. An API lookup for "Pedro Franceschi" returns a LinkedIn profile. The brain returns the full picture including private context.
|
||||
5. **`gbrain doctor` catches silent failures.** Embedding pipelines can stall, sync can fail silently, database connections can drop. The daily heartbeat catches these before they compound into data loss.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. Send a message mentioning a person with a brain page. Confirm the agent detects the entity and adds a timeline entry to their page (`gbrain get_timeline <slug>`).
|
||||
2. Ask the agent about someone in the brain. Confirm it runs `gbrain search` or `gbrain get` BEFORE reaching for external APIs (check the tool call order).
|
||||
3. Write a new page with `gbrain put`, then immediately run `gbrain search` for it. Confirm it appears in results (verifies sync ran).
|
||||
4. Run `gbrain doctor`. Confirm it returns a health report with database status, page count, and any flagged issues.
|
||||
5. After a dream cycle runs, check a page that had unlinked entity mentions. Confirm new links were added (`gbrain get_links <slug>`).
|
||||
|
||||
---
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,87 +0,0 @@
|
||||
# The Originals Folder
|
||||
|
||||
## Goal
|
||||
Capture the user's original thinking with their exact phrasing, deep cross-links, and full provenance -- so intellectual capital compounds instead of evaporating.
|
||||
|
||||
## What the User Gets
|
||||
Without this: the user generates a brilliant framework in conversation and it vanishes when the session ends. Six months later, they vaguely remember the idea but can't find it, can't recall the exact phrasing, and can't trace what influenced it. With this: every original observation, thesis, framework, and hot take is captured verbatim in `brain/originals/`, cross-linked to the people, companies, and media that shaped it, and searchable forever.
|
||||
|
||||
## Implementation
|
||||
|
||||
```
|
||||
on user_message(message):
|
||||
# Detect original thinking in every message
|
||||
if contains_original_thinking(message):
|
||||
# The authorship test:
|
||||
# User generated the idea? -> originals/{slug}.md
|
||||
# User's unique synthesis of someone else's? -> originals/ (synthesis IS original)
|
||||
# World concept someone else coined? -> concepts/{slug}.md
|
||||
# Product or business idea? -> ideas/{slug}.md
|
||||
|
||||
# Step 1: Use the user's EXACT phrasing for the slug
|
||||
# "meatsuit-maintenance-tax"
|
||||
# NOT "biological-needs-maintenance-overhead"
|
||||
# The vividness IS the concept.
|
||||
slug = slugify(user_exact_phrase)
|
||||
|
||||
# Step 2: Create the originals page
|
||||
gbrain put originals/{slug} --content """
|
||||
# {User's Exact Phrase}
|
||||
|
||||
## The Idea
|
||||
{User's original thinking, captured in their own words.
|
||||
Do NOT paraphrase. Do NOT clean up the language.
|
||||
The raw phrasing is the intellectual artifact.}
|
||||
|
||||
## Context
|
||||
{What triggered this thinking. Meeting? Article? Conversation?
|
||||
Include the source that sparked it.}
|
||||
[Source: User, {context}, {date} {time} {tz}]
|
||||
|
||||
## Connections
|
||||
- Related to: [[{person_slug}]] -- {how they connect}
|
||||
- Emerged from: [[{meeting_slug}]] -- {what was discussed}
|
||||
- Influenced by: [[{book_or_media_slug}]] -- {what resonated}
|
||||
- Builds on: [[{other_original_slug}]] -- {how ideas cluster}
|
||||
"""
|
||||
|
||||
# Step 3: Cross-link to everything that shaped the thinking
|
||||
for entity in idea.influences:
|
||||
gbrain add_link originals/{slug} <entity_slug>
|
||||
gbrain add_link <entity_slug> originals/{slug}
|
||||
|
||||
# Step 4: Sync
|
||||
gbrain sync
|
||||
|
||||
# What counts as original thinking:
|
||||
# - Novel frameworks ("the meatsuit maintenance tax")
|
||||
# - Hot takes on someone else's work (synthesis IS original)
|
||||
# - Pattern recognition across multiple entities
|
||||
# - Predictions or bets about the future
|
||||
# - Contrarian positions with reasoning
|
||||
|
||||
# What does NOT go in originals/:
|
||||
# - Facts about the world (-> entity pages)
|
||||
# - Concepts someone else coined (-> concepts/)
|
||||
# - Product ideas (-> ideas/)
|
||||
# - Preferences (-> agent memory)
|
||||
```
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Naming: the vividness IS the concept.** `meatsuit-maintenance-tax` not `biological-needs-maintenance-overhead`. `ambition-debt` not `deferred-career-risk-accumulation`. The user's colorful phrasing is the intellectual artifact. Never sanitize it into corporate-speak.
|
||||
2. **Synthesis IS original.** The user's take on Peter Thiel's zero-to-one framework goes in `originals/`, not `concepts/`. The original part is the user's synthesis, interpretation, or disagreement -- even though the underlying ideas came from someone else.
|
||||
3. **An original without cross-links is a dead original.** The connections ARE the intelligence. An idea about "ambition debt" that doesn't link to the people who exemplify it, the meeting where it was discussed, and the book that influenced it is just a note in a graveyard. Cross-link aggressively.
|
||||
4. **Originals form clusters.** Over time, the user's ideas connect to each other. "Meatsuit maintenance tax" connects to "ambition debt" connects to "founder energy budget." Link originals to other originals. The cluster IS the user's worldview.
|
||||
5. **Capture the trigger context.** What conversation, meeting, article, or moment sparked this idea? The context often matters as much as the idea itself for future retrieval. Include it in the page.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. Generate an original idea in conversation (e.g., "I call this the 'ambition debt' problem -- every year you delay going big, the compound interest works against you"). Confirm a new page appears at `brain/originals/ambition-debt` with `gbrain get originals/ambition-debt`.
|
||||
2. Check that the page uses the user's exact phrasing for the title and slug -- not a sanitized version.
|
||||
3. Run `gbrain get_links originals/ambition-debt`. Confirm cross-links exist to related people, meetings, or other originals.
|
||||
4. Express a take on someone else's idea (e.g., "I think Thiel's contrarian question is wrong because..."). Confirm it goes to `originals/` (synthesis is original), not `concepts/`.
|
||||
5. Run `gbrain search "ambition debt"`. Confirm the originals page appears in search results and is discoverable.
|
||||
|
||||
---
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,163 +0,0 @@
|
||||
# Plugin authors guide (v0.15)
|
||||
|
||||
`gbrain` discovers subagent definitions from outside this repo via
|
||||
`GBRAIN_PLUGIN_PATH`. If you maintain a downstream agent (your OpenClaw
|
||||
deployment, a workflow host, a private tool) and want to ship custom
|
||||
subagents alongside it, drop a plugin directory on that env path.
|
||||
|
||||
This guide is for plugin authors. The CLI user doesn't need to read it.
|
||||
|
||||
## Minimum viable plugin
|
||||
|
||||
```
|
||||
/path/to/my-plugin/
|
||||
├── gbrain.plugin.json
|
||||
└── subagents/
|
||||
└── my-summarizer.md
|
||||
```
|
||||
|
||||
`gbrain.plugin.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "my-plugin",
|
||||
"version": "1.0.0",
|
||||
"plugin_version": "gbrain-plugin-v1"
|
||||
}
|
||||
```
|
||||
|
||||
`subagents/my-summarizer.md`:
|
||||
|
||||
```markdown
|
||||
---
|
||||
name: my-summarizer
|
||||
model: claude-sonnet-4-6
|
||||
allowed_tools:
|
||||
- brain_search
|
||||
- brain_get_page
|
||||
---
|
||||
|
||||
You are a brain page summarizer. Given a slug, fetch the page and produce
|
||||
a 3-sentence summary.
|
||||
```
|
||||
|
||||
## Turning it on
|
||||
|
||||
```bash
|
||||
export GBRAIN_PLUGIN_PATH="/path/to/my-plugin"
|
||||
gbrain jobs work # worker startup prints the plugin load line
|
||||
gbrain agent run "summarize meetings/2026-04-20" --subagent-def my-summarizer
|
||||
```
|
||||
|
||||
Multiple plugins: colon-separated, just like `$PATH`.
|
||||
|
||||
```bash
|
||||
export GBRAIN_PLUGIN_PATH="/path/to/plugin-a:/path/to/plugin-b"
|
||||
```
|
||||
|
||||
## Rules (strict by design)
|
||||
|
||||
**Path policy.** Absolute paths only. Relative paths, `~`-prefixed paths,
|
||||
and URL-style paths (`https://`, `file://`) are rejected with a warning.
|
||||
You control where your plugin lives on disk; `gbrain` doesn't guess.
|
||||
|
||||
**Collision policy.** If two plugins ship a subagent with the same `name`,
|
||||
the one listed FIRST in `GBRAIN_PLUGIN_PATH` wins. The other is dropped
|
||||
with a warning naming both sources.
|
||||
|
||||
**Trust policy.** Plugins ship subagent definitions ONLY in v0.15:
|
||||
|
||||
- You **cannot** declare new tools.
|
||||
- You **cannot** extend the brain tool allow-list.
|
||||
- You **cannot** override any `agentSafe` or similar flag.
|
||||
- Your `allowed_tools:` frontmatter field MUST subset the derived brain
|
||||
tool registry. Names not in the registry are rejected at plugin load
|
||||
time (worker startup), NOT at subagent dispatch time — so a typo in
|
||||
your plugin gives you a loud startup error, not a silent "tool never
|
||||
fires" at 3am.
|
||||
|
||||
v0.16+ may open up plugin-declared tools with a separate contract. Don't
|
||||
expect it.
|
||||
|
||||
## `gbrain.plugin.json`
|
||||
|
||||
| field | type | required | notes |
|
||||
|------------------|--------|----------|--------------------------------------------------------------------|
|
||||
| `name` | string | yes | Human-readable plugin id. Shows up in warnings and collision logs. |
|
||||
| `version` | string | yes | Your plugin's semver. Informational. |
|
||||
| `plugin_version` | string | yes | Contract lock. Must equal `"gbrain-plugin-v1"` for v0.15. |
|
||||
| `subagents` | string | no | Subdir name (default `subagents`). Escape-attempts are rejected. |
|
||||
| `description` | string | no | Shown in future `gbrain plugin list`. |
|
||||
|
||||
## Subagent definition files
|
||||
|
||||
Plain markdown with YAML frontmatter. The body is the system prompt. The
|
||||
frontmatter controls runtime behavior.
|
||||
|
||||
Recognized frontmatter fields:
|
||||
|
||||
| field | type | required | notes |
|
||||
|-----------------|----------|----------|-----------------------------------------------------------------------------------------|
|
||||
| `name` | string | no | Subagent identifier used as `--subagent-def`. Defaults to the file basename. |
|
||||
| `model` | string | no | Anthropic model id. Defaults to the handler default (sonnet). |
|
||||
| `max_turns` | number | no | Cap on assistant turns. Defaults to 20. |
|
||||
| `allowed_tools` | string[] | no | Whitelist of tool names. Must subset the derived brain registry. Rejected on mismatch. |
|
||||
|
||||
Unknown frontmatter fields are preserved but ignored by the handler. v0.16
|
||||
may consume more of them.
|
||||
|
||||
## Caveats that will bite you
|
||||
|
||||
1. **Plugin definitions can't change during a run.** The loader reads the
|
||||
disk once at worker startup. Editing a subagent def doesn't re-take
|
||||
effect until you restart the worker. This is deliberate — live
|
||||
reloads would break crash-resumable replay.
|
||||
|
||||
2. **`~/.gbrain/audit/subagent-jobs-*.jsonl` is local only.** If your
|
||||
worker runs on a different host than the `gbrain agent logs` caller,
|
||||
the CLI won't see heartbeats from that worker. v0.16 will unify this;
|
||||
for now assume worker + CLI share a filesystem.
|
||||
|
||||
3. **Tool calls always run with `ctx.remote = true`.** Even on local CLI
|
||||
invocation. Tools that gate on `remote=true` (file_upload's strict
|
||||
confinement, put_page's namespace check) will apply. Good default; a
|
||||
subagent definition that wants local-filesystem reach beyond the brain
|
||||
can't have it.
|
||||
|
||||
4. **`put_page` writes are namespace-scoped.** A subagent with id 42 can
|
||||
only write under `wiki/agents/42/...`. This is enforced both in the
|
||||
tool schema (the slug pattern shown to the model) AND server-side in
|
||||
the `put_page` operation (fail-closed if `viaSubagent=true`). Don't
|
||||
try to route around it; you'll get `permission_denied`.
|
||||
|
||||
## Example: a downstream-OpenClaw plugin
|
||||
|
||||
```
|
||||
~/your-openclaw/
|
||||
└── gbrain-plugin/
|
||||
├── gbrain.plugin.json
|
||||
└── subagents/
|
||||
├── meeting-ingestion.md
|
||||
├── signal-detector.md
|
||||
└── daily-task-prep.md
|
||||
```
|
||||
|
||||
`~/your-openclaw/gbrain-plugin/gbrain.plugin.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "your-openclaw",
|
||||
"version": "2026.4.20",
|
||||
"plugin_version": "gbrain-plugin-v1",
|
||||
"description": "Your OpenClaw's personal-brain subagents"
|
||||
}
|
||||
```
|
||||
|
||||
Environment:
|
||||
|
||||
```bash
|
||||
export GBRAIN_PLUGIN_PATH="$HOME/your-openclaw/gbrain-plugin"
|
||||
```
|
||||
|
||||
Then your OpenClaw calls `gbrain agent run --subagent-def meeting-ingestion
|
||||
--fanout-by transcript ...` and its definitions load automatically.
|
||||
@@ -1,137 +0,0 @@
|
||||
# Plugin handlers — registering host-specific Minion handlers
|
||||
|
||||
GBrain's Minion worker ships with seven built-in handlers: `sync`,
|
||||
`embed`, `lint`, `import`, `extract`, `backlinks`, `autopilot-cycle`.
|
||||
These cover every background operation the gbrain CLI itself performs.
|
||||
|
||||
Host platforms (OpenClaw deployments, future hosts) register their own
|
||||
handlers via a plugin bootstrap that imports
|
||||
`gbrain/minions`. No `handlers.json`-style data file — handlers are
|
||||
code, loaded by the worker, with the same trust model as any other
|
||||
code in the host's repo.
|
||||
|
||||
## Why code, not data
|
||||
|
||||
An earlier design draft shipped `~/.claude/gbrain-handlers.json` where
|
||||
each entry was a shell command the worker would exec on job claim.
|
||||
Codex flagged this as a durable RCE surface: an agent-writable data
|
||||
file that spawns arbitrary shell. We dropped the data-file approach;
|
||||
handlers are code that the host imports explicitly and ships through
|
||||
code review.
|
||||
|
||||
## The plugin contract
|
||||
|
||||
A host worker bootstrap looks like this (TypeScript):
|
||||
|
||||
```ts
|
||||
import { MinionQueue, MinionWorker } from 'gbrain/minions';
|
||||
import type { BrainEngine } from 'gbrain/engine';
|
||||
|
||||
async function main() {
|
||||
const engine: BrainEngine = /* your engine setup */;
|
||||
await engine.connect({});
|
||||
|
||||
const worker = new MinionWorker(engine, { queue: 'default' });
|
||||
|
||||
// Register every host-specific handler the host's cron manifest references.
|
||||
// Each handler returns a plain object (serialized as the job result).
|
||||
// Throw on failure — the worker catches and retries per max_attempts.
|
||||
|
||||
worker.register('ea-inbox-sweep', async (ctx) => {
|
||||
const slot = ctx.data.slot ?? new Date().toISOString();
|
||||
// Host-specific agent turn: call your LLM, scan the inbox, write
|
||||
// brain pages, return a summary. ctx.signal.aborted indicates the
|
||||
// worker wants you to cooperate with shutdown — honor it.
|
||||
return { swept: true, slot };
|
||||
});
|
||||
|
||||
worker.register('morning-briefing', async (ctx) => {
|
||||
/* host logic */
|
||||
return { briefed: true };
|
||||
});
|
||||
|
||||
// Call start() AFTER every handler is registered. The worker's
|
||||
// stall-detector ignores jobs whose name is not in the registered set.
|
||||
await worker.start();
|
||||
}
|
||||
|
||||
main().catch(err => { console.error(err); process.exit(1); });
|
||||
```
|
||||
|
||||
Ship this as a separate binary in the host repo (e.g. `your-openclaw-worker`)
|
||||
or as a side-effect module that the stock `gbrain jobs work` command
|
||||
auto-loads on startup (configurable via a host-provided entry point).
|
||||
|
||||
## Handler contract
|
||||
|
||||
Every handler receives a `MinionJobContext`:
|
||||
|
||||
```ts
|
||||
interface MinionJobContext {
|
||||
data: Record<string, unknown>; // job params (whatever the cron submit passed)
|
||||
job: MinionJob; // full job row (id, queue, attempts, etc.)
|
||||
signal: AbortSignal; // set to aborted when the worker is shutting down
|
||||
inbox: MinionInbox; // read messages sent to this job while it runs
|
||||
}
|
||||
```
|
||||
|
||||
Return a serializable object on success. Throw on failure (the worker
|
||||
will log + retry per `max_attempts`).
|
||||
|
||||
**Abort cooperation.** When `ctx.signal.aborted` becomes true, finish
|
||||
gracefully. The worker will wait 30s for you to return before SIGKILL.
|
||||
Long-running LLM calls should pass the signal through to whatever
|
||||
network library they use.
|
||||
|
||||
**Idempotency.** The queue enforces unique `idempotency_key` at the DB
|
||||
layer, so you don't need to worry about double-submits from a cron that
|
||||
fires while the previous invocation is still running.
|
||||
|
||||
## Gbrain's migration flow
|
||||
|
||||
The v0.11.0 migration orchestrator (run by `gbrain apply-migrations`)
|
||||
detects cron entries whose handler name is NOT in GBrain's builtin set
|
||||
and emits a structured TODO to `~/.gbrain/migrations/pending-host-work.jsonl`.
|
||||
Each TODO has shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "cron-handler-needs-host-registration",
|
||||
"handler": "ea-inbox-sweep",
|
||||
"cron_schedule": "0 */30 * * *",
|
||||
"manifest_path": "/path/to/cron/jobs.json",
|
||||
"current_cmd": "agentTurn ea-inbox-sweep",
|
||||
"recommendation": "Add a handler registration for `ea-inbox-sweep` in your host worker bootstrap per docs/guides/plugin-handlers.md. Once registered, re-run `gbrain apply-migrations` to auto-rewrite this entry.",
|
||||
"status": "pending"
|
||||
}
|
||||
```
|
||||
|
||||
The host agent walks these entries using `skills/migrations/v0.11.0.md`:
|
||||
|
||||
1. Read `~/.gbrain/migrations/pending-host-work.jsonl`.
|
||||
2. For each `cron-handler-needs-host-registration` row, ship a handler
|
||||
registration in the host's worker bootstrap following the pattern
|
||||
above.
|
||||
3. Deploy the updated worker.
|
||||
4. Re-run `gbrain apply-migrations --yes`. The orchestrator now
|
||||
recognizes the newly-registerable handler (worker writes the
|
||||
registered names to a discovery file on startup) and rewrites the
|
||||
cron entry to use `gbrain jobs submit`. The JSONL row is marked
|
||||
`status: "complete"`.
|
||||
|
||||
## Trust boundary
|
||||
|
||||
Handler code runs inside the worker process with the same privileges
|
||||
as the rest of the host binary. There is no elevation. But there is
|
||||
also no runtime sandbox — handlers can read + write anywhere the
|
||||
worker user can. Review handler PRs the same way you review any other
|
||||
code that touches production data.
|
||||
|
||||
## Related
|
||||
|
||||
- `skills/conventions/cron-via-minions.md` — the rewrite convention
|
||||
for cron manifests.
|
||||
- `skills/migrations/v0.11.0.md` — how the migration orchestrator
|
||||
drives the host agent through this work.
|
||||
- `skills/minion-orchestrator/SKILL.md` — patterns for submitting,
|
||||
monitoring, steering, and replaying jobs once the handler is live.
|
||||
@@ -1,76 +0,0 @@
|
||||
# Queue operations runbook
|
||||
|
||||
"My queue looks wedged — what do I run?" The commands below are in the order
|
||||
you probably want them. Shipped with v0.19.1 after a production incident
|
||||
where the queue held for 90+ minutes before the operator noticed.
|
||||
|
||||
## First signal: jobs aren't running
|
||||
|
||||
```bash
|
||||
gbrain doctor --json | jq '.checks[] | select(.name == "queue_health")'
|
||||
```
|
||||
|
||||
`queue_health` flags two patterns:
|
||||
|
||||
- **stalled-forever**: active job whose `started_at` is older than 1h.
|
||||
- **waiting-depth**: any per-name queue deeper than 10 (override via
|
||||
`GBRAIN_QUEUE_WAITING_THRESHOLD`). Signals a missing `maxWaiting`.
|
||||
|
||||
## Triage commands
|
||||
|
||||
```bash
|
||||
# Who's active right now?
|
||||
gbrain jobs list --status active
|
||||
|
||||
# Who's waiting, biggest pile first?
|
||||
gbrain jobs list --status waiting --limit 50
|
||||
|
||||
# What's wrong with a specific job?
|
||||
gbrain jobs get <id>
|
||||
```
|
||||
|
||||
## Rescue actions (in order of escalation)
|
||||
|
||||
```bash
|
||||
# Force-kill a single stuck job:
|
||||
gbrain jobs cancel <id>
|
||||
|
||||
# Clear a specific job entirely (last resort):
|
||||
gbrain jobs delete <id>
|
||||
|
||||
# Health smoke on the mechanism itself:
|
||||
gbrain jobs smoke --wedge-rescue
|
||||
```
|
||||
|
||||
## What each subcheck means
|
||||
|
||||
- **stalled-forever** — A worker claimed a job, started executing, and has
|
||||
held the row for over an hour. The wall-clock sweep evicts jobs past
|
||||
2× `timeout_ms`; if one's still active, either no `timeout_ms` was set
|
||||
or the sweep is newly deployed and this job predates it. Cancel it.
|
||||
- **waiting-depth** — Submitters are piling up jobs faster than workers
|
||||
drain them. Set `--max-waiting N` on the submission or on the programmatic
|
||||
`queue.add()` call. If you want a taller pile, raise the threshold via
|
||||
`GBRAIN_QUEUE_WAITING_THRESHOLD=50 gbrain doctor`.
|
||||
|
||||
## Self-check: is a worker even running?
|
||||
|
||||
```bash
|
||||
# If you're running autopilot with --no-worker, check that your external
|
||||
# worker (systemd / Docker / OpenClaw service-manager) is alive:
|
||||
gbrain jobs list --status active | head -5
|
||||
```
|
||||
|
||||
If the list is empty AND your submissions keep piling up, no worker is
|
||||
claiming. Start one:
|
||||
|
||||
```bash
|
||||
GBRAIN_ALLOW_SHELL_JOBS=1 gbrain jobs work --concurrency 4
|
||||
```
|
||||
|
||||
## Follow-ups tracked for v0.20+
|
||||
|
||||
- B7 — `minion_workers` heartbeat table for ground-truth liveness (the
|
||||
`--no-worker` probe and the dropped `queue_health` worker-heartbeat
|
||||
subcheck both need this).
|
||||
- B3 — `gbrain doctor --fix` learns to rescue queue wedges.
|
||||
@@ -1,165 +0,0 @@
|
||||
# Quiet Hours and Timezone-Aware Delivery
|
||||
|
||||
## Goal
|
||||
|
||||
Hold all notifications during sleep hours, merge held messages into the morning briefing, and adjust automatically when the user travels.
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: 3 AM pings from cron jobs. One bad notification and the user
|
||||
disables the entire system.
|
||||
|
||||
With this: the brain works overnight (dream cycle, collectors, enrichment)
|
||||
but notifications are held until morning. Travel to Tokyo? The system adjusts
|
||||
automatically from your calendar, no config change needed.
|
||||
|
||||
## Implementation
|
||||
|
||||
### Quiet Hours Gate
|
||||
|
||||
Every cron job that sends notifications must check quiet hours FIRST.
|
||||
|
||||
```
|
||||
QUIET_START = 23 // 11 PM local time
|
||||
QUIET_END = 8 // 8 AM local time
|
||||
|
||||
is_quiet(local_hour):
|
||||
return local_hour >= QUIET_START OR local_hour < QUIET_END
|
||||
```
|
||||
|
||||
**Before sending any notification:**
|
||||
1. Determine user's current timezone (from config or heartbeat state)
|
||||
2. Convert current UTC time to local time
|
||||
3. If quiet hours: hold the message, don't send
|
||||
|
||||
### Held Messages
|
||||
|
||||
During quiet hours, output goes to a held directory instead of being sent:
|
||||
|
||||
```
|
||||
if is_quiet():
|
||||
mkdir -p /tmp/cron-held/
|
||||
write("/tmp/cron-held/{job-name}.md", output)
|
||||
exit // don't send
|
||||
else:
|
||||
send(output)
|
||||
```
|
||||
|
||||
The morning briefing picks up held messages:
|
||||
|
||||
```
|
||||
morning_briefing():
|
||||
held_files = list("/tmp/cron-held/*.md")
|
||||
if held_files:
|
||||
briefing += "## Overnight Updates\n\n"
|
||||
for file in held_files:
|
||||
briefing += read(file)
|
||||
delete(file)
|
||||
```
|
||||
|
||||
This way nothing is lost. Overnight cron results get folded into the
|
||||
first thing the user sees in the morning.
|
||||
|
||||
### Timezone Awareness
|
||||
|
||||
The agent should know what timezone the user is in. Store it in
|
||||
the agent's operational state:
|
||||
|
||||
```json
|
||||
{
|
||||
"currentLocation": {
|
||||
"timezone": "US/Pacific",
|
||||
"city": "San Francisco"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Update the timezone when:**
|
||||
- Calendar shows the user flying somewhere (check for airline/hotel events)
|
||||
- User mentions being in a different city
|
||||
- User's active hours shift (they're responding at 3 AM PT = they're probably traveling)
|
||||
|
||||
**All times shown to the user should be in their LOCAL timezone.** Never
|
||||
show UTC or a timezone the user isn't in.
|
||||
|
||||
### Shell Implementation
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# quiet-hours-gate.sh — run before any notification
|
||||
|
||||
TIMEZONE="${USER_TIMEZONE:-US/Pacific}"
|
||||
LOCAL_HOUR=$(TZ="$TIMEZONE" date +%H)
|
||||
|
||||
if [ "$LOCAL_HOUR" -ge 23 ] || [ "$LOCAL_HOUR" -lt 8 ]; then
|
||||
echo "QUIET_HOURS=true"
|
||||
exit 1 # don't send
|
||||
fi
|
||||
|
||||
echo "QUIET_HOURS=false"
|
||||
exit 0 # ok to send
|
||||
```
|
||||
|
||||
**In cron job scripts:**
|
||||
```bash
|
||||
# Check quiet hours first
|
||||
if ! bash scripts/quiet-hours-gate.sh; then
|
||||
mkdir -p /tmp/cron-held
|
||||
echo "$OUTPUT" > /tmp/cron-held/$(basename "$0" .sh).md
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Not quiet hours — send normally
|
||||
send_notification "$OUTPUT"
|
||||
```
|
||||
|
||||
### Configurable Hours
|
||||
|
||||
Some users want different quiet hours. Store the config:
|
||||
|
||||
```json
|
||||
{
|
||||
"quiet_hours": {
|
||||
"start": 23,
|
||||
"end": 8,
|
||||
"enabled": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Set `enabled: false` to disable quiet hours entirely (e.g., for 24/7 monitoring).
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Gate on EVERY job.** The quiet hours check must run before every single
|
||||
cron job that produces notifications. If even one job skips the gate, the
|
||||
user gets a 3 AM ping and loses trust in the entire system. No exceptions.
|
||||
|
||||
2. **Held messages MUST be picked up.** If the morning briefing doesn't read
|
||||
`/tmp/cron-held/`, overnight results vanish silently. Verify the briefing
|
||||
skill reads and clears the held directory. Orphaned held files mean the
|
||||
pickup integration is broken.
|
||||
|
||||
3. **Timezone auto-detection is fragile.** Calendar-based timezone detection
|
||||
relies on the user having airline/hotel events with location data. If the
|
||||
user books travel without calendar entries, the system won't detect the
|
||||
move. Fall back to activity-hour analysis (responding at 3 AM PT = probably
|
||||
not in PT anymore) and ask the user if uncertain.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Set quiet hours to the current hour.** Temporarily set `QUIET_START` to
|
||||
one hour before now and `QUIET_END` to one hour after. Trigger a cron job.
|
||||
Verify the output goes to `/tmp/cron-held/` instead of being sent.
|
||||
|
||||
2. **Check held message pickup.** After step 1, run or simulate the morning
|
||||
briefing. Verify the held message appears in the "Overnight Updates"
|
||||
section and the file is deleted from `/tmp/cron-held/`.
|
||||
|
||||
3. **Verify timezone adjustment.** Change the timezone config to a zone where
|
||||
it's currently quiet hours. Trigger a notification. Verify it's held. Change
|
||||
back to your real timezone during active hours. Trigger again. Verify it sends.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,158 +0,0 @@
|
||||
# Two-Repo Architecture: Agent Behavior vs World Knowledge
|
||||
|
||||
## Goal
|
||||
|
||||
Separate agent behavior (replaceable) from world knowledge (permanent) into two repos with strict boundaries.
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: agent config and world knowledge are mixed together. Switch agents
|
||||
and you lose your knowledge. Switch knowledge tools and you lose your agent setup.
|
||||
|
||||
With this: your brain (14,700+ files of people, companies, meetings, ideas)
|
||||
survives any agent swap. Your agent config survives any knowledge tool swap.
|
||||
|
||||
## Implementation
|
||||
|
||||
### The Boundary Test
|
||||
|
||||
**"Is this about how the agent operates, or is this knowledge about the world?"**
|
||||
|
||||
| Question | If YES -> Agent Repo | If YES -> Brain Repo |
|
||||
|----------|---------------------|---------------------|
|
||||
| Would this file transfer if you switched AI agents? | YES | -- |
|
||||
| Would this file transfer if you switched to a different person? | -- | YES |
|
||||
| Is this about how the agent behaves? | YES | -- |
|
||||
| Is this about a person, company, deal, meeting, or idea? | -- | YES |
|
||||
|
||||
### Quick Decision Tree
|
||||
|
||||
```
|
||||
New file to create?
|
||||
|-- About a person, company, deal, project, meeting, idea? -> brain/
|
||||
|-- A spec, research doc, or strategic analysis? -> brain/
|
||||
|-- An original idea or observation? -> brain/originals/
|
||||
|-- A daily session log or heartbeat state? -> agent-repo/
|
||||
|-- A skill, config, cron, or ops file? -> agent-repo/
|
||||
|-- A task or todo? -> agent-repo/tasks/
|
||||
```
|
||||
|
||||
### Agent Repo (operational config)
|
||||
|
||||
How the agent works. Identity, configuration, operational state.
|
||||
|
||||
```
|
||||
agent-repo/
|
||||
├── AGENTS.md # Agent identity + operational rules
|
||||
├── SOUL.md # Persona, voice, values
|
||||
├── USER.md # User preferences + context
|
||||
├── HEARTBEAT.md # Daily ops flow
|
||||
├── TOOLS.md # Available tools + credentials
|
||||
├── MEMORY.md # Operational memory (preferences, decisions)
|
||||
├── skills/ # Agent capabilities (SKILL.md files)
|
||||
│ ├── ingest/SKILL.md
|
||||
│ ├── query/SKILL.md
|
||||
│ ├── enrich/SKILL.md
|
||||
│ └── ...
|
||||
├── cron/ # Scheduled jobs
|
||||
│ └── jobs.json
|
||||
├── tasks/ # Current task list
|
||||
│ └── current.md
|
||||
├── hooks/ # Event hooks + transforms
|
||||
├── scripts/ # Operational scripts (collectors, gates)
|
||||
└── memory/ # Session logs, state files
|
||||
├── heartbeat-state.json
|
||||
└── YYYY-MM-DD.md # Daily session logs
|
||||
```
|
||||
|
||||
### Brain Repo (world knowledge)
|
||||
|
||||
What you know. People, companies, deals, meetings, ideas, media.
|
||||
This is the repo GBrain indexes.
|
||||
|
||||
```
|
||||
brain/
|
||||
├── people/ # Person dossiers (compiled truth + timeline)
|
||||
├── companies/ # Company profiles
|
||||
├── deals/ # Deal tracking
|
||||
├── meetings/ # Meeting transcripts + analysis
|
||||
├── originals/ # YOUR original thinking (highest value)
|
||||
├── concepts/ # World concepts and frameworks
|
||||
├── ideas/ # Product and business ideas
|
||||
├── media/ # Video transcripts, books, articles
|
||||
│ ├── youtube/
|
||||
│ ├── podcasts/
|
||||
│ └── articles/
|
||||
├── sources/ # Source material summaries
|
||||
├── daily/ # Daily data (calendar, logs)
|
||||
│ └── calendar/
|
||||
│ └── YYYY/
|
||||
│ └── YYYY-MM-DD.md
|
||||
├── projects/ # Project specs and docs
|
||||
├── writing/ # Essays, drafts, published work
|
||||
├── diligence/ # Investment diligence materials
|
||||
│ └── company-name/
|
||||
│ ├── index.md
|
||||
│ ├── pitch-deck.md
|
||||
│ └── .raw/ # Original PDFs/files
|
||||
└── Apple Notes/ # Imported Apple Notes archive
|
||||
```
|
||||
|
||||
### The Hard Rule
|
||||
|
||||
**Never write knowledge to the agent repo.** If a skill, sub-agent, or cron
|
||||
job needs to create a file about a person, company, deal, meeting, project,
|
||||
or idea, it MUST write to the brain repo, never to the agent repo.
|
||||
|
||||
The brain is the permanent record. The agent repo is replaceable.
|
||||
|
||||
### Why Two Repos
|
||||
|
||||
**Independence.** You can switch AI agents (OpenClaw -> Hermes -> custom) without
|
||||
losing your knowledge. You can switch knowledge tools (GBrain -> something else)
|
||||
without losing your agent setup.
|
||||
|
||||
**Scale.** The brain grows large (10,000+ files). The agent repo stays small
|
||||
(< 100 files). Different backup strategies, different sync cadences.
|
||||
|
||||
**Privacy.** The brain contains sensitive information (people, deals, personal
|
||||
notes). The agent repo contains operational config. Different access controls.
|
||||
|
||||
**GBrain indexes the brain repo.** Run `gbrain sync --repo ~/brain/` to keep
|
||||
the search index current. The agent repo is never indexed by GBrain.
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Never write knowledge to the agent repo.** This is the most common
|
||||
violation. A skill that creates a person page, a cron job that saves
|
||||
meeting notes, a sub-agent that captures an idea -- all of these MUST
|
||||
write to the brain repo. If it's about the world, it goes in the brain.
|
||||
|
||||
2. **The brain is the permanent record.** When in doubt, ask: "Would this
|
||||
file survive switching to a completely different AI agent?" If yes, it
|
||||
belongs in the brain. Agent configs, skills, cron jobs, and operational
|
||||
state are replaceable. People, companies, ideas, and meetings are not.
|
||||
|
||||
3. **Don't index the agent repo.** GBrain indexes the brain repo only.
|
||||
Running `gbrain sync` against the agent repo pollutes search results
|
||||
with operational config instead of world knowledge.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Check file placement.** After any skill or cron job creates a file,
|
||||
verify it landed in the correct repo. Person/company/idea/meeting files
|
||||
should be in `brain/`. Skill/config/cron/state files should be in the
|
||||
agent repo. Any knowledge file in the agent repo is a boundary violation.
|
||||
|
||||
2. **Run the boundary test.** Pick 5 recently created files and ask: "Would
|
||||
this transfer if I switched AI agents?" and "Would this transfer if I
|
||||
switched to a different person?" If the answers don't match the file's
|
||||
location, it's in the wrong repo.
|
||||
|
||||
3. **Verify GBrain only indexes brain.** Run `gbrain stats` and check the
|
||||
indexed paths. None should point to the agent repo directory. If agent
|
||||
config files appear in search results, the sync target is misconfigured.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,154 +0,0 @@
|
||||
# RLS and you
|
||||
|
||||
Short version: every table in your gbrain's `public` schema needs Row Level
|
||||
Security enabled. If one doesn't, `gbrain doctor` now fails, not warns, and the
|
||||
process exits 1.
|
||||
|
||||
This guide explains why, what to do when you hit the check, and the escape hatch
|
||||
for the cases where you really do want a table to stay readable by the anon key.
|
||||
|
||||
## Why RLS matters
|
||||
|
||||
Supabase exposes everything in the `public` schema via PostgREST. Whatever's
|
||||
there is reachable by the anon key, which is a client-side secret by design.
|
||||
If RLS is off on a public table, the anon key can read it. On anything sensitive
|
||||
(auth tokens, chat history, financial data) that's an exfiltration vector, not
|
||||
a footgun.
|
||||
|
||||
gbrain's service-role connection holds `BYPASSRLS`, so enabling RLS without
|
||||
policies does NOT break gbrain itself. It just blocks the anon key's default
|
||||
read. That's the security posture: deny-by-default to anon, full access for
|
||||
the service role.
|
||||
|
||||
## What to do when doctor fails
|
||||
|
||||
Doctor's message names every table missing RLS and gives you a `ALTER TABLE`
|
||||
line per table:
|
||||
|
||||
```
|
||||
1 table(s) WITHOUT Row Level Security: expenses_ramp.
|
||||
Fix: ALTER TABLE "public"."expenses_ramp" ENABLE ROW LEVEL SECURITY;
|
||||
If a table should stay readable by the anon key on purpose, see
|
||||
docs/guides/rls-and-you.md for the GBRAIN:RLS_EXEMPT comment escape hatch.
|
||||
```
|
||||
|
||||
99% of the time, you want the fix. Run the SQL. Re-run `gbrain doctor`. Done.
|
||||
|
||||
## The 1% case: deliberate exemption
|
||||
|
||||
Sometimes a public table is supposed to be readable by the anon key. An
|
||||
analytics view backing a public dashboard. A read-only reference table. A
|
||||
plugin that ships its own frontend and intentionally uses the anon key for
|
||||
reads.
|
||||
|
||||
gbrain has an escape hatch for these. It is deliberately painful to set up.
|
||||
That is the feature.
|
||||
|
||||
### The format
|
||||
|
||||
```sql
|
||||
-- In psql, connected as a BYPASSRLS role (e.g. postgres):
|
||||
COMMENT ON TABLE public.your_table IS
|
||||
'GBRAIN:RLS_EXEMPT reason=<why this is anon-readable on purpose>';
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- The comment value MUST start with `GBRAIN:RLS_EXEMPT` (case-sensitive).
|
||||
- It MUST include `reason=` followed by at least 4 characters of justification.
|
||||
- No other prefix, no checkbox in a config file, no environment variable. Only
|
||||
a Postgres table comment counts.
|
||||
- If RLS is also off on the table (which it must be for the anon key to
|
||||
actually read), you also need `ALTER TABLE ... DISABLE ROW LEVEL SECURITY;`
|
||||
explicitly. Disabling alone is not enough; the comment is what tells doctor
|
||||
this is intentional.
|
||||
|
||||
### Example
|
||||
|
||||
```sql
|
||||
ALTER TABLE public.expenses_ramp DISABLE ROW LEVEL SECURITY;
|
||||
COMMENT ON TABLE public.expenses_ramp IS
|
||||
'GBRAIN:RLS_EXEMPT reason=analytics-only, anon-readable ok, owner=garry, 2026-04-22';
|
||||
```
|
||||
|
||||
After that, `gbrain doctor` reports:
|
||||
|
||||
```
|
||||
rls: ok — RLS enabled on 20/21 public tables (1 explicitly exempt: expenses_ramp)
|
||||
```
|
||||
|
||||
Note that every subsequent run re-enumerates your exemptions by name. That's
|
||||
intentional. The escape hatch is not a one-time sign-off, it's a recurring
|
||||
reminder. If you ever want to know which tables are open, run `gbrain doctor`.
|
||||
|
||||
## Why SQL and not a CLI subcommand
|
||||
|
||||
gbrain does NOT ship a `gbrain rls-exempt add <table>` command. A CLI command
|
||||
would make it easy for an agent to silently open a table to anon reads. The
|
||||
comment-in-psql requirement forces the operator to type the justification
|
||||
in SQL, which is:
|
||||
|
||||
- Visible in shell history.
|
||||
- Visible in a git-tracked schema dump.
|
||||
- Visible in `pg_dump` output the next time you restore.
|
||||
- Visible in `gbrain doctor` output on every run.
|
||||
|
||||
An agent CAN still run the SQL, but it can't do it without the user seeing the
|
||||
action. That's the "write it in blood" design.
|
||||
|
||||
## Auditing exemptions later
|
||||
|
||||
To see every exemption in the current DB:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
c.relname AS table_name,
|
||||
obj_description(c.oid, 'pg_class') AS comment
|
||||
FROM pg_class c
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public'
|
||||
AND c.relkind = 'r'
|
||||
AND obj_description(c.oid, 'pg_class') LIKE 'GBRAIN:RLS_EXEMPT%';
|
||||
```
|
||||
|
||||
If that list is longer than you remember signing off on, that's the signal.
|
||||
|
||||
## Removing an exemption
|
||||
|
||||
Just drop the comment and re-enable RLS:
|
||||
|
||||
```sql
|
||||
ALTER TABLE public.expenses_ramp ENABLE ROW LEVEL SECURITY;
|
||||
COMMENT ON TABLE public.expenses_ramp IS NULL;
|
||||
```
|
||||
|
||||
`gbrain doctor` stops listing the table as exempt and goes back to checking
|
||||
it like any other.
|
||||
|
||||
## PGLite
|
||||
|
||||
If you're on PGLite (the zero-config default), doctor skips this check
|
||||
entirely: PGLite is embedded, single-user, and has no PostgREST in front of
|
||||
it. The public-schema-exposure risk doesn't exist. You'll see:
|
||||
|
||||
```
|
||||
rls: ok — Skipped (PGLite — no PostgREST exposure, RLS not applicable)
|
||||
```
|
||||
|
||||
If you migrate to Supabase or self-hosted Postgres later, the check starts
|
||||
running and will flag any table that came over without RLS.
|
||||
|
||||
## Self-hosted Postgres
|
||||
|
||||
If you're running Postgres without PostgREST in front, the anon-key exposure
|
||||
doesn't apply. But gbrain still fails the check on missing RLS, because:
|
||||
|
||||
- The framing is "RLS on all public tables" is a gbrain security invariant,
|
||||
not a Supabase-specific workaround.
|
||||
- The `ALTER TABLE ... ENABLE RLS` fix is harmless on any Postgres: it only
|
||||
constrains non-bypass roles, which gbrain doesn't use.
|
||||
- If you ever put PostgREST or a similar tool in front later, the guard is
|
||||
already in place.
|
||||
|
||||
If this framing doesn't fit your deployment, file an issue with the specifics
|
||||
so we can decide whether a self-hosted-exempt mode is justified.
|
||||
@@ -1,78 +0,0 @@
|
||||
# Search Modes
|
||||
|
||||
## Goal
|
||||
Know which search command to use and when -- keyword, hybrid, or direct -- so every lookup is fast and returns the right result.
|
||||
|
||||
## What the User Gets
|
||||
Without this: the agent fumbles between search commands, returns chunks when full pages are needed, runs expensive semantic searches when a direct get would do, or misses results entirely. With this: every lookup uses the optimal mode, token budgets are respected, and the user gets the right information in the fewest calls.
|
||||
|
||||
## Implementation
|
||||
|
||||
```
|
||||
on user_asks_about(topic):
|
||||
# Decision tree: pick the right search mode
|
||||
|
||||
if know_exact_slug(topic):
|
||||
# MODE 3: Direct get -- instant, no search overhead
|
||||
result = gbrain get <slug>
|
||||
# e.g., "Tell me about Pedro" -> gbrain get pedro-franceschi
|
||||
# Returns the FULL page -- compiled truth + timeline
|
||||
|
||||
elif topic.is_exact_name or topic.is_keyword:
|
||||
# MODE 1: Keyword search -- fast, no embeddings needed, day-one ready
|
||||
results = gbrain search "{name_or_keyword}"
|
||||
# e.g., "Find anything about Series A" -> gbrain search "Series A"
|
||||
# Returns CHUNKS, not full pages
|
||||
|
||||
# IMPORTANT: keyword search returns chunks
|
||||
# If the chunk confirms relevance, THEN load the full page:
|
||||
if chunk.confirms_relevance:
|
||||
full_page = gbrain get <slug_from_chunk>
|
||||
|
||||
elif topic.is_semantic_question:
|
||||
# MODE 2: Hybrid search -- semantic + keyword, needs embeddings
|
||||
results = gbrain query "{natural language question}"
|
||||
# e.g., "Who do I know at fintech companies?" -> gbrain query "fintech contacts"
|
||||
# Returns ranked chunks via vector + keyword + RRF
|
||||
|
||||
# Same rule: chunks first, then get full page if needed
|
||||
if chunk.confirms_relevance:
|
||||
full_page = gbrain get <slug_from_chunk>
|
||||
|
||||
# Quick reference:
|
||||
# | Mode | Command | Needs Embeddings | Speed | Best For |
|
||||
# |---------|----------------------|------------------|---------|---------------------------------|
|
||||
# | Keyword | gbrain search "term" | No | Fastest | Known names, exact matches |
|
||||
# | Hybrid | gbrain query "..." | Yes | Fast | Semantic questions, fuzzy match |
|
||||
# | Direct | gbrain get <slug> | No | Instant | When you know the slug |
|
||||
|
||||
# Progression over time:
|
||||
# Day 1: keyword search (works without embeddings)
|
||||
# After first embed: hybrid search unlocked
|
||||
# Once you know slugs: direct get for speed
|
||||
|
||||
# Precedence for conflicting information within a page:
|
||||
# 1. User's direct statements (always wins)
|
||||
# 2. Compiled truth sections (synthesized from evidence)
|
||||
# 3. Timeline entries (raw signal, reverse chronological)
|
||||
# 4. External sources (web search, APIs)
|
||||
```
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Search returns chunks, not full pages.** After `gbrain search` or `gbrain query`, you get excerpts. Always run `gbrain get <slug>` to load the full page when the chunk confirms relevance. Don't answer questions from chunks alone when the full context matters.
|
||||
2. **Keyword search works without embeddings.** On day one before any embedding run, `gbrain search` still works. Don't tell the user "search isn't available yet" -- keyword search is always available.
|
||||
3. **Don't use hybrid search for known names.** `gbrain query "Pedro Franceschi"` wastes embedding compute. Use `gbrain search "Pedro Franceschi"` or better yet `gbrain get pedro-franceschi` if you know the slug.
|
||||
4. **Token budget awareness.** A full page via `gbrain get` can be large. Read the search chunks first to confirm relevance before pulling the full page. "Did anyone mention the Series A?" -- search results (chunks) are probably enough. "Tell me everything about Pedro" -- get the full page.
|
||||
5. **Hybrid search needs embeddings to have been run.** If `gbrain query` returns nothing but `gbrain search` finds results, the embeddings haven't been generated yet. Run the embedding pipeline first.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. Run `gbrain search "Pedro"` -- confirm it returns chunks with matching text and slug references.
|
||||
2. Run `gbrain query "who works at fintech companies"` -- confirm it returns semantically relevant results (not just keyword matches on "fintech").
|
||||
3. Run `gbrain get pedro-franceschi` -- confirm it returns the full page with compiled truth and timeline.
|
||||
4. Compare: search for the same entity using all three modes. Keyword should be fastest, hybrid should surface conceptual matches, direct should return the complete page.
|
||||
5. After a search returns a chunk, run `gbrain get` on the slug from that chunk. Confirm the full page contains more context than the chunk alone.
|
||||
|
||||
---
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,131 +0,0 @@
|
||||
# Skill Development Cycle
|
||||
|
||||
## Goal
|
||||
|
||||
Turn every repeating task into a durable, automated skill so that if you ask twice, it should already be running on a cron.
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: ad-hoc work that the agent forgets how to do. You ask "enrich
|
||||
this person" and the agent invents a new process each time. Quality varies.
|
||||
|
||||
With this: every capability is codified, tested, and scheduled. Enrichment
|
||||
runs the same way every time. New patterns get skill-ified within a day.
|
||||
|
||||
## Implementation
|
||||
|
||||
**The Rule:** If you have to ask your agent for something twice, it should
|
||||
already be a skill running on a cron. First time is discovery. Second time
|
||||
is system failure.
|
||||
|
||||
### The 5-Step Cycle
|
||||
|
||||
**Step 1: Concept the Process.**
|
||||
Describe what needs to happen in plain language:
|
||||
- What's the input? What's the output? What triggers it?
|
||||
- What data sources does it touch?
|
||||
- How often should it run?
|
||||
|
||||
**Step 2: Run Manually for 3-10 Items.**
|
||||
Actually do the work by hand on a small batch. This is the prototype phase.
|
||||
Do NOT write a SKILL.md yet. Just do the work and observe:
|
||||
- What does the output actually look like?
|
||||
- What edge cases appear?
|
||||
- What quality bar is right?
|
||||
|
||||
**Step 3: Evaluate Output.**
|
||||
Show the user the results. Get feedback.
|
||||
- Does output look good? Is quality right?
|
||||
- Did you miss anything? Over-engineer?
|
||||
- Revise the process based on what you learned.
|
||||
|
||||
**Step 4: Codify into a Skill.**
|
||||
Write the SKILL.md. Either:
|
||||
- **New skill** -- genuinely new capability
|
||||
- **Add to existing skill** -- variation of something that exists (parameterize it)
|
||||
|
||||
The skill must be:
|
||||
- **Durable** -- works tomorrow, next week, next month without manual intervention
|
||||
- **MECE** -- doesn't overlap with other skills (see below)
|
||||
- **Parameterized** -- handles variations through parameters, not separate skills
|
||||
|
||||
**Step 5: Add to Cron (if recurring).**
|
||||
If the process should run automatically:
|
||||
- Add to existing cron job if it fits naturally
|
||||
- Create new cron job if it has a distinct scheduling concern
|
||||
- Monitor the first 2-3 automated runs for quality
|
||||
- Fix issues that emerge at scale
|
||||
|
||||
### MECE Discipline
|
||||
|
||||
Skills should be **Mutually Exclusive, Collectively Exhaustive**:
|
||||
- Each entity type has exactly ONE owner skill
|
||||
- Each signal source has exactly ONE owner skill
|
||||
- Two skills creating the same brain page = MECE violation
|
||||
|
||||
**Example ownership (no overlap):**
|
||||
|
||||
| Signal Source | Owner Skill | Creates |
|
||||
|--------------|-------------|---------|
|
||||
| Meeting transcripts | meeting-ingestion | brain/meetings/ pages |
|
||||
| Email messages | executive-assistant | brain/people/ timeline entries |
|
||||
| X/Twitter posts | x-collector | brain/media/ pages |
|
||||
| Person enrichment | enrich | brain/people/ compiled truth |
|
||||
| Calendar events | calendar-sync | brain/daily/calendar/ pages |
|
||||
| Video/podcast content | media-ingest | brain/media/ pages |
|
||||
|
||||
### Quality Bar Checklist
|
||||
|
||||
A skill is ready when:
|
||||
|
||||
- [ ] Ran successfully on 3-10 real items with good output
|
||||
- [ ] User reviewed output and approved
|
||||
- [ ] SKILL.md is under 500 lines (use references for overflow)
|
||||
- [ ] Checks notability before creating brain pages (don't create pages for nobodies)
|
||||
- [ ] Has citation enforcement (every fact has a source)
|
||||
- [ ] Doesn't overlap with existing skills (MECE)
|
||||
- [ ] If recurring: on a cron with appropriate schedule
|
||||
- [ ] If it creates brain pages: checks notability first
|
||||
|
||||
### What This Means in Practice
|
||||
|
||||
- Don't do ad-hoc brain enrichment, use the enrich skill
|
||||
- Don't manually check social media, use an automated cron
|
||||
- Don't manually ingest meeting notes, use the meeting-sync recipe
|
||||
- Don't manually create entity pages, use the entity detector
|
||||
- If a new pattern emerges, prototype it, skill-ify it, cron-ify it
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **MECE violations compound silently.** Two skills that both create
|
||||
`brain/people/` pages will produce duplicates and conflicting data.
|
||||
Before creating a new skill, check the ownership table. If an existing
|
||||
skill already owns that entity type, extend it with parameters instead
|
||||
of creating a new skill.
|
||||
|
||||
2. **The quality bar is real.** Don't ship a skill that hasn't been tested
|
||||
on 3-10 real items with user approval. A skill that produces bad output
|
||||
is worse than no skill -- it creates bad brain pages at scale on a cron.
|
||||
|
||||
3. **Don't create stubs.** A SKILL.md with "TODO: implement" is not a skill.
|
||||
Every skill must be complete enough to run end-to-end on real data. If
|
||||
you can't finish it, don't create the file. Keep it as manual work until
|
||||
you can do it right.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Run the skill on 3 real items.** Execute the skill against live data
|
||||
(not test data). Check that the output matches the quality bar: citations
|
||||
present, notability checked, no stubs created.
|
||||
|
||||
2. **Check MECE against existing skills.** Review the ownership table. Does
|
||||
this new skill create pages in a directory already owned by another skill?
|
||||
If yes, it's a MECE violation. Merge or parameterize instead.
|
||||
|
||||
3. **Verify the quality bar checklist.** Walk through every item in the
|
||||
Quality Bar Checklist above. If any item is unchecked, the skill isn't
|
||||
ready for cron deployment.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,75 +0,0 @@
|
||||
# Source Attribution
|
||||
|
||||
## Goal
|
||||
Every fact in the brain traces to where it came from -- who said it, in what context, and when.
|
||||
|
||||
## What the User Gets
|
||||
Without this: six months from now, someone reads a brain page and has no idea if "Pedro co-founded Brex" came from Pedro himself, a LinkedIn scrape, or a hallucination. With this: every claim is auditable, conflicts are surfaced, and the brain is a court-admissible record of reality.
|
||||
|
||||
## Implementation
|
||||
|
||||
```
|
||||
on brain_write(page, fact):
|
||||
# EVERY fact gets a citation -- compiled truth AND timeline
|
||||
citation = format_citation(source)
|
||||
# format: [Source: {who}, {channel/context}, {date} {time} {tz}]
|
||||
|
||||
# Category-specific formats:
|
||||
if source.type == "direct":
|
||||
# [Source: User, direct message, 2026-04-07 12:33 PM PT]
|
||||
elif source.type == "meeting":
|
||||
# [Source: Meeting notes "Team Sync" #12345, 2026-04-03 12:11 PM PT]
|
||||
elif source.type == "api_enrichment":
|
||||
# [Source: Crustdata LinkedIn enrichment, 2026-04-07 12:35 PM PT]
|
||||
elif source.type == "social_media":
|
||||
# MUST include full URL -- not just @handle
|
||||
# [Source: X/@pedroh96 tweet, product launch, 2026-04-07](https://x.com/pedroh96/status/...)
|
||||
elif source.type == "email":
|
||||
# [Source: email from Sarah Chen re Q2 board deck, 2026-04-05 2:30 PM PT]
|
||||
elif source.type == "workspace":
|
||||
# [Source: Slack #engineering, Keith re deploy schedule, 2026-04-06 11:45 AM PT]
|
||||
elif source.type == "web":
|
||||
# [Source: Happenstance research, 2026-04-07 12:35 PM PT]
|
||||
elif source.type == "published":
|
||||
# [Source: [Wall Street Journal, 2026-04-05](https://wsj.com/...)]
|
||||
elif source.type == "funding":
|
||||
# [Source: Captain API funding data, 2026-04-07 2:00 PM PT]
|
||||
|
||||
# Attach citation inline with the fact
|
||||
gbrain put <slug> --content "...fact [Source: ...]..."
|
||||
|
||||
# When sources conflict, note BOTH -- never silently pick one
|
||||
if conflicts_exist(fact, existing_page):
|
||||
append_to_compiled_truth(
|
||||
"Conflict: Source A says X, Source B says Y. "
|
||||
"[Source: A] [Source: B]"
|
||||
)
|
||||
|
||||
# Source hierarchy for conflict resolution (highest authority first):
|
||||
SOURCE_PRIORITY = [
|
||||
"User direct statements", # 1 -- always wins
|
||||
"Primary sources", # 2 -- meetings, emails, direct conversations
|
||||
"Enrichment APIs", # 3 -- Crustdata, Happenstance, Captain
|
||||
"Web search results", # 4
|
||||
"Social media posts", # 5
|
||||
]
|
||||
```
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Compiled truth is NOT exempt from citations.** "Pedro co-founded Brex" in the synthesis section needs `[Source: ...]` just as much as a timeline entry does. Most agents skip citations above the bar.
|
||||
2. **Tweet URLs are mandatory.** `[Source: X/@handle tweet, topic, date]` without a URL is a broken citation. Hundreds of brain pages end up with unreachable tweet references when the URL is omitted. Always: `[Source: X/@handle tweet, topic, date](https://x.com/handle/status/ID)`.
|
||||
3. **"User said it" isn't enough.** WHERE, ABOUT WHAT, WHEN. `[Source: User, direct message, 2026-04-07 12:33 PM PT]` -- not just `[Source: User]`.
|
||||
4. **Don't silently resolve conflicts.** When the user says one thing and an API says another, note the contradiction in compiled truth with both citations. Let the reader decide.
|
||||
5. **Timeline entries need sources too.** Every append to the timeline carries provenance. A timeline entry without a source is an orphan fact.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. Open any brain page with `gbrain get <slug>`. Read the compiled truth section above the bar. Every factual claim should have an inline `[Source: ...]` citation.
|
||||
2. Search for tweet references: `gbrain search "X/@"`. Every result should have a full URL, not just an @handle.
|
||||
3. Find a page with data from multiple sources (e.g., a person enriched via API + mentioned in a meeting). Confirm both sources are cited independently.
|
||||
4. Check timeline entries on 3 random pages. Each entry should have a source citation with date and context.
|
||||
5. Look for a page where the user stated something that contradicts an API result. Confirm the contradiction is noted, not silently resolved.
|
||||
|
||||
---
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,122 +0,0 @@
|
||||
# Sub-Agent Model Routing
|
||||
|
||||
## Goal
|
||||
|
||||
Route sub-agents to the cheapest model that can do the job, saving 10-40x on costs without sacrificing quality.
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: every sub-agent runs on Opus ($15/MTok). Entity detection on
|
||||
every message costs $3-5/day. Research tasks cost $10+ each.
|
||||
|
||||
With this: entity detection runs on Sonnet ($3/MTok, 5x cheaper). Research
|
||||
runs on DeepSeek ($0.50/MTok, 30x cheaper). Main session stays on Opus for
|
||||
quality. Total cost drops 70-80%.
|
||||
|
||||
## Implementation
|
||||
|
||||
### Routing Table
|
||||
|
||||
| Task Type | Recommended Model | Why |
|
||||
|-----------|------------------|-----|
|
||||
| Main session / complex instructions | Opus-class (default) | Best reasoning and instruction following |
|
||||
| Research / synthesis / analysis | DeepSeek V3 or equivalent | 25-40x cheaper, strong on exploratory work |
|
||||
| Structured output / long context | Large context model (Qwen, Gemini) | 200K+ context, reliable JSON output |
|
||||
| Fast lightweight sub-agents | Fast inference model (Groq) | 500 tok/s, cheap, good for quick tasks |
|
||||
| Deep reasoning (use sparingly) | Reasoning model (DeepSeek-R1, o3) | Best for hard problems, expensive |
|
||||
| Entity detection (signal detector) | Sonnet-class | Fast, cheap, sufficient quality for detection |
|
||||
|
||||
### The Signal Detector Pattern
|
||||
|
||||
Spawn a lightweight sub-agent on EVERY inbound message. This is mandatory.
|
||||
|
||||
```
|
||||
on_every_message(text):
|
||||
// Spawn async — don't block the response
|
||||
spawn_subagent({
|
||||
task: `SIGNAL DETECTION — scan this message:
|
||||
"${text}"
|
||||
|
||||
1. IDEAS FIRST: Is the user expressing an original thought?
|
||||
If yes -> create/update brain/originals/ with EXACT phrasing
|
||||
2. ENTITIES: Extract person names, company names, media titles
|
||||
For each -> check brain, create/enrich if notable
|
||||
3. FACTS: New info about existing entities -> update timeline
|
||||
4. CITATIONS: Every fact needs [Source: ...] attribution
|
||||
5. Sync changes to brain repo`,
|
||||
model: "sonnet-class", // fast + cheap
|
||||
timeout: 120s
|
||||
})
|
||||
```
|
||||
|
||||
**Why Sonnet-class for detection:** Entity detection is pattern matching, not
|
||||
deep reasoning. Sonnet is 5-10x cheaper than Opus and fast enough for async
|
||||
detection. The main session continues on Opus while detection runs in parallel.
|
||||
|
||||
### Research Pipeline Pattern
|
||||
|
||||
For research-heavy tasks, use a multi-model pipeline:
|
||||
|
||||
```
|
||||
1. PLANNING (Opus): Write research brief, identify what to look for
|
||||
2. EXECUTION (DeepSeek): Sub-agent does the actual research (web, APIs, docs)
|
||||
3. SYNTHESIS (Opus): Read research output, add strategic analysis
|
||||
```
|
||||
|
||||
**Why this works:** The planning and synthesis steps need taste and judgment
|
||||
(Opus). The execution step is mechanical data gathering (DeepSeek at 25-40x
|
||||
lower cost). You get Opus-quality output at DeepSeek-level cost for 80% of
|
||||
the work.
|
||||
|
||||
### When to Spawn Sub-Agents
|
||||
|
||||
| Situation | Spawn? | Model |
|
||||
|-----------|--------|-------|
|
||||
| Every inbound message | YES (mandatory) | Sonnet |
|
||||
| Research request | YES | DeepSeek for execution |
|
||||
| Quick lookup / fact check | YES | Fast model (Groq) |
|
||||
| Complex analysis | NO -- handle in main session | Opus |
|
||||
| Writing / editing | NO -- handle in main session | Opus |
|
||||
|
||||
### Cost Optimization
|
||||
|
||||
The main session runs on your best model. Everything else runs on the
|
||||
cheapest model that can do the job. In practice, 60-70% of sub-agent
|
||||
work is entity detection (Sonnet) and research execution (DeepSeek),
|
||||
which are 10-40x cheaper than the main session model.
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Sonnet, not Opus, for detection.** The most common mistake is running
|
||||
entity detection on Opus. Detection is pattern matching, not deep reasoning.
|
||||
Sonnet is 5-10x cheaper and fast enough. Reserve Opus for the main session
|
||||
where reasoning quality matters.
|
||||
|
||||
2. **Don't block the main thread.** Sub-agents must run asynchronously. If the
|
||||
signal detector runs synchronously, the user waits 30-120 seconds for every
|
||||
message while entity detection completes. Spawn and forget. The user sees
|
||||
a response immediately.
|
||||
|
||||
3. **Cost optimization is multiplicative.** Entity detection runs on every
|
||||
single message. If you use Opus at $15/MTok for detection across 50
|
||||
messages/day, that's $3-5/day just for detection. Sonnet at $3/MTok brings
|
||||
that to $0.60-1.00/day. Over a month, the wrong model choice costs $100+
|
||||
more than necessary.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Spawn a signal detector and check the model.** Send a message and verify
|
||||
the sub-agent was spawned on Sonnet-class, not Opus. Check the model field
|
||||
in the sub-agent config or logs.
|
||||
|
||||
2. **Check cost per day.** After running for a day with sub-agent routing,
|
||||
compare total API costs against the previous day without routing. You
|
||||
should see a 50-80% reduction in total cost.
|
||||
|
||||
3. **Verify async execution.** Send a message and measure response time. The
|
||||
response should arrive in under 5 seconds. If it takes 30+ seconds, the
|
||||
signal detector is running synchronously and blocking the main thread.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
@@ -1,182 +0,0 @@
|
||||
# Upgrades and Auto-Update Notifications
|
||||
|
||||
## Goal
|
||||
|
||||
Users get notified of new GBrain features conversationally, and the agent walks them through upgrading with post-upgrade migrations that make the new version actually work.
|
||||
|
||||
## What the User Gets
|
||||
|
||||
Without this: GBrain ships updates but nobody knows. The user stays on an old
|
||||
version with stale skills and missing features. Or worse, someone runs
|
||||
`gbrain upgrade` but skips the post-upgrade steps, leaving new code with old
|
||||
agent behavior.
|
||||
|
||||
With this: the agent checks for updates daily, sells the upgrade with punchy
|
||||
benefit-focused bullets, waits for explicit permission, then runs the full
|
||||
upgrade flow including re-reading skills, running migrations, and syncing
|
||||
schema. The user gets new capabilities automatically.
|
||||
|
||||
## Implementation
|
||||
|
||||
### The Check (cron-initiated)
|
||||
|
||||
```
|
||||
check_for_update():
|
||||
result = run("gbrain check-update --json")
|
||||
|
||||
if not result.update_available:
|
||||
exit_silently() // do NOT message the user
|
||||
|
||||
// Sell the upgrade — lead with what they can DO, not what changed
|
||||
message = compose_upgrade_message(
|
||||
current: result.current_version,
|
||||
latest: result.latest_version,
|
||||
changelog: result.changelog
|
||||
)
|
||||
send_to_user(message, respect_quiet_hours=true)
|
||||
```
|
||||
|
||||
### The Upgrade Message
|
||||
|
||||
Sell the upgrade. The user should feel "hell yeah, I want that." Lead with
|
||||
what they can DO now that they couldn't before, not what files changed.
|
||||
|
||||
```
|
||||
> **GBrain v0.5.0 is available** (you're on v0.4.0)
|
||||
>
|
||||
> What's new:
|
||||
> - Your brain never falls behind. Live sync keeps the vector DB current
|
||||
> automatically, so edits show up in search within minutes
|
||||
> - New verification runbook catches silent failures before they bite you
|
||||
> - New installs set up live sync automatically. No more manual setup step
|
||||
>
|
||||
> Want me to upgrade? I'll update everything and refresh my playbook.
|
||||
>
|
||||
> (Reply **yes** to upgrade, **not now** to skip, **weekly** to check
|
||||
> less often, or **stop** to turn off update checks)
|
||||
```
|
||||
|
||||
### Handling Responses
|
||||
|
||||
| User says | Action |
|
||||
|-----------|--------|
|
||||
| yes / y / sure / ok / do it / upgrade | Run the full upgrade flow (below) |
|
||||
| not now / later / skip / snooze | Acknowledge, check again next cycle |
|
||||
| weekly | Store preference, switch cron to weekly |
|
||||
| daily | Store preference, switch cron back to daily |
|
||||
| stop / unsubscribe / no more | Disable the cron. Tell user how to resume |
|
||||
|
||||
**Never auto-upgrade.** Always wait for explicit confirmation.
|
||||
|
||||
### The Full Upgrade Flow (after user says yes)
|
||||
|
||||
```
|
||||
full_upgrade():
|
||||
// Step 1: Update the binary/package
|
||||
run("gbrain upgrade")
|
||||
|
||||
// Step 2: Re-read all updated skills
|
||||
for skill in find("skills/*/SKILL.md"):
|
||||
read_and_internalize(skill) // updated skills = better agent behavior
|
||||
|
||||
// Step 3: Re-read production reference docs
|
||||
read("docs/GBRAIN_SKILLPACK.md")
|
||||
read("docs/GBRAIN_RECOMMENDED_SCHEMA.md")
|
||||
|
||||
// Step 4: Check for version-specific migration directives
|
||||
for version in range(old_version, new_version):
|
||||
migration = find(f"skills/migrations/v{version}.md")
|
||||
if migration exists:
|
||||
read_and_execute(migration) // in order, don't skip
|
||||
|
||||
// Step 5: Schema sync — suggest new, respect declined
|
||||
state = read("~/.gbrain/update-state.json")
|
||||
for recommendation in new_schema_recommendations:
|
||||
if recommendation not in state.declined:
|
||||
suggest_to_user(recommendation)
|
||||
update(state, new_choices)
|
||||
|
||||
// Step 6: Report what changed
|
||||
summarize_to_user(actions_taken)
|
||||
```
|
||||
|
||||
### Migration Files
|
||||
|
||||
Migration files live at `skills/migrations/vX.Y.Z.md`. They contain agent
|
||||
instructions (not scripts) for post-upgrade actions that make the new version
|
||||
work for existing users. Example: v0.5.0 migration sets up live sync and
|
||||
runs the verification runbook.
|
||||
|
||||
The agent reads migration files in version order and executes them step by
|
||||
step. Without migrations, the agent has new code but the user's environment
|
||||
hasn't changed.
|
||||
|
||||
### Cron Registration
|
||||
|
||||
```
|
||||
Name: gbrain-update-check
|
||||
Default schedule: 0 9 * * * (daily 9 AM)
|
||||
Weekly schedule: 0 9 * * 1 (Monday 9 AM)
|
||||
Prompt: "Run gbrain check-update --json. If update_available is true,
|
||||
summarize the changelog and message me asking if I'd like to upgrade.
|
||||
If false, stay silent."
|
||||
```
|
||||
|
||||
### Frequency Preferences
|
||||
|
||||
Default: daily. Store in agent memory as `gbrain_update_frequency: daily|weekly|off`.
|
||||
Also persist in `~/.gbrain/update-state.json` so it survives agent context resets.
|
||||
|
||||
### Standalone Skillpack Users
|
||||
|
||||
If you loaded this SKILLPACK directly (copied or read from GitHub) without
|
||||
installing gbrain, you can still stay current. Both GBRAIN_SKILLPACK.md and
|
||||
GBRAIN_RECOMMENDED_SCHEMA.md have version markers:
|
||||
|
||||
```bash
|
||||
curl -s https://raw.githubusercontent.com/garrytan/gbrain/master/docs/GBRAIN_SKILLPACK.md | head -1
|
||||
# Returns: <!-- skillpack-version: X.Y.Z -->
|
||||
```
|
||||
|
||||
If the remote version is newer, fetch the full file and replace your local
|
||||
copy. Set up a weekly cron to check automatically.
|
||||
|
||||
## Tricky Spots
|
||||
|
||||
1. **Never auto-install.** The upgrade must always wait for the user's explicit
|
||||
"yes." Even if the cron detects an update at 9 AM and the changelog looks
|
||||
great, the agent messages the user and waits. Auto-installing can break
|
||||
workflows, introduce breaking changes, or interrupt work in progress.
|
||||
|
||||
2. **Migration files are agent instructions, not scripts.** They tell the agent
|
||||
what to do step by step in plain language. They are NOT bash scripts to
|
||||
execute blindly. The agent reads them, understands the context, and adapts
|
||||
to the user's specific environment (e.g., skip a step if the user already
|
||||
has live sync configured).
|
||||
|
||||
3. **check-update should run on a daily cron.** Don't rely on the user
|
||||
remembering to check for updates. The cron runs `gbrain check-update --json`
|
||||
daily at 9 AM (respecting quiet hours). If there's nothing new, it stays
|
||||
completely silent. The user only hears about updates when there IS something
|
||||
worth upgrading to.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Run check-update and verify detection.** Execute
|
||||
`gbrain check-update --json`. Verify it returns the current version and
|
||||
correctly reports whether an update is available. If `update_available`
|
||||
is false, verify the version matches the latest release on GitHub.
|
||||
|
||||
2. **Verify migration files are readable.** List `skills/migrations/` and
|
||||
check that each file follows the naming convention `vX.Y.Z.md`. Open one
|
||||
and verify it contains step-by-step agent instructions, not raw scripts.
|
||||
The agent should be able to read and execute each step.
|
||||
|
||||
3. **Test the full upgrade flow end-to-end.** If an update is available, say
|
||||
"yes" and watch the agent execute the full flow: upgrade, re-read skills,
|
||||
run migrations, sync schema, report. Verify each step completes and the
|
||||
agent reports what changed.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md).*
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 220 KiB |
@@ -1,119 +0,0 @@
|
||||
# Getting Data Into Your Brain
|
||||
|
||||
GBrain is the retrieval layer. But retrieval is only as good as what you put in.
|
||||
This directory covers how to get data flowing into your brain automatically.
|
||||
|
||||
## How Data Flows In
|
||||
|
||||
```
|
||||
Signal arrives (phone call, email, tweet, calendar event)
|
||||
↓
|
||||
Collector captures it (deterministic code, reliable)
|
||||
↓
|
||||
Agent analyzes it (LLM, judgment, entity detection)
|
||||
↓
|
||||
Brain pages created/updated (compiled truth + timeline)
|
||||
↓
|
||||
GBrain indexes it (chunking, embedding, search-ready)
|
||||
↓
|
||||
Next query is smarter (the compounding effect)
|
||||
```
|
||||
|
||||
## Available Integrations
|
||||
|
||||
### Self-Installing Recipes
|
||||
|
||||
These are integration recipes your agent can set up for you. Run
|
||||
`gbrain integrations` to see what's available and their status.
|
||||
|
||||
| Recipe | Category | Requires | What It Does | Setup Time |
|
||||
|--------|----------|----------|-------------|------------|
|
||||
| [ngrok-tunnel](../../recipes/ngrok-tunnel.md) | Infra | — | Fixed public URL for MCP + voice ($8/mo) | 10 min |
|
||||
| [credential-gateway](../../recipes/credential-gateway.md) | Infra | — | Gmail + Calendar access (ClawVisor or Google OAuth) | 15 min |
|
||||
| [voice-to-brain](../../recipes/twilio-voice-brain.md) | Sense | ngrok-tunnel | Phone calls create brain pages via Twilio + OpenAI Realtime | 30 min |
|
||||
| [email-to-brain](../../recipes/email-to-brain.md) | Sense | credential-gateway | Gmail messages flow into entity pages via deterministic collector | 20 min |
|
||||
| [x-to-brain](../../recipes/x-to-brain.md) | Sense | — | Twitter timeline, mentions, keyword monitoring with deletion detection | 15 min |
|
||||
| [calendar-to-brain](../../recipes/calendar-to-brain.md) | Sense | credential-gateway | Google Calendar events become searchable daily brain pages | 20 min |
|
||||
| [meeting-sync](../../recipes/meeting-sync.md) | Sense | — | Circleback meeting transcripts auto-import with attendee propagation | 15 min |
|
||||
|
||||
### Manual Integration Guides
|
||||
|
||||
These require manual setup (no self-installing recipe yet):
|
||||
|
||||
| Guide | What It Does |
|
||||
|-------|-------------|
|
||||
| [Credential Gateway](credential-gateway.md) | Set up ClawVisor or Hermes for Gmail, Calendar, Contacts access |
|
||||
| [Meeting & Call Webhooks](meeting-webhooks.md) | Circleback meeting transcripts + Quo/OpenPhone SMS/calls |
|
||||
|
||||
## How to Read a Recipe
|
||||
|
||||
Integration recipes are markdown files with YAML frontmatter. Your agent reads
|
||||
the recipe and walks you through setup.
|
||||
|
||||
```yaml
|
||||
---
|
||||
id: voice-to-brain # unique identifier
|
||||
name: Voice-to-Brain # human-readable name
|
||||
version: 0.7.0 # recipe version
|
||||
description: Phone calls... # what it does
|
||||
category: sense # sense (data input) or reflex (automated response)
|
||||
requires: [] # other recipes that must be set up first
|
||||
secrets: # API keys and credentials needed
|
||||
- name: TWILIO_ACCOUNT_SID
|
||||
description: Twilio account SID
|
||||
where: https://console.twilio.com # exact URL to get this key
|
||||
health_checks: # typed DSL to verify the integration is working
|
||||
- type: http
|
||||
url: "https://api.twilio.com/2010-04-01/Accounts/$TWILIO_ACCOUNT_SID.json"
|
||||
auth: basic
|
||||
auth_user: "$TWILIO_ACCOUNT_SID"
|
||||
auth_token: "$TWILIO_AUTH_TOKEN"
|
||||
label: "Twilio account"
|
||||
setup_time: 30 min # estimated time to complete setup
|
||||
---
|
||||
|
||||
[Setup instructions the agent follows step by step...]
|
||||
```
|
||||
|
||||
**The recipe IS the installer.** Your agent (OpenClaw, Hermes, Claude Code) reads
|
||||
the markdown body and executes the setup steps. It asks you for API keys, validates
|
||||
each one, configures the integration, and runs a smoke test.
|
||||
|
||||
### Recipe trust boundary
|
||||
|
||||
Only recipes shipped inside the gbrain package itself (the `recipes/` directory in
|
||||
a source install, or the global install copy) are trusted. Recipes discovered at
|
||||
runtime from `$GBRAIN_RECIPES_DIR` or a cwd-local `./recipes/` are marked untrusted:
|
||||
they cannot run `command` health checks, cannot run `http` health checks (SSRF
|
||||
defense), and cannot use the deprecated string health_check form. Untrusted recipes
|
||||
can still use `env_exists` and `any_of` compositions. To ship a recipe that runs
|
||||
live checks, contribute it upstream so it becomes package-bundled.
|
||||
|
||||
## The Deterministic Collector Pattern
|
||||
|
||||
When an LLM keeps failing at a mechanical task despite repeated prompt fixes,
|
||||
stop fighting the LLM. Move the mechanical work to code.
|
||||
|
||||
**Code for data. LLMs for judgment.**
|
||||
|
||||
- Email collection: code pulls emails with baked-in links (100% reliable).
|
||||
LLM reads the digest, classifies, enriches brain entries (judgment).
|
||||
- Tweet collection: code pulls timeline, detects deletions, tracks engagement
|
||||
(deterministic). LLM extracts entities, writes brain updates (judgment).
|
||||
- Calendar sync: code pulls events and attendees (deterministic). LLM enriches
|
||||
attendee brain pages (judgment).
|
||||
|
||||
This pattern prevents the "LLM forgot the links" failure mode. Mechanical work
|
||||
must be 100% reliable. Judgment work is where LLMs shine.
|
||||
|
||||
See [Deterministic Collectors](../guides/deterministic-collectors.md) for the
|
||||
full pattern.
|
||||
|
||||
## Architecture
|
||||
|
||||
For details on the shared infrastructure that all integrations build on
|
||||
(import pipeline, chunking, embedding, search), see the
|
||||
[Infrastructure Layer](../architecture/infra-layer.md).
|
||||
|
||||
For the philosophy behind thin harness + fat skills, see
|
||||
[Thin Harness, Fat Skills](../ethos/THIN_HARNESS_FAT_SKILLS.md).
|
||||
@@ -1,52 +0,0 @@
|
||||
# Credential Gateway (ClawVisor / Hermes)
|
||||
|
||||
|
||||
Three integrations that make the agent real. Without these, the brain is a static
|
||||
database. With them, it's alive.
|
||||
|
||||
### 14a. Credential Gateway (ClawVisor / Hermes Gateway)
|
||||
|
||||
The EA workflow needs Gmail, Calendar, Contacts, and messaging access. The agent
|
||||
should never hold API keys directly. Use a credential gateway that enforces policies
|
||||
and injects credentials at request time.
|
||||
|
||||
**OpenClaw: ClawVisor.** [ClawVisor](https://clawvisor.com) is a credential vaulting
|
||||
and authorization gateway with task-scoped authorization.
|
||||
|
||||
**Services:** Gmail (list, read, send, draft), Google Calendar (CRUD), Google Drive
|
||||
(list, search, read), Google Contacts (list, search), Apple iMessage (list, read,
|
||||
search, send), GitHub, Slack.
|
||||
|
||||
**Task-scoped authorization:** Every request must include a `task_id` from an approved
|
||||
standing task. Tasks declare: purpose (verbose, 2-3 sentences), authorized actions with
|
||||
expected use patterns, auto-execute flag, lifetime (standing vs ephemeral).
|
||||
|
||||
**Why this matters for GBrain:** The EA workflow needs Gmail (sender lookup before
|
||||
triage), Calendar (meeting prep, attendee pages), Contacts (enrichment trigger), and
|
||||
iMessage (direct instructions). ClawVisor gives the agent access without giving it
|
||||
raw credentials.
|
||||
|
||||
**Setup:**
|
||||
|
||||
1. Create agent in ClawVisor dashboard, copy agent token
|
||||
2. Set `CLAWVISOR_URL` and `CLAWVISOR_AGENT_TOKEN` in env
|
||||
3. Activate services (Google, iMessage, etc.) in the dashboard
|
||||
4. Create standing tasks with expansive scopes (narrow purposes cause false blocks)
|
||||
5. Store standing task IDs in agent memory for reuse
|
||||
|
||||
**Critical scoping rule:** Be expansive in task purposes. "Full executive assistant
|
||||
email management including inbox triage, searching by any criteria, reading emails,
|
||||
tracking threads" works. "Email triage" gets rejected. The intent verification model
|
||||
uses the purpose to judge whether each request is consistent -- if your purpose is
|
||||
narrow, legitimate requests fail verification.
|
||||
|
||||
**Hermes Agent: Built-in gateway.** Hermes has multi-platform messaging (Telegram,
|
||||
Discord, Slack, WhatsApp, Signal, Email) and tool access built into its gateway. Use
|
||||
`config.yaml` to configure API credentials. The gateway daemon manages connections
|
||||
and routes webhooks to agent sessions. For Google services, configure OAuth credentials
|
||||
in the gateway config. Hermes's scheduled automations can run the same EA workflows
|
||||
(email triage, calendar prep, contact enrichment) through the gateway's tool system.
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md). See also: [Getting Data In](README.md)*
|
||||
@@ -1,63 +0,0 @@
|
||||
# Meeting & Call Webhooks
|
||||
|
||||
### 14b. Circleback -- Meeting Ingestion via Webhooks
|
||||
|
||||
[Circleback](https://circleback.ai) records meetings, generates transcripts with
|
||||
speaker diarization, and fires webhooks on completion.
|
||||
|
||||
**Webhook setup:**
|
||||
|
||||
1. In Circleback dashboard -> Automations -> add webhook
|
||||
2. URL: `{your_agent_gateway}/hooks/circleback-meetings`
|
||||
3. Circleback provides a signing secret for HMAC-SHA256 signature verification
|
||||
4. Store the signing secret in your webhook transform for verification
|
||||
|
||||
**Webhook payload:** Meeting JSON with id, name, attendees, notes, action items, full
|
||||
transcript, calendar event context.
|
||||
|
||||
**Signature verification:** Header `X-Circleback-Signature` contains `sha256=<hex>`.
|
||||
Verify with `HMAC-SHA256(body, signing_secret)`. Reject unverified webhooks.
|
||||
|
||||
**OAuth for API access:** Circleback uses dynamic client registration (OAuth 2.0).
|
||||
Access tokens expire in ~24h, auto-refresh via refresh token. Store credentials in
|
||||
agent memory.
|
||||
|
||||
**Flow:** Webhook fires -> transform validates signature + normalizes -> agent wakes ->
|
||||
pulls full transcript via API -> creates brain meeting page -> propagates to entity
|
||||
pages -> commits to brain repo -> `gbrain sync`.
|
||||
|
||||
### 14c. Quo (OpenPhone) -- SMS and Call Integration
|
||||
|
||||
[Quo](https://openphone.com) (formerly OpenPhone) provides business phone numbers with
|
||||
SMS, calls, voicemail, and AI transcripts.
|
||||
|
||||
**Webhook setup:**
|
||||
|
||||
1. In Quo dashboard -> Integrations -> Webhooks
|
||||
2. Register webhooks for: `message.received`, `call.completed`, `call.summary.completed`, `call.transcript.completed`
|
||||
3. Point all to: `{your_agent_gateway}/hooks/quo-events`
|
||||
4. Store registered webhook IDs in agent memory
|
||||
|
||||
**How inbound texts work:**
|
||||
|
||||
- Webhook fires with sender phone, message text, conversation context
|
||||
- Agent looks up sender in brain by phone number
|
||||
- Surfaces to user's messaging platform with sender identity + brain context
|
||||
- Drafts reply for approval (never auto-replies without explicit permission)
|
||||
|
||||
**How inbound calls work:**
|
||||
|
||||
- `call.completed` fires -> if duration > 30s, fetch transcript + AI summary via API
|
||||
- Ingest to brain (meeting-style page at `meetings/`)
|
||||
- Update relevant people and company pages
|
||||
|
||||
**API auth:** Bare API key in `Authorization` header (no Bearer prefix).
|
||||
|
||||
**Key endpoints:** `POST /v1/messages` (send SMS), `GET /v1/messages` (list),
|
||||
`GET /v1/call-transcripts/{id}`, `GET /v1/conversations`.
|
||||
|
||||
---
|
||||
|
||||
---
|
||||
|
||||
*Part of the [GBrain Skillpack](../GBRAIN_SKILLPACK.md). See also: [Getting Data In](README.md)*
|
||||
@@ -1,105 +0,0 @@
|
||||
# Pre-commit hook for brain repos (v0.22.4+)
|
||||
|
||||
`gbrain frontmatter install-hook` installs a git pre-commit hook in your
|
||||
brain source's repo that runs `gbrain frontmatter validate` against staged
|
||||
`.md` and `.mdx` files. Malformed frontmatter blocks the commit. Bypass with
|
||||
`git commit --no-verify`.
|
||||
|
||||
## What the hook catches
|
||||
|
||||
The same seven validation classes the `frontmatter-guard` skill and
|
||||
`gbrain doctor`'s `frontmatter_integrity` subcheck report:
|
||||
|
||||
| Code | What it catches |
|
||||
|-------------------|---------------------------------------------------------------------|
|
||||
| `MISSING_OPEN` | File doesn't start with `---` |
|
||||
| `MISSING_CLOSE` | No closing `---` before first heading |
|
||||
| `YAML_PARSE` | YAML failed to parse (syntax or structure) |
|
||||
| `SLUG_MISMATCH` | `slug:` in frontmatter doesn't match path-derived slug |
|
||||
| `NULL_BYTES` | Binary corruption (`\x00`) anywhere in the content |
|
||||
| `NESTED_QUOTES` | `title: "outer "inner" outer"` shape that breaks YAML |
|
||||
| `EMPTY_FRONTMATTER` | `---` ... `---` with nothing meaningful between |
|
||||
|
||||
## Install
|
||||
|
||||
For all registered sources that are git repos:
|
||||
|
||||
```bash
|
||||
gbrain frontmatter install-hook
|
||||
```
|
||||
|
||||
For one source:
|
||||
|
||||
```bash
|
||||
gbrain frontmatter install-hook --source <id>
|
||||
```
|
||||
|
||||
For force-overwrite of an existing pre-commit hook (writes a `.bak`):
|
||||
|
||||
```bash
|
||||
gbrain frontmatter install-hook --force
|
||||
```
|
||||
|
||||
The hook lands at `<source>/.githooks/pre-commit`. If `core.hooksPath` is
|
||||
unset, the install also runs `git config core.hooksPath .githooks` so the
|
||||
hook is picked up without manual git config.
|
||||
|
||||
## Bypass
|
||||
|
||||
Standard git escape hatch:
|
||||
|
||||
```bash
|
||||
git commit --no-verify
|
||||
```
|
||||
|
||||
This skips ALL pre-commit hooks. Use sparingly — the next time the user
|
||||
runs `gbrain doctor`, the issues will surface.
|
||||
|
||||
## Uninstall
|
||||
|
||||
```bash
|
||||
gbrain frontmatter install-hook --uninstall
|
||||
```
|
||||
|
||||
If a `.bak` was saved during install, it's restored as the active hook.
|
||||
Otherwise the hook is removed cleanly.
|
||||
|
||||
## Behavior on machines without gbrain installed
|
||||
|
||||
The hook script checks for `gbrain` on `$PATH`. When missing, it prints a
|
||||
one-line warning to stderr and exits 0 — commits aren't blocked just because
|
||||
a developer hasn't installed gbrain locally. Once gbrain is installed, the
|
||||
hook resumes blocking malformed pages.
|
||||
|
||||
## For downstream agent forks
|
||||
|
||||
If your OpenClaw wraps gbrain in a host repo
|
||||
that's not the brain repo itself, you may want a separate hook strategy:
|
||||
|
||||
- **Brain repo IS the host repo** (gbrain skills + brain pages in one repo):
|
||||
install via `gbrain frontmatter install-hook` as above.
|
||||
- **Brain repo is a separate registered source** (e.g. `~/brain` registered
|
||||
as a source, host repo is `~/agent-fork`): install in the brain repo only;
|
||||
agent-fork code doesn't need this hook.
|
||||
- **Brain repo is auto-generated** (e.g. by a sync daemon writing to a
|
||||
bucket): skip the hook entirely; gate at the writer instead via
|
||||
`import { writeBrainPage } from 'gbrain/brain-writer'` (planned in a
|
||||
later release; currently the CLI is the surface).
|
||||
|
||||
## How it fits into the broader frontmatter pipeline
|
||||
|
||||
```
|
||||
agent writes a page git commit doctor scan
|
||||
↓ ↓ ↓
|
||||
[source content] → [pre-commit hook validates] → [frontmatter_integrity check]
|
||||
↓ ↓ ↓
|
||||
raw file on disk blocks malformed commits surfaces existing issues
|
||||
↓
|
||||
`gbrain frontmatter validate
|
||||
<source-path> --fix`
|
||||
(writes .bak backups)
|
||||
```
|
||||
|
||||
The hook is the write-time gate; doctor is the audit gate; the CLI is the
|
||||
fix tool. They share `parseMarkdown(..., {validate:true})` as the single
|
||||
source of truth for what counts as malformed.
|
||||
@@ -1,66 +0,0 @@
|
||||
# Reliability repair (v0.12.2)
|
||||
|
||||
If you ran v0.12.0 on real Postgres or Supabase, two bugs may have corrupted
|
||||
data already in your brain. v0.12.1 fixed the code going forward.
|
||||
v0.12.2 adds detection in `gbrain doctor` and a standalone `gbrain repair-jsonb`
|
||||
command for the mechanically fixable class. PGLite users are not affected.
|
||||
|
||||
## What got corrupted
|
||||
|
||||
**JSONB double-encode.** Four write sites used
|
||||
`${JSON.stringify(x)}::jsonb` with postgres.js, which stored a JSONB
|
||||
*string literal* instead of an object. `frontmatter ->> 'key'` returns NULL;
|
||||
GIN indexes are ineffective. Affected: `pages.frontmatter`,
|
||||
`raw_data.data`, `ingest_log.pages_updated`, `files.metadata`.
|
||||
|
||||
**Markdown body truncation.** `splitBody()` treated `---` horizontal rules
|
||||
as a body/timeline delimiter, dropping everything after the first rule.
|
||||
Wiki-style pages with multiple `##`/`###` sections lost the bulk of their
|
||||
content at import time.
|
||||
|
||||
## Detect
|
||||
|
||||
```
|
||||
gbrain doctor
|
||||
```
|
||||
|
||||
Reports two new checks:
|
||||
|
||||
- `jsonb_integrity` — counts double-encoded rows per table and points you
|
||||
at `gbrain repair-jsonb`.
|
||||
- `markdown_body_completeness` — heuristic for pages whose `compiled_truth`
|
||||
is suspiciously short compared to `raw_data.data ->> 'content'`.
|
||||
|
||||
## Repair
|
||||
|
||||
For JSONB (mechanically fixable):
|
||||
|
||||
```
|
||||
gbrain repair-jsonb
|
||||
```
|
||||
|
||||
Runs `UPDATE <table> SET <col> = (<col>#>>'{}')::jsonb WHERE jsonb_typeof(<col>) = 'string'`
|
||||
across every affected column. Idempotent. Second run reports 0 rows. Use
|
||||
`--dry-run` to preview, `--json` for structured output. The `v0_12_2`
|
||||
migration runs this automatically on `gbrain upgrade`.
|
||||
|
||||
For truncated markdown bodies (source-dependent):
|
||||
|
||||
```
|
||||
gbrain sync --force
|
||||
# or per-page
|
||||
gbrain import <slug> --force
|
||||
```
|
||||
|
||||
v0.12.2 cannot recover content that was already lost if you no longer have
|
||||
the source markdown file. `gbrain doctor` tells you which pages look short;
|
||||
you decide whether to re-import from source or accept the truncation.
|
||||
|
||||
## Verify
|
||||
|
||||
```
|
||||
gbrain doctor
|
||||
```
|
||||
|
||||
All four `jsonb_integrity` rows should read zero. `markdown_body_completeness`
|
||||
should match your expectations for the corpus.
|
||||
@@ -1,67 +0,0 @@
|
||||
# Remote MCP Deployment Options
|
||||
|
||||
GBrain's MCP server runs via `gbrain serve` (stdio transport). To make it
|
||||
accessible from other devices and AI clients, run `gbrain serve --http`
|
||||
(built-in HTTP transport with bearer auth, Postgres-only ... see
|
||||
[DEPLOY.md](DEPLOY.md)) behind a public tunnel. Here are your tunnel options.
|
||||
|
||||
## ngrok (recommended)
|
||||
|
||||
[ngrok](https://ngrok.com) provides instant public tunnels. The Hobby tier
|
||||
($8/mo) gives you a fixed domain that never changes.
|
||||
|
||||
```bash
|
||||
# 1. Install ngrok
|
||||
brew install ngrok
|
||||
|
||||
# 2. Start the built-in HTTP transport
|
||||
gbrain serve --http --port 8787
|
||||
# See docs/mcp/DEPLOY.md for token setup
|
||||
|
||||
# 3. Expose via ngrok
|
||||
ngrok http 8787 --url your-brain.ngrok.app
|
||||
```
|
||||
|
||||
See the [ngrok-tunnel recipe](../../recipes/ngrok-tunnel.md) for full setup
|
||||
including auth token configuration and fixed domain setup.
|
||||
|
||||
## Tailscale Funnel
|
||||
|
||||
[Tailscale Funnel](https://tailscale.com/kb/1223/tailscale-funnel) gives you
|
||||
a permanent public HTTPS URL with automatic TLS. Free tier available. Best for
|
||||
private networks where you control both endpoints.
|
||||
|
||||
```bash
|
||||
# 1. Install Tailscale
|
||||
brew install tailscale
|
||||
|
||||
# 2. Expose your MCP server
|
||||
tailscale funnel 8787
|
||||
# Your brain is now at https://your-machine.ts.net
|
||||
```
|
||||
|
||||
## Fly.io / Railway (always-on)
|
||||
|
||||
For production deployments that need to run 24/7 without your machine:
|
||||
|
||||
- **Fly.io:** $5-10/mo, global edge, `fly deploy`
|
||||
- **Railway:** $5/mo, git push deploy
|
||||
|
||||
Both run Bun natively. No bundling, no Deno, no cold start, no timeout limits.
|
||||
|
||||
## Comparison
|
||||
|
||||
| | ngrok | Tailscale | Fly.io/Railway |
|
||||
|--|---|---|---|
|
||||
| Cost | $8/mo (Hobby) | Free | $5-10/mo |
|
||||
| Fixed URL | Yes (Hobby) | Yes | Yes |
|
||||
| Works when laptop is off | No | No | Yes |
|
||||
| Cold start | None | None | None |
|
||||
| Timeout limits | None | None | None |
|
||||
| All 30 operations | Yes | Yes | Yes |
|
||||
| Setup time | 5 min | 10 min | 15 min |
|
||||
|
||||
**Note:** `gbrain serve --http` is the built-in HTTP transport (v0.22.7+). Bearer auth
|
||||
against the `access_tokens` table, default-deny CORS, two-bucket rate limit, body cap,
|
||||
per-request audit log. Postgres-only by design (PGLite is local-only). See
|
||||
[DEPLOY.md](DEPLOY.md) and [SECURITY.md](../../SECURITY.md) for env vars and tunables.
|
||||
@@ -1,40 +0,0 @@
|
||||
# Connect GBrain to Claude Code
|
||||
|
||||
## Option 1: Local (recommended, zero server needed)
|
||||
|
||||
```bash
|
||||
claude mcp add gbrain -- gbrain serve
|
||||
```
|
||||
|
||||
That's it. Claude Code spawns `gbrain serve` as a stdio subprocess. No server, no
|
||||
tunnel, no token needed. Works with both PGLite and Supabase engines.
|
||||
|
||||
## Option 2: Remote (access from any machine)
|
||||
|
||||
If you have GBrain running on a server with a public tunnel (see
|
||||
[ngrok-tunnel recipe](../../recipes/ngrok-tunnel.md)):
|
||||
|
||||
```bash
|
||||
claude mcp add gbrain -t http \
|
||||
https://YOUR-DOMAIN.ngrok.app/mcp \
|
||||
-H "Authorization: Bearer YOUR_TOKEN"
|
||||
```
|
||||
|
||||
Replace `YOUR-DOMAIN` with your ngrok domain and `YOUR_TOKEN` with a token
|
||||
from `gbrain auth create "claude-code"`.
|
||||
|
||||
## Verify
|
||||
|
||||
In Claude Code, try:
|
||||
|
||||
```
|
||||
search for [any topic in your brain]
|
||||
```
|
||||
|
||||
You should see results from your GBrain knowledge base.
|
||||
|
||||
## Remove
|
||||
|
||||
```bash
|
||||
claude mcp remove gbrain
|
||||
```
|
||||
@@ -1,33 +0,0 @@
|
||||
# Connect GBrain to Claude Cowork
|
||||
|
||||
Two ways to get GBrain into Cowork sessions:
|
||||
|
||||
## Option 1: Remote (via self-hosted server + tunnel)
|
||||
|
||||
For Team/Enterprise plans, an org Owner adds the connector:
|
||||
|
||||
1. Go to **Organization Settings > Connectors**
|
||||
2. Add a new connector with the MCP server URL:
|
||||
```
|
||||
https://YOUR-DOMAIN.ngrok.app/mcp
|
||||
```
|
||||
3. Add Bearer token authentication in Advanced Settings
|
||||
(create one with `gbrain auth create "cowork"`)
|
||||
4. Save
|
||||
|
||||
Note: Cowork connects from Anthropic's cloud, not your device. Your server
|
||||
must be publicly reachable (ngrok, Tailscale Funnel, or cloud-hosted).
|
||||
|
||||
## Option 2: Local Bridge (via Claude Desktop)
|
||||
|
||||
If you already have GBrain configured in Claude Desktop (via `gbrain serve`
|
||||
stdio or a remote integration), Cowork gets access automatically. Claude
|
||||
Desktop bridges local MCP servers into Cowork via its SDK layer.
|
||||
|
||||
This means: if `gbrain serve` is running and configured in Claude Desktop,
|
||||
you don't need a separate server for Cowork.
|
||||
|
||||
## Which to use?
|
||||
|
||||
- **Remote server:** works even when your laptop is closed, available to all org members
|
||||
- **Local Bridge:** zero extra setup if Claude Desktop already has GBrain, but requires your machine to be running
|
||||
@@ -1,39 +0,0 @@
|
||||
# Connect GBrain to Claude Desktop
|
||||
|
||||
**Important:** Claude Desktop does NOT connect to remote MCP servers via
|
||||
`claude_desktop_config.json`. That file only works for local stdio servers.
|
||||
Remote HTTP servers must be added through the GUI.
|
||||
|
||||
## Setup
|
||||
|
||||
1. Open Claude Desktop
|
||||
2. Go to **Settings > Integrations**
|
||||
3. Click **Add Integration** (or **Add Connector**)
|
||||
4. Enter the MCP server URL:
|
||||
```
|
||||
https://YOUR-DOMAIN.ngrok.app/mcp
|
||||
```
|
||||
Replace `YOUR-DOMAIN` with your ngrok domain (see
|
||||
[ngrok-tunnel recipe](../../recipes/ngrok-tunnel.md) for setup).
|
||||
5. Set authentication to **Bearer Token** and paste your token
|
||||
(create one with `gbrain auth create "claude-desktop"`)
|
||||
6. Save
|
||||
|
||||
## Verify
|
||||
|
||||
Start a new conversation and try:
|
||||
|
||||
```
|
||||
Search my brain for [any topic]
|
||||
```
|
||||
|
||||
Claude Desktop will use your GBrain tools automatically.
|
||||
|
||||
## Common Mistakes
|
||||
|
||||
**Using claude_desktop_config.json for remote servers** — this silently fails
|
||||
with no error message. The JSON config only works for local stdio MCP servers.
|
||||
Remote HTTP servers must be added via Settings > Integrations in the GUI.
|
||||
|
||||
**Using the wrong URL** — make sure the URL ends with `/mcp` (not `/health`
|
||||
or just the base domain).
|
||||
@@ -1,129 +0,0 @@
|
||||
# Deploy GBrain Remote MCP Server
|
||||
|
||||
> **v0.22.7+:** Use `gbrain serve --http` for remote access. It includes built-in
|
||||
> bearer token auth, default-deny CORS, two-bucket rate limiting, body cap, and
|
||||
> per-request audit log. **Postgres-only** (PGLite is local-only by design).
|
||||
> See [SECURITY.md](../../SECURITY.md) for env vars and tunable defaults.
|
||||
|
||||
Access your brain from any device, any AI client. GBrain's MCP server runs locally
|
||||
via `gbrain serve` (stdio). For remote access, expose it via the built-in HTTP
|
||||
transport behind a public tunnel.
|
||||
|
||||
## Two Paths
|
||||
|
||||
### Local (zero setup)
|
||||
|
||||
```bash
|
||||
gbrain serve
|
||||
```
|
||||
|
||||
Works with Claude Code, Cursor, Windsurf, and any MCP client that supports stdio.
|
||||
No server, no tunnel, no token needed. Works on both PGLite and Postgres engines.
|
||||
|
||||
### Remote (any device, any AI client) — Postgres only
|
||||
|
||||
```
|
||||
Your AI client (Claude Desktop, Perplexity, etc.)
|
||||
→ ngrok tunnel (https://YOUR-DOMAIN.ngrok.app)
|
||||
→ gbrain serve --http (built-in transport with bearer auth)
|
||||
→ Postgres (pooler connection or self-hosted)
|
||||
```
|
||||
|
||||
This requires:
|
||||
1. A Postgres-backed brain (the `access_tokens` table only exists on Postgres;
|
||||
running `gbrain serve --http` against a PGLite install fails fast at startup)
|
||||
2. A machine running `gbrain serve --http`
|
||||
3. A public tunnel (ngrok, Tailscale, or cloud host)
|
||||
4. A bearer token created via `gbrain auth create <name>`
|
||||
|
||||
## Remote Setup
|
||||
|
||||
### 1. Set up the tunnel
|
||||
|
||||
See the [ngrok-tunnel recipe](../../recipes/ngrok-tunnel.md) for full setup.
|
||||
Quick version:
|
||||
|
||||
```bash
|
||||
brew install ngrok
|
||||
ngrok config add-authtoken YOUR_TOKEN
|
||||
ngrok http 8787 --url your-brain.ngrok.app # Hobby tier for fixed domain
|
||||
```
|
||||
|
||||
### 2. Create access tokens
|
||||
|
||||
```bash
|
||||
# Create a token for each client
|
||||
gbrain auth create "claude-desktop"
|
||||
|
||||
# List all tokens
|
||||
gbrain auth list
|
||||
|
||||
# Revoke a token
|
||||
gbrain auth revoke "claude-desktop"
|
||||
```
|
||||
|
||||
Tokens are per-client. Create one for each device/app. Revoke individually
|
||||
if compromised. Tokens are stored SHA-256 hashed in your database.
|
||||
|
||||
### 3. Connect your AI client
|
||||
|
||||
- **Claude Code:** [setup guide](CLAUDE_CODE.md)
|
||||
- **Claude Desktop:** [setup guide](CLAUDE_DESKTOP.md) (must use GUI, not JSON config)
|
||||
- **Claude Cowork:** [setup guide](CLAUDE_COWORK.md)
|
||||
- **Perplexity:** [setup guide](PERPLEXITY.md)
|
||||
|
||||
### 4. Verify
|
||||
|
||||
```bash
|
||||
gbrain auth test \
|
||||
https://YOUR-DOMAIN.ngrok.app/mcp \
|
||||
--token YOUR_TOKEN
|
||||
```
|
||||
|
||||
## Operations
|
||||
|
||||
All 30 GBrain operations are available remotely, including `sync_brain` and
|
||||
`file_upload` (no timeout limits with self-hosted server).
|
||||
|
||||
**Security note on `file_upload`:** remote MCP callers are confined to the working
|
||||
directory where `gbrain serve` was launched. Symlinks, `..` traversal, and absolute
|
||||
paths outside cwd are rejected. Page slugs and filenames are allowlist-validated
|
||||
(alphanumeric + hyphens; no control chars, RTL overrides, or backslashes). Local
|
||||
CLI callers (`gbrain file upload ...`) keep unrestricted filesystem access since
|
||||
the user owns the machine.
|
||||
|
||||
## Deployment Options
|
||||
|
||||
See [ALTERNATIVES.md](ALTERNATIVES.md) for a comparison of ngrok, Tailscale
|
||||
Funnel, and cloud hosts (Fly.io, Railway).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"missing_auth" error**
|
||||
Include the Authorization header: `Authorization: Bearer YOUR_TOKEN`
|
||||
|
||||
**"invalid_token" error**
|
||||
Run `gbrain auth list` to see active tokens.
|
||||
|
||||
**"service_unavailable" error**
|
||||
Database connection failed. Check your Supabase dashboard for outages.
|
||||
|
||||
**Claude Desktop doesn't connect**
|
||||
Remote servers must be added via Settings > Integrations, NOT
|
||||
`claude_desktop_config.json`. See [CLAUDE_DESKTOP.md](CLAUDE_DESKTOP.md).
|
||||
|
||||
## Expected Latencies
|
||||
|
||||
| Operation | Typical Latency | Notes |
|
||||
|-----------|----------------|-------|
|
||||
| get_page | < 100ms | Single DB query |
|
||||
| list_pages | < 200ms | DB query with filters |
|
||||
| search (keyword) | 100-300ms | Full-text search |
|
||||
| query (hybrid) | 1-3s | Embedding + vector + keyword + RRF |
|
||||
| put_page | 100-500ms | Write + trigger search_vector update |
|
||||
| get_stats | < 100ms | Aggregate query |
|
||||
|
||||
**Note:** `gbrain serve --http` (built-in HTTP transport) is planned but not yet
|
||||
implemented. Currently, remote MCP requires a custom HTTP wrapper. See the
|
||||
production deployment pattern in the [voice recipe](../../recipes/twilio-voice-brain.md)
|
||||
for a reference implementation.
|
||||
@@ -1,31 +0,0 @@
|
||||
# Connect GBrain to Perplexity Computer
|
||||
|
||||
Perplexity Computer supports remote MCP servers with bearer token authentication.
|
||||
|
||||
## Setup
|
||||
|
||||
1. Open Perplexity (requires Pro subscription)
|
||||
2. Go to **Settings > Connectors** (or **MCP Servers**)
|
||||
3. Add a new remote connector:
|
||||
- **URL:** `https://YOUR-DOMAIN.ngrok.app/mcp`
|
||||
- **Authentication:** API Key / Bearer Token
|
||||
- **Token:** your GBrain access token
|
||||
(create one with `gbrain auth create "perplexity"`)
|
||||
4. Save
|
||||
|
||||
Replace `YOUR-DOMAIN` with your ngrok domain (see
|
||||
[ngrok-tunnel recipe](../../recipes/ngrok-tunnel.md) for setup).
|
||||
|
||||
## Verify
|
||||
|
||||
In a Perplexity conversation, ask it to use your brain:
|
||||
|
||||
```
|
||||
Use my GBrain to search for [topic]
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- Perplexity Computer is available to Pro subscribers
|
||||
- Both the Perplexity Mac app and web version support MCP connectors
|
||||
- The Mac app also supports local MCP servers if you prefer `gbrain serve` (stdio)
|
||||
@@ -1,191 +0,0 @@
|
||||
# Progress events
|
||||
|
||||
Canonical reference for the JSONL progress stream that `gbrain` writes to
|
||||
`stderr` when a bulk command runs with `--progress-json`. Stable from
|
||||
v0.15.2. Additive changes only; no renames or removals without a major
|
||||
version bump.
|
||||
|
||||
Most humans won't read this page. Agents parsing progress will.
|
||||
|
||||
## When do I get these events?
|
||||
|
||||
Any of these commands stream events when `--progress-json` is set:
|
||||
|
||||
- `gbrain doctor` (DB checks, JSONB integrity, markdown body completeness,
|
||||
integrity sample)
|
||||
- `gbrain orphans`
|
||||
- `gbrain embed`
|
||||
- `gbrain files sync`
|
||||
- `gbrain export`
|
||||
- `gbrain extract [links|timeline|all]` (fs or db source)
|
||||
- `gbrain import`
|
||||
- `gbrain sync`
|
||||
- `gbrain migrate --to …`
|
||||
- `gbrain repair-jsonb`
|
||||
- `gbrain check-backlinks`
|
||||
- `gbrain lint`
|
||||
- `gbrain integrity auto`
|
||||
- `gbrain eval`
|
||||
- `gbrain apply-migrations` (the orchestrator + every child command)
|
||||
|
||||
Non-bulk commands (`stats`, `graph-query`, `get`, `put`, etc.) don't emit
|
||||
events — they return in under a second.
|
||||
|
||||
## Channel
|
||||
|
||||
- Progress events: **`stderr`**, one JSON object per line, `\n`-terminated.
|
||||
- Data results (`--json` payloads from each command): **`stdout`**.
|
||||
- Final human summaries: **`stdout`**.
|
||||
|
||||
Agents can safely capture stdout for their result parsing and read stderr
|
||||
separately for progress.
|
||||
|
||||
## Flags
|
||||
|
||||
| Flag | Behavior |
|
||||
|---|---|
|
||||
| *(none)* | Auto. TTY: `\r`-rewriting single line. Non-TTY: plain line-per-event on stderr. |
|
||||
| `--progress-json` | Force JSON-lines mode on stderr (this doc). |
|
||||
| `--quiet` | Suppress progress entirely. Warnings and final output still print. |
|
||||
| `--progress-interval=<ms>` | Override the minimum interval between tick emits (default 1000). |
|
||||
|
||||
Global flags: parsed by `src/core/cli-options.ts` before command dispatch,
|
||||
so `gbrain --progress-json doctor` works the same as
|
||||
`gbrain doctor --progress-json` (the latter also works — per-command
|
||||
parsers see the flag via the shared `CliOptions` singleton).
|
||||
|
||||
## Event types
|
||||
|
||||
Every event is a single-line JSON object with these common fields:
|
||||
|
||||
| Field | Type | Notes |
|
||||
|---|---|---|
|
||||
| `event` | string | One of: `start`, `tick`, `heartbeat`, `finish`, `abort`. |
|
||||
| `phase` | string | Machine-stable snake_case, dot-separated. See "Phase names" below. |
|
||||
| `ts` | ISO 8601 UTC string | Event emission time. |
|
||||
| `elapsed_ms` | number | Ms since the phase started. Present on `tick`/`heartbeat`/`finish`/`abort`. |
|
||||
|
||||
### `start`
|
||||
|
||||
Emitted when a phase begins.
|
||||
|
||||
```json
|
||||
{"event":"start","phase":"doctor.db_checks","ts":"2026-04-20T12:34:56.789Z"}
|
||||
{"event":"start","phase":"import.files","total":52000,"ts":"2026-04-20T12:34:56.789Z"}
|
||||
```
|
||||
|
||||
Optional fields:
|
||||
|
||||
- `total` — the total item count if known at start.
|
||||
|
||||
### `tick`
|
||||
|
||||
Emitted periodically during iteration. Time- and item-gated: the reporter
|
||||
won't emit more often than `minIntervalMs` (default 1000) and
|
||||
`minItems` (default `max(10, ceil(total/100))`).
|
||||
|
||||
```json
|
||||
{"event":"tick","phase":"orphans.scan","done":15000,"total":52000,"pct":28.8,"elapsed_ms":4200,"eta_ms":10300,"ts":"..."}
|
||||
```
|
||||
|
||||
Fields:
|
||||
|
||||
- `done` — items completed in this phase.
|
||||
- `total` — total items, if known. Omitted when the scan doesn't have a
|
||||
total up front (e.g. a streaming iterator).
|
||||
- `pct` — `done/total * 100`, one decimal. Omitted when `total` is unknown.
|
||||
- `eta_ms` — projected ms until `done === total`, from the observed rate.
|
||||
Omitted when `total` is unknown.
|
||||
- `note` — optional string with the current item (e.g. a slug or filename).
|
||||
|
||||
### `heartbeat`
|
||||
|
||||
Emitted for long-running single operations that don't iterate
|
||||
(e.g. `SELECT` against a 50K-row table). No `done`, no `total` — just a
|
||||
signal that work is still happening.
|
||||
|
||||
```json
|
||||
{"event":"heartbeat","phase":"doctor.markdown_body_completeness","note":"scanning pages for truncation…","elapsed_ms":1000,"ts":"..."}
|
||||
```
|
||||
|
||||
### `finish`
|
||||
|
||||
Emitted when a phase completes normally.
|
||||
|
||||
```json
|
||||
{"event":"finish","phase":"import.files","done":52000,"total":52000,"elapsed_ms":187000,"ts":"..."}
|
||||
```
|
||||
|
||||
### `abort`
|
||||
|
||||
Emitted by a single process-level SIGINT/SIGTERM handler that tracks every
|
||||
live phase. After `abort`, no further events emit for that phase.
|
||||
|
||||
```json
|
||||
{"event":"abort","phase":"doctor.markdown_body_completeness","reason":"SIGINT","elapsed_ms":5300,"ts":"..."}
|
||||
```
|
||||
|
||||
## Phase names
|
||||
|
||||
Phases use `snake_case.dot.path` naming. A fresh reporter starts at the
|
||||
root; `child()` composition appends to the parent's current phase, so a
|
||||
sync that calls import emits `sync.import.<file>`, not `import.<file>`.
|
||||
|
||||
Stable phase names shipped in v0.15.2:
|
||||
|
||||
- `doctor.db_checks` (umbrella for all DB-side doctor checks)
|
||||
- `orphans.scan`
|
||||
- `embed.pages`
|
||||
- `extract.links_fs`, `extract.timeline_fs`, `extract.links_db`, `extract.timeline_db`
|
||||
- `import.files`
|
||||
- `sync.deletes`, `sync.renames`, `sync.imports`
|
||||
- `migrate.copy_pages`, `migrate.copy_links`
|
||||
- `repair_jsonb.run`, `repair_jsonb.<table>.<column>`
|
||||
- `backlinks.scan`
|
||||
- `lint.pages`
|
||||
- `integrity.auto`
|
||||
- `eval.single`, `eval.ab`
|
||||
- `export.pages`
|
||||
- `files.sync`
|
||||
|
||||
Sub-phases exposed via `child()`:
|
||||
|
||||
- `sync.import.files` — nested inside a sync
|
||||
- `apply_migrations.v0_12_2.jsonb_repair` — nested inside the orchestrator
|
||||
|
||||
## Subprocess inheritance
|
||||
|
||||
When a parent CLI spawns `gbrain …` child processes (mostly in
|
||||
`src/commands/migrations/*`), global flags (`--quiet`, `--progress-json`,
|
||||
`--progress-interval`) are propagated to the child's argv via the
|
||||
`childGlobalFlags()` helper in `src/core/cli-options.ts`. Child stderr
|
||||
passes straight through `stdio: 'inherit'` so the event stream is one
|
||||
merged JSONL feed on the parent's stderr.
|
||||
|
||||
One exception: the orchestrator phase in `migrations/v0_12_2.ts` that
|
||||
captures child stdout (`repair-jsonb --dry-run --json` for verification)
|
||||
does not pass `--progress-json` to avoid any risk of stdout pollution
|
||||
breaking the orchestrator's `JSON.parse`. Its stdio is explicit:
|
||||
`['ignore', 'pipe', 'inherit']` so stderr still flows through.
|
||||
|
||||
## Minion jobs
|
||||
|
||||
`gbrain jobs work` (the Minion worker daemon) keeps progress in the DB,
|
||||
not on stderr. Each Minion handler that runs a bulk core (embed, sync,
|
||||
extract, import, backlinks) calls `job.updateProgress({done, total,
|
||||
…})` per iteration. Agents read per-job progress via the
|
||||
`get_job_progress` MCP operation or `gbrain jobs get <id>`.
|
||||
|
||||
The `jobs work` daemon itself emits coarse one-line-per-job stderr output
|
||||
for liveness only. Per-page detail lives in the DB.
|
||||
|
||||
## Compatibility
|
||||
|
||||
- **Added**: only. A new event type, a new field, a new phase name — all
|
||||
safe. Agents must ignore unknown fields and unknown event types.
|
||||
- **Removed/renamed**: never without a major version bump.
|
||||
- **Schema changes**: announced in `CHANGELOG.md` and in
|
||||
`skills/migrations/v<next>.md`.
|
||||
|
||||
If your agent depends on this schema and something surprises you, open
|
||||
an issue with the event you received and what you expected.
|
||||
@@ -1,210 +0,0 @@
|
||||
# Storage Tiering: db-tracked vs db-only directories
|
||||
|
||||
## Overview
|
||||
|
||||
GBrain supports storage tiering to separate version-controlled content from bulk machine-generated data. This prevents git repositories from becoming bloated with large amounts of automatically generated content while still preserving it in the database.
|
||||
|
||||
> Note on naming: prior to v0.22.11 the keys were `git_tracked` / `supabase_only`. The canonical names are now `db_tracked` / `db_only` (engine-agnostic — works on both PGLite and Postgres). The deprecated keys still load with a once-per-process warning. Run `gbrain doctor --fix` for an automated rename when that path lands.
|
||||
|
||||
## Configuration
|
||||
|
||||
Add a `storage` section to your `gbrain.yml` file in the brain repository root:
|
||||
|
||||
```yaml
|
||||
storage:
|
||||
# Directories that are version-controlled (human-edited, committed to git).
|
||||
db_tracked:
|
||||
- people/
|
||||
- companies/
|
||||
- deals/
|
||||
- concepts/
|
||||
- yc/
|
||||
- ideas/
|
||||
- projects/
|
||||
|
||||
# Directories persisted via the brain database only (bulk machine-generated
|
||||
# content). Written to disk as a local cache but not committed to git;
|
||||
# `gbrain sync` auto-manages .gitignore for these paths. `gbrain export
|
||||
# --restore-only` repopulates missing files from the database.
|
||||
db_only:
|
||||
- media/x/
|
||||
- media/articles/
|
||||
- meetings/transcripts/
|
||||
```
|
||||
|
||||
Path requirements:
|
||||
|
||||
- Each directory must end with `/` for canonical form. The validator auto-normalizes missing trailing slashes (one-time info note shows what changed).
|
||||
- A directory cannot appear in both tiers — that's a tier-overlap error and `loadStorageConfig` throws `StorageConfigError`. Edit `gbrain.yml` to remove the overlap and try again.
|
||||
|
||||
## Behavior Changes
|
||||
|
||||
### 1. `gbrain sync` — automatic .gitignore management
|
||||
|
||||
When storage configuration is present, `gbrain sync` automatically manages `.gitignore` entries on every successful sync:
|
||||
|
||||
- Adds missing `db_only` directory patterns to `.gitignore`.
|
||||
- Idempotent — re-running adds no duplicate entries.
|
||||
- Stable comment header so the managed block is grep-able.
|
||||
- Skipped on `--dry-run` (don't mutate disk in preview mode).
|
||||
- Skipped on `blocked_by_failures` status (sync state is inconsistent).
|
||||
- Skipped when the repo is a git submodule (`.git` is a file, not a directory) — submodule .gitignore changes don't survive parent updates. A warning explains.
|
||||
- Skipped entirely when `GBRAIN_NO_GITIGNORE=1` is set (escape hatch for shared-repo setups where a maintainer wants gbrain to leave .gitignore alone).
|
||||
- Failures (write permission denied, etc.) are caught and logged, never crash sync.
|
||||
|
||||
Example `.gitignore` addition:
|
||||
|
||||
```gitignore
|
||||
# Auto-managed by gbrain (db_only directories)
|
||||
media/x/
|
||||
media/articles/
|
||||
meetings/transcripts/
|
||||
```
|
||||
|
||||
### 2. `gbrain export --restore-only` — repopulate missing db_only files
|
||||
|
||||
```bash
|
||||
# Restore only missing db_only files from the database.
|
||||
gbrain export --restore-only --repo /path/to/brain
|
||||
|
||||
# Filter by page type.
|
||||
gbrain export --restore-only --type media --repo /path/to/brain
|
||||
|
||||
# Filter by slug prefix.
|
||||
gbrain export --restore-only --slug-prefix media/x/ --repo /path/to/brain
|
||||
|
||||
# Combine filters.
|
||||
gbrain export --restore-only --type media --slug-prefix media/x/ --repo /path/to/brain
|
||||
```
|
||||
|
||||
The `--restore-only` flag:
|
||||
|
||||
- Resolves repoPath via the chain `--repo` → typed `sources.getDefault()` → hard error.
|
||||
Never falls through to the current directory.
|
||||
- Only exports pages that match `db_only` patterns AND are missing from disk.
|
||||
- Ideal for container restart recovery and fresh clones.
|
||||
|
||||
### 3. `gbrain storage status` — storage-tier health dashboard
|
||||
|
||||
```bash
|
||||
# Human-readable status.
|
||||
gbrain storage status --repo /path/to/brain
|
||||
|
||||
# JSON output for scripts and orchestrators.
|
||||
gbrain storage status --repo /path/to/brain --json
|
||||
```
|
||||
|
||||
Output includes:
|
||||
|
||||
- Total page counts by storage tier.
|
||||
- Disk usage breakdown by tier.
|
||||
- Missing files that need restoration (top 10 shown; full list in `--json`).
|
||||
- Configuration validation warnings.
|
||||
- Current tier directory listing.
|
||||
|
||||
Example output:
|
||||
|
||||
```
|
||||
Storage Status
|
||||
==============
|
||||
|
||||
Repository: /data/brain
|
||||
Total pages: 15,243
|
||||
|
||||
Storage Tiers:
|
||||
-------------
|
||||
DB tracked: 2,156 pages
|
||||
DB only: 12,887 pages
|
||||
Unspecified: 200 pages
|
||||
|
||||
Disk Usage:
|
||||
-----------
|
||||
DB tracked: 45.2 MB
|
||||
DB only: 2.1 GB
|
||||
|
||||
Missing Files (need restore):
|
||||
-----------------------------
|
||||
media/x/tweet-1234567890
|
||||
media/x/tweet-0987654321
|
||||
... and 47 more
|
||||
|
||||
Use: gbrain export --restore-only --repo "/data/brain"
|
||||
|
||||
Configuration:
|
||||
--------------
|
||||
DB tracked directories:
|
||||
- people/
|
||||
- companies/
|
||||
- deals/
|
||||
|
||||
DB-only directories:
|
||||
- media/x/
|
||||
- media/articles/
|
||||
- meetings/transcripts/
|
||||
```
|
||||
|
||||
## Validation
|
||||
|
||||
`loadStorageConfig` runs `normalizeAndValidateStorageConfig` after parsing:
|
||||
|
||||
- Auto-fixes (silent, with one-time info note showing what changed):
|
||||
- Missing trailing `/` is added: `'media/x'` → `'media/x/'`.
|
||||
- Throws `StorageConfigError` (caller sees a clean exit-1 with actionable message):
|
||||
- Same directory in both `db_tracked` and `db_only` (ambiguous routing).
|
||||
|
||||
## Use cases
|
||||
|
||||
### Brain repository scaling
|
||||
|
||||
Perfect for brain repositories crossing 50K-200K+ files where:
|
||||
|
||||
- Core knowledge (people, companies, deals) remains git-tracked.
|
||||
- Bulk data (tweets, articles, transcripts) moves to db_only.
|
||||
- Development stays fast with smaller git repos.
|
||||
- Full data remains available via the database.
|
||||
|
||||
### Container-based deployments
|
||||
|
||||
Essential for ephemeral container environments:
|
||||
|
||||
- Git repo contains only essential files.
|
||||
- Container restarts don't lose db_only data.
|
||||
- `gbrain export --restore-only` quickly restores bulk files when needed.
|
||||
- Local disk acts as a cache layer.
|
||||
|
||||
### Multi-environment consistency
|
||||
|
||||
Enables consistent data access across environments:
|
||||
|
||||
- Development: small git clone, restore bulk data on demand.
|
||||
- Production: full dataset via the database, selective local caching.
|
||||
- CI/CD: fast tests with git-tracked data only.
|
||||
|
||||
## Migration strategy
|
||||
|
||||
1. **Assess current repository**: use `gbrain storage status` to understand current distribution.
|
||||
2. **Plan directory structure**: identify which directories should be db_tracked vs db_only.
|
||||
3. **Create `gbrain.yml`**: add storage configuration to the repository root.
|
||||
4. **Test with dry-run**: `gbrain sync --dry-run` to verify behavior; `.gitignore` is NOT touched on dry-run.
|
||||
5. **Run a real sync**: `gbrain sync` updates `.gitignore` automatically on success.
|
||||
6. **Verify restore**: test `gbrain export --restore-only --repo .` against a small db_only directory.
|
||||
|
||||
## Best practices
|
||||
|
||||
- **Directory naming**: end storage paths with `/` (canonical form). The validator normalizes if you forget.
|
||||
- **Start small**: begin with clearly machine-generated directories in `db_only`.
|
||||
- **Address validation errors**: tier overlap is an error, not a warning. Fix it before sync.
|
||||
- **Test restore**: regularly test `--restore-only` in staging environments.
|
||||
- **Document decisions**: comment your `gbrain.yml` to explain tier choices.
|
||||
|
||||
## PGLite engine note
|
||||
|
||||
On the PGLite engine (gbrain's local-only embedded Postgres), the "DB" your db_only pages live in IS the local file gbrain uses for everything else. The `.gitignore` housekeeping still helps (keeps bulk content out of git history), but the offload-to-DB promise is technically vacuous. A once-per-process soft-warn explains when the engine is detected. To get full tiering, migrate to Postgres with `gbrain migrate --to supabase`.
|
||||
|
||||
## Compatibility
|
||||
|
||||
- **Backward compatible**: systems without `gbrain.yml` work unchanged.
|
||||
- **Progressive enhancement**: add configuration when needed.
|
||||
- **Database unchanged**: all data remains in Postgres regardless of tier.
|
||||
- **Existing workflows**: all existing `sync` and `export` behavior preserved.
|
||||
- **Deprecated keys**: `git_tracked` / `supabase_only` still load with a once-per-process warning.
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
storage:
|
||||
# Directories that are version-controlled — human-curated, edited by hand.
|
||||
db_tracked:
|
||||
- people/
|
||||
- companies/
|
||||
- deals/
|
||||
- concepts/
|
||||
- yc/
|
||||
- ideas/
|
||||
- projects/
|
||||
|
||||
# Directories persisted via the brain database only — bulk machine-generated
|
||||
# content. .gitignored automatically by `gbrain sync`. Restorable from the DB
|
||||
# via `gbrain export --restore-only`.
|
||||
db_only:
|
||||
- media/x/
|
||||
- media/articles/
|
||||
- meetings/transcripts/
|
||||
-5478
File diff suppressed because it is too large
Load Diff
@@ -1,52 +0,0 @@
|
||||
# GBrain
|
||||
|
||||
> GBrain is a personal knowledge brain and GStack mod for agent platforms. Pluggable engines (PGLite default, Postgres+pgvector for scale), contract-first operations, 26 fat-markdown skills. Teaches agents brain ops, ingestion, enrichment, scheduling, identity, and access control.
|
||||
|
||||
Repo: https://github.com/garrytan/gbrain
|
||||
|
||||
## Core entry points
|
||||
|
||||
- [AGENTS.md](https://raw.githubusercontent.com/garrytan/gbrain/master/AGENTS.md): Start here if you are not Claude Code. Install order, trust boundary, skill resolver, config/debug/migration pointers.
|
||||
- [CLAUDE.md](https://raw.githubusercontent.com/garrytan/gbrain/master/CLAUDE.md): Architecture reference. Key files, trust boundaries, engine factory, test layout.
|
||||
- [INSTALL_FOR_AGENTS.md](https://raw.githubusercontent.com/garrytan/gbrain/master/INSTALL_FOR_AGENTS.md): 9-step agent installation.
|
||||
- [skills/RESOLVER.md](https://raw.githubusercontent.com/garrytan/gbrain/master/skills/RESOLVER.md): Skill dispatcher. Read first for any task.
|
||||
- [README.md](https://raw.githubusercontent.com/garrytan/gbrain/master/README.md): Project overview, benchmarks, 30-minute setup.
|
||||
|
||||
## Configuration
|
||||
|
||||
- [docs/ENGINES.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/ENGINES.md): PGLite vs Postgres trade-off and when to migrate.
|
||||
- [docs/GBRAIN_RECOMMENDED_SCHEMA.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/GBRAIN_RECOMMENDED_SCHEMA.md): MECE directory structure (people/, companies/, concepts/).
|
||||
- [docs/guides/live-sync.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/guides/live-sync.md): Incremental markdown sync setup.
|
||||
- [docs/guides/cron-schedule.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/guides/cron-schedule.md): Recurring job scheduling.
|
||||
- [docs/guides/minions-deployment.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/guides/minions-deployment.md): Deploying the gbrain jobs worker: crontab + watchdog, inline --follow, systemd/Procfile/fly.toml, upgrade checklist.
|
||||
- [docs/guides/quiet-hours.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/guides/quiet-hours.md): Notification hold + timezone-aware delivery.
|
||||
- [docs/mcp/DEPLOY.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/mcp/DEPLOY.md): MCP server deployment.
|
||||
|
||||
## Debugging
|
||||
|
||||
- [docs/GBRAIN_VERIFY.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/GBRAIN_VERIFY.md): 7-check post-setup verification. Start here when something feels off.
|
||||
- [docs/guides/minions-fix.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/guides/minions-fix.md): Troubleshooting the Minions job queue.
|
||||
- [docs/integrations/reliability-repair.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/integrations/reliability-repair.md): Data integrity recovery.
|
||||
|
||||
## Migrations
|
||||
|
||||
- [docs/UPGRADING_DOWNSTREAM_AGENTS.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/UPGRADING_DOWNSTREAM_AGENTS.md): Patches for downstream agent skill forks. One section per release.
|
||||
- [skills/migrations/](https://raw.githubusercontent.com/garrytan/gbrain/master/skills/migrations/): Per-version (v0.5.0 - v0.14.1) agent-executable migration instructions.
|
||||
- [CHANGELOG.md](https://raw.githubusercontent.com/garrytan/gbrain/master/CHANGELOG.md): Release-summary voice + itemized changes + self-repair block per version.
|
||||
|
||||
## Philosophy
|
||||
|
||||
- [docs/ethos/THIN_HARNESS_FAT_SKILLS.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/ethos/THIN_HARNESS_FAT_SKILLS.md): Why skills live in markdown.
|
||||
- [docs/ethos/MARKDOWN_SKILLS_AS_RECIPES.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/ethos/MARKDOWN_SKILLS_AS_RECIPES.md): Homebrew for Personal AI.
|
||||
|
||||
## Optional
|
||||
|
||||
- [docs/designs/](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/designs/): Forward-looking designs.
|
||||
- [docs/architecture/infra-layer.md](https://raw.githubusercontent.com/garrytan/gbrain/master/docs/architecture/infra-layer.md): Shared infra patterns.
|
||||
|
||||
## Operational tips
|
||||
|
||||
- `gbrain doctor [--json] [--fast] [--fix]` - built-in health checks.
|
||||
- `gbrain orphans [--json]` - pages with zero inbound wikilinks.
|
||||
- `gbrain repair-jsonb [--dry-run]` - repair v0.12.0 double-encoded JSONB rows.
|
||||
- `gbrain upgrade` runs post-upgrade + apply-migrations.
|
||||
@@ -1,69 +0,0 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.19.0",
|
||||
"description": "Personal knowledge brain with Postgres + pgvector hybrid search",
|
||||
"family": "bundle-plugin",
|
||||
"configSchema": {
|
||||
"database_url": {
|
||||
"type": "string",
|
||||
"required": true,
|
||||
"description": "PostgreSQL connection URL (Supabase recommended)",
|
||||
"uiHints": { "sensitive": true }
|
||||
},
|
||||
"openai_api_key": {
|
||||
"type": "string",
|
||||
"required": false,
|
||||
"description": "OpenAI API key for embeddings (uses OPENAI_API_KEY env var if not set)",
|
||||
"uiHints": { "sensitive": true }
|
||||
}
|
||||
},
|
||||
"mcpServers": {
|
||||
"gbrain": {
|
||||
"command": "./bin/gbrain",
|
||||
"args": ["serve"]
|
||||
}
|
||||
},
|
||||
"skills": [
|
||||
"skills/brain-ops",
|
||||
"skills/briefing",
|
||||
"skills/citation-fixer",
|
||||
"skills/cross-modal-review",
|
||||
"skills/cron-scheduler",
|
||||
"skills/daily-task-manager",
|
||||
"skills/daily-task-prep",
|
||||
"skills/data-research",
|
||||
"skills/enrich",
|
||||
"skills/idea-ingest",
|
||||
"skills/ingest",
|
||||
"skills/maintain",
|
||||
"skills/media-ingest",
|
||||
"skills/meeting-ingestion",
|
||||
"skills/minion-orchestrator",
|
||||
"skills/query",
|
||||
"skills/reports",
|
||||
"skills/repo-architecture",
|
||||
"skills/signal-detector",
|
||||
"skills/skill-creator",
|
||||
"skills/skillify",
|
||||
"skills/skillpack-check",
|
||||
"skills/soul-audit",
|
||||
"skills/testing",
|
||||
"skills/webhook-transforms"
|
||||
],
|
||||
"shared_deps": [
|
||||
"skills/conventions",
|
||||
"skills/_brain-filing-rules.md",
|
||||
"skills/_brain-filing-rules.json",
|
||||
"skills/_output-rules.md"
|
||||
],
|
||||
"excluded_from_install": [
|
||||
"skills/setup",
|
||||
"skills/migrate",
|
||||
"skills/publish"
|
||||
],
|
||||
"openclaw": {
|
||||
"compat": {
|
||||
"pluginApi": ">=2026.4.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.25.0",
|
||||
"description": "Postgres-native personal knowledge brain with hybrid RAG search",
|
||||
"type": "module",
|
||||
"main": "src/core/index.ts",
|
||||
"bin": {
|
||||
"gbrain": "src/cli.ts"
|
||||
},
|
||||
"exports": {
|
||||
".": "./src/core/index.ts",
|
||||
"./engine": "./src/core/engine.ts",
|
||||
"./types": "./src/core/types.ts",
|
||||
"./operations": "./src/core/operations.ts",
|
||||
"./minions": "./src/core/minions/index.ts",
|
||||
"./engine-factory": "./src/core/engine-factory.ts",
|
||||
"./pglite-engine": "./src/core/pglite-engine.ts",
|
||||
"./link-extraction": "./src/core/link-extraction.ts",
|
||||
"./import-file": "./src/core/import-file.ts",
|
||||
"./transcription": "./src/core/transcription.ts",
|
||||
"./embedding": "./src/core/embedding.ts",
|
||||
"./config": "./src/core/config.ts",
|
||||
"./markdown": "./src/core/markdown.ts",
|
||||
"./backoff": "./src/core/backoff.ts",
|
||||
"./search/hybrid": "./src/core/search/hybrid.ts",
|
||||
"./search/expansion": "./src/core/search/expansion.ts",
|
||||
"./extract": "./src/commands/extract.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "bun run src/cli.ts",
|
||||
"build": "bun build --compile --outfile bin/gbrain src/cli.ts",
|
||||
"build:all": "bun build --compile --target=bun-darwin-arm64 --outfile bin/gbrain-darwin-arm64 src/cli.ts && bun build --compile --target=bun-linux-x64 --outfile bin/gbrain-linux-x64 src/cli.ts",
|
||||
"build:schema": "bash scripts/build-schema.sh",
|
||||
"build:llms": "bun run scripts/build-llms.ts",
|
||||
"build:pglite-snapshot": "bun run scripts/build-pglite-snapshot.ts",
|
||||
"test": "scripts/check-privacy.sh && scripts/check-jsonb-pattern.sh && scripts/check-progress-to-stdout.sh && scripts/check-trailing-newline.sh && scripts/check-wasm-embedded.sh && scripts/check-exports-count.sh && bun run typecheck && bun test --timeout=60000",
|
||||
"check:wasm": "scripts/check-wasm-embedded.sh",
|
||||
"check:newlines": "scripts/check-trailing-newline.sh",
|
||||
"test:e2e": "bash scripts/run-e2e.sh",
|
||||
"test:slow": "bash scripts/run-slow-tests.sh",
|
||||
"test:profile": "bash scripts/profile-tests.sh",
|
||||
"ci:local": "bash scripts/ci-local.sh",
|
||||
"ci:local:diff": "bash scripts/ci-local.sh --diff",
|
||||
"ci:select-e2e": "bun run scripts/select-e2e.ts",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"check:jsonb": "scripts/check-jsonb-pattern.sh",
|
||||
"check:privacy": "scripts/check-privacy.sh",
|
||||
"check:progress": "scripts/check-progress-to-stdout.sh",
|
||||
"check:exports-count": "scripts/check-exports-count.sh",
|
||||
"postinstall": "command -v gbrain >/dev/null 2>&1 && gbrain apply-migrations --yes --non-interactive || echo '[gbrain] postinstall skipped. If installed via bun install -g github:...: run `gbrain doctor` and `gbrain apply-migrations --yes` manually. See https://github.com/garrytan/gbrain/issues/218' 1>&2",
|
||||
"prepublish:clawhub": "bun run build:all",
|
||||
"publish:clawhub": "clawhub package publish . --family bundle-plugin"
|
||||
},
|
||||
"openclaw": {
|
||||
"compat": {
|
||||
"pluginApi": ">=2026.4.0"
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "^0.30.0",
|
||||
"@aws-sdk/client-s3": "^3.1028.0",
|
||||
"@dqbd/tiktoken": "^1.0.22",
|
||||
"@electric-sql/pglite": "0.4.3",
|
||||
"@modelcontextprotocol/sdk": "^1.0.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
"marked": "^18.0.0",
|
||||
"openai": "^4.0.0",
|
||||
"pgvector": "^0.2.0",
|
||||
"postgres": "^3.4.0",
|
||||
"tree-sitter-wasms": "0.1.13",
|
||||
"web-tree-sitter": "0.22.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"bun-types": "^1.3.13",
|
||||
"typescript": "^5.6.0"
|
||||
},
|
||||
"trustedDependencies": [
|
||||
"@electric-sql/pglite"
|
||||
],
|
||||
"license": "MIT"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
<!-- gbrain-plugin-tree-stamp: 0.46.11.0 -->
|
||||
# gbrain plugin skill tree (generated — do not hand-edit)
|
||||
|
||||
This tree is the curated skill set for the gbrain Codex and Claude Code
|
||||
plugins. Regenerate with `bun run scripts/generate-plugin-tree.ts --out plugin`;
|
||||
curation lives in `skills/plugin-lanes.json` (one recorded decision per
|
||||
addition/exclusion).
|
||||
|
||||
## MCP surface note (read once)
|
||||
|
||||
The plugin's MCP server runs `gbrain serve --surface starter` — the 26-op
|
||||
daily-driver surface (the seven memory verbs + daily brain ops). 21
|
||||
bundled skills reference gbrain operations beyond that surface; every one of
|
||||
them has a first-class `gbrain` CLI path, which is the primary way skills
|
||||
drive gbrain. When a skill step names an operation your MCP tool list doesn't
|
||||
carry, run the equivalent `gbrain` CLI command, or widen this machine's
|
||||
plugin surface with `GBRAIN_SURFACE=full` (the launcher honors it; new
|
||||
sessions pick it up).
|
||||
|
||||
## Requirements
|
||||
|
||||
- gbrain CLI installed: `bun install -g github:garrytan/gbrain#latest-stable`
|
||||
(the npm package named `gbrain` is unrelated — never `npm install -g gbrain`).
|
||||
- A brain: `gbrain init` (the bundled `setup` skill walks the full path).
|
||||
@@ -0,0 +1,148 @@
|
||||
# Agent onboarding — what to do with the files in this directory
|
||||
|
||||
You (the agent) are running on a host that scaffolded gbrain skills here. This
|
||||
file is the operating contract. Read it on every cold start. It is short on
|
||||
purpose.
|
||||
|
||||
## What lives in this directory
|
||||
|
||||
```
|
||||
skills/
|
||||
_AGENT_README.md ← you are here
|
||||
_brain-filing-rules.md ← where to file brain pages (read on every write)
|
||||
_output-rules.md ← output quality standards (no LLM slop, exact phrasing)
|
||||
_friction-protocol.md ← log friction the user hits to ~/.gstack/friction/
|
||||
conventions/ ← cross-cutting rules every skill defers to
|
||||
<skill-name>/
|
||||
SKILL.md ← the skill's contract + workflow
|
||||
routing-eval.jsonl ← (optional) test fixtures for routing-eval
|
||||
script.ts ← (optional) deterministic code, if any
|
||||
```
|
||||
|
||||
Other files in the host repo's `src/`, `docs/`, `recipes/` etc. are owned by the
|
||||
host, not by gbrain. Don't treat them as gbrain artifacts.
|
||||
|
||||
## Routing — your first job
|
||||
|
||||
Discover skills at runtime by walking every `skills/<slug>/SKILL.md` here and
|
||||
parsing the YAML frontmatter. Each skill declares one or more `triggers:`
|
||||
strings; they are the user-facing phrases that route to that skill.
|
||||
|
||||
```yaml
|
||||
---
|
||||
name: book-mirror
|
||||
triggers:
|
||||
- "personalized version of this book"
|
||||
- "mirror this book"
|
||||
- "two-column book analysis"
|
||||
---
|
||||
```
|
||||
|
||||
On every user message, match the message against every skill's `triggers:`
|
||||
array. Substring match is the baseline. Semantic similarity (embedding or
|
||||
keyword expansion) is fine on top. When a trigger matches strongly, invoke the
|
||||
skill — read its SKILL.md body in full and follow the workflow described there.
|
||||
|
||||
**The routing contract:** frontmatter `triggers:` are authoritative.
|
||||
`skills/RESOLVER.md` is the human-readable dispatch map of the same routing —
|
||||
useful for scanning every skill and its trigger phrases in one place, and it
|
||||
carries the disambiguation rules for overlapping matches. If the two disagree,
|
||||
frontmatter wins. (There is no machine-managed block inside `RESOLVER.md` or
|
||||
`AGENTS.md`; that pattern was retired.)
|
||||
|
||||
## When the user invokes a skill
|
||||
|
||||
Read the entire `skills/<slug>/SKILL.md` file. Follow its `## Phases`,
|
||||
`## Workflow`, or equivalent step-by-step section. If the skill has a
|
||||
`mutating: true` frontmatter and declares `writes_pages:` / `writes_to:`,
|
||||
those are the brain-side write surfaces — consult `_brain-filing-rules.md`
|
||||
to confirm the file path is sanctioned.
|
||||
|
||||
If the SKILL.md frontmatter declares `sources:` (paired source files), those
|
||||
live at their mirror path in the host repo (e.g. `src/commands/<slug>.ts`).
|
||||
They are reference code that the gbrain CLI calls. You do not run them
|
||||
directly unless the SKILL.md tells you to.
|
||||
|
||||
## Updates — when gbrain ships a new version
|
||||
|
||||
The user runs `gbrain upgrade`. Skill files DO NOT change automatically.
|
||||
gbrain becomes a reference library you compare against.
|
||||
|
||||
On every cold start, or any time the user mentions an upgrade, run:
|
||||
|
||||
```bash
|
||||
gbrain skillpack reference --all
|
||||
```
|
||||
|
||||
That sweeps every bundled skill and reports per-skill `identical / differs /
|
||||
missing` counts. For each `differs`:
|
||||
|
||||
```bash
|
||||
gbrain skillpack reference <slug>
|
||||
```
|
||||
|
||||
This prints a unified diff between gbrain's bundle and the local file. Read
|
||||
it, then decide per file:
|
||||
|
||||
- **Local edit was intentional.** Keep your version. gbrain is reference, not
|
||||
law.
|
||||
- **Local edit was accidental drift** (e.g. you wrote stale content into the
|
||||
skill body). Either patch by hand, or run
|
||||
`gbrain skillpack reference <slug> --apply-clean-hunks` (read the WARNING
|
||||
about two-way merge below first).
|
||||
- **Genuinely new gbrain change in a section you don't care about.** Skip or
|
||||
apply per your judgment.
|
||||
|
||||
For `missing` files (gbrain added a new bundled skill since you scaffolded),
|
||||
run `gbrain skillpack scaffold <new-slug>` to bring it in.
|
||||
|
||||
### `reference --apply-clean-hunks` — two-way merge warning
|
||||
|
||||
This command does a two-way diff against gbrain's current bundle. It does
|
||||
NOT have access to the version you originally scaffolded. Consequence: if
|
||||
the user's local file differs from gbrain in ANY section (including
|
||||
intentional user edits), those sections WILL be aligned to gbrain.
|
||||
|
||||
Always run plain `gbrain skillpack reference <slug>` first to inspect.
|
||||
Use `--apply-clean-hunks` only when you're confident the local edits were
|
||||
accidental or you want to fully reset to gbrain's current bundle.
|
||||
|
||||
## Removing a scaffolded skill
|
||||
|
||||
There is no `uninstall` command (`gbrain skillpack uninstall` exits with an
|
||||
error pointing here). The files are yours.
|
||||
|
||||
```bash
|
||||
rm -rf skills/<slug>
|
||||
# if the skill declared paired source files:
|
||||
rm src/commands/<slug>.ts
|
||||
```
|
||||
|
||||
Consult the skill's frontmatter `sources:` array for the full paired-file
|
||||
list before deleting.
|
||||
|
||||
## When in doubt
|
||||
|
||||
The single source of truth for the model is
|
||||
`docs/guides/skillpacks-as-scaffolding.md` in the gbrain repo. The skill
|
||||
files you scaffolded are the source of truth for individual skill behavior.
|
||||
This file (`_AGENT_README.md`) is the routing contract — keep it short.
|
||||
|
||||
## Frontmatter contract notes
|
||||
|
||||
- **`upstream: <donor-skill>@<short-sha>`** — the provenance pin: which
|
||||
donor skill (by slug) and which commit of it this skill was ported from.
|
||||
Multi-source ports pin every donor, either as a YAML list or plus-joined
|
||||
(`upstream: skill-a@abc1234 + skill-b@def5678`). To resolve a drift or
|
||||
behavior question, diff the current SKILL.md against the pinned source
|
||||
commit — the pin is what makes that diff possible.
|
||||
- **Optional keys are omitted, not zeroed.** Omit `writes_to` entirely when
|
||||
the skill writes no pages (an empty list implies "writes pages, nowhere",
|
||||
which is a contradiction). `brain_first: exempt` is allowed only with an
|
||||
adjacent comment justifying WHY the skill is exempt from the brain-first
|
||||
lookup chain — an unexplained exemption is a conformance failure.
|
||||
- **`priority:` is NOT part of the routing contract.** Nothing in the routing
|
||||
path consumes it — matching is substring-over-`triggers:` (see "Routing"
|
||||
above), with `RESOLVER.md` disambiguation for overlaps. A `priority:` key is
|
||||
inert; don't add one expecting it to reorder matches. Encode precedence in
|
||||
trigger specificity and the resolver's disambiguation rules instead.
|
||||
@@ -81,11 +81,65 @@
|
||||
"examples": ["logistics", "family"],
|
||||
"description": "Personal-life content — kept separate from work."
|
||||
},
|
||||
{
|
||||
"kind": "idea",
|
||||
"directory": "ideas/",
|
||||
"examples": ["product ideas", "essay seeds", "back-of-envelope concepts"],
|
||||
"description": "Generative ideas the user might build, write, or expand later. Stub-shaped pages that mature over time. voice-note-ingest, archive-crawler, and similar capture-flavored skills file here when content is something to potentially act on."
|
||||
},
|
||||
{
|
||||
"kind": "research",
|
||||
"directory": "research/",
|
||||
"examples": ["web-research deltas", "freshness checks", "citation-verified claims"],
|
||||
"description": "Web-research output: what is NEW vs already-known about a topic, citation-checked claims, freshness deltas. perplexity-research and academic-verify file here."
|
||||
},
|
||||
{
|
||||
"kind": "original",
|
||||
"directory": "originals/",
|
||||
"examples": ["the user's own theses", "frameworks the user generated", "novel observations the user expressed"],
|
||||
"description": "Pages where the user is the primary author of the idea — original thinking, not summarizations of someone else's work. voice-note-ingest, archive-crawler, signal-detector route content here when the user is the originator."
|
||||
},
|
||||
{
|
||||
"kind": "voice-note",
|
||||
"directory": "voice-notes/",
|
||||
"examples": ["raw transcripts", "audio capture pages"],
|
||||
"description": "Voice-note transcript holders, especially when the content is a random thought that doesn't cleanly fit originals/, concepts/, or another subject directory. voice-note-ingest is the primary writer."
|
||||
},
|
||||
{
|
||||
"kind": "openclaw",
|
||||
"directory": "openclaw/",
|
||||
"examples": ["agent-state notes"],
|
||||
"description": "Notes about the host OpenClaw agent itself, not the underlying entities."
|
||||
},
|
||||
{
|
||||
"kind": "synthesis-output",
|
||||
"directory": "media/books/",
|
||||
"examples": ["personalized book mirrors", "two-column chapter analyses"],
|
||||
"description": "Sanctioned exception to 'file by primary subject' for sui generis synthesized output that is one-of-one to a single book and a specific reader. Format-prefixed under media/<format>/ is allowed for synthesis output only, never for raw ingest. See _brain-filing-rules.md."
|
||||
},
|
||||
{
|
||||
"kind": "synthesis-output",
|
||||
"directory": "media/articles/",
|
||||
"examples": ["personalized article reads", "long-form content tailored to reader"],
|
||||
"description": "Same sanctioned exception as media/books/. One-of-one synthesis output of an article personalized for the reader. Distinct from raw article ingest, which goes to the article's primary-subject directory."
|
||||
},
|
||||
{
|
||||
"kind": "daily",
|
||||
"directory": "daily/",
|
||||
"examples": ["daily/calendar/YYYY-MM-DD.md", "daily/notes/YYYY-MM-DD.md"],
|
||||
"description": "Date-keyed pages for events, calendar entries, or daily notes. Calendar imports land at daily/calendar/YYYY-MM-DD.md with attendees cross-linked to people/. Use when the primary subject is the date itself, not a person or topic."
|
||||
},
|
||||
{
|
||||
"kind": "media-format",
|
||||
"directory": "media/",
|
||||
"examples": ["media/x/{handle}/", "media/audio/", "media/video/"],
|
||||
"description": "Format-prefixed parent for media-by-source-format ingest. Subdirectories like media/x/{handle}/ hold X/Twitter archives, media/audio/ holds podcast/voice captures. The format-prefix lives only when the content is sui generis to the source format AND lacks a clean primary-subject directory. Prefer subject-by-subject filing; fall through to media/ only when the source format IS the unifying frame."
|
||||
},
|
||||
{
|
||||
"kind": "conversation",
|
||||
"directory": "conversations/",
|
||||
"examples": ["conversations/chatgpt/{thread-slug}.md", "conversations/claude/{thread-slug}.md"],
|
||||
"description": "Imported chat exports (ChatGPT, Claude, etc.) where the conversation itself is the artifact. Cross-link concepts and people from the conversation; the conversation page is the source-of-truth for the dialog. Distinct from voice-notes/ (which holds raw voice capture)."
|
||||
}
|
||||
],
|
||||
"sources_dir": {
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user