mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-16 09:52:22 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bff3066985 |
@@ -28,5 +28,5 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: rhysd/actionlint@393031adb9afb225ee52ae2ccd7a5af5525e03e8 # v1.7.11
|
||||
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -116,7 +116,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
id-token: write # for attest-build-provenance (Sigstore OIDC)
|
||||
attestations: write # for attest-build-provenance
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
with:
|
||||
path: artifacts
|
||||
- name: Create release
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
|
||||
with:
|
||||
files: |
|
||||
artifacts/gbrain-darwin-arm64/gbrain-darwin-arm64
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
container:
|
||||
image: semgrep/semgrep:1.170.0@sha256:c98f8829eea377274ee4b10656458b078b88232469b2ff913f091c2317347c9d
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
# Non-blocking initially (continue-on-error): the first runs establish a
|
||||
# baseline without failing unrelated PRs. Graduation path: once the
|
||||
# baseline findings are triaged (fixed or `# nosemgrep`'d), remove
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
hit: ${{ steps.lookup.outputs.cache-hit }}
|
||||
hash: ${{ steps.compute.outputs.hash }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- name: Compute content hash
|
||||
id: compute
|
||||
run: |
|
||||
@@ -84,7 +84,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: gitleaks/gitleaks-action@dcedce43c6f43de0b836d1fe38946645c9c638dc # v2
|
||||
@@ -103,7 +103,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 12
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -124,7 +124,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -149,7 +149,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 12
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -172,7 +172,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 12
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -206,17 +206,13 @@ jobs:
|
||||
needs: cache-check
|
||||
if: needs.cache-check.outputs.hit != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
# 22, not 15: under parallel PR load the PGLite WASM cold-starts stretch a
|
||||
# shard past 15 min while every test is still passing — the timeout then
|
||||
# cancels the job and the test-status gate reads it as a failure. 13 runs
|
||||
# died this way on 2026-07-21/22 alone.
|
||||
timeout-minutes: 22
|
||||
timeout-minutes: 15
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
|
||||
@@ -35,11 +35,6 @@ export/
|
||||
# .context/test-shards/. Workspace-local by design — never committed.
|
||||
.context/
|
||||
|
||||
# Local agent instruction overrides (CLAUDE.local.md / AGENTS.local.md) — personal,
|
||||
# per-clone, loaded after the committed CLAUDE.md/AGENTS.md. Never committed.
|
||||
CLAUDE.local.md
|
||||
AGENTS.local.md
|
||||
|
||||
# Tier 3 PGLite snapshot fixture (built on demand by build:pglite-snapshot)
|
||||
test/fixtures/pglite-snapshot.tar
|
||||
test/fixtures/pglite-snapshot.version
|
||||
|
||||
-168
@@ -2,174 +2,6 @@
|
||||
|
||||
All notable changes to GBrain will be documented in this file.
|
||||
|
||||
## [0.42.65.0] - 2026-07-23
|
||||
|
||||
**A large maintenance release: 93 verified fixes and small features merged since v0.42.64.0, most of them community contributions.**
|
||||
|
||||
If you use gbrain day to day, this release makes the boring parts trustworthy. Importing and syncing notes is safer: a failed pull no longer pretends everything is up to date, imported pages are read back after writing to confirm they landed, and a page with real content can no longer be silently overwritten by an empty one. Search answers get better inputs: the think command now picks excerpts that actually match your question, and results respect your federated source settings. Background enrichment (the "dream" cycle) wastes less money and retries properly when an AI provider is down. Spending caps now fail closed, so a billing hiccup can never turn into an uncapped spend. And `gbrain doctor` is quieter, with several false alarms removed and real problems (like an embedding backlog with no worker running) now flagged.
|
||||
|
||||
More AI providers work out of the box, including OpenRouter prompt caching, MiniMax and Zhipu GLM recipes, Ollama Matryoshka embedding dimensions, and llama-server batch limits.
|
||||
|
||||
## To take advantage of v0.42.65.0
|
||||
|
||||
`gbrain upgrade` should do this automatically. No new schema migrations ship in this release.
|
||||
|
||||
1. **Upgrade and verify:**
|
||||
```bash
|
||||
gbrain upgrade
|
||||
gbrain doctor
|
||||
gbrain stats
|
||||
```
|
||||
2. **If `gbrain doctor` reports new findings after upgrading,** that is the quieter, more accurate check set working as intended. Each finding names its fix.
|
||||
3. **If any step fails,** please file an issue at https://github.com/garrytan/gbrain/issues with the output of `gbrain doctor` and `~/.gbrain/upgrade-errors.jsonl` if it exists.
|
||||
|
||||
### Itemized changes
|
||||
|
||||
#### Security
|
||||
|
||||
- MCP source scoping for remote callers got a hardening pass, so agent-facing connections stay confined to the sources they were granted. (#2881, contributed by @spinsirr)
|
||||
- Paid MCP spend accounting is now atomic and fails closed, and resolver spend is recorded before a cap error is raised, so caps cannot be raced past or undercounted. (#3203, #3204, contributed by @caterpillarC15)
|
||||
- The OAuth token endpoint rate limit on the HTTP server is now configurable via env for deployments behind shared IPs. (#3114, contributed by @time-attack)
|
||||
- `WWW-Authenticate` responses now carry `resource_metadata` per the MCP spec and RFC 9728, so conforming clients can discover the auth server. (#1410, contributed by @rayers)
|
||||
|
||||
#### Search, retrieval, and think
|
||||
|
||||
- `think` selects query-relevant excerpts instead of generic ones. (#3197, contributed by @Y0lan)
|
||||
- Unqualified local CLI `search`/`query` now honors `sources.config.federated` read visibility. (#2561, #3141, contributed by @time-attack)
|
||||
- Email citation metadata is projected into search results. (#2873, contributed by @amtagrwl)
|
||||
- The `think` Gaps section renders once instead of twice. (#1662, contributed by @howwohmm)
|
||||
- Fuzzy entity lookup threads the caller's source scope and skips soft-deleted entities. (#1508, contributed by @tim404x)
|
||||
- `code-def` surfaces method, constructor, field, and struct definitions, not just top-level symbols. (#1628, contributed by @rayers)
|
||||
- Briefing pages are excluded from their own Brain Pulse salience. (#1202, contributed by @rwbaker)
|
||||
- Reranker calls with missing auth are classified as configuration errors before falling back. (#2059, #3139, contributed by @time-attack)
|
||||
|
||||
#### Import, sync, and ingestion
|
||||
|
||||
- A failed git pull with zero imports reports `partial (pull_failed)` instead of `up_to_date`. (#3068, #3253, contributed by @Masashi-Ono0611)
|
||||
- Imports run a post-write read-back verification with a durable ingest-log record. (#2869, contributed by @Andredsouza1984)
|
||||
- `put` refuses to overwrite a non-empty page with empty content. (#2708, contributed by @symmetric-matthew)
|
||||
- `putPage` restores soft-deleted rows instead of colliding with them. (#2779, contributed by @RerankerGuo)
|
||||
- Mixed-case slugs are normalized before chunk upsert, ending duplicate-chunk churn. (#430, #3143, contributed by @time-attack)
|
||||
- Imports fall back to the body H1 for the title when frontmatter lacks `title:`. (#2446, #3072, contributed by @time-attack)
|
||||
- YAML comments inside the frontmatter fence are no longer treated as markdown headings. (#3225, #3247, contributed by @Masashi-Ono0611)
|
||||
- Write-through guards case-insensitive filesystem collisions before the atomic write. (#2831, #3119, contributed by @time-attack)
|
||||
- Path-qualified wikilinks outside the known directory pattern resolve on the DB/put_page path. (#2866, contributed by @paul-0320)
|
||||
- CJK slugs are supported in the slug registry and dream-cycle summary slugs. (#782, #738, #3083, contributed by @time-attack)
|
||||
- Three ingest/sync/serve singleton fixes: page-type round-trip, deleted-slug embed noise, and a stateless width guard. (#3140, contributed by @time-attack)
|
||||
- Sync honors the `embedding_disabled` sentinel as an implicit `--no-embed`. (#2879, contributed by @gawievanblerk)
|
||||
- Verified sync head sentinels are cleared correctly. (#2734, contributed by @symmetric-matthew)
|
||||
- Resumed syncs report the pinned commit they actually landed on. (#3202, contributed by @caterpillarC15)
|
||||
- The expected `discover_git_root` probe failure stays off stderr. (#3232, contributed by @Masashi-Ono0611)
|
||||
- `extract --stale` runs the real resolver so basename resolution reaches stale pages, and clears pre-version-bump pages. (#2576, #2717, contributed by @paul-0320; #1791, contributed by @Nazim22)
|
||||
- Oversized code chunks are capped so they stay embeddable, and code-chunk metadata survives re-embeds. (#1675, contributed by @lubosxyz; #769, #1232, contributed by @rayers)
|
||||
|
||||
#### Background cycle, dream, and facts
|
||||
|
||||
- Path-derived dream sources are stamped, and the engine closes cleanly on autopilot shutdown. (#3178, contributed by @time-attack)
|
||||
- All-provider-failed atom drains propagate so durable jobs retry instead of silently dropping work. (#3218, #3248, contributed by @Masashi-Ono0611)
|
||||
- Atom extraction raises `maxTokens` and case-normalizes `atom_type` for Gemini models. (#3211, contributed by @alexey-metaengage)
|
||||
- The conversation extractor gates anonymous-speaker self-attribution instead of guessing. (#3228, contributed by @asenkovskiy)
|
||||
- Incremental dream extraction stamps its watermark so re-runs stop reprocessing. (#2636, #3115, contributed by @time-attack)
|
||||
- `dream --dry-run --json` keeps stdout clean of embed summaries. (#394, #3109, contributed by @time-attack)
|
||||
- Synthesized dream pages require a self-contained opening summary. (#2770, contributed by @Masashi-Ono0611)
|
||||
- PGLite inline synth subagent drains complete, and `lint` gains `--exclude`. (#2699, #2649, #3162, contributed by @time-attack)
|
||||
- Live context reads the documented "P1 Today" heading form with plain checkbox tasks, matching the daily-task-manager skill's output format. (#2186, #3124, contributed by @time-attack)
|
||||
- Queued AI jobs refresh gateway config at execution time instead of using a stale snapshot. (#2125, contributed by @maxpetrusenkoagent)
|
||||
- `brainstorm`/`propose_takes` honor configured models: cost preview uses the configured model, the judge reads its config key, provider probes are skipped when unneeded, and page projection is narrowed. (#3120, contributed by @time-attack)
|
||||
- Backlog hardening wave: x-to-brain health check, propose_takes deadlines, capture title truncation, extract_atoms backlog handling, and pooler direct-URL routing. (#3165, contributed by @time-attack)
|
||||
- `skillopt` emits `proposed.md` in no-mutate mode. (#2635, #3182, contributed by @time-attack)
|
||||
- Nightly quality probe enable path and conversation-parser probe are wired up. (#2629, #2630, #3094, contributed by @time-attack)
|
||||
|
||||
#### Doctor, health, and maintenance
|
||||
|
||||
- New safe maintenance automation with a shared orphan-exclusion policy, so routine cleanup runs without risking linked content. (#3015, #3023, contributed by @time-attack)
|
||||
- `orphan_ratio` excludes the chronicle volume under `life/events/`. (#2264, #3214, contributed by @asenkovskiy)
|
||||
- `brain_score` orphan/timeline components use the orphans-audit linkable scope. (#3155, contributed by @time-attack)
|
||||
- Entity timeline coverage is measured separately from whole-brain density. (#2761, contributed by @TurgutKural)
|
||||
- Doctor flags embed backfills queued with no worker running. (#2696, contributed by @javieraldape)
|
||||
- Two doctor false-positive/timeout fixes: the drift walk skips `node_modules`, and the bare-tweet check skips inline code and cited lines. (#1772, contributed by @sonlndv)
|
||||
- A dead `llm_fallback_enabled` recommendation is dropped from conversation format coverage. (#1903, contributed by @ElliotDrel)
|
||||
- Skill triggers with CRLF line endings parse on Windows. (#1149, contributed by @samporter-31)
|
||||
- Onboard check names are registered in doctor categories, ending unknown-check warnings, and onboard-check remediations survive the `--apply --auto` path. (#3075, #3097, contributed by @time-attack)
|
||||
- Dead slug prefixes are counted by slug. (#2697, contributed by @RerankerGuo)
|
||||
- The backlinks worker defaults to check, not fix, and `check-backlinks` honors its positional directory argument. (#1853, contributed by @choomz; #3076, contributed by @time-attack)
|
||||
- Calibration resolves the owner holder via config, defaulting to `self`. (#3077, contributed by @time-attack)
|
||||
- Memory throttling on Linux reads `/proc/meminfo` MemAvailable. (#556, contributed by @chengzehsu)
|
||||
|
||||
#### AI providers and gateway
|
||||
|
||||
- OpenRouter gets family-scoped prompt caching, and query expansion works on chat-capable openai-compat recipes. (#3152, contributed by @time-attack)
|
||||
- MiniMax recipe: embedding wire-shape compat fetch plus a chat touchpoint. (#1977, #3089, contributed by @time-attack)
|
||||
- The Zhipu recipe gains a chat touchpoint so GLM subagents work. (#1157, #3084, contributed by @time-attack)
|
||||
- Tier-configured models reach the recipe allowlist, Anthropic model lists are refreshed, tier resolutions are registered, and probe labels are honest. (#2800, contributed by @p3ob7o)
|
||||
- Provider base URL config merges from the DB. (#1676, contributed by @TheLordArgus)
|
||||
- The gateway falls back to the pooler when the derived direct host is unreachable. (#1641, #3088, contributed by @time-attack)
|
||||
- Config-plane `voyage_api_key` folds into `VOYAGE_API_KEY` like the other hosted keys. (#3236, contributed by @Masashi-Ono0611)
|
||||
- The `zeroentropyai:zerank-2` reranker has a pricing entry so the budget tracker can meter it. (#3223, #3233, contributed by @Masashi-Ono0611)
|
||||
- llama-server embedding batches are capped at its 32-input request limit. (#1281, contributed by @mmekkaoui)
|
||||
- Matryoshka dimensions thread through for Qwen3-Embedding on Ollama. (#1072, contributed by @mgandal)
|
||||
- `init` seeds AI options from env on cold install, and `whoami` reports the stdio transport. (#3091, contributed by @time-attack)
|
||||
- The `models` dispatch subcommand reads its first argument correctly. (#1428, contributed by @BenjaminDSmithy)
|
||||
- Synopsis generation tail-truncates document text for small-model chat handlers. (#1427, contributed by @BenjaminDSmithy)
|
||||
- The contradiction judge token cap is raised for thinking models. (#3210, contributed by @alexey-metaengage)
|
||||
|
||||
#### Schema, migrations, and storage engines
|
||||
|
||||
- Engine migration counts and surfaces per-page copy failures instead of silently advancing. (#3241, contributed by @Masashi-Ono0611)
|
||||
- Invalid `CONCURRENTLY`-build index remnants are dropped without a DO block. (#3191, contributed by @Masashi-Ono0611)
|
||||
- Unsupported large-dimension HNSW indexes are skipped instead of failing schema setup. (#1734, #3080, contributed by @time-attack)
|
||||
- The v0.32.2 migration dirty-check scopes to targeted sources and surfaces failed phase detail. (#3093, contributed by @time-attack)
|
||||
- Schema packs merge the full `extends` chain and `borrow_from` into the resolved manifest. (#1749, #3181, contributed by @time-attack)
|
||||
- The schema-pack stats catch-all is narrowed so masked errors surface instead of fake zero-page counts. (#2466, #3133, contributed by @time-attack)
|
||||
- Bundled schema-pack inspection reports the pack actually shipped in the binary, and minion subagent auth resolves through config. (#3110, contributed by @time-attack)
|
||||
- PGLite `putPage` guards against zero-row RETURNING. (#1649, contributed by @alexhawkins)
|
||||
|
||||
#### MCP server and CLI surface
|
||||
|
||||
- `list_pages` rows include `source_id`. (#3209, contributed by @alexey-metaengage)
|
||||
- Running CLI commands while `gbrain serve` (MCP) holds the brain now notifies about the conflict instead of failing confusingly. (#3243, contributed by @fdefitte)
|
||||
- The OpenClaw plugin manifest entry is declared so the plugin loads. (#2551, #3185, contributed by @time-attack)
|
||||
|
||||
#### For contributors
|
||||
|
||||
- CI scanner roots are normalized on macOS. (#3198, contributed by @caterpillarC15)
|
||||
- CI shard timeout raised to 22 minutes plus a delta-assert reporter leak test. (#3231, contributed by @time-attack)
|
||||
- E2E suite hardening: flaky tests, no-op assertions, and cross-test coupling removed. (#1704, contributed by @auroracapital)
|
||||
- `mechanical.test.ts` isolates `$HOME` so the E2E suite stops clobbering user config. (#434, contributed by @lloydarmbrust)
|
||||
- The lint code-fence-wrap detector and fixer regex now agree. (#1597, contributed by @chungty)
|
||||
- README project links for OpenClaw and Hermes are corrected. (#1961, #3179, contributed by @time-attack)
|
||||
- A completed TODOS entry is dropped. (#3229, contributed by @Masashi-Ono0611)
|
||||
|
||||
## [0.42.64.0] - 2026-07-20
|
||||
|
||||
### Fixed
|
||||
|
||||
- Confidential OAuth clients can now revoke access tokens through the standard revocation endpoint when client secrets are stored as hashes. Invalid credentials fail closed, malformed or mixed authentication is rejected, backend failures remain retryable, and discovery metadata accurately advertises supported authentication methods.
|
||||
|
||||
No schema migrations.
|
||||
## [0.42.63.0] - 2026-07-20
|
||||
|
||||
**Schema commands now open the local brain you actually configured.**
|
||||
|
||||
If your PGLite brain lives at a custom path, commands such as `gbrain schema stats` previously ignored that path and could inspect the default brain instead. That made a healthy configured brain look empty or report the wrong schema counts. Schema commands now use the same complete database configuration as the rest of GBrain. PostgreSQL behavior is unchanged, and no migration is required.
|
||||
|
||||
### How to use it
|
||||
|
||||
Upgrade, then run the schema command normally:
|
||||
|
||||
```bash
|
||||
gbrain upgrade
|
||||
gbrain schema stats --json
|
||||
```
|
||||
|
||||
The reported page and type counts now come from the `database_path` in `~/.gbrain/config.json` when the engine is PGLite.
|
||||
|
||||
### Itemized changes
|
||||
|
||||
#### Fixed
|
||||
- **Schema CLI commands preserve configured PGLite paths.** Engine construction and connection now receive the canonical complete engine configuration, including both `database_path` and `database_url` where applicable.
|
||||
- **CLI tests are isolated from ambient database URLs.** Schema subprocess tests explicitly clear inherited PostgreSQL URL variables, and a persistent-PGLite regression test proves `schema stats` reads the configured database rather than the default brain.
|
||||
|
||||
## [0.42.62.0] - 2026-07-17
|
||||
|
||||
**If your brain holds more than one source, everything now lands in the right one. Link extraction, timeline extraction, background cycles, and webhook captures used to quietly file some of their output under the default source; all of those paths now carry the correct source identity. Background agent jobs got tougher too: a failed database reconnect can no longer wedge the engine, and workers recover from dropped connections instead of crash-looping. If you run the admin dashboard behind a reverse proxy, the live activity panel finally connects. Long agent conversations cost less because repeated context is reused between turns on Anthropic calls. Local LiteLLM proxies work out of the box. Nested sources scan correctly again instead of reporting zero files. And the project's automated checks now include dependency vulnerability scanning, static code-security analysis, and signed provenance for release builds. Thirty merged changes in all, the largest batch to date, each one reviewed and verified against the live codebase before landing.**
|
||||
|
||||
@@ -71,8 +71,8 @@ GBrain is designed to be installed and operated by an AI agent. The fastest path
|
||||
|
||||
If you don't already have an AI agent platform running, start with one of these. Both are designed to read GBrain's install protocol and execute it:
|
||||
|
||||
- **[OpenClaw](https://github.com/openclaw/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/NousResearch/hermes-agent)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
- **[OpenClaw](https://github.com/openclawagents/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/openclawagents/hermes)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
|
||||
Then paste this into your agent:
|
||||
|
||||
@@ -258,24 +258,6 @@ The whole loop is described in [`docs/architecture/topologies.md`](docs/architec
|
||||
|
||||
**Job queue (Minions).** BullMQ-shaped, Postgres-native job queue. Durable subagents (LLM tool loops that survive crashes via two-phase pending→done persistence), shell jobs with audit, child jobs with cascading timeouts, rate leases for outbound providers, attachments via S3/Supabase storage. Replaces "spawn subagent as fire-and-forget Promise" with something that recovers from anything.
|
||||
|
||||
**Non-English brains (FTS language config).** The Postgres full-text search tokenizer is configurable via `GBRAIN_FTS_LANGUAGE`. Defaults to `english`. Set it to any text-search configuration that exists in your Postgres instance:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=portuguese # uses built-in portuguese stemmer
|
||||
export GBRAIN_FTS_LANGUAGE=spanish # built-in spanish stemmer
|
||||
export GBRAIN_FTS_LANGUAGE=pt_br # custom config (e.g. unaccent + portuguese)
|
||||
```
|
||||
|
||||
List available configs: `psql -c "SELECT cfgname FROM pg_ts_config"`. Both the **query side** (`websearch_to_tsquery`) and the **write side** (the trigger functions that populate `pages.search_vector` and `content_chunks.search_vector`) honor `GBRAIN_FTS_LANGUAGE`. On first install (or upgrade), the `configurable_fts_language` schema migration reads the env var and creates trigger functions in the configured language; subsequent inserts/updates tokenize using that setting. To change language on a brain that has already run the migration, use the dedicated CLI command:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=portuguese
|
||||
gbrain reindex-search-vector --dry-run # preview row counts
|
||||
gbrain reindex-search-vector --yes # recreate triggers + backfill
|
||||
```
|
||||
|
||||
The command is idempotent (re-running with the same language is a no-op for vector content) and uses the same recreate-and-backfill primitives as the migration. For accent-insensitive Portuguese (`pt_br`), see [docs/guides/multi-language-fts.md](docs/guides/multi-language-fts.md) for the `unaccent` + portuguese stemmer recipe.
|
||||
|
||||
**43 curated skills.** Routing lives in [`skills/RESOLVER.md`](skills/RESOLVER.md). Covers signal capture, ingest (idea / media / meeting), enrichment, querying, brain ops, citation fixing, daily task management, cron scheduling, reports, voice, soul audit, skill creation, eval framework, and migrations. Skills are markdown files (tool-agnostic), packaged as a single skillpack the installer drops into your agent workspace.
|
||||
|
||||
**Eval framework.** `gbrain eval longmemeval` runs the public [LongMemEval](https://huggingface.co/datasets/xiaowu0162/longmemeval) benchmark against your hybrid retrieval. `gbrain eval export` + `gbrain eval replay` capture real queries and replay them against code changes (set `GBRAIN_CONTRIBUTOR_MODE=1`). `gbrain eval cross-modal` cross-checks an output against the task using three different-provider frontier models. `gbrain eval retrieval-quality` runs NamedThingBench, which hard-gates the named-thing retrieval families (title-substring, alias-synonym, generic-to-named, multi-chunk-dilution) so a regression in "find the page this query names" fails CI loudly. Full methodology in [`docs/eval/SEARCH_MODE_METHODOLOGY.md`](docs/eval/SEARCH_MODE_METHODOLOGY.md).
|
||||
|
||||
-12
@@ -135,18 +135,6 @@ the PGLite schema. Local agents continue to use stdio (`gbrain serve`).
|
||||
Running `--http` against a PGLite-backed install fails fast with a clear
|
||||
error message at startup.
|
||||
|
||||
### Docker network isolation (self-hosted Postgres)
|
||||
|
||||
OAuth and source scoping enforce isolation on the `serve --http` path only.
|
||||
Raw Postgres reachability bypasses both: a container that shares Docker's
|
||||
default `bridge` network with the brain's Postgres can open a direct DB
|
||||
session without any token and read every source. Put the brain's Postgres on
|
||||
a user-defined Docker network with nothing untrusted on it, publish its port
|
||||
loopback-only (if at all), and never put `DATABASE_URL` or a Postgres
|
||||
password in untrusted agent containers — those should reach the brain
|
||||
exclusively via OAuth against `serve --http`. Full operator checklist:
|
||||
[docs/mcp/DEPLOY.md — Co-located Docker workloads](docs/mcp/DEPLOY.md#co-located-docker-workloads-self-hosted-postgres).
|
||||
|
||||
### CORS
|
||||
|
||||
Default-deny: no `Access-Control-Allow-Origin` header is sent unless an
|
||||
|
||||
@@ -2,6 +2,14 @@
|
||||
|
||||
## community fix-wave follow-ups (filed v0.42.60.0)
|
||||
|
||||
- [ ] **P1 — take-writes source scoping fails open when source resolution errors (#2684 residual).**
|
||||
`resolveTakesSourceId` (src/commands/takes.ts) swallows resolution errors and returns
|
||||
`undefined`, which falls back to the unscoped slug-only page lookup — so an invalid
|
||||
`GBRAIN_SOURCE` (or a broken dotfile chain) silently restores the pre-#2698 cross-source
|
||||
write behavior on multi-source brains. Decide fail-closed semantics: error out when a
|
||||
source was explicitly requested but doesn't resolve; keep the unscoped fallback only for
|
||||
brains with no source configuration at all. Add a regression test for the invalid-source
|
||||
path. Found by cross-model adversarial review during the v0.42.60.0 release ship.
|
||||
- [ ] **P2 — cherry-pick #2112's uncovered doctor.ts hunk.** Fix-wave A (#2820) superseded
|
||||
most of #2112 but not its `checkSubagentCapability` fix (check explicit `models.subagent`
|
||||
before `models.tier.subagent`). Refile or cherry-pick; the rest of that PR is covered.
|
||||
@@ -2279,25 +2287,10 @@ at plan time and got carved out:
|
||||
via `buildPerSourceBindings`. Document workaround: register
|
||||
source-scoped OAuth clients.
|
||||
|
||||
- [x] **v0.41+: T20 — extends-chain merging in registry.ts.** DONE (#1749).
|
||||
`resolvePack` now merges parent → child (child-wins) for the six
|
||||
ingest/query-shaping fields (`page_types`, `link_types`,
|
||||
`frontmatter_links`, `enrichable_types`, `filing_rules`, `takes_kinds`)
|
||||
plus `borrow_from` materialization, in `src/core/schema-pack/merge.ts`.
|
||||
The cascade was transparent (consumers already read `resolved.manifest`),
|
||||
not per-consumer. `phases`/`calibration_domains` deliberately excluded —
|
||||
see the P3 follow-up below.
|
||||
|
||||
- [ ] **P3: explicit opt-in to inherit `phases` / `calibration_domains`.**
|
||||
T20 excludes these two from the child-wins merge because they gate real
|
||||
cycle execution (`cycle.ts` `packDeclaresPhase`) and the manifest
|
||||
contract says each pack declares its own participation explicitly —
|
||||
auto-inheriting would silently make a child run cycle phases it never
|
||||
requested. Multi-level lens packs (`gbrain-everything`) therefore still
|
||||
re-declare them by hand. If that redeclaration becomes painful, add an
|
||||
explicit manifest flag (e.g. `inherit_phases: true`) so a pack author
|
||||
opts in consciously. Depends on: T20 (landed). Start in
|
||||
`src/core/schema-pack/merge.ts` (`mergeInheritedManifest`).
|
||||
- [ ] **v0.41+: T20 — extends-chain merging in registry.ts.**
|
||||
`registry.ts:167` documents the gap. Implementing full child-wins
|
||||
merge cascades through every consumer of `manifest.page_types`. ~1
|
||||
day CC.
|
||||
|
||||
- [ ] **v0.41+: T21 — comment-preserving YAML emitter.**
|
||||
v0.40.7.0 emitter does NOT preserve comments. Authors who care
|
||||
|
||||
@@ -51,9 +51,8 @@
|
||||
"@electric-sql/pglite",
|
||||
],
|
||||
"overrides": {
|
||||
"@hono/node-server": "^2.0.5",
|
||||
"body-parser": "^2.3.0",
|
||||
"fast-uri": "^3.1.4",
|
||||
"@hono/node-server": "^1.19.13",
|
||||
"fast-uri": "^3.1.2",
|
||||
"fast-xml-builder": "^1.1.7",
|
||||
"fast-xml-parser": "^5.7.0",
|
||||
"form-data": "^4.0.6",
|
||||
@@ -163,7 +162,7 @@
|
||||
|
||||
"@electric-sql/pglite": ["@electric-sql/pglite@0.4.3", "", {}, "sha512-ichuWTgtd4mOM1G4SpyGJa5trT03lWbMypDV0fUXUCXg5hiHqVAz/bZyV68NqmkLB7WcYmj1RMJVSp8HV/v/ZQ=="],
|
||||
|
||||
"@hono/node-server": ["@hono/node-server@2.0.11", "", { "peerDependencies": { "hono": "^4" } }, "sha512-bjD221KPLoJTWUwso1J6fGKiTXEUFedG/s0visavY4zakFPkeGURMRNly+FhBHs7T8Dz4qHaZIMX9ZoJHSJtKA=="],
|
||||
"@hono/node-server": ["@hono/node-server@1.19.14", "", { "peerDependencies": { "hono": "^4" } }, "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw=="],
|
||||
|
||||
"@jsquash/avif": ["@jsquash/avif@2.1.1", "", { "dependencies": { "wasm-feature-detect": "^1.2.11" } }, "sha512-LMRxd0fMgfCLtobDh0/sFYJMMiRJTNYSEEWvRDKXlAeZ08t3gI5V+1thIT0XjXJ+SVG7Zug9B0XPyx0Ti5VRNA=="],
|
||||
|
||||
@@ -327,7 +326,7 @@
|
||||
|
||||
"asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
|
||||
|
||||
"body-parser": ["body-parser@2.3.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^2.0.0", "debug": "^4.4.3", "http-errors": "^2.0.1", "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", "qs": "^6.15.2", "raw-body": "^3.0.2", "type-is": "^2.1.0" } }, "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw=="],
|
||||
"body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="],
|
||||
|
||||
"bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="],
|
||||
|
||||
@@ -401,7 +400,7 @@
|
||||
|
||||
"fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="],
|
||||
|
||||
"fast-uri": ["fast-uri@3.1.4", "", {}, "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw=="],
|
||||
"fast-uri": ["fast-uri@3.1.3", "", {}, "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg=="],
|
||||
|
||||
"fast-xml-builder": ["fast-xml-builder@1.3.0", "", { "dependencies": { "path-expression-matcher": "^1.6.2", "xml-naming": "^0.3.0" } }, "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ=="],
|
||||
|
||||
@@ -615,10 +614,6 @@
|
||||
|
||||
"@types/bun/bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
|
||||
|
||||
"body-parser/content-type": ["content-type@2.0.0", "", {}, "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ=="],
|
||||
|
||||
"body-parser/type-is": ["type-is@2.1.0", "", { "dependencies": { "content-type": "^2.0.0", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA=="],
|
||||
|
||||
"es-set-tostringtag/hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
|
||||
|
||||
"eventsource/eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
|
||||
|
||||
+1
-6
@@ -13,9 +13,4 @@ timeout = 60_000
|
||||
# fixtures still match the schema. v0.37's production default is ZE/1280;
|
||||
# tests that want the new default call configureGateway() explicitly in
|
||||
# their own beforeAll.
|
||||
#
|
||||
# #2823: redirect GBRAIN_AUDIT_DIR to a per-run scratch dir BEFORE any test
|
||||
# runs, so audit-emitting code paths (content-sanity, shell-audit, etc.)
|
||||
# can't leak fixture events into the operator's real ~/.gbrain/audit/. See
|
||||
# test/helpers/audit-dir-preload.ts for the full rationale.
|
||||
preload = ["./test/helpers/legacy-embedding-preload.ts", "./test/helpers/audit-dir-preload.ts"]
|
||||
preload = ["./test/helpers/legacy-embedding-preload.ts"]
|
||||
|
||||
@@ -148,51 +148,6 @@ RRF fusion, multi-query expansion, and 4-layer dedup are engine-agnostic. They o
|
||||
|
||||
**Why not self-hosted for v0:** The brain should be infrastructure agents use, not something you maintain. Self-hosted Postgres with Docker is a welcome community PR, but v0 optimizes for zero ops.
|
||||
|
||||
### Opt-in RLS source-scope binding (`GBRAIN_RLS_SCOPE_BINDING`)
|
||||
|
||||
Defense-in-depth layer for Postgres deployments that want the database itself
|
||||
to enforce source isolation, in addition to the mandatory app-layer filters
|
||||
(`sourceScopeOpts` — layer 1, always on).
|
||||
|
||||
**Mechanism.** With `GBRAIN_RLS_SCOPE_BINDING=1` (or `true`), the engine's
|
||||
source-scoped read methods wrap their queries in a transaction that first runs
|
||||
`SELECT set_config('app.scopes', $1, true)` — the value is a bound parameter
|
||||
(federated `sourceIds` CSV > scalar `sourceId` > `'*'` for unscoped internal
|
||||
reads), transaction-local (equivalent to `SET LOCAL`, which itself can't take
|
||||
bound params). An RLS policy can then filter rows by
|
||||
`current_setting('app.scopes', true)`.
|
||||
|
||||
**Default off.** With the env var unset, reads call through on the shared pool
|
||||
exactly as before — no per-read transaction, no pool-slot hold (the search
|
||||
methods keep the transaction they always had for their `SET LOCAL
|
||||
statement_timeout`). Existing operators see zero behavior change.
|
||||
|
||||
**Enabling it** (operator-managed SQL; gbrain ships no DDL for this):
|
||||
|
||||
```sql
|
||||
ALTER TABLE pages ENABLE ROW LEVEL SECURITY;
|
||||
CREATE POLICY pages_scope_filter ON pages
|
||||
USING (current_setting('app.scopes', true) = '*'
|
||||
OR source_id = ANY(string_to_array(current_setting('app.scopes', true), ',')));
|
||||
|
||||
-- Required: connections that don't run through the scoped read helper
|
||||
-- (admin, autopilot, cycle, writes) must default to unscoped, or they
|
||||
-- see zero rows once the policy exists:
|
||||
ALTER ROLE <runtime-role> SET app.scopes = '*';
|
||||
|
||||
-- If the runtime role OWNS the table, RLS is skipped for it unless forced:
|
||||
ALTER TABLE pages FORCE ROW LEVEL SECURITY;
|
||||
```
|
||||
|
||||
Safe to enable in either order: the env var without a policy is a no-op
|
||||
setting; a policy without the env var is enforced only via the role default.
|
||||
|
||||
**Honest caveat:** only read paths routed through the scoped helper carry a
|
||||
per-request scope binding — unwrapped paths (writes, admin/maintenance reads)
|
||||
run under the role default and are not backstopped per caller. This is layer 2;
|
||||
the app-layer source filters remain layer 1 and stay mandatory. Behavioral pins
|
||||
live in `test/postgres-engine-rls-scope.test.ts`.
|
||||
|
||||
## PGLiteEngine (v0.7, ships)
|
||||
|
||||
**Dependencies:** `@electric-sql/pglite` (v0.4.4+)
|
||||
|
||||
@@ -3,8 +3,6 @@
|
||||
On-demand reference (see CLAUDE.md Reference map). Current behavior + invariants
|
||||
only.
|
||||
|
||||
`test/e2e/serve-http-oauth.test.ts` additionally pins confidential POST/Basic revocation, public-client SDK fallthrough, malformed/mixed authentication rejection, cross-client isolation, unknown-token opacity, metadata auth methods, no-store responses, strict post-revoke `401`, and retryable backend `503` semantics.
|
||||
|
||||
### Test command tiers
|
||||
|
||||
Seven test command tiers, each with a clear scope:
|
||||
@@ -189,7 +187,6 @@ Unit tests and what they cover:
|
||||
- `test/orphans.test.ts` — orphans command: detection, pseudo filtering, text/json/count outputs, MCP op.
|
||||
- `test/postgres-engine.test.ts` — `statement_timeout` scoping: `sql.begin` + `SET LOCAL` shape, source-level grep guardrail against a reintroduced bare `SET statement_timeout`.
|
||||
- `test/sync.test.ts` — sync logic + regression guard asserting top-level `engine.transaction` is not called.
|
||||
- `test/sync-pull-failed-anchor.serial.test.ts` — #3068 regression: a failed internal `git pull` (local-path origin vs `protocol.file.allow=never`) with zero imports returns `partial`/`pull_failed` (not `up_to_date`), freezes `last_commit` + `last_sync_at`, recovers after a manual pull; fall-through import of local commits preserved. Serial: pins `GBRAIN_HOME` to a temp dir for the whole file.
|
||||
- `test/sync-concurrency.test.ts` — `autoConcurrency()` thresholds + PGLite-forces-serial + explicit-override clamping; `shouldRunParallel()` explicit-bypasses-floor contract; `parseWorkers()` validation rejecting `'0'`/`'-3'`/`'foo'`/`'1.5'`/trailing chars.
|
||||
- `test/sync-parallel.test.ts` — PGLite-routed coverage of the bookmark gate under concurrency, head-drift gate, vanished-file failure capture, PGLite-stays-serial, and the `gbrain-sync` writer-lock contract.
|
||||
- `test/sync-failures.test.ts` — `classifyErrorCode` regex coverage for all 12 codes against literal production message strings from `markdown.ts` and `import-file.ts`; `summarizeFailuresByCode` sort + pre-classified-honor; `recordSyncFailures` code-field persistence; `acknowledgeSyncFailures` `AcknowledgeResult` shape + backfill on legacy entries.
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -75,15 +75,6 @@ Meta-pack stacking creator + investor + engineer via the v0.38
|
||||
preserved — this IS the active pack; the registry walks extends +
|
||||
borrow to materialize the merged view.
|
||||
|
||||
**Merge contract (T20 / #1749).** `resolvePack` merges parent → child
|
||||
(child-wins) for the six ingest/query-shaping fields: `page_types`,
|
||||
`link_types`, `frontmatter_links`, `enrichable_types`, `filing_rules`,
|
||||
and `takes_kinds` (unioned — a child cannot narrow it). `phases` and
|
||||
`calibration_domains` are **NOT** inherited: they gate cycle execution,
|
||||
so each pack must declare its own participation explicitly. That is why
|
||||
`gbrain-everything` re-declares all its phases and all 7
|
||||
`calibration_domains` — inheritance does not carry them.
|
||||
|
||||
Activate via `gbrain config set schema_pack gbrain-everything` and
|
||||
calibration_profile produces all 7 domain scorecards in one JSONB.
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ api_version: gbrain-schema-pack-v1
|
||||
name: my-pack
|
||||
version: 0.0.1
|
||||
gbrain_min_version: 0.39.0
|
||||
extends: gbrain-base # inherits base's TYPES (see Merge contract below); add overrides
|
||||
extends: gbrain-base # inherits everything from base; add overrides below
|
||||
description: |
|
||||
My personal pack.
|
||||
|
||||
@@ -170,34 +170,6 @@ enrichable_types: []
|
||||
filing_rules: []
|
||||
```
|
||||
|
||||
## Merge contract (`extends` + `borrow_from`)
|
||||
|
||||
`resolvePack` composes a pack against its `extends` chain (and any
|
||||
`borrow_from` targets) into the `resolved.manifest` every consumer reads
|
||||
(T20 / #1749). The rules:
|
||||
|
||||
- **Six fields inherit, child-wins:** `page_types`, `link_types`,
|
||||
`frontmatter_links`, `enrichable_types`, `filing_rules`, and `takes_kinds`.
|
||||
A child value with the same key (type name, link name, etc.) overrides the
|
||||
parent's; keys the child doesn't declare come through from the parent.
|
||||
- **`page_types` ordering:** overrides of a base type keep the base's declared
|
||||
position (base's `inferType` prefix priority is authoritative); a genuinely
|
||||
new type — from the child, a `borrow_from`, or a middle pack in the chain —
|
||||
is prepended nearest-first, so a more-derived type's `path_prefix` wins
|
||||
regardless of how deep the chain is.
|
||||
- **`takes_kinds` is UNION, not replace** — it carries a Zod default, so an
|
||||
omitted field is indistinguishable from an explicit one. A child can ADD
|
||||
kinds but **cannot narrow** `takes_kinds` below base ∪ parent. If you need a
|
||||
smaller set, don't `extends` a pack that declares the larger one.
|
||||
- **`phases` and `calibration_domains` are NOT inherited** (child-only). They
|
||||
gate real cycle execution, so each pack must declare its own participation
|
||||
explicitly — inheriting them would silently make a child run phases it never
|
||||
requested. This is why `gbrain-everything` re-declares all its phases and
|
||||
calibration domains by hand. See `lens-packs.md` for the worked example.
|
||||
- **`borrow_from` is selective + non-transitive + fail-closed:** it pulls only
|
||||
the named `types`/`link_types` from the target's OWN declarations (omitting a
|
||||
category borrows none of it); a missing target throws `UnknownPackError`.
|
||||
|
||||
## Recovery + revert
|
||||
|
||||
The single-PR cathedral is hard to revert atomically. Per codex finding
|
||||
|
||||
@@ -21,17 +21,14 @@ GBrain is tuned for the Supabase **Transaction pooler** (port 6543): it
|
||||
auto-disables prepared statements there and routes `engine.transaction()`
|
||||
(migrations, DDL, sync imports) to a derived **direct** connection
|
||||
(`db.<ref>.supabase.co:5432`). That direct host is IPv6-only, so on an
|
||||
IPv4-only host it is unreachable. When that happens gbrain now falls back to
|
||||
the pooler automatically (one stderr warning, then single-pool mode for the
|
||||
rest of the process) — but the pooler's ~2-min statement timeout can truncate
|
||||
very long migrations or bulk imports.
|
||||
IPv4-only host, reads work but sync **silently skips most pages**. This is the
|
||||
number one cause of "sync ran but nothing happened."
|
||||
|
||||
Fix: make the direct connection reachable over IPv4. Either set
|
||||
`GBRAIN_DIRECT_DATABASE_URL` to the **Session pooler** string (port 5432 on the
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on.
|
||||
`GBRAIN_DISABLE_DIRECT_POOL=1` skips the direct pool (and the fallback warning)
|
||||
entirely. Verify by running `gbrain sync` and checking that the page count in
|
||||
`gbrain stats` matches the syncable file count in the repo.
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on. Verify by
|
||||
running `gbrain sync` and checking that the page count in `gbrain stats` matches
|
||||
the syncable file count in the repo.
|
||||
|
||||
### The Primitives
|
||||
|
||||
@@ -134,16 +131,6 @@ hashes match. If both a cron and `--watch` fire simultaneously, no conflict.
|
||||
history rewrite still hard-blocks even with `--skip-failed`. Run
|
||||
`gbrain sync --skip-failed` to acknowledge a known-bad set yourself.
|
||||
|
||||
5. **Import checkpoints name the import target, not the caller's CWD.**
|
||||
Interrupted `gbrain import <dir>` runs may leave
|
||||
`~/.gbrain/import-checkpoint.json` so the next import can resume. The
|
||||
checkpoint `dir` is the absolute, resolved import target captured when
|
||||
import starts. It is not a cleanup instruction and it must not be
|
||||
re-derived from the process working directory. Checkpoints written by
|
||||
gbrain include `schema_version: 1`, `owner: "gbrain"`, and
|
||||
`kind: "import"` so downstream tools can validate the contract before
|
||||
deciding whether to resume.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Edit a file and search for the change.** Edit a brain markdown file,
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
# Multi-language full-text search
|
||||
|
||||
GBrain's keyword search arm uses Postgres full-text search (tsvector/tsquery).
|
||||
The tokenizer language is configurable via the `GBRAIN_FTS_LANGUAGE`
|
||||
environment variable. Default: `english`.
|
||||
|
||||
## How it works
|
||||
|
||||
Postgres text-search configurations control stemming and stop-word removal.
|
||||
`GBRAIN_FTS_LANGUAGE` is read by `src/core/fts-language.ts` and applied on
|
||||
both sides of the search:
|
||||
|
||||
- **Query side** — `websearch_to_tsquery('<lang>', $query)` in both engines
|
||||
(Postgres and PGLite).
|
||||
- **Write side** — the `update_page_search_vector` and
|
||||
`update_chunk_search_vector` trigger functions that populate
|
||||
`pages.search_vector` and `content_chunks.search_vector`.
|
||||
|
||||
The value is validated against `/^[a-z][a-z0-9_]*$/` before it is ever
|
||||
interpolated into SQL (tsvector functions don't accept parameterized config
|
||||
names). Invalid values fall back to `english` with a warning.
|
||||
|
||||
## Built-in languages
|
||||
|
||||
Set the env var to any configuration your Postgres instance ships:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=portuguese
|
||||
export GBRAIN_FTS_LANGUAGE=spanish
|
||||
export GBRAIN_FTS_LANGUAGE=german
|
||||
```
|
||||
|
||||
List what's available:
|
||||
|
||||
```sql
|
||||
SELECT cfgname FROM pg_ts_config;
|
||||
```
|
||||
|
||||
PGLite (the embedded default engine) ships the same built-in snowball
|
||||
configurations as stock Postgres.
|
||||
|
||||
## First install vs. changing language later
|
||||
|
||||
On first install (or upgrade), the `configurable_fts_language` schema
|
||||
migration reads `GBRAIN_FTS_LANGUAGE` and stamps the trigger functions with
|
||||
that language. After the migration has run, changing the env var alone does
|
||||
NOT retokenize existing rows — the migration shows as applied and is skipped.
|
||||
Use the explicit command:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=portuguese
|
||||
gbrain reindex-search-vector --dry-run # preview: language + row counts
|
||||
gbrain reindex-search-vector --yes # recreate triggers + backfill
|
||||
```
|
||||
|
||||
The command recreates both trigger functions under the new language and
|
||||
backfills every existing `pages` and `content_chunks` row in batches,
|
||||
streaming progress to stderr. It is idempotent: re-running with the same
|
||||
language produces identical vectors. `--json` prints a machine-readable
|
||||
result envelope but still requires `--yes` (or an interactive confirm).
|
||||
|
||||
## Recipe: accent-insensitive Portuguese (`pt_br`)
|
||||
|
||||
Brazilian Portuguese content often mixes accented and unaccented spellings
|
||||
("São Paulo" vs "Sao Paulo"). Build a custom config that folds accents via
|
||||
the `unaccent` extension, then stems with the portuguese snowball dictionary:
|
||||
|
||||
```sql
|
||||
CREATE EXTENSION IF NOT EXISTS unaccent;
|
||||
|
||||
CREATE TEXT SEARCH CONFIGURATION pt_br (COPY = portuguese);
|
||||
|
||||
ALTER TEXT SEARCH CONFIGURATION pt_br
|
||||
ALTER MAPPING FOR hword, hword_part, word
|
||||
WITH unaccent, portuguese_stem;
|
||||
```
|
||||
|
||||
Then point GBrain at it:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=pt_br
|
||||
gbrain reindex-search-vector --yes
|
||||
```
|
||||
|
||||
Note: custom configurations require a real Postgres instance (e.g. the
|
||||
Supabase engine). The config must exist BEFORE the migration or the reindex
|
||||
command runs, or Postgres will reject the trigger recreation with
|
||||
`text search configuration "pt_br" does not exist`.
|
||||
|
||||
## Caveats
|
||||
|
||||
- One language per brain: the setting is global to the database, not
|
||||
per-source. Mixed-language brains should pick the dominant language (the
|
||||
vector-search arm is language-agnostic and covers the rest).
|
||||
- Keep `GBRAIN_FTS_LANGUAGE` set consistently in every environment that
|
||||
writes to the brain (CLI shells, MCP server, cron jobs) — a writer without
|
||||
the env var tokenizes new rows in `english` until the next reindex.
|
||||
@@ -114,11 +114,8 @@ Flip later with `gbrain sources federate <id>` / `unfederate <id>`.
|
||||
Full subcommand reference:
|
||||
|
||||
```
|
||||
gbrain sources add <id> --path <p> [--name <n>] [--federated|--no-federated] [--force]
|
||||
gbrain sources add <id> --path <p> [--name <n>] [--federated|--no-federated]
|
||||
Register a source. id: [a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?
|
||||
--path must be a git repo (or a subdirectory of one) — see
|
||||
"The git requirement for --path sources" below. --force
|
||||
skips that check to register before git-init exists.
|
||||
gbrain sources list [--json] List all sources with page counts + federation state.
|
||||
gbrain sources remove <id> [--yes] [--dry-run] [--keep-storage]
|
||||
Cascade-delete a source (pages, chunks, timeline).
|
||||
@@ -131,47 +128,6 @@ gbrain sources federate <id>
|
||||
gbrain sources unfederate <id>
|
||||
```
|
||||
|
||||
## The git requirement for --path sources
|
||||
|
||||
Every `--path` source must be a git repository (or live inside one — a
|
||||
subdirectory of a git repo works too) with at least one committed, tracked
|
||||
file under that path. `gbrain sources add` validates this at registration
|
||||
time and refuses a directory that doesn't qualify — no `.git` at all, a
|
||||
`git init` with no commit yet, or a commit made before `git add` — with an
|
||||
actionable error instead of silently registering a source that will fail
|
||||
(or worse, "succeed" while importing nothing) on its first `gbrain sync`.
|
||||
Fix it with:
|
||||
|
||||
```bash
|
||||
git -C <path> init
|
||||
git -C <path> add -A
|
||||
git -C <path> commit -m "initial import"
|
||||
gbrain sources add <id> --path <path>
|
||||
```
|
||||
|
||||
Two details that are easy to miss:
|
||||
|
||||
- **Files must actually be committed, not just present.** The sync walker
|
||||
reads files through git objects, so `git init` alone — even followed by an
|
||||
empty commit (`git commit --allow-empty`) — isn't enough. Registration
|
||||
checks for real tracked content (`git ls-tree HEAD` scoped to the path),
|
||||
not just a resolvable `HEAD`, so this footgun is caught immediately
|
||||
instead of surfacing later as a sync that imports nothing.
|
||||
- **`--force` registers the source anyway**, skipping the check. Use this if
|
||||
you're registering a path before an automated pipeline gets around to
|
||||
`git init`-ing it. GBrain never auto-`git init`s a `--path` source for
|
||||
you — it's your directory, not a gbrain-managed clone (same consent
|
||||
boundary as sync-time self-heal, which also never mutates a `--path`
|
||||
source without an explicit ask).
|
||||
|
||||
**If sync ever reports a problem with the sync anchor** (`last_commit`) —
|
||||
after a force-push, a history rewrite, or a from-scratch `git init` on a
|
||||
directory that was synced before — you do not need to reset anything by
|
||||
hand. `gbrain sync` detects an unreachable or non-ancestor anchor
|
||||
automatically and recovers: either a full reimport (anchor object missing)
|
||||
or a direct tree-to-tree diff against the orphaned bookmark (anchor present
|
||||
but rewritten), advancing the anchor to the new HEAD when it completes.
|
||||
|
||||
## Citation format for agents
|
||||
|
||||
When agents receive multi-source results they MUST cite pages in
|
||||
|
||||
@@ -131,9 +131,7 @@ into gbrain so other clients can scaffold it. Default behavior:
|
||||
`~/.gbrain/harvest-private-patterns.txt` plus built-in defaults
|
||||
(canonical private fork name, common email regex, Slack channel pattern). Any
|
||||
match → rollback (delete the harvested files) and exit non-zero.
|
||||
- `openclaw.plugin.json` updated with the new slug, sorted. Harvest must preserve
|
||||
the top-level OpenClaw-native plugin fields (`id`, `configSchema`, `contracts`)
|
||||
because OpenClaw validates those before it can install the package.
|
||||
- `openclaw.plugin.json` updated with the new slug, sorted.
|
||||
- `--no-lint` bypasses the linter (after a manual editorial scrub).
|
||||
|
||||
Use the `skillpack-harvest` skill (its companion editorial workflow)
|
||||
|
||||
@@ -258,43 +258,6 @@ the user owns the machine.
|
||||
See [ALTERNATIVES.md](ALTERNATIVES.md) for a comparison of ngrok, Tailscale
|
||||
Funnel, and cloud hosts (Fly.io, Railway).
|
||||
|
||||
### Co-located Docker workloads (self-hosted Postgres)
|
||||
|
||||
OAuth scopes and source scoping guard the `gbrain serve --http` path. They do
|
||||
NOT guard raw Postgres. If the brain's Postgres runs as a container on the same
|
||||
Docker host as other workloads (agent runtimes, n8n, staging fixtures), any
|
||||
container sharing Docker's default `bridge` network can open a direct DB
|
||||
session — no OAuth token required — and read every source. That silently
|
||||
recreates a privileged path underneath the isolation you configured at the MCP
|
||||
layer.
|
||||
|
||||
Network-zone the host so untrusted containers can never reach Postgres:
|
||||
|
||||
```
|
||||
Docker host
|
||||
├── gbrain-net ← ONLY the brain's Postgres (+ gbrain serve, if containerized)
|
||||
├── agent-<id>-net ← each untrusted agent runtime, isolated
|
||||
└── default bridge ← no secret-bearing databases
|
||||
```
|
||||
|
||||
Operator checklist:
|
||||
|
||||
```text
|
||||
[ ] Postgres is on a user-defined Docker network, not the default bridge
|
||||
(or nothing else runs on that bridge)
|
||||
[ ] If Postgres publishes a host port at all, it binds loopback only
|
||||
(`-p 127.0.0.1:5432:5432`, never `0.0.0.0`)
|
||||
[ ] Untrusted agent containers have no DATABASE_URL or Postgres password
|
||||
[ ] Untrusted agents reach the brain via OAuth/Bearer against serve --http only
|
||||
(host loopback via host.docker.internal / host gateway — never gbrain-net)
|
||||
[ ] OAuth clients are least-privilege: scoped --source / --federated-read,
|
||||
pre-minted short-lived tokens preferred over long-lived client secrets
|
||||
[ ] Isolation verified: a team-scoped client cannot read internal-only sources
|
||||
```
|
||||
|
||||
Optional defense-in-depth: a dedicated Postgres role (or RLS) limited to the
|
||||
allowed `source_id`s, so even a leaked connection string can't read everything.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"missing_auth" error**
|
||||
|
||||
@@ -91,18 +91,3 @@ First full takes extraction run on a ~100K-page brain:
|
||||
4. **Self-reported ≠ verified.** "Reports 7 figures" → holder=person, weight=0.75, NOT world/1.0
|
||||
5. **No false precision.** Use 0.05 increments (0.35, 0.55, 0.75), not 0.74 or 0.82
|
||||
6. **"So what" test.** Skip Twitter handles, follower counts, obvious metadata
|
||||
|
||||
## Owner-holder canonicalization
|
||||
|
||||
"The brain owner" is, by convention, the holder string **`self`** — the value the
|
||||
dream `consolidate` phase stamps when it promotes the owner's hot facts into cold
|
||||
takes. Calibration, `think`, and the `doctor` calibration check resolve the owner
|
||||
holder through `resolveOwnerHolder` (`src/core/owner-holder.ts`): explicit override
|
||||
> `emotional_weight.user_holder` config > `self`.
|
||||
|
||||
Known limitation (tracked in garrytan/gbrain#2465): the owner can also
|
||||
appear under `brain` (a take the owner asserts, via `propose_takes`) and
|
||||
`people/<owner>` (extraction that names the owner). The resolver selects the
|
||||
*default* canonical owner string for reads; it does not merge those other
|
||||
strings. Per-take attribution for other people (e.g. `people/george`) is
|
||||
unaffected and correct.
|
||||
|
||||
@@ -484,10 +484,6 @@ Returns a per-source dashboard: when each source last synced, how many pages, ho
|
||||
|
||||
The admin dashboard at `https://brain.acme-co.com/admin` shows live request volume, registered OAuth clients, recent activity, and brain stats. Use the admin bootstrap token from Part 4 to log in the first time, then register additional admin users from inside the dashboard.
|
||||
|
||||
### If agents run as containers on the same Docker host
|
||||
|
||||
OAuth source scoping only guards the HTTP MCP path. If the brain's Postgres and your teammates' agent runtimes are containers on the same Docker host, make sure the agents can't reach Postgres directly over Docker's default bridge network — a direct DB session skips OAuth entirely. Put Postgres on its own user-defined network, publish it loopback-only if at all, and never hand agent containers a `DATABASE_URL`. The copy-paste operator checklist lives in [docs/mcp/DEPLOY.md — Co-located Docker workloads](../mcp/DEPLOY.md#co-located-docker-workloads-self-hosted-postgres).
|
||||
|
||||
---
|
||||
|
||||
## Part 13: Cost and speed expectations
|
||||
|
||||
@@ -233,14 +233,13 @@ keep it or `git checkout` to throw it away. Nothing is committed for you.
|
||||
|
||||
**For a skill that ships with gbrain** (anything under the gbrain repo's own
|
||||
`skills/`): SkillOpt refuses to overwrite it by default and writes the winner to
|
||||
`skills/<name>/skillopt/proposed.md` instead (while keeping `best.md` as the
|
||||
optimizer's current-best pointer), so an optimization pass can never silently
|
||||
mutate a skill other people depend on. Two ways to handle that:
|
||||
`skills/<name>/skillopt/best.md` instead, so an optimization pass can never
|
||||
silently mutate a skill other people depend on. Two ways to handle that:
|
||||
|
||||
```bash
|
||||
# See the proposed improvement without touching SKILL.md (works for ANY skill):
|
||||
gbrain skillopt meeting-prep --split 1:1:1 --no-mutate
|
||||
# → writes skills/meeting-prep/skillopt/proposed.md, updates best.md, and prints the proposal path.
|
||||
# → writes skills/meeting-prep/skillopt/best.md (the proposed rewrite), prints its path. Copy what you want.
|
||||
|
||||
# Actually rewrite a bundled skill (explicit opt-in + an independent held-out set):
|
||||
gbrain skillopt brain-ops --split 1:1:1 --allow-mutate-bundled \
|
||||
|
||||
+7
-120
@@ -1565,8 +1565,8 @@ GBrain is designed to be installed and operated by an AI agent. The fastest path
|
||||
|
||||
If you don't already have an AI agent platform running, start with one of these. Both are designed to read GBrain's install protocol and execute it:
|
||||
|
||||
- **[OpenClaw](https://github.com/openclaw/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/NousResearch/hermes-agent)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
- **[OpenClaw](https://github.com/openclawagents/openclaw)** — deploy [AlphaClaw on Render](https://render.com/deploy?repo=https://github.com/chrysb/alphaclaw) (one click, 8GB+ RAM)
|
||||
- **[Hermes](https://github.com/openclawagents/hermes)** — deploy on [Railway](https://github.com/praveen-ks-2001/hermes-agent-template) (one click)
|
||||
|
||||
Then paste this into your agent:
|
||||
|
||||
@@ -1752,24 +1752,6 @@ The whole loop is described in [`docs/architecture/topologies.md`](docs/architec
|
||||
|
||||
**Job queue (Minions).** BullMQ-shaped, Postgres-native job queue. Durable subagents (LLM tool loops that survive crashes via two-phase pending→done persistence), shell jobs with audit, child jobs with cascading timeouts, rate leases for outbound providers, attachments via S3/Supabase storage. Replaces "spawn subagent as fire-and-forget Promise" with something that recovers from anything.
|
||||
|
||||
**Non-English brains (FTS language config).** The Postgres full-text search tokenizer is configurable via `GBRAIN_FTS_LANGUAGE`. Defaults to `english`. Set it to any text-search configuration that exists in your Postgres instance:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=portuguese # uses built-in portuguese stemmer
|
||||
export GBRAIN_FTS_LANGUAGE=spanish # built-in spanish stemmer
|
||||
export GBRAIN_FTS_LANGUAGE=pt_br # custom config (e.g. unaccent + portuguese)
|
||||
```
|
||||
|
||||
List available configs: `psql -c "SELECT cfgname FROM pg_ts_config"`. Both the **query side** (`websearch_to_tsquery`) and the **write side** (the trigger functions that populate `pages.search_vector` and `content_chunks.search_vector`) honor `GBRAIN_FTS_LANGUAGE`. On first install (or upgrade), the `configurable_fts_language` schema migration reads the env var and creates trigger functions in the configured language; subsequent inserts/updates tokenize using that setting. To change language on a brain that has already run the migration, use the dedicated CLI command:
|
||||
|
||||
```bash
|
||||
export GBRAIN_FTS_LANGUAGE=portuguese
|
||||
gbrain reindex-search-vector --dry-run # preview row counts
|
||||
gbrain reindex-search-vector --yes # recreate triggers + backfill
|
||||
```
|
||||
|
||||
The command is idempotent (re-running with the same language is a no-op for vector content) and uses the same recreate-and-backfill primitives as the migration. For accent-insensitive Portuguese (`pt_br`), see [docs/guides/multi-language-fts.md](docs/guides/multi-language-fts.md) for the `unaccent` + portuguese stemmer recipe.
|
||||
|
||||
**43 curated skills.** Routing lives in [`skills/RESOLVER.md`](skills/RESOLVER.md). Covers signal capture, ingest (idea / media / meeting), enrichment, querying, brain ops, citation fixing, daily task management, cron scheduling, reports, voice, soul audit, skill creation, eval framework, and migrations. Skills are markdown files (tool-agnostic), packaged as a single skillpack the installer drops into your agent workspace.
|
||||
|
||||
**Eval framework.** `gbrain eval longmemeval` runs the public [LongMemEval](https://huggingface.co/datasets/xiaowu0162/longmemeval) benchmark against your hybrid retrieval. `gbrain eval export` + `gbrain eval replay` capture real queries and replay them against code changes (set `GBRAIN_CONTRIBUTOR_MODE=1`). `gbrain eval cross-modal` cross-checks an output against the task using three different-provider frontier models. `gbrain eval retrieval-quality` runs NamedThingBench, which hard-gates the named-thing retrieval families (title-substring, alias-synonym, generic-to-named, multi-chunk-dilution) so a regression in "find the page this query names" fails CI loudly. Full methodology in [`docs/eval/SEARCH_MODE_METHODOLOGY.md`](docs/eval/SEARCH_MODE_METHODOLOGY.md).
|
||||
@@ -2113,51 +2095,6 @@ RRF fusion, multi-query expansion, and 4-layer dedup are engine-agnostic. They o
|
||||
|
||||
**Why not self-hosted for v0:** The brain should be infrastructure agents use, not something you maintain. Self-hosted Postgres with Docker is a welcome community PR, but v0 optimizes for zero ops.
|
||||
|
||||
### Opt-in RLS source-scope binding (`GBRAIN_RLS_SCOPE_BINDING`)
|
||||
|
||||
Defense-in-depth layer for Postgres deployments that want the database itself
|
||||
to enforce source isolation, in addition to the mandatory app-layer filters
|
||||
(`sourceScopeOpts` — layer 1, always on).
|
||||
|
||||
**Mechanism.** With `GBRAIN_RLS_SCOPE_BINDING=1` (or `true`), the engine's
|
||||
source-scoped read methods wrap their queries in a transaction that first runs
|
||||
`SELECT set_config('app.scopes', $1, true)` — the value is a bound parameter
|
||||
(federated `sourceIds` CSV > scalar `sourceId` > `'*'` for unscoped internal
|
||||
reads), transaction-local (equivalent to `SET LOCAL`, which itself can't take
|
||||
bound params). An RLS policy can then filter rows by
|
||||
`current_setting('app.scopes', true)`.
|
||||
|
||||
**Default off.** With the env var unset, reads call through on the shared pool
|
||||
exactly as before — no per-read transaction, no pool-slot hold (the search
|
||||
methods keep the transaction they always had for their `SET LOCAL
|
||||
statement_timeout`). Existing operators see zero behavior change.
|
||||
|
||||
**Enabling it** (operator-managed SQL; gbrain ships no DDL for this):
|
||||
|
||||
```sql
|
||||
ALTER TABLE pages ENABLE ROW LEVEL SECURITY;
|
||||
CREATE POLICY pages_scope_filter ON pages
|
||||
USING (current_setting('app.scopes', true) = '*'
|
||||
OR source_id = ANY(string_to_array(current_setting('app.scopes', true), ',')));
|
||||
|
||||
-- Required: connections that don't run through the scoped read helper
|
||||
-- (admin, autopilot, cycle, writes) must default to unscoped, or they
|
||||
-- see zero rows once the policy exists:
|
||||
ALTER ROLE <runtime-role> SET app.scopes = '*';
|
||||
|
||||
-- If the runtime role OWNS the table, RLS is skipped for it unless forced:
|
||||
ALTER TABLE pages FORCE ROW LEVEL SECURITY;
|
||||
```
|
||||
|
||||
Safe to enable in either order: the env var without a policy is a no-op
|
||||
setting; a policy without the env var is enforced only via the role default.
|
||||
|
||||
**Honest caveat:** only read paths routed through the scoped helper carry a
|
||||
per-request scope binding — unwrapped paths (writes, admin/maintenance reads)
|
||||
run under the role default and are not backstopped per caller. This is layer 2;
|
||||
the app-layer source filters remain layer 1 and stay mandatory. Behavioral pins
|
||||
live in `test/postgres-engine-rls-scope.test.ts`.
|
||||
|
||||
## PGLiteEngine (v0.7, ships)
|
||||
|
||||
**Dependencies:** `@electric-sql/pglite` (v0.4.4+)
|
||||
@@ -2720,17 +2657,14 @@ GBrain is tuned for the Supabase **Transaction pooler** (port 6543): it
|
||||
auto-disables prepared statements there and routes `engine.transaction()`
|
||||
(migrations, DDL, sync imports) to a derived **direct** connection
|
||||
(`db.<ref>.supabase.co:5432`). That direct host is IPv6-only, so on an
|
||||
IPv4-only host it is unreachable. When that happens gbrain now falls back to
|
||||
the pooler automatically (one stderr warning, then single-pool mode for the
|
||||
rest of the process) — but the pooler's ~2-min statement timeout can truncate
|
||||
very long migrations or bulk imports.
|
||||
IPv4-only host, reads work but sync **silently skips most pages**. This is the
|
||||
number one cause of "sync ran but nothing happened."
|
||||
|
||||
Fix: make the direct connection reachable over IPv4. Either set
|
||||
`GBRAIN_DIRECT_DATABASE_URL` to the **Session pooler** string (port 5432 on the
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on.
|
||||
`GBRAIN_DISABLE_DIRECT_POOL=1` skips the direct pool (and the fallback warning)
|
||||
entirely. Verify by running `gbrain sync` and checking that the page count in
|
||||
`gbrain stats` matches the syncable file count in the repo.
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on. Verify by
|
||||
running `gbrain sync` and checking that the page count in `gbrain stats` matches
|
||||
the syncable file count in the repo.
|
||||
|
||||
### The Primitives
|
||||
|
||||
@@ -2833,16 +2767,6 @@ hashes match. If both a cron and `--watch` fire simultaneously, no conflict.
|
||||
history rewrite still hard-blocks even with `--skip-failed`. Run
|
||||
`gbrain sync --skip-failed` to acknowledge a known-bad set yourself.
|
||||
|
||||
5. **Import checkpoints name the import target, not the caller's CWD.**
|
||||
Interrupted `gbrain import <dir>` runs may leave
|
||||
`~/.gbrain/import-checkpoint.json` so the next import can resume. The
|
||||
checkpoint `dir` is the absolute, resolved import target captured when
|
||||
import starts. It is not a cleanup instruction and it must not be
|
||||
re-derived from the process working directory. Checkpoints written by
|
||||
gbrain include `schema_version: 1`, `owner: "gbrain"`, and
|
||||
`kind: "import"` so downstream tools can validate the contract before
|
||||
deciding whether to resume.
|
||||
|
||||
## How to Verify
|
||||
|
||||
1. **Edit a file and search for the change.** Edit a brain markdown file,
|
||||
@@ -3905,43 +3829,6 @@ the user owns the machine.
|
||||
See [ALTERNATIVES.md](ALTERNATIVES.md) for a comparison of ngrok, Tailscale
|
||||
Funnel, and cloud hosts (Fly.io, Railway).
|
||||
|
||||
### Co-located Docker workloads (self-hosted Postgres)
|
||||
|
||||
OAuth scopes and source scoping guard the `gbrain serve --http` path. They do
|
||||
NOT guard raw Postgres. If the brain's Postgres runs as a container on the same
|
||||
Docker host as other workloads (agent runtimes, n8n, staging fixtures), any
|
||||
container sharing Docker's default `bridge` network can open a direct DB
|
||||
session — no OAuth token required — and read every source. That silently
|
||||
recreates a privileged path underneath the isolation you configured at the MCP
|
||||
layer.
|
||||
|
||||
Network-zone the host so untrusted containers can never reach Postgres:
|
||||
|
||||
```
|
||||
Docker host
|
||||
├── gbrain-net ← ONLY the brain's Postgres (+ gbrain serve, if containerized)
|
||||
├── agent-<id>-net ← each untrusted agent runtime, isolated
|
||||
└── default bridge ← no secret-bearing databases
|
||||
```
|
||||
|
||||
Operator checklist:
|
||||
|
||||
```text
|
||||
[ ] Postgres is on a user-defined Docker network, not the default bridge
|
||||
(or nothing else runs on that bridge)
|
||||
[ ] If Postgres publishes a host port at all, it binds loopback only
|
||||
(`-p 127.0.0.1:5432:5432`, never `0.0.0.0`)
|
||||
[ ] Untrusted agent containers have no DATABASE_URL or Postgres password
|
||||
[ ] Untrusted agents reach the brain via OAuth/Bearer against serve --http only
|
||||
(host loopback via host.docker.internal / host gateway — never gbrain-net)
|
||||
[ ] OAuth clients are least-privilege: scoped --source / --federated-read,
|
||||
pre-minted short-lived tokens preferred over long-lived client secrets
|
||||
[ ] Isolation verified: a team-scoped client cannot read internal-only sources
|
||||
```
|
||||
|
||||
Optional defense-in-depth: a dedicated Postgres role (or RLS) limited to the
|
||||
allowed `source_id`s, so even a leaked connection string can't read everything.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"missing_auth" error**
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
{
|
||||
"id": "gbrain-context-engine",
|
||||
"name": "gbrain",
|
||||
"version": "0.32.3.0",
|
||||
"description": "Personal knowledge brain with Postgres + pgvector hybrid search",
|
||||
|
||||
+3
-4
@@ -144,11 +144,10 @@
|
||||
"bun": ">=1.3.10"
|
||||
},
|
||||
"license": "MIT",
|
||||
"version": "0.42.65.0",
|
||||
"version": "0.42.62.0",
|
||||
"overrides": {
|
||||
"@hono/node-server": "^2.0.5",
|
||||
"fast-uri": "^3.1.4",
|
||||
"body-parser": "^2.3.0",
|
||||
"@hono/node-server": "^1.19.13",
|
||||
"fast-uri": "^3.1.2",
|
||||
"fast-xml-builder": "^1.1.7",
|
||||
"fast-xml-parser": "^5.7.0",
|
||||
"form-data": "^4.0.6",
|
||||
|
||||
+8
-13
@@ -1,7 +1,7 @@
|
||||
---
|
||||
id: x-to-brain
|
||||
name: X-to-Brain
|
||||
version: 0.8.2
|
||||
version: 0.8.1
|
||||
description: Twitter timeline, mentions, and keyword monitoring flow into brain pages. Tracks deletions, engagement velocity, OCR on images, and real-time alerts.
|
||||
category: sense
|
||||
requires: []
|
||||
@@ -9,12 +9,9 @@ secrets:
|
||||
- name: X_BEARER_TOKEN
|
||||
description: X API v2 Bearer token (Basic tier minimum, $200/mo for full archive search)
|
||||
where: https://developer.x.com/en/portal/dashboard — create a project + app, copy the Bearer Token from "Keys and tokens"
|
||||
- name: X_HANDLE
|
||||
description: Your X username without the @ (used for the app-only health check — /users/me requires user-context OAuth, which app-only bearer tokens don't have)
|
||||
where: Your X profile — the handle in your profile URL, e.g. x.com/yourhandle → yourhandle
|
||||
health_checks:
|
||||
- type: http
|
||||
url: "https://api.x.com/2/users/by/username/$X_HANDLE"
|
||||
url: "https://api.x.com/2/users/me"
|
||||
auth: bearer
|
||||
auth_token: "$X_BEARER_TOKEN"
|
||||
label: "X API"
|
||||
@@ -113,17 +110,15 @@ Tell the user:
|
||||
4. Inside the project, create a new App
|
||||
5. Go to the app's 'Keys and tokens' tab
|
||||
6. Under 'Bearer Token', click 'Generate' (or 'Regenerate')
|
||||
7. Copy the Bearer Token and paste it to me, along with your X handle (without the @)
|
||||
7. Copy the Bearer Token and paste it to me
|
||||
|
||||
Note: Free tier gives read-only access with low limits. Basic tier ($200/mo)
|
||||
gives search/recent endpoint and higher limits. Pro tier gets full archive search."
|
||||
|
||||
Set both `X_BEARER_TOKEN` and `X_HANDLE` in the environment. Validate immediately
|
||||
(app-only bearer tokens cannot call `/users/me` — that endpoint requires
|
||||
user-context OAuth — so validation uses the by-username lookup):
|
||||
Validate immediately:
|
||||
```bash
|
||||
curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \
|
||||
"https://api.x.com/2/users/by/username/$X_HANDLE" \
|
||||
"https://api.x.com/2/users/me" \
|
||||
&& echo "PASS: X API connected" \
|
||||
|| echo "FAIL: X API token invalid"
|
||||
```
|
||||
@@ -139,10 +134,10 @@ starting with 'AAA...', (3) if you just created the app, the token is valid imme
|
||||
```bash
|
||||
# Look up the user's X user ID from their handle
|
||||
curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \
|
||||
"https://api.x.com/2/users/by/username/$X_HANDLE" | grep -o '"id":"[^"]*"'
|
||||
"https://api.x.com/2/users/by/username/USERNAME" | grep -o '"id":"[^"]*"'
|
||||
```
|
||||
|
||||
Look up the user ID from the handle collected in Step 1.
|
||||
Ask the user for their X handle (e.g., @yourhandle). Look up their user ID.
|
||||
Save it — the collector needs the numeric ID, not the handle.
|
||||
|
||||
### Step 3: Configure the Collector
|
||||
@@ -210,7 +205,7 @@ The agent should review collected data 2-3x daily and run enrichment.
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.gbrain/integrations/x-to-brain
|
||||
echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.2","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl
|
||||
echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.1","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl
|
||||
```
|
||||
|
||||
## Production Patterns (v0.8.1)
|
||||
|
||||
@@ -70,7 +70,7 @@ PATTERN='import[[:space:]]+(\*[[:space:]]+as[[:space:]]+[a-zA-Z_$][a-zA-Z0-9_$]*
|
||||
FOUND_FILES=""
|
||||
while IFS= read -r f; do
|
||||
[ -n "$f" ] && FOUND_FILES="$FOUND_FILES$f"$'\n'
|
||||
done < <(grep -rlE --include='*.ts' "$PATTERN" src 2>/dev/null | sort -u || true)
|
||||
done < <(grep -rlE --include='*.ts' "$PATTERN" src/ 2>/dev/null | sort -u || true)
|
||||
|
||||
FAIL=0
|
||||
|
||||
|
||||
@@ -100,9 +100,9 @@ IFS='|' eval 'PATTERN="${PATTERN_PARTS[*]}"'
|
||||
|
||||
# Find tool.
|
||||
if command -v rg >/dev/null 2>&1; then
|
||||
matches="$(rg -niH --no-heading -t ts "$PATTERN" test 2>/dev/null || true)"
|
||||
matches="$(rg -niH --no-heading -t ts "$PATTERN" test/ 2>/dev/null || true)"
|
||||
elif command -v grep >/dev/null 2>&1; then
|
||||
matches="$(grep -rniE --include='*.test.ts' "$PATTERN" test 2>/dev/null || true)"
|
||||
matches="$(grep -rniE --include='*.test.ts' "$PATTERN" test/ 2>/dev/null || true)"
|
||||
else
|
||||
echo "check-test-real-names: ERROR: neither rg nor grep available." >&2
|
||||
exit 2
|
||||
|
||||
@@ -133,7 +133,6 @@ for i in $(seq 1 "$N"); do
|
||||
env SHARD="$i/$N" \
|
||||
bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \
|
||||
> "$SHARD_LOG" 2>&1
|
||||
rc=$?
|
||||
else
|
||||
env SHARD="$i/$N" \
|
||||
bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \
|
||||
@@ -143,20 +142,10 @@ for i in $(seq 1 "$N"); do
|
||||
sleep 5 && kill -KILL "$pid" 2>/dev/null ) &
|
||||
cap_pid=$!
|
||||
wait "$pid" 2>/dev/null
|
||||
# Capture the shard's exit code from ITS `wait`, before any watchdog
|
||||
# teardown runs. The teardown commands below overwrite $? — the killed
|
||||
# watchdog reports 143 — which used to get stamped into every shard's
|
||||
# sentinel on machines with no gtimeout/timeout: every run "failed"
|
||||
# with rc=143 summaries even when all tests passed.
|
||||
rc=$?
|
||||
# Reap the watchdog's `sleep` child too (pkill -P), then the watchdog.
|
||||
# Killing only the subshell leaves the sleep orphaned until
|
||||
# $SHARD_TIMEOUT elapses — same quirk the heartbeat cleanup below works
|
||||
# around; CI's orphan-process sweep flags those.
|
||||
pkill -P "$cap_pid" 2>/dev/null
|
||||
kill "$cap_pid" 2>/dev/null
|
||||
wait "$cap_pid" 2>/dev/null
|
||||
fi
|
||||
rc=$?
|
||||
echo "$rc" > "$LOG_DIR/shard-$i.exit"
|
||||
[ "$rc" = "124" ] && echo "WEDGED" > "$LOG_DIR/shard-$i.wedged"
|
||||
) &
|
||||
|
||||
@@ -126,7 +126,6 @@ for c in "${CHECKS[@]}"; do
|
||||
(
|
||||
if [ -n "$TIMEOUT_BIN" ]; then
|
||||
"$TIMEOUT_BIN" "${TIMEOUT}s" bun run "$c" > "$LOG_FILE" 2>&1
|
||||
rc=$?
|
||||
else
|
||||
bun run "$c" > "$LOG_FILE" 2>&1 &
|
||||
pid=$!
|
||||
@@ -134,20 +133,10 @@ for c in "${CHECKS[@]}"; do
|
||||
sleep 5 && kill -KILL "$pid" 2>/dev/null ) &
|
||||
cap_pid=$!
|
||||
wait "$pid" 2>/dev/null
|
||||
# Capture the check's exit code from ITS `wait`, before any watchdog
|
||||
# teardown runs. The teardown commands below overwrite $? — the killed
|
||||
# watchdog reports 143 — which used to get stamped into every sentinel
|
||||
# on machines with no gtimeout/timeout: verify reported pass=0
|
||||
# fail=<all> while every per-check log said OK.
|
||||
rc=$?
|
||||
# Reap the watchdog's `sleep` child too (pkill -P), then the watchdog.
|
||||
# Killing only the subshell leaves the sleep orphaned until $TIMEOUT
|
||||
# elapses — same quirk the heartbeat cleanup in run-unit-parallel.sh
|
||||
# works around; CI's orphan-process sweep flags those.
|
||||
pkill -P "$cap_pid" 2>/dev/null
|
||||
kill "$cap_pid" 2>/dev/null
|
||||
wait "$cap_pid" 2>/dev/null
|
||||
fi
|
||||
rc=$?
|
||||
echo "$rc" > "$EXIT_FILE"
|
||||
) &
|
||||
PIDS+=($!)
|
||||
|
||||
@@ -62,7 +62,7 @@ gbrain capture "..." --json # structured output for agents
|
||||
- **Slug:** `inbox/YYYY-MM-DD-<hash8>` (stable for same content; the daemon's 24h dedup catches re-captures).
|
||||
- **Type:** `note` (override with `--type idea` etc.).
|
||||
- **Frontmatter stamps:** `captured_via: capture-cli`, `captured_at: <ISO>`.
|
||||
- **Title:** first non-empty line of the body, capped at 80 chars (truncation appends `…`).
|
||||
- **Title:** first non-empty line of the body, capped at 80 chars.
|
||||
|
||||
## Output Format
|
||||
|
||||
|
||||
@@ -266,5 +266,4 @@ editorial pass.
|
||||
(e.g. `src/commands/<slug>.ts` if the host SKILL.md declares it
|
||||
in frontmatter)
|
||||
- gbrain's `openclaw.plugin.json` — adds the slug to `skills:`
|
||||
array, sorted alphabetically, without removing OpenClaw-native plugin fields
|
||||
like `id`, `configSchema`, or `contracts`
|
||||
array, sorted alphabetically
|
||||
|
||||
@@ -57,8 +57,6 @@ This mode guarantees:
|
||||
- `skills/manifest.json` lists every skill directory
|
||||
- `skills/RESOLVER.md` references every skill in the manifest
|
||||
- `openclaw.plugin.json` `skills[]` round-trips with both
|
||||
- `openclaw.plugin.json` keeps OpenClaw install-required native plugin fields
|
||||
(`id`, object `configSchema`, and `contracts.contextEngines` when applicable)
|
||||
- No MECE violations (duplicate triggers across skills)
|
||||
|
||||
### Phases
|
||||
@@ -74,7 +72,7 @@ This mode guarantees:
|
||||
### Automation
|
||||
|
||||
```bash
|
||||
bun test test/skills-conformance.test.ts test/resolver.test.ts test/openclaw-plugin-manifest.test.ts
|
||||
bun test test/skills-conformance.test.ts test/resolver.test.ts
|
||||
```
|
||||
|
||||
The CI-gated check is the package.json `test` script.
|
||||
|
||||
+5
-131
@@ -24,7 +24,6 @@ import type { GBrainConfig } from './core/config.ts';
|
||||
import type { AIGatewayConfig } from './core/ai/types.ts';
|
||||
import type { BrainEngine } from './core/engine.ts';
|
||||
import { operations, OperationError } from './core/operations.ts';
|
||||
import { resolveSourceIdEngineFree } from './core/source-resolver.ts';
|
||||
import { formatVolunteeredPage } from './core/context/volunteer.ts';
|
||||
import type { Operation, OperationContext } from './core/operations.ts';
|
||||
import { shouldForceExitAfterMain, finishCliTeardown, flushThenExit, currentExitCode, setCliExitVerdict } from './core/cli-force-exit.ts';
|
||||
@@ -55,7 +54,7 @@ export function bigintToStringReplacer(_key: string, value: unknown): unknown {
|
||||
}
|
||||
|
||||
// CLI-only commands that bypass the operation layer
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'maintain', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'reconcile-links', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector']);
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch']);
|
||||
// CLI-only commands whose handlers print their own --help text. These are
|
||||
// excluded from the generic short-circuit so detailed per-command and
|
||||
// per-subcommand usage stays reachable.
|
||||
@@ -79,8 +78,6 @@ const CLI_ONLY_SELF_HELP = new Set([
|
||||
'capture',
|
||||
// v0.42 self-upgrade ships its own usage (flags + the agent-skill story).
|
||||
'self-upgrade',
|
||||
// maintain (#3015) prints its own usage block (modes + not-auto-applied list).
|
||||
'maintain',
|
||||
// v0.43 (#2095): watch ships WATCH_HELP (flags + the stdin-turn protocol).
|
||||
'watch',
|
||||
// v0.37 fix wave (Lane D.4 + CDX2-12): sync's --no-embed flag was
|
||||
@@ -385,15 +382,6 @@ async function main() {
|
||||
if (op.localOnly) {
|
||||
refuseThinClient(command, cfgPre!.remote_mcp!.mcp_url);
|
||||
}
|
||||
// #2098: the local path resolves --source / GBRAIN_SOURCE / .gbrain-source
|
||||
// inside makeContext (ctx.sourceId), which this route never reaches — so
|
||||
// scope must be mapped onto the op's source_id wire param before the call.
|
||||
try {
|
||||
applyThinClientSourceScope(op, params);
|
||||
} catch (e: unknown) {
|
||||
console.error(e instanceof Error ? e.message : String(e));
|
||||
process.exit(1);
|
||||
}
|
||||
await runThinClientRouted(op, params, cfgPre!, cliOpts);
|
||||
return;
|
||||
}
|
||||
@@ -814,80 +802,18 @@ export function parseOpArgs(op: Operation, args: string[]): Record<string, unkno
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* #2098: thin-client source scoping. Locally, --source / GBRAIN_SOURCE /
|
||||
* .gbrain-source resolve to ctx.sourceId in makeContext; the thin-client
|
||||
* route short-circuits before that, so `gbrain query --source X` against a
|
||||
* remote brain silently searched unscoped. This runs the engine-free tiers
|
||||
* (flag → env → dotfile; the DB-backed tiers can't run without an engine —
|
||||
* the server's grant scoping covers the rest) and maps the result onto the
|
||||
* op's `source_id` wire param.
|
||||
*
|
||||
* Ops that declare their OWN `source` param (facts add, etc.) are left
|
||||
* untouched — their --source is an op param, not scope. An explicit --source
|
||||
* on an op with no source_id wire param throws (loud beats silent drop);
|
||||
* ambient env/dotfile scope with nowhere to send it is ignored, matching the
|
||||
* pre-fix behavior for non-scopeable ops. Exported for tests.
|
||||
*/
|
||||
// Ops whose `source_id` wire param is NOT read-scope semantics: get_skill's
|
||||
// source_id flips the lookup from host catalog to brain-resident-pack
|
||||
// (getResidentSkillDetail). Ambient env/dotfile scope must never leak into
|
||||
// these; an explicit --source-id still passes through untouched above.
|
||||
const NON_SCOPE_SOURCE_ID_OPS = new Set(['get_skill']);
|
||||
|
||||
export function applyThinClientSourceScope(
|
||||
op: Operation,
|
||||
params: Record<string, unknown>,
|
||||
cwd?: string,
|
||||
): void {
|
||||
if ('source' in op.params) return; // the op owns --source; not a scope flag
|
||||
const explicit = typeof params.source === 'string' && params.source.length > 0
|
||||
? (params.source as string)
|
||||
: null;
|
||||
delete params.source; // never a wire param on these ops — don't leak it
|
||||
// Explicit per-call scope already on the wire wins over ambient tiers.
|
||||
if (params.source_id !== undefined || params.all_sources === true) {
|
||||
if (explicit) {
|
||||
throw new Error('Pass either --source or --source-id/--all-sources, not both.');
|
||||
}
|
||||
return;
|
||||
}
|
||||
const resolved = resolveSourceIdEngineFree(explicit, cwd);
|
||||
if (!resolved) return;
|
||||
if (!('source_id' in op.params) || NON_SCOPE_SOURCE_ID_OPS.has(op.name)) {
|
||||
if (explicit) {
|
||||
const hint = NON_SCOPE_SOURCE_ID_OPS.has(op.name)
|
||||
? `(its source_id parameter is not a scope filter; pass --source-id explicitly if you mean it)`
|
||||
: `(the remote op has no source_id parameter; the server scopes it to your grant)`;
|
||||
throw new Error(
|
||||
`gbrain ${op.cliHints?.name || op.name} does not accept --source on a thin-client install ${hint}.`,
|
||||
);
|
||||
}
|
||||
return; // ambient env/dotfile scope with nowhere to send it
|
||||
}
|
||||
params.source_id = resolved;
|
||||
}
|
||||
|
||||
async function makeContext(engine: BrainEngine, params: Record<string, unknown>): Promise<OperationContext> {
|
||||
// v0.31.8 (D11): resolve sourceId via the canonical 6-tier chain. Honors
|
||||
// --source / GBRAIN_SOURCE / .gbrain-source / path-match / brain default /
|
||||
// 'default'. Wrapped in try/catch so a doctor / single-source brain that
|
||||
// never set up sources still returns 'default' silently.
|
||||
let sourceId: string | undefined;
|
||||
// #2561: when the source resolved via a NON-explicit tier (path-match /
|
||||
// brain default / sole-non-default / seed default), unqualified search-shaped
|
||||
// reads span every `config.federated = true` source. Computed here (the
|
||||
// trusted local boundary) and consumed by federatedSearchScope in
|
||||
// operations.ts, which additionally gates on ctx.remote === false.
|
||||
let localFederated: string[] | undefined;
|
||||
try {
|
||||
const { resolveSourceWithTier, localFederatedSourceIds } = await import('./core/source-resolver.ts');
|
||||
const { resolveSourceId } = await import('./core/source-resolver.ts');
|
||||
// params.source is set when a CLI flag was parsed for the op (rare; most
|
||||
// CLI ops don't take --source). Falls through to env/dotfile/path-match.
|
||||
const explicit = (params.source as string | undefined) ?? null;
|
||||
const resolved = await resolveSourceWithTier(engine, explicit);
|
||||
sourceId = resolved.source_id;
|
||||
localFederated = await localFederatedSourceIds(engine, resolved.source_id, resolved.tier);
|
||||
sourceId = await resolveSourceId(engine, explicit);
|
||||
} catch {
|
||||
// Source resolution failed (e.g. sources table doesn't exist on a fresh
|
||||
// pre-init brain). Leave sourceId unset; engine read methods fall through
|
||||
@@ -908,7 +834,6 @@ async function makeContext(engine: BrainEngine, params: Record<string, unknown>)
|
||||
// table). Matches dispatch.ts's auto-fill so the contract holds across
|
||||
// every transport.
|
||||
sourceId: sourceId ?? 'default',
|
||||
...(localFederated ? { localFederatedSourceIds: localFederated } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1010,10 +935,7 @@ export function formatResult(opName: string, result: unknown): string {
|
||||
lines.push(`Link coverage (entities): ${(h.link_coverage * 100).toFixed(1)}%`);
|
||||
}
|
||||
if (h.timeline_coverage !== undefined) {
|
||||
lines.push(`Timeline coverage (entity pages): ${(h.timeline_coverage * 100).toFixed(1)}%`);
|
||||
}
|
||||
if (h.timeline_coverage_score !== undefined) {
|
||||
lines.push(`Timeline density (all pages): ${h.timeline_coverage_score}/15 (whole-brain brain-score component)`);
|
||||
lines.push(`Timeline coverage (entities): ${(h.timeline_coverage * 100).toFixed(1)}%`);
|
||||
}
|
||||
if (Array.isArray(h.most_connected) && h.most_connected.length > 0) {
|
||||
lines.push('Most connected entities:');
|
||||
@@ -1076,13 +998,6 @@ const THIN_CLIENT_REFUSED_COMMANDS = new Set([
|
||||
// - `code-def`/`code-refs`/`code-callers`/`code-callees` have NO MCP ops
|
||||
// in operations.ts:2630-2671; cannot be "fixed by routing" yet
|
||||
'pages', 'files', 'eval', 'code-def', 'code-refs', 'code-callers', 'code-callees',
|
||||
// scratch-DB audit: `config` get/set operate on the host brain's config
|
||||
// plane (DB rows / host file-plane). On a thin client they fabricated an
|
||||
// ephemeral local PGLite (full migration replay per call) and read/wrote
|
||||
// config nobody would ever see. NOTE: `jobs` is deliberately NOT here —
|
||||
// it gets a partial dispatch (list/get route over MCP engine-free, the
|
||||
// rest refuse) in the main dispatch before connectEngine().
|
||||
'config',
|
||||
]);
|
||||
|
||||
/**
|
||||
@@ -1120,9 +1035,6 @@ const THIN_CLIENT_REFUSE_HINTS: Record<string, string> = {
|
||||
'code-refs': '`code-refs` has no MCP op yet. Run on the host.',
|
||||
'code-callers': '`code-callers` has no MCP op yet. Run on the host.',
|
||||
'code-callees': '`code-callees` has no MCP op yet. Run on the host.',
|
||||
// scratch-DB audit additions
|
||||
config: "config reads/writes the host brain's config plane. Edit the host's .gbrain/config.json (file-plane keys) or run on the host with GBRAIN_HOME set.",
|
||||
jobs: '`jobs list` and `jobs get <id>` are thin-client routable; this subcommand runs against the host queue. Use the submit_job / list_jobs / get_job MCP tools from your agent, or run on the host with GBRAIN_HOME set.',
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -1681,27 +1593,6 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
}
|
||||
}
|
||||
|
||||
// Thin-client `jobs` dispatch: `list` and `get` route over MCP (v0.32
|
||||
// routing branches in commands/jobs.ts) and never touch a local engine —
|
||||
// but falling through to connectEngine() below fabricates an empty
|
||||
// scratch PGLite in the thin-client GBRAIN_HOME and replays the entire
|
||||
// migration chain on every invocation before the remote call even runs.
|
||||
// Dispatch them engine-free here; every other jobs subcommand is
|
||||
// host-queue-bound, so refuse with a pinpoint hint instead of building
|
||||
// the scratch store.
|
||||
if (command === 'jobs') {
|
||||
const cfgJobs = loadConfig();
|
||||
if (isThinClient(cfgJobs)) {
|
||||
const jobsSub = args[0];
|
||||
if (jobsSub === 'list' || jobsSub === 'get') {
|
||||
const { runJobs } = await import('./commands/jobs.ts');
|
||||
await runJobs(null, args);
|
||||
return;
|
||||
}
|
||||
refuseThinClient('jobs', cfgJobs!.remote_mcp!.mcp_url);
|
||||
}
|
||||
}
|
||||
|
||||
// All remaining CLI-only commands need a DB connection
|
||||
const engine = await connectEngine();
|
||||
try {
|
||||
@@ -1835,11 +1726,6 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runOrphans(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'maintain': {
|
||||
const { runMaintain } = await import('./commands/maintain.ts');
|
||||
await runMaintain(engine, args);
|
||||
break;
|
||||
}
|
||||
// v0.32.7 CJK wave — post-upgrade markdown re-chunk sweep.
|
||||
// v0.36 Phase 3 wave — `gbrain reindex --multimodal` re-embeds content_chunks
|
||||
// into the unified Voyage multimodal-3 column.
|
||||
@@ -2115,15 +2001,6 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runReindexCodeCli(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'reindex-search-vector': {
|
||||
// Explicit recreate of FTS trigger functions + batched backfill,
|
||||
// honoring GBRAIN_FTS_LANGUAGE. Use after changing the language
|
||||
// env var on a brain that already ran the configurable_fts_language
|
||||
// migration.
|
||||
const { runReindexSearchVectorCli } = await import('./commands/reindex-search-vector.ts');
|
||||
await runReindexSearchVectorCli(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'reindex-frontmatter': {
|
||||
// v0.29.1: recovery / explicit-rebuild path for pages.effective_date.
|
||||
// Mirror of reindex-code shape. Wraps the shared library function in
|
||||
@@ -2372,7 +2249,7 @@ IMPORT/EXPORT
|
||||
import <dir> [--no-embed] Import markdown directory
|
||||
sync [--repo <path>] [flags] Git-to-brain incremental sync
|
||||
sync --watch [--interval N] Continuous sync (loops until stopped)
|
||||
See also: autopilot --install (continuous daemon).
|
||||
sync --install-cron Install persistent sync daemon
|
||||
export [--dir ./out/] Export to markdown
|
||||
export --restore-only [--repo <p>] Restore missing supabase-only files
|
||||
[--type T] [--slug-prefix S] With optional filters
|
||||
@@ -2459,9 +2336,6 @@ CODE INDEXING (v0.19.0 / v0.20.0 Cathedral II)
|
||||
query <q> --symbol-kind <k> Filter to symbol type (function|class|method|...) (v0.20.0)
|
||||
reconcile-links [--dry-run] Batch-recompute doc↔impl edges (v0.20.0)
|
||||
reindex-code [--source id] [--yes] Explicit code-page reindex (v0.20.0)
|
||||
reindex-search-vector [--dry-run] [--yes] [--json]
|
||||
Recreate FTS triggers + backfill under
|
||||
$GBRAIN_FTS_LANGUAGE (default 'english')
|
||||
sync --strategy code Sync code files into the brain
|
||||
|
||||
JOBS (Minions)
|
||||
|
||||
@@ -133,15 +133,14 @@ function indexCompleted(entries: CompletedMigrationEntry[]): CompletedIndex {
|
||||
* Returns the resolved status for a migration based on its entries.
|
||||
*
|
||||
* Semantics (Bug 3 — keep "complete wins" safety):
|
||||
* - If the latest entry is `retry`, the version is pending. This is the
|
||||
* explicit escape hatch written by `--force-retry`, and it overrides an
|
||||
* earlier `complete` entry without hand-editing the ledger.
|
||||
* - Otherwise, if any entry is `complete`, the version is complete.
|
||||
* - If any entry is `complete`, the version is complete. Terminal state.
|
||||
* - Otherwise, if the latest entry is `retry`, the version is pending
|
||||
* (user requested a fresh attempt).
|
||||
* - Otherwise, if any entry is `partial`, the version is partial.
|
||||
* - Otherwise, pending.
|
||||
*
|
||||
* `complete` never regresses accidentally. A later `partial` append cannot
|
||||
* undo a completed migration; only a trailing, explicit `retry` marker can.
|
||||
* `complete` never regresses. A later accidental `partial` append cannot
|
||||
* undo a completed migration.
|
||||
*/
|
||||
function statusForVersion(
|
||||
version: string,
|
||||
@@ -149,9 +148,9 @@ function statusForVersion(
|
||||
): 'complete' | 'partial' | 'pending' | 'wedged' {
|
||||
const entries = idx.byVersion.get(version) ?? [];
|
||||
if (entries.length === 0) return 'pending';
|
||||
if (entries.some(e => e.status === 'complete')) return 'complete';
|
||||
const latest = entries[entries.length - 1];
|
||||
if (latest.status === 'retry') return 'pending';
|
||||
if (entries.some(e => e.status === 'complete')) return 'complete';
|
||||
// Bug 3 attempt cap — count consecutive partials from the end (stopping
|
||||
// at any 'retry' or 'complete'). If we hit MAX_CONSECUTIVE_PARTIALS,
|
||||
// the migration is wedged and needs explicit --force-retry to try again.
|
||||
@@ -439,13 +438,6 @@ export async function runApplyMigrations(args: string[]): Promise<void> {
|
||||
const result = await m.orchestrator(orchestratorOptsFrom(cli));
|
||||
if (result.status === 'failed') {
|
||||
console.error(`Migration v${m.version} reported status=failed.`);
|
||||
// Surface each failed phase's detail — the ledger records it, but
|
||||
// the operator needs it on stderr to act (#921).
|
||||
for (const p of result.phases) {
|
||||
if (p.status === 'failed') {
|
||||
console.error(` phase ${p.name}: ${p.detail ?? '(no detail)'}`);
|
||||
}
|
||||
}
|
||||
// Record the attempt as 'partial' (not 'complete') so the cap counts
|
||||
// it. Don't let a failed orchestrator look like it never ran.
|
||||
try {
|
||||
|
||||
@@ -515,60 +515,6 @@ async function registerClient(name: string, args: string[]) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.42.x (#1914): rescope an existing OAuth client's write source and/or
|
||||
* federated read scope. This is the operator surface the DCR registration
|
||||
* comment promised ("rescope via the CLI later") — DCR clients land with
|
||||
* source_id='default' / federated_read=['default'] and must not self-widen,
|
||||
* so widening happens here (trusted local CLI) or via the requireAdmin
|
||||
* /admin/api/rescope-client endpoint.
|
||||
*/
|
||||
async function rescopeClient(clientId: string, args: string[]) {
|
||||
const usage = 'Usage: auth rescope-client <client_id> [--source SOURCE] [--federated-read SRC1,SRC2,...]';
|
||||
if (!clientId) {
|
||||
console.error(usage);
|
||||
process.exit(1);
|
||||
}
|
||||
let sourceId: string | undefined;
|
||||
let federatedRead: string[] | undefined;
|
||||
for (let i = 0; i < args.length; i += 2) {
|
||||
const flag = args[i];
|
||||
const value = args[i + 1];
|
||||
if (value === undefined || value.startsWith('--')) {
|
||||
console.error(`Error: ${flag} requires a value`);
|
||||
console.error(usage);
|
||||
process.exit(1);
|
||||
}
|
||||
if (flag === '--source') sourceId = value;
|
||||
else if (flag === '--federated-read') {
|
||||
federatedRead = value.split(',').map(s => s.trim()).filter(Boolean);
|
||||
} else {
|
||||
console.error(`Error: Unknown flag: ${flag}`);
|
||||
console.error(usage);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
if (sourceId === undefined && federatedRead === undefined) {
|
||||
console.error('Error: pass --source and/or --federated-read');
|
||||
console.error(usage);
|
||||
process.exit(1);
|
||||
}
|
||||
try {
|
||||
await withConfiguredSql(async (sql) => {
|
||||
const { GBrainOAuthProvider } = await import('../core/oauth-provider.ts');
|
||||
const provider = new GBrainOAuthProvider({ sql });
|
||||
const result = await provider.rescopeClient(clientId, { sourceId, federatedRead });
|
||||
console.log(`OAuth client rescoped: "${result.clientName}" (${result.clientId})\n`);
|
||||
console.log(` Write source: ${result.sourceId}`);
|
||||
console.log(` Federated reads: ${result.federatedRead.join(', ') || '<none>'}`);
|
||||
console.log('\nTakes effect on the client\'s next request (existing tokens included).');
|
||||
});
|
||||
} catch (e: any) {
|
||||
console.error('Error:', e.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Entry point for the `gbrain auth` CLI subcommand. Also reused by the
|
||||
* direct-script path (see bottom of file) so `bun run src/commands/auth.ts`
|
||||
@@ -610,7 +556,6 @@ export async function runAuth(args: string[]): Promise<void> {
|
||||
return;
|
||||
}
|
||||
case 'register-client': await registerClient(rest[0], rest.slice(1)); return;
|
||||
case 'rescope-client': await rescopeClient(rest[0], rest.slice(1)); return;
|
||||
case 'revoke-client': await revokeClient(rest[0]); return;
|
||||
case 'test': {
|
||||
const tokenIdx = rest.indexOf('--token');
|
||||
@@ -648,11 +593,6 @@ Usage:
|
||||
--bound-slug-prefixes <prefix1,prefix2> Bind submit_agent writes to slug prefixes
|
||||
--bound-max-concurrent <n> Bound submit_agent concurrency (default: 1)
|
||||
--budget-usd-per-day <usd> Bound submit_agent daily spend cap
|
||||
gbrain auth rescope-client <client_id> [options] Change an existing client's source scope (e.g. a DCR
|
||||
client stuck on the 'default' source). Only the flags
|
||||
you pass change; the other axis is left as-is.
|
||||
--source <id> New write source
|
||||
--federated-read <id1,id2,...> New read-scope source list
|
||||
gbrain auth revoke-client <client_id> Hard-delete an OAuth 2.1 client (cascades to tokens + codes)
|
||||
gbrain auth test <url> --token <token> Smoke-test a remote MCP server
|
||||
`);
|
||||
|
||||
+14
-181
@@ -17,7 +17,7 @@
|
||||
* gbrain autopilot --status [--json]
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync, writeFileSync, mkdirSync, appendFileSync, utimesSync, unlinkSync, chmodSync } from 'fs';
|
||||
import { existsSync, readFileSync, writeFileSync, mkdirSync, appendFileSync, utimesSync, unlinkSync } from 'fs';
|
||||
import { setCliExitVerdict } from '../core/cli-force-exit.ts';
|
||||
import { join } from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
@@ -38,7 +38,6 @@ import { logSelfUpgrade } from '../core/audit/self-upgrade-audit.ts';
|
||||
import { detectInstallMethod } from './upgrade.ts';
|
||||
import { evaluateQuietHours } from '../core/minions/quiet-hours.ts';
|
||||
import { inspectLock } from '../core/db-lock.ts';
|
||||
import { registerCleanup } from '../core/process-cleanup.ts';
|
||||
|
||||
/**
|
||||
* v0.37.7.0 #1162 — classify autopilot reconnect-loop errors.
|
||||
@@ -110,21 +109,7 @@ function logError(phase: string, e: unknown) {
|
||||
*/
|
||||
export function resolveGbrainCliPath(): string {
|
||||
try {
|
||||
// #2747: `env: process.env` is required under Bun. Bun's execSync
|
||||
// snapshots process.env at Bun's OWN startup, not at call time — a
|
||||
// runtime PATH mutation (dotenv/config loading, shell-profile sourcing
|
||||
// in a wrapper, etc.) happening between Bun boot and this call is
|
||||
// invisible to `which` without explicitly forwarding the current env.
|
||||
// This is why "which gbrain" succeeds when run standalone (fresh Bun
|
||||
// process, no prior mutation) but can fail from inside autopilot's own
|
||||
// process at this exact call site. Same fix already applied to
|
||||
// detectTini() in spawn-helpers.ts (see its comment) — this call site
|
||||
// was missed.
|
||||
const which = execSync('which gbrain', {
|
||||
encoding: 'utf-8',
|
||||
stdio: ['ignore', 'pipe', 'ignore'],
|
||||
env: process.env,
|
||||
}).trim();
|
||||
const which = execSync('which gbrain', { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
||||
if (which) return which;
|
||||
} catch { /* not on $PATH — fall through */ }
|
||||
|
||||
@@ -138,14 +123,7 @@ export function resolveGbrainCliPath(): string {
|
||||
return arg1;
|
||||
}
|
||||
|
||||
// #2747: include what we actually saw so an operator (or a future bug
|
||||
// report) doesn't have to guess whether PATH/execPath/argv[1] looked
|
||||
// sane at the moment of failure.
|
||||
throw new Error(
|
||||
'Could not resolve the gbrain CLI path. Install gbrain so it is on $PATH ' +
|
||||
'(e.g. /usr/local/bin/gbrain), or run autopilot from the compiled binary directly. ' +
|
||||
`Debug: PATH=${JSON.stringify(process.env.PATH ?? '')} execPath=${JSON.stringify(exec)} argv1=${JSON.stringify(arg1)}`,
|
||||
);
|
||||
throw new Error('Could not resolve the gbrain CLI path. Install gbrain so it is on $PATH (e.g. /usr/local/bin/gbrain), or run autopilot from the compiled binary directly.');
|
||||
}
|
||||
|
||||
export function shouldSpawnAutopilotWorker(args: string[]): boolean {
|
||||
@@ -434,37 +412,6 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
let stopping = false;
|
||||
let childSupervisor: ChildWorkerSupervisor | null = null;
|
||||
|
||||
// #1872: graceful engine shutdown. On PGLite the cycle steps run INLINE in
|
||||
// this process, so a hard `process.exit` mid-write (systemctl stop →
|
||||
// SIGTERM) kills WASM Postgres with the WAL dirty and can corrupt the
|
||||
// brain. Two exit paths must both close the engine:
|
||||
// - autopilot's own shutdown() below (owns SIGINT + internal stops like
|
||||
// max_crashes / cycle-failure-cap), and
|
||||
// - process-cleanup's SIGTERM handler (installed at cli.ts module load;
|
||||
// it runs the cleanup registry with a 3s deadline and then exits) —
|
||||
// which is why closeEngine is ALSO registered there.
|
||||
// closeEngine aborts the in-flight inline cycle (runCycle checks the
|
||||
// signal between phases and threads it into phase sub-work), gives it a
|
||||
// short bounded window to wind down, then disconnects. PGLite's
|
||||
// disconnect() drains the pending query and checkpoints before closing;
|
||||
// a second call is a no-op (disconnect snapshots + nulls the handle), so
|
||||
// both paths firing is safe.
|
||||
const shutdownAbort = new AbortController();
|
||||
let inflightInlineCycle: Promise<unknown> | null = null;
|
||||
const closeEngine = async () => {
|
||||
shutdownAbort.abort(new Error('autopilot shutdown'));
|
||||
if (inflightInlineCycle) {
|
||||
// ponytail: 2s cap keeps us inside process-cleanup's 3s deadline; a
|
||||
// between-phase abort resolves instantly, a mid-phase one may not.
|
||||
await Promise.race([
|
||||
inflightInlineCycle.catch(() => { /* cycle errors already logged by the loop */ }),
|
||||
new Promise((r) => setTimeout(r, 2_000)),
|
||||
]);
|
||||
}
|
||||
try { await engine.disconnect(); } catch { /* best-effort */ }
|
||||
};
|
||||
const deregisterEngineClose = registerCleanup('autopilot-engine-close', closeEngine);
|
||||
|
||||
if (spawnManagedWorker) {
|
||||
const cliPath = resolveGbrainCliPath();
|
||||
// Cgroup-aware auto-sized RSS watchdog cap (issue #1678). The old flat
|
||||
@@ -552,10 +499,6 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
childSupervisor.killChild('SIGKILL');
|
||||
}
|
||||
}
|
||||
// #1872: abort the in-flight inline cycle and close the engine BEFORE
|
||||
// process.exit — a hard exit mid-write corrupts PGLite's WASM Postgres.
|
||||
await closeEngine();
|
||||
deregisterEngineClose();
|
||||
try { unlinkSync(lockPath); } catch { /* already gone */ }
|
||||
process.exit(0);
|
||||
};
|
||||
@@ -563,9 +506,6 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
process.on('SIGINT', () => { void shutdown('SIGINT'); });
|
||||
|
||||
let consecutiveErrors = 0;
|
||||
// Parser-probe fixture warning is once-per-process, not once-per-cycle
|
||||
// (compiled-binary installs have no source tree; don't spam the log).
|
||||
let parserProbeFixtureWarned = false;
|
||||
// v0.37.7.0 #1162 — counter for consecutive reconnect failures.
|
||||
// Reset on every successful health probe or reconnect. Threshold
|
||||
// controlled by GBRAIN_AUTOPILOT_MAX_RECONNECT_FAILS env (default 30).
|
||||
@@ -864,10 +804,7 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
{
|
||||
queue: 'default',
|
||||
idempotency_key: idemKey,
|
||||
// issue #3218: the handler now throws on an
|
||||
// all-provider-failed batch, so give the queue's
|
||||
// backoff a chance (was 1 — dead-lettered instantly).
|
||||
max_attempts: 3,
|
||||
max_attempts: 1,
|
||||
timeout_ms: timeoutMs,
|
||||
},
|
||||
{ allowProtectedSubmit: true },
|
||||
@@ -907,19 +844,9 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
} catch {
|
||||
embeddingModel = (await engine.getConfig('embedding_model')) ?? undefined;
|
||||
}
|
||||
// #2662 (codex round-3): HOSTED_EMBED_KEY_CONFIG entries are keys
|
||||
// buildGatewayConfig folds from the FILE plane only — `gbrain config
|
||||
// set <key> X` writes the DB plane, which never reaches the gateway
|
||||
// for these fields. Reading via engine.getConfig() here (DB plane)
|
||||
// would report a provider "configured" from a DB-only key that the
|
||||
// gateway can never actually use, dispatching a doomed embed job.
|
||||
// Read the same file-plane source context.ts (doctor) reads instead,
|
||||
// so autopilot and doctor agree with what the gateway can see.
|
||||
const { loadConfigFileOnly } = await import('../core/config.ts');
|
||||
const fileCfg = loadConfigFileOnly() as Record<string, unknown> | null;
|
||||
const embedKeyCfg: Record<string, unknown> = {};
|
||||
const embedKeyCfg: Record<string, string | null> = {};
|
||||
for (const field of Object.values(HOSTED_EMBED_KEY_CONFIG)) {
|
||||
embedKeyCfg[field] = fileCfg?.[field];
|
||||
embedKeyCfg[field] = await engine.getConfig(field);
|
||||
}
|
||||
const ctx = {
|
||||
repoPath,
|
||||
@@ -1060,21 +987,16 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
// path's phase set). Now both converge on the same primitive.
|
||||
try {
|
||||
const { runCycle } = await import('../core/cycle.ts');
|
||||
// #1872: track the promise so closeEngine can drain it on shutdown,
|
||||
// and pass the abort signal so the cycle winds down between phases.
|
||||
const cyclePromise = runCycle(engine, {
|
||||
const report = await runCycle(engine, {
|
||||
brainDir: repoPath,
|
||||
// Autopilot daemon path: pulls by default (matches
|
||||
// pre-v0.17 autopilot behavior). CLI dream defaults false
|
||||
// for cron safety; that choice is scoped to dream only.
|
||||
pull: true,
|
||||
signal: shutdownAbort.signal,
|
||||
yieldBetweenPhases: async () => {
|
||||
await new Promise(r => setImmediate(r));
|
||||
},
|
||||
});
|
||||
inflightInlineCycle = cyclePromise;
|
||||
const report = await cyclePromise.finally(() => { inflightInlineCycle = null; });
|
||||
// Only 'failed' (every attempted phase failed) trips the autopilot
|
||||
// circuit breaker. 'partial' means at least one phase warned or
|
||||
// failed while others ran — that's a soft signal, not a fatal
|
||||
@@ -1130,36 +1052,17 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
// loop. Probe runs even when cycleOk=false (probe may surface signal
|
||||
// explaining why the cycle is failing).
|
||||
try {
|
||||
const { resolveProbeEnabled, resolveProbeMaxUsd, runNightlyQualityProbe } = await import('../core/cycle/nightly-quality-probe.ts');
|
||||
// Dual-plane read: `gbrain config set` (what the doctor enable hint
|
||||
// prints) writes the DB plane; ~/.gbrain/config.json is the fallback.
|
||||
let dbEnabled: string | null = null;
|
||||
let dbMaxUsd: string | null = null;
|
||||
try {
|
||||
dbEnabled = await engine.getConfig('autopilot.nightly_quality_probe.enabled');
|
||||
dbMaxUsd = await engine.getConfig('autopilot.nightly_quality_probe.max_usd');
|
||||
} catch { /* DB unavailable → file plane only */ }
|
||||
const probeEnabled = resolveProbeEnabled(dbEnabled, cfg?.autopilot?.nightly_quality_probe?.enabled);
|
||||
const probeEnabled = cfg?.autopilot?.nightly_quality_probe?.enabled === true;
|
||||
if (probeEnabled) {
|
||||
const { runNightlyQualityProbe } = await import('../core/cycle/nightly-quality-probe.ts');
|
||||
const { runLongMemEvalForProbe, runCrossModalBatchForProbe } = await import('../core/cycle/nightly-probe-adapters.ts');
|
||||
const { isAvailable } = await import('../core/ai/gateway.ts');
|
||||
const { existsSync } = await import('node:fs');
|
||||
const { fileURLToPath } = await import('node:url');
|
||||
const { join } = await import('node:path');
|
||||
const maxUsd = resolveProbeMaxUsd(dbMaxUsd, cfg?.autopilot?.nightly_quality_probe?.max_usd);
|
||||
// The committed fixture (test/fixtures/longmemeval-nightly.jsonl)
|
||||
// lives in the gbrain PACKAGE, not the brain repo — repoPath is
|
||||
// sync.repo_path (the user's brain), where the fixture never
|
||||
// exists, so the probe error'd on every real install. Resolve the
|
||||
// package root from the module location; keep repoPath as the
|
||||
// fallback for setups that vendor the fixture into the brain repo.
|
||||
const pkgRoot = fileURLToPath(new URL('../..', import.meta.url));
|
||||
const fixtureAtPkgRoot = existsSync(join(pkgRoot, 'test', 'fixtures', 'longmemeval-nightly.jsonl'));
|
||||
const maxUsd = Number(cfg?.autopilot?.nightly_quality_probe?.max_usd ?? 5);
|
||||
await runNightlyQualityProbe({
|
||||
isEnabled: () => true, // already gated above; phase re-checks for defense-in-depth
|
||||
hasEmbeddingProvider: () => isAvailable('embedding'),
|
||||
resolveMaxUsd: () => maxUsd,
|
||||
resolveRepoRoot: () => (fixtureAtPkgRoot ? pkgRoot : repoPath ?? gbrainHomePath('.')),
|
||||
resolveRepoRoot: () => repoPath ?? gbrainHomePath('.'),
|
||||
runLongMemEval: runLongMemEvalForProbe,
|
||||
runCrossModalBatch: runCrossModalBatchForProbe,
|
||||
now: () => new Date(),
|
||||
@@ -1171,62 +1074,6 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
// informational; autopilot loop continues.
|
||||
}
|
||||
|
||||
// 4.6 — Nightly conversation-parser probe (v0.41.16.0 phase module;
|
||||
// the scheduler wire-up was deferred at ship and is added here). Same
|
||||
// posture as 4.5: the phase owns its gates (enabled/mode-gate, LLM
|
||||
// key), the wiring owns invocation + the audit row, and a probe
|
||||
// failure NEVER crashes the autopilot loop. Per D10 the probe is
|
||||
// default-ON for search.mode=tokenmax, opt-in otherwise.
|
||||
try {
|
||||
const { runConversationParserNightlyProbe } = await import('../core/conversation-parser/nightly-probe.ts');
|
||||
const { logParserProbeEvent, parserProbeRanWithin } = await import('../core/audit-parser-probe.ts');
|
||||
const { isAvailable } = await import('../core/ai/gateway.ts');
|
||||
const { existsSync } = await import('node:fs');
|
||||
const { fileURLToPath } = await import('node:url');
|
||||
const { join } = await import('node:path');
|
||||
// Flag reads dual-plane: the DB row (`gbrain config set …`) wins,
|
||||
// ~/.gbrain/config.json is the fallback. search.mode lives on the
|
||||
// DB plane only (mode.ts owns it).
|
||||
let parserDbEnabled: string | null = null;
|
||||
let dbSearchMode: string | null = null;
|
||||
try {
|
||||
parserDbEnabled = await engine.getConfig('autopilot.conversation_parser_probe.enabled');
|
||||
dbSearchMode = await engine.getConfig('search.mode');
|
||||
} catch { /* DB unavailable → file plane only */ }
|
||||
const parserEnabled = parserDbEnabled != null
|
||||
? parserDbEnabled === 'true'
|
||||
: cfg?.autopilot?.conversation_parser_probe?.enabled === true;
|
||||
const searchMode = dbSearchMode ?? '';
|
||||
// Fixtures are committed in the gbrain package (test/fixtures/…),
|
||||
// NOT the brain repo — resolve from the module location. Compiled
|
||||
// binaries carry no source tree: skip quietly instead of writing
|
||||
// failure rows that would flip doctor to WARN on every binary install.
|
||||
const pkgRoot = fileURLToPath(new URL('../..', import.meta.url));
|
||||
const fixturePath = join(pkgRoot, 'test', 'fixtures', 'conversation-formats', 'all.jsonl');
|
||||
const adversarialPath = join(pkgRoot, 'test', 'fixtures', 'conversation-formats', 'adversarial.jsonl');
|
||||
const shouldInvoke = parserEnabled || searchMode === 'tokenmax';
|
||||
if (shouldInvoke && existsSync(fixturePath) && existsSync(adversarialPath)) {
|
||||
const result = await runConversationParserNightlyProbe({
|
||||
isEnabled: () => parserEnabled,
|
||||
searchMode: () => searchMode,
|
||||
hasLlmKey: () => isAvailable('chat'),
|
||||
resolveFixturePath: () => fixturePath,
|
||||
resolveAdversarialPath: () => adversarialPath,
|
||||
now: () => new Date(),
|
||||
shouldSkipForRateLimit: () => parserProbeRanWithin(24 * 60 * 60 * 1000),
|
||||
});
|
||||
// rate_limited is a non-run: the loop ticks every few minutes, so
|
||||
// logging every skip would flood the audit file with no-signal rows.
|
||||
if (result.outcome !== 'rate_limited') logParserProbeEvent(result);
|
||||
} else if (shouldInvoke && !parserProbeFixtureWarned) {
|
||||
parserProbeFixtureWarned = true;
|
||||
console.error(`[parser-probe] fixtures not found under ${pkgRoot}; skipping (probe needs a source-checkout install)`);
|
||||
}
|
||||
} catch (e) {
|
||||
logError('autopilot.parser_probe', e);
|
||||
// Informational, like 4.5: do NOT bump consecutiveErrors.
|
||||
}
|
||||
|
||||
// Wait for next cycle
|
||||
await new Promise(r => setTimeout(r, interval * 1000));
|
||||
}
|
||||
@@ -1395,14 +1242,7 @@ function installLaunchd(wrapperPath: string, home: string, repoPath: string) {
|
||||
try {
|
||||
const agentsDir = join(home, 'Library', 'LaunchAgents');
|
||||
mkdirSync(agentsDir, { recursive: true });
|
||||
writeFileSync(plistPath(), plist, { mode: 0o644 });
|
||||
// launchd rejects group/world-writable agent plists: bootstrap/load fails
|
||||
// with the opaque "Bootstrap failed: 5: Input/output error" and the login
|
||||
// scan skips the file silently. writeFileSync's mode only applies on
|
||||
// create — a reinstall over an existing plist keeps the old bits (a 0666
|
||||
// plist written under an umask-0 parent stays 0666 forever) — so
|
||||
// normalize unconditionally.
|
||||
chmodSync(plistPath(), 0o644);
|
||||
writeFileSync(plistPath(), plist);
|
||||
execSync(`launchctl load "${plistPath()}"`, { stdio: 'pipe' });
|
||||
console.log('Installed launchd service: com.gbrain.autopilot');
|
||||
console.log(` Repo: ${repoPath}`);
|
||||
@@ -1492,11 +1332,7 @@ export function migrateSystemdUnitToRestartAlways(): { rewritten: boolean; reaso
|
||||
return { rewritten: false, reason: 'hand-edited' };
|
||||
}
|
||||
try {
|
||||
writeFileSync(unitPath, generateSystemdUnit(execMatch![1]), { mode: 0o644 });
|
||||
// This path always rewrites an EXISTING unit, so writeFileSync's mode
|
||||
// never applies — chmod is the only thing that normalizes a unit born
|
||||
// 0666 under a umask-0 parent (systemd warns on world-writable units).
|
||||
chmodSync(unitPath, 0o644);
|
||||
writeFileSync(unitPath, generateSystemdUnit(execMatch![1]));
|
||||
try {
|
||||
execSync('systemctl --user daemon-reload', { stdio: 'pipe', timeout: 10_000 });
|
||||
} catch {
|
||||
@@ -1513,10 +1349,7 @@ function installSystemd(wrapperPath: string, repoPath: string) {
|
||||
try {
|
||||
const unitPath = systemdUnitPath();
|
||||
mkdirSync(join(process.env.HOME || '', '.config', 'systemd', 'user'), { recursive: true });
|
||||
writeFileSync(unitPath, unit, { mode: 0o644 });
|
||||
// Same umask-0 hardening as the launchd path (systemd warns on
|
||||
// world-writable units); mode only applies on create, so normalize.
|
||||
chmodSync(unitPath, 0o644);
|
||||
writeFileSync(unitPath, unit);
|
||||
execSync('systemctl --user daemon-reload', { stdio: 'pipe', timeout: 10_000 });
|
||||
execSync('systemctl --user enable --now gbrain-autopilot.service', { stdio: 'pipe', timeout: 15_000 });
|
||||
console.log('Installed systemd user service: gbrain-autopilot.service');
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
* checks if back-links exist, and optionally creates them.
|
||||
*
|
||||
* Usage:
|
||||
* gbrain check-backlinks check [dir] [--dir <brain-dir>] # report missing back-links
|
||||
* gbrain check-backlinks fix [dir] [--dir <brain-dir>] # create missing back-links
|
||||
* gbrain check-backlinks check [--dir <brain-dir>] # report missing back-links
|
||||
* gbrain check-backlinks fix [--dir <brain-dir>] # create missing back-links
|
||||
* gbrain check-backlinks fix --dry-run # preview fixes
|
||||
*/
|
||||
|
||||
@@ -201,40 +201,6 @@ export interface BacklinksResult {
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
export interface ParsedBacklinksArgs {
|
||||
subcommand: string | undefined;
|
||||
brainDir: string;
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
export function parseBacklinksArgs(args: string[]): ParsedBacklinksArgs {
|
||||
const subcommand = args[0];
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const dirIdx = args.indexOf('--dir');
|
||||
const flagDir = dirIdx >= 0 && args[dirIdx + 1] && !args[dirIdx + 1].startsWith('--')
|
||||
? args[dirIdx + 1]
|
||||
: undefined;
|
||||
|
||||
let positionalDir: string | undefined;
|
||||
for (let i = 1; i < args.length; i++) {
|
||||
const arg = args[i];
|
||||
if (arg === '--dir') {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (arg === '--dry-run') continue;
|
||||
if (arg.startsWith('--')) continue;
|
||||
positionalDir = arg;
|
||||
break;
|
||||
}
|
||||
|
||||
return {
|
||||
subcommand,
|
||||
brainDir: flagDir ?? positionalDir ?? '.',
|
||||
dryRun,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Library-level backlinks check/fix. Throws on validation errors; returns a
|
||||
* structured result so Minions handlers + autopilot-cycle can surface counts.
|
||||
@@ -270,14 +236,16 @@ export async function runBacklinksCore(opts: BacklinksOpts): Promise<BacklinksRe
|
||||
}
|
||||
|
||||
export async function runBacklinks(args: string[]) {
|
||||
const { subcommand, brainDir, dryRun } = parseBacklinksArgs(args);
|
||||
const subcommand = args[0];
|
||||
const dirIdx = args.indexOf('--dir');
|
||||
const brainDir = dirIdx >= 0 ? args[dirIdx + 1] : '.';
|
||||
const dryRun = args.includes('--dry-run');
|
||||
|
||||
if (!subcommand || !['check', 'fix'].includes(subcommand)) {
|
||||
console.error('Usage: gbrain check-backlinks <check|fix> [dir] [--dir <brain-dir>] [--dry-run]');
|
||||
console.error('Usage: gbrain check-backlinks <check|fix> [--dir <brain-dir>] [--dry-run]');
|
||||
console.error(' check Report missing back-links');
|
||||
console.error(' fix Create missing back-links (appends to Timeline)');
|
||||
console.error(' dir Brain directory (default: current directory)');
|
||||
console.error(' --dir Brain directory override');
|
||||
console.error(' --dir Brain directory (default: current directory)');
|
||||
console.error(' --dry-run Preview fixes without writing');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@ import { runPhaseCalibrationProfile } from '../core/cycle/calibration-profile.ts
|
||||
import { sourceScopeOpts, type OperationContext } from '../core/operations.ts';
|
||||
import type { GBrainConfig } from '../core/config.ts';
|
||||
import { GBrainError } from '../core/types.ts';
|
||||
import { resolveOwnerHolder } from '../core/owner-holder.ts';
|
||||
|
||||
export interface CalibrationProfileRow {
|
||||
/** BIGSERIAL → string (postgres.js int8 wire shape; never Number() — int8
|
||||
@@ -168,10 +167,7 @@ export async function runCalibration(
|
||||
config: GBrainConfig,
|
||||
): Promise<void> {
|
||||
const { opts } = parseArgs(args);
|
||||
const holder = resolveOwnerHolder({
|
||||
override: opts.holder,
|
||||
configValue: await engine.getConfig('emotional_weight.user_holder'),
|
||||
});
|
||||
const holder = opts.holder ?? 'garry';
|
||||
// Resolve --source / GBRAIN_SOURCE / .gbrain-source so the (now reachable, #2035)
|
||||
// calibration command targets the right source in a multi-source brain instead
|
||||
// of always reading `default`. No signal → 'default' (prior behavior).
|
||||
@@ -257,15 +253,12 @@ export async function getCalibrationProfileOp(
|
||||
ctx: OperationContext,
|
||||
params: { holder?: string },
|
||||
): Promise<CalibrationProfileRow | null> {
|
||||
const holder = resolveOwnerHolder({
|
||||
override: params.holder,
|
||||
configValue: await ctx.engine.getConfig('emotional_weight.user_holder'),
|
||||
});
|
||||
const holder = params.holder ?? 'garry';
|
||||
if (typeof holder !== 'string' || holder.length === 0) {
|
||||
throw new GBrainError(
|
||||
'INVALID_HOLDER',
|
||||
'get_calibration_profile.holder must be a non-empty string',
|
||||
'pass holder="<slug>" or omit to default to the owner holder (config emotional_weight.user_holder, else "self")',
|
||||
'pass holder="<slug>" or omit to default to "garry"',
|
||||
);
|
||||
}
|
||||
const scope = sourceScopeOpts(ctx);
|
||||
|
||||
@@ -233,18 +233,14 @@ export function maybeRewriteSourceFkError(err: unknown, sourceId: string | undef
|
||||
|
||||
/**
|
||||
* Derive a title from the first non-empty, non-`---` line of the body,
|
||||
* stripping leading markdown heading marks, capped at 80 chars. Truncation
|
||||
* is codepoint-aware (never splits an astral surrogate pair) and appends an
|
||||
* ellipsis so a cut title is visibly cut.
|
||||
* stripping leading markdown heading marks, capped at 80 chars.
|
||||
* Falls back to 'Capture' when no usable line exists.
|
||||
*/
|
||||
function deriveTitle(rawBody: string): string {
|
||||
const firstLine = rawBody
|
||||
.split('\n')
|
||||
.find((l) => l.trim().length > 0 && l.trim() !== '---') ?? '';
|
||||
const stripped = firstLine.replace(/^#+\s*/, '');
|
||||
const cps = [...stripped];
|
||||
return (cps.length > 80 ? cps.slice(0, 79).join('') + '…' : stripped) || 'Capture';
|
||||
return firstLine.replace(/^#+\s*/, '').slice(0, 80) || 'Capture';
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -37,19 +37,9 @@ export async function findCodeDef(
|
||||
// trigger) are first-class definitions in the SQL sense. The chunker's
|
||||
// normalizeSymbolType maps create_table → 'table' etc, so adding the SQL
|
||||
// kinds here is what makes `gbrain code-def users` work against SQL.
|
||||
// Method-level + member definitions. normalizeSymbolType only canonicalizes
|
||||
// some node types; the rest fall through `type.replace(/_/g, ' ')`, so
|
||||
// tree-sitter's method_declaration → 'method declaration', struct_specifier →
|
||||
// 'struct specifier', protocol_declaration → 'protocol declaration', etc.
|
||||
// Without these, code-def is blind to every method, constructor, field, C
|
||||
// struct, and Swift protocol — which is most of an OO codebase. The plain
|
||||
// 'struct' entry above never matched for the same reason (C emits the
|
||||
// 'struct specifier' fallback form).
|
||||
const DEF_TYPES = [
|
||||
'function', 'class', 'interface', 'type', 'enum', 'struct', 'trait', 'module', 'contract',
|
||||
'table', 'view', 'index', 'procedure', 'schema', 'database', 'trigger',
|
||||
'method declaration', 'method definition', 'constructor declaration',
|
||||
'field declaration', 'field definition', 'struct specifier', 'protocol declaration',
|
||||
];
|
||||
const params: unknown[] = [symbol, limit];
|
||||
let whereLang = '';
|
||||
|
||||
+221
-418
@@ -28,7 +28,6 @@ import type { DbUrlSource } from '../core/config.ts';
|
||||
import { gbrainPath, loadConfig } from '../core/config.ts';
|
||||
import { reflexEnabled } from '../core/context/reflex.ts';
|
||||
import { resolveSocketPath } from '../core/context/resolve-ipc.ts';
|
||||
import { resolveOwnerHolder } from '../core/owner-holder.ts';
|
||||
import { homedir } from 'os';
|
||||
import { dirname, isAbsolute, join, resolve as resolvePath } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
@@ -529,61 +528,6 @@ export async function childTableOrphansCheck(engine: BrainEngine): Promise<Check
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Raw-source persistence guarantee (#1978, warn-only v1).
|
||||
*
|
||||
* Invariant: every synthesized/derived page (dream_generated:true frontmatter
|
||||
* or type:synthesis) must either carry a raw trace or declare an explicit
|
||||
* exemption. Accepted traces:
|
||||
* - frontmatter key `raw_trace` / `raw_source` / `source_uri`
|
||||
* - an attached `raw_data` row
|
||||
* - `synthesis_evidence` rows (think-op citations)
|
||||
* - explicit `raw_trace_exempt: true` (reason in `raw_trace_exempt_reason`)
|
||||
*
|
||||
* v1 is deliberately warn-only — no write path is blocked. Escalation to
|
||||
* fail-closed enforcement in the synthesis/import write paths is the v2
|
||||
* follow-up once real brains run clean.
|
||||
*
|
||||
* Pure helper (engine.executeRaw only) for parity with
|
||||
* childTableOrphansCheck so tests can target it directly.
|
||||
*/
|
||||
export async function rawProvenanceCheck(engine: BrainEngine): Promise<Check> {
|
||||
const where = `
|
||||
p.deleted_at IS NULL
|
||||
AND (COALESCE(p.frontmatter->>'dream_generated', '') = 'true' OR p.type = 'synthesis')
|
||||
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ?| ARRAY['raw_trace', 'raw_source', 'source_uri', 'raw_trace_exempt'])
|
||||
AND NOT EXISTS (SELECT 1 FROM raw_data rd WHERE rd.page_id = p.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM synthesis_evidence se WHERE se.synthesis_page_id = p.id)`;
|
||||
try {
|
||||
const rows = await engine.executeRaw<{ n: string | number }>(
|
||||
`SELECT COUNT(*)::int AS n FROM pages p WHERE ${where}`,
|
||||
);
|
||||
const n = Number(rows[0]?.n ?? 0);
|
||||
if (n === 0) {
|
||||
return {
|
||||
name: 'raw_provenance',
|
||||
status: 'ok',
|
||||
message: 'All synthesized pages carry a raw trace or explicit exemption',
|
||||
};
|
||||
}
|
||||
const sample = await engine.executeRaw<{ slug: string }>(
|
||||
`SELECT p.slug FROM pages p WHERE ${where} ORDER BY p.slug LIMIT 5`,
|
||||
);
|
||||
const slugs = sample.map(r => r.slug).join(', ');
|
||||
return {
|
||||
name: 'raw_provenance',
|
||||
status: 'warn',
|
||||
message:
|
||||
`${n} synthesized page(s) lack a raw trace (no raw_trace/raw_source/source_uri frontmatter, ` +
|
||||
`raw_data row, or synthesis evidence) and carry no raw_trace_exempt marker. e.g. ${slugs}. ` +
|
||||
`Fix: stamp raw_source (path/URI of the source material) or raw_trace_exempt: true + ` +
|
||||
`raw_trace_exempt_reason in frontmatter. Warn-only (#1978).`,
|
||||
};
|
||||
} catch {
|
||||
return { name: 'raw_provenance', status: 'warn', message: 'Could not check raw provenance (older schema?)' };
|
||||
}
|
||||
}
|
||||
|
||||
export async function doctorReportRemote(engine: BrainEngine): Promise<DoctorReport> {
|
||||
const checks: Check[] = [];
|
||||
|
||||
@@ -813,7 +757,31 @@ export async function doctorReportRemote(engine: BrainEngine): Promise<DoctorRep
|
||||
|
||||
// 5. Queue health (Postgres-only). PGLite has no minion_jobs in the same
|
||||
// shape; skip the check there with an informational message.
|
||||
checks.push(await computeQueueHealthCheck(engine));
|
||||
if (engine.kind === 'postgres') {
|
||||
try {
|
||||
// issue #1801: column is `status`, not `state` (schema.sql:780). The
|
||||
// pre-fix query errored every run and the catch silently returned "No
|
||||
// queue activity," so this remote/thin-client check was a no-op.
|
||||
const rows = await engine.executeRaw<{ stalled: string | number }>(
|
||||
`SELECT COUNT(*) AS stalled FROM minion_jobs
|
||||
WHERE status = 'active'
|
||||
AND started_at IS NOT NULL
|
||||
AND started_at < NOW() - INTERVAL '1 hour'`,
|
||||
);
|
||||
const stalled = Number(rows[0]?.stalled ?? 0);
|
||||
checks.push({
|
||||
name: 'queue_health',
|
||||
status: stalled === 0 ? 'ok' : 'warn',
|
||||
message: stalled === 0
|
||||
? 'No stalled active jobs'
|
||||
: `${stalled} active job(s) stalled > 1h — \`gbrain jobs cancel <id>\` or \`gbrain jobs retry <id>\` on the host`,
|
||||
});
|
||||
} catch {
|
||||
checks.push({ name: 'queue_health', status: 'ok', message: 'No queue activity' });
|
||||
}
|
||||
} else {
|
||||
checks.push({ name: 'queue_health', status: 'ok', message: 'PGLite — no queue to check' });
|
||||
}
|
||||
|
||||
// issue #1801 — wedged_queue (cross-surface parity with buildChecks).
|
||||
checks.push(await computeWedgedQueueCheck(engine));
|
||||
@@ -1319,19 +1287,14 @@ export async function checkAbandonedThreads(engine: BrainEngine): Promise<Check>
|
||||
|
||||
/**
|
||||
* calibration_freshness: warns when the active calibration profile is
|
||||
* older than 7 days (configurable). Default holder resolves via resolveOwnerHolder
|
||||
* (config emotional_weight.user_holder, else 'self'). Multi-source
|
||||
* older than 7 days (configurable). Default holder 'garry'. Multi-source
|
||||
* brains see one row per source; this check uses the most recent across
|
||||
* all sources.
|
||||
*/
|
||||
export async function checkCalibrationFreshness(engine: BrainEngine): Promise<Check> {
|
||||
try {
|
||||
const ownerHolder = resolveOwnerHolder({
|
||||
configValue: await engine.getConfig('emotional_weight.user_holder'),
|
||||
});
|
||||
const rows = await engine.executeRaw<{ generated_at: Date | null }>(
|
||||
`SELECT MAX(generated_at) AS generated_at FROM calibration_profiles WHERE holder = $1`,
|
||||
[ownerHolder],
|
||||
`SELECT MAX(generated_at) AS generated_at FROM calibration_profiles WHERE holder = 'garry'`,
|
||||
);
|
||||
const generated = rows[0]?.generated_at;
|
||||
if (!generated) {
|
||||
@@ -1588,24 +1551,6 @@ export async function checkRerankerHealth(engine: BrainEngine): Promise<Check> {
|
||||
};
|
||||
}
|
||||
|
||||
// Historical #2059 rows were logged as `unknown` before missing reranker
|
||||
// auth was classified at the gateway. Surface repeated unknowns instead of
|
||||
// reporting "ok" while every rerank fails open.
|
||||
const unknownFails = failures.filter((f) => f.reason === 'unknown');
|
||||
if (unknownFails.length >= 3) {
|
||||
const setupHint = unknownFails.some((f) => {
|
||||
const summary = String(f.error_summary ?? '');
|
||||
return summary.includes('ZEROENTROPY_API_KEY') || summary.toLowerCase().includes('api key');
|
||||
})
|
||||
? ' Fix: verify ZEROENTROPY_API_KEY and run `gbrain models doctor`.'
|
||||
: '';
|
||||
return {
|
||||
name: 'reranker_health',
|
||||
status: 'warn',
|
||||
message: `${unknownFails.length} unknown reranker failure(s) in last 7 days.${setupHint}`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'reranker_health',
|
||||
status: 'ok',
|
||||
@@ -1640,174 +1585,6 @@ export async function checkRerankerHealth(engine: BrainEngine): Promise<Check> {
|
||||
* Also surfaces (codex M-10): runs resolveBulkRetryOpts(process.env) at
|
||||
* startup so bad GBRAIN_BULK_* config fails at doctor time, not first-retry.
|
||||
*/
|
||||
/**
|
||||
* queue_health: Postgres Minion queue diagnostics.
|
||||
*
|
||||
* Includes the original stalled/depth/memory/prompt checks plus the #2557
|
||||
* no-worker signal: old `embed-backfill` jobs waiting on a queue with no live
|
||||
* registered worker for that queue. That catches the default deployment shape
|
||||
* where `sync` enqueues deferred embedding work but the operator never started
|
||||
* `gbrain jobs work` or a supervisor.
|
||||
*/
|
||||
export async function computeQueueHealthCheck(
|
||||
engine: BrainEngine,
|
||||
opts: {
|
||||
waitingDepthThreshold?: number;
|
||||
oldWaitingHours?: number;
|
||||
readWorkers?: () => Array<{ queue: string }>;
|
||||
} = {},
|
||||
): Promise<Check> {
|
||||
if (engine.kind === 'pglite') {
|
||||
return {
|
||||
name: 'queue_health',
|
||||
status: 'ok',
|
||||
message: 'Skipped (PGLite — no multi-process worker surface)',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
// issue #1801: column is `status`, not `state` (schema.sql:780).
|
||||
const stalledRows: Array<{ id: number; name: string; started_at: string }> =
|
||||
await engine.executeRaw(
|
||||
`SELECT id, name, started_at::text AS started_at
|
||||
FROM minion_jobs
|
||||
WHERE status = 'active'
|
||||
AND started_at IS NOT NULL
|
||||
AND started_at < now() - interval '1 hour'
|
||||
ORDER BY started_at ASC
|
||||
LIMIT 5`,
|
||||
);
|
||||
|
||||
const threshold = opts.waitingDepthThreshold
|
||||
?? _resolveEnvNumber('GBRAIN_QUEUE_WAITING_THRESHOLD', 10);
|
||||
const depthRows: Array<{ name: string; queue: string; depth: number }> =
|
||||
await engine.executeRaw(
|
||||
`SELECT name, queue, count(*)::int AS depth
|
||||
FROM minion_jobs
|
||||
WHERE status = 'waiting'
|
||||
GROUP BY name, queue
|
||||
HAVING count(*) > $1
|
||||
ORDER BY depth DESC
|
||||
LIMIT 5`,
|
||||
[threshold],
|
||||
);
|
||||
|
||||
const rssKillRows: Array<{ cnt: number }> = await engine.executeRaw(
|
||||
`SELECT count(*)::int AS cnt
|
||||
FROM minion_jobs
|
||||
WHERE status IN ('dead', 'failed')
|
||||
AND finished_at > now() - interval '24 hours'
|
||||
AND error_text = 'aborted: watchdog'`,
|
||||
);
|
||||
const rssKillCount = Number(rssKillRows[0]?.cnt ?? 0);
|
||||
|
||||
const promptTooLongRows: Array<{ cnt: number }> = await engine.executeRaw(
|
||||
`SELECT count(*)::int AS cnt
|
||||
FROM minion_jobs
|
||||
WHERE name = 'subagent'
|
||||
AND status = 'dead'
|
||||
AND finished_at > now() - interval '24 hours'
|
||||
AND error_text LIKE 'prompt_too_long:%'`,
|
||||
);
|
||||
const promptTooLongCount = Number(promptTooLongRows[0]?.cnt ?? 0);
|
||||
|
||||
const oldWaitingHours = opts.oldWaitingHours
|
||||
?? _resolveEnvNumber('GBRAIN_QUEUE_NO_WORKER_WARN_HOURS', 1);
|
||||
const oldWaitingRows: Array<{
|
||||
name: string;
|
||||
queue: string;
|
||||
depth: number;
|
||||
oldest_age_seconds: number;
|
||||
}> = await engine.executeRaw(
|
||||
`SELECT name,
|
||||
queue,
|
||||
count(*)::int AS depth,
|
||||
EXTRACT(EPOCH FROM (now() - min(created_at)))::int AS oldest_age_seconds
|
||||
FROM minion_jobs
|
||||
WHERE status = 'waiting'
|
||||
AND name = 'embed-backfill'
|
||||
GROUP BY name, queue
|
||||
HAVING min(created_at) < now() - ($1::text::interval)
|
||||
ORDER BY oldest_age_seconds DESC
|
||||
LIMIT 5`,
|
||||
[`${oldWaitingHours} hours`],
|
||||
);
|
||||
|
||||
let liveWorkerQueues = new Set<string>();
|
||||
if (oldWaitingRows.length > 0) {
|
||||
const workers = opts.readWorkers
|
||||
? opts.readWorkers()
|
||||
: (await import('../core/minions/worker-registry.ts')).readWorkers();
|
||||
liveWorkerQueues = new Set(workers.map((w) => w.queue));
|
||||
}
|
||||
|
||||
const problems: string[] = [];
|
||||
if (stalledRows.length > 0) {
|
||||
const sample = stalledRows
|
||||
.map(r => `#${r.id}(${r.name})`)
|
||||
.join(', ');
|
||||
problems.push(
|
||||
`${stalledRows.length} stalled-forever job(s): ${sample}. ` +
|
||||
`Fix: gbrain jobs get <id> to inspect; gbrain jobs cancel <id> to force-kill.`
|
||||
);
|
||||
}
|
||||
if (depthRows.length > 0) {
|
||||
const sample = depthRows
|
||||
.map(r => `${r.name}@${r.queue}=${r.depth}`)
|
||||
.join(', ');
|
||||
problems.push(
|
||||
`waiting-queue depth exceeds ${threshold} for: ${sample}. ` +
|
||||
`Fix: set maxWaiting on the submitter (or raise GBRAIN_QUEUE_WAITING_THRESHOLD).`
|
||||
);
|
||||
}
|
||||
for (const row of oldWaitingRows) {
|
||||
if (liveWorkerQueues.has(row.queue)) continue;
|
||||
const hours = Math.max(1, Math.round(Number(row.oldest_age_seconds ?? 0) / 3600));
|
||||
problems.push(
|
||||
`${row.depth} ${row.name} job(s) have waited on queue '${row.queue}' for up to ${hours}h ` +
|
||||
`and no live worker is registered for that queue. ` +
|
||||
`Start one with \`gbrain jobs work --queue ${row.queue}\` or ` +
|
||||
`\`gbrain jobs supervisor start --queue ${row.queue}\`.`
|
||||
);
|
||||
}
|
||||
if (rssKillCount > 0) {
|
||||
problems.push(
|
||||
`${rssKillCount} job(s) dead-lettered for RSS-watchdog memory-limit kills in last 24h. ` +
|
||||
`Fix: raise the limit (e.g. \`gbrain jobs work --max-rss 4096\`) or opt out (\`--max-rss 0\`). ` +
|
||||
`→ see worker_oom_loop for the cap + fix (the authoritative OOM-loop signal).`
|
||||
);
|
||||
}
|
||||
if (promptTooLongCount > 0) {
|
||||
problems.push(
|
||||
`${promptTooLongCount} subagent job(s) dead-lettered with prompt_too_long in last 24h. ` +
|
||||
`Dream/synthesize transcripts exceeded the model's input context. ` +
|
||||
`Fix: \`gbrain dream --phase synthesize --dry-run --json\` to identify fat transcripts; ` +
|
||||
`set \`dream.synthesize.max_prompt_tokens\` to bound the per-chunk budget, or use a ` +
|
||||
`larger-context model (Opus 4.7 = 1M tokens vs Sonnet 4.6 = 200K).`
|
||||
);
|
||||
}
|
||||
|
||||
if (problems.length === 0) {
|
||||
return {
|
||||
name: 'queue_health',
|
||||
status: 'ok',
|
||||
message: `No stalled-forever jobs; no queue over depth ${threshold}; no old embed-backfill jobs without a worker.`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
name: 'queue_health',
|
||||
status: 'warn',
|
||||
message: problems.join(' '),
|
||||
};
|
||||
} catch (e) {
|
||||
return {
|
||||
name: 'queue_health',
|
||||
status: 'warn',
|
||||
message: `queue_health scan skipped: ${e instanceof Error ? e.message : String(e)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* issue #1801 — `wedged_queue` check. Surfaces the alive-but-wedged-worker
|
||||
* signature (a queue with claimable work waiting, zero live-lock active jobs,
|
||||
@@ -3183,54 +2960,6 @@ function _resolveSyncFreshnessHours(varName: string, fallback: number): number {
|
||||
* branch (disabled / enabled-no-events / enabled-all-pass / enabled-with-failures)
|
||||
* without spinning up the audit JSONL or a real config file.
|
||||
*/
|
||||
/**
|
||||
* Pure function form of the conversation_parser_probe_health check.
|
||||
* Mirrors computeNightlyQualityProbeHealthCheck: skip-with-hint when the
|
||||
* probe is off and silent, surface the last 7 days of audit events when
|
||||
* it has run, WARN on any non-pass outcome.
|
||||
*
|
||||
* `effectiveEnabled` folds the D10 mode-gate in: explicitly enabled OR
|
||||
* search.mode=tokenmax (where the probe is default-on).
|
||||
*/
|
||||
export function computeConversationParserProbeHealthCheck(
|
||||
effectiveEnabled: boolean,
|
||||
events: ReadonlyArray<{ outcome: string; ts: string; reason?: string }>,
|
||||
): Check {
|
||||
const name = 'conversation_parser_probe_health';
|
||||
if (!effectiveEnabled && events.length === 0) {
|
||||
return {
|
||||
name,
|
||||
status: 'ok',
|
||||
message:
|
||||
'disabled (opt-in; default-on only for search.mode=tokenmax). Enable with: ' +
|
||||
'`gbrain config set autopilot.conversation_parser_probe.enabled true`',
|
||||
};
|
||||
}
|
||||
if (events.length === 0) {
|
||||
return {
|
||||
name,
|
||||
status: 'ok',
|
||||
message: 'enabled but no probe events in the last 7 days (next run by autopilot; fixtures require a source-checkout install).',
|
||||
};
|
||||
}
|
||||
const bad = events.filter(e => e.outcome !== 'pass');
|
||||
const latest = events[events.length - 1]!;
|
||||
if (bad.length > 0) {
|
||||
return {
|
||||
name,
|
||||
status: 'warn',
|
||||
message:
|
||||
`${bad.length}/${events.length} probe run(s) in the last 7 days did not pass; ` +
|
||||
`latest: ${latest.outcome}${latest.reason ? ` (${latest.reason})` : ''}`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
name,
|
||||
status: 'ok',
|
||||
message: `${events.length} probe run(s) in the last 7 days, all pass (latest ${latest.ts}).`,
|
||||
};
|
||||
}
|
||||
|
||||
export function computeNightlyQualityProbeHealthCheck(
|
||||
probeEnabled: boolean,
|
||||
events: ReadonlyArray<{ outcome: string; ts: string; detail?: string }>,
|
||||
@@ -3327,7 +3056,7 @@ export async function computeConversationFactsBacklogCheck(
|
||||
const typesRaw = await engine.getConfig(
|
||||
'cycle.conversation_facts_backfill.types',
|
||||
);
|
||||
let types = ['conversation', 'meeting', 'slack', 'email', 'imessage', 'imessage-daily'];
|
||||
let types = ['conversation', 'meeting', 'slack', 'email'];
|
||||
if (typesRaw) {
|
||||
try {
|
||||
const parsed = JSON.parse(typesRaw);
|
||||
@@ -4616,7 +4345,7 @@ export async function buildChecks(
|
||||
|
||||
// 2. Skill conformance (SKILL group — gated)
|
||||
if (scope === 'all' && skillsDir) {
|
||||
const conformanceResult = skillConformanceCheck(skillsDir);
|
||||
const conformanceResult = checkSkillConformance(skillsDir);
|
||||
checks.push(conformanceResult);
|
||||
}
|
||||
|
||||
@@ -5114,17 +4843,10 @@ export async function buildChecks(
|
||||
try {
|
||||
const { readRecentQualityProbeEvents } = await import('../core/audit-quality-probe.ts');
|
||||
const { loadConfig } = await import('../core/config.ts');
|
||||
const { resolveProbeEnabled } = await import('../core/cycle/nightly-quality-probe.ts');
|
||||
let probeEnabled = false;
|
||||
try {
|
||||
// Dual-plane read, matching the autopilot gate: the DB row (what the
|
||||
// enable hint's `gbrain config set` writes) wins; file plane fallback.
|
||||
let dbVal: string | null = null;
|
||||
try {
|
||||
dbVal = engine ? await engine.getConfig('autopilot.nightly_quality_probe.enabled') : null;
|
||||
} catch { /* DB unavailable → file plane only */ }
|
||||
const cfg = loadConfig();
|
||||
probeEnabled = resolveProbeEnabled(dbVal, (cfg as any)?.autopilot?.nightly_quality_probe?.enabled);
|
||||
probeEnabled = Boolean((cfg as any)?.autopilot?.nightly_quality_probe?.enabled);
|
||||
} catch { /* config unavailable → treat as disabled */ }
|
||||
const events = readRecentQualityProbeEvents(7);
|
||||
const check = computeNightlyQualityProbeHealthCheck(probeEnabled, events);
|
||||
@@ -5205,8 +4927,8 @@ export async function buildChecks(
|
||||
try {
|
||||
const { readConversationBodyForParsing } = await import('../core/conversation-parser/body.ts');
|
||||
const { parseConversation } = await import('../core/conversation-parser/parse.ts');
|
||||
const allowedTypes = ['conversation', 'meeting', 'slack', 'email', 'imessage', 'imessage-daily'] as const;
|
||||
// PageFilters supports singular `type` only; iterate the allowed types
|
||||
const allowedTypes = ['conversation', 'meeting', 'slack', 'email'] as const;
|
||||
// PageFilters supports singular `type` only; iterate the 4 types
|
||||
// and cap at ~50/each to land at ~200 total max.
|
||||
const sample: import('../core/types.ts').Page[] = [];
|
||||
for (const t of allowedTypes) {
|
||||
@@ -5241,7 +4963,8 @@ export async function buildChecks(
|
||||
message:
|
||||
`${unmatched}/${sample.length} conversation pages (${unmatchedPct.toFixed(1)}%) match NO built-in pattern. ` +
|
||||
`Breakdown: ${breakdown}. ` +
|
||||
`Investigate: gbrain conversation-parser scan <slug>`,
|
||||
`Investigate: gbrain conversation-parser scan <slug> | ` +
|
||||
`Enable LLM fallback (opt-in): gbrain config set conversation_parser.llm_fallback_enabled true`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
@@ -5307,29 +5030,19 @@ export async function buildChecks(
|
||||
|
||||
// 3d.5 v0.41.13.0 — conversation_parser_probe_health. Mode-gated
|
||||
// per D10: ON when search.mode=tokenmax, opt-in for other modes.
|
||||
// Surfaces the last 7 days of nightly-probe audit events; warn on any
|
||||
// non-pass outcome (fail / budget_exceeded / adversarial_false_positive).
|
||||
// (Until the autopilot wire-up this was a hardcoded "Skipped" stub.)
|
||||
try {
|
||||
const { readRecentParserProbeEvents } = await import('../core/audit-parser-probe.ts');
|
||||
let parserProbeEnabled = false;
|
||||
try {
|
||||
let dbVal: string | null = null;
|
||||
let dbMode: string | null = null;
|
||||
try {
|
||||
dbVal = engine ? await engine.getConfig('autopilot.conversation_parser_probe.enabled') : null;
|
||||
dbMode = engine ? await engine.getConfig('search.mode') : null;
|
||||
} catch { /* DB unavailable → file plane only */ }
|
||||
const { loadConfig } = await import('../core/config.ts');
|
||||
const fileVal = (loadConfig() as any)?.autopilot?.conversation_parser_probe?.enabled;
|
||||
const flagOn = dbVal != null ? dbVal === 'true' : fileVal === true;
|
||||
parserProbeEnabled = flagOn || dbMode === 'tokenmax';
|
||||
} catch { /* config unavailable → treat as disabled */ }
|
||||
const parserEvents = readRecentParserProbeEvents(7);
|
||||
checks.push(computeConversationParserProbeHealthCheck(parserProbeEnabled, parserEvents));
|
||||
} catch {
|
||||
// Best-effort; audit-log read failure shouldn't stop doctor.
|
||||
}
|
||||
// Surface the last 7 days of nightly-probe events; warn on FAIL /
|
||||
// BUDGET_EXCEEDED / adversarial_false_positive.
|
||||
//
|
||||
// v0.41.13.0 ships the probe as opt-in (autopilot wiring deferred
|
||||
// to T7 in the cathedral plan); this check skips with an enable
|
||||
// hint until the probe has at least one audit event written.
|
||||
checks.push({
|
||||
name: 'conversation_parser_probe_health',
|
||||
status: 'ok',
|
||||
message:
|
||||
'Skipped (nightly probe is opt-in; enable with ' +
|
||||
'`gbrain config set autopilot.conversation_parser_probe.enabled true`)',
|
||||
});
|
||||
|
||||
// 3e. home_dir_in_worktree (v0.35.8.0). Walks up from `gbrainPath()`
|
||||
// looking for a `.git` directory OR file. If found, warns: `~/.gbrain/`
|
||||
@@ -6155,12 +5868,12 @@ export async function buildChecks(
|
||||
message: `Only code/test fixture entity pages found (${entityCount}); graph_coverage not applicable`,
|
||||
});
|
||||
} else if (linkCoverage >= 0.5 && timelineCoverage >= 0.5) {
|
||||
checks.push({ name: 'graph_coverage', status: 'ok', message: `Entity link coverage ${linkPct}%, entity timeline coverage ${timelinePct}%` });
|
||||
checks.push({ name: 'graph_coverage', status: 'ok', message: `Entity link coverage ${linkPct}%, timeline ${timelinePct}%` });
|
||||
} else {
|
||||
checks.push({
|
||||
name: 'graph_coverage',
|
||||
status: 'warn',
|
||||
message: `Entity link coverage ${linkPct}%, entity timeline coverage ${timelinePct}% (${eligibleEntityCount} entity pages). Run: gbrain extract all`,
|
||||
message: `Entity link coverage ${linkPct}%, timeline ${timelinePct}% (${eligibleEntityCount} entity pages). Run: gbrain extract all`,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -6172,7 +5885,7 @@ export async function buildChecks(
|
||||
const parts = [
|
||||
`embed ${health.embed_coverage_score}/35`,
|
||||
`links ${health.link_density_score}/25`,
|
||||
`timeline density (all pages) ${health.timeline_coverage_score}/15`,
|
||||
`timeline ${health.timeline_coverage_score}/15`,
|
||||
`orphans ${health.no_orphans_score}/15`,
|
||||
`dead-links ${health.no_dead_links_score}/10`,
|
||||
];
|
||||
@@ -6345,12 +6058,6 @@ export async function buildChecks(
|
||||
progress.heartbeat('child_table_orphans');
|
||||
checks.push(await childTableOrphansCheck(engine));
|
||||
|
||||
// 10d. Raw-source persistence guarantee (#1978, warn-only v1).
|
||||
// Every synthesized/derived page must carry a raw trace or an explicit
|
||||
// exemption. Warn-only in v1 — surfaces violations, blocks nothing.
|
||||
progress.heartbeat('raw_provenance');
|
||||
checks.push(await rawProvenanceCheck(engine));
|
||||
|
||||
// v0.33: whoknows_health — fixture presence + row count. The eval
|
||||
// gate itself runs via `gbrain eval whoknows`; this check is the
|
||||
// "did you do the assignment?" signal.
|
||||
@@ -7209,12 +6916,159 @@ export async function buildChecks(
|
||||
}
|
||||
}
|
||||
|
||||
// 11b. Queue health (v0.19.1 queue-resilience wave).
|
||||
// Postgres-only because PGLite has no multi-process worker surface. Two
|
||||
// subchecks, both cheap (single SELECT each, status-index-covered):
|
||||
//
|
||||
// 1. stalled-forever: any active job whose started_at is > 1h old. The
|
||||
// incident that motivated this release ran 90+ min before surfacing.
|
||||
// Surface the ID so the operator can `gbrain jobs get <id>` to inspect
|
||||
// or `gbrain jobs cancel <id>` to force-kill.
|
||||
//
|
||||
// 2. backpressure-missed: per-name waiting depth exceeds the threshold
|
||||
// (default 10, override via GBRAIN_QUEUE_WAITING_THRESHOLD env). Signal
|
||||
// that a submitter probably needs maxWaiting set. Bounded by per-name
|
||||
// aggregation so a single name's pile shows up clearly instead of
|
||||
// getting lost in the total.
|
||||
//
|
||||
// Not included in v0.19.1 (tracked as B7 follow-up): worker-heartbeat
|
||||
// staleness. It needs a minion_workers table; the lock_until-on-active-jobs
|
||||
// proxy can't distinguish "no worker" from "worker idle," and a check that
|
||||
// cries wolf erodes trust in every other doctor check.
|
||||
progress.heartbeat('queue_health');
|
||||
const queueHealthHb = startHeartbeat(progress, 'scanning queue health…');
|
||||
try {
|
||||
checks.push(await computeQueueHealthCheck(engine));
|
||||
} finally {
|
||||
queueHealthHb();
|
||||
if (engine.kind === 'pglite') {
|
||||
checks.push({
|
||||
name: 'queue_health',
|
||||
status: 'ok',
|
||||
message: 'Skipped (PGLite — no multi-process worker surface)',
|
||||
});
|
||||
} else {
|
||||
const queueHealthHb = startHeartbeat(progress, 'scanning queue health…');
|
||||
try {
|
||||
const sql = db.getConnection();
|
||||
// Subcheck 1: stalled-forever active jobs (>1h wall-clock).
|
||||
const stalledRows: Array<{ id: number; name: string; started_at: string }> = await sql`
|
||||
SELECT id, name, started_at::text AS started_at
|
||||
FROM minion_jobs
|
||||
WHERE status = 'active'
|
||||
AND started_at IS NOT NULL
|
||||
AND started_at < now() - interval '1 hour'
|
||||
ORDER BY started_at ASC
|
||||
LIMIT 5
|
||||
`;
|
||||
// Subcheck 2: per-name waiting depth exceeds threshold.
|
||||
const rawThreshold = process.env.GBRAIN_QUEUE_WAITING_THRESHOLD;
|
||||
const parsedThreshold = rawThreshold ? parseInt(rawThreshold, 10) : 10;
|
||||
const threshold = Number.isFinite(parsedThreshold) && parsedThreshold >= 1
|
||||
? parsedThreshold
|
||||
: 10;
|
||||
const depthRows: Array<{ name: string; queue: string; depth: number }> = await sql`
|
||||
SELECT name, queue, count(*)::int AS depth
|
||||
FROM minion_jobs
|
||||
WHERE status = 'waiting'
|
||||
GROUP BY name, queue
|
||||
HAVING count(*) > ${threshold}
|
||||
ORDER BY depth DESC
|
||||
LIMIT 5
|
||||
`;
|
||||
// Subcheck 3 (v0.22.14): RSS-watchdog kills in the last 24h. Bare workers
|
||||
// newly default to --max-rss 2048 (was 0); operators who run large embed
|
||||
// or import jobs may see kills that didn't happen pre-v0.22.14. We surface
|
||||
// a hint when this signature appears so the upgrade path is obvious.
|
||||
// Signature: when the watchdog trips, gracefulShutdown('watchdog') aborts
|
||||
// in-flight jobs with `new Error('watchdog')`. The worker's failJob path
|
||||
// (worker.ts:660-664) writes `error_text = 'aborted: watchdog'` for any
|
||||
// job in-flight at the moment of the kill.
|
||||
//
|
||||
// We deliberately DO NOT do a loose `ILIKE '%watchdog%'`:
|
||||
// 1. Parent jobs that inherit `on_child_fail='fail_parent'` get
|
||||
// `"child job N failed: aborted: watchdog"` — counting that
|
||||
// double-counts (child + parent) for one watchdog event.
|
||||
// 2. Any user error_text containing the word "watchdog" matches.
|
||||
// Match the exact prefix `'aborted: watchdog'` to scope this purely to
|
||||
// the worker's own kill signature.
|
||||
const rssKillRows: Array<{ cnt: number }> = await sql`
|
||||
SELECT count(*)::int AS cnt
|
||||
FROM minion_jobs
|
||||
WHERE status IN ('dead', 'failed')
|
||||
AND finished_at > now() - interval '24 hours'
|
||||
AND error_text = 'aborted: watchdog'
|
||||
`;
|
||||
const rssKillCount = rssKillRows[0]?.cnt ?? 0;
|
||||
|
||||
// Subcheck 4 (v0.30.2): prompt_too_long terminal failures on subagent
|
||||
// jobs in the last 24h. The dream/synthesize phase classifies Anthropic
|
||||
// 400 "prompt is too long" responses as UnrecoverableError so they
|
||||
// dead-letter on first attempt instead of clogging the queue with
|
||||
// max_stalled retries. Surface count + fix hint when present.
|
||||
const promptTooLongRows: Array<{ cnt: number }> = await sql`
|
||||
SELECT count(*)::int AS cnt
|
||||
FROM minion_jobs
|
||||
WHERE name = 'subagent'
|
||||
AND status = 'dead'
|
||||
AND finished_at > now() - interval '24 hours'
|
||||
AND error_text LIKE 'prompt_too_long:%'
|
||||
`;
|
||||
const promptTooLongCount = promptTooLongRows[0]?.cnt ?? 0;
|
||||
|
||||
const problems: string[] = [];
|
||||
if (stalledRows.length > 0) {
|
||||
const sample = stalledRows
|
||||
.map(r => `#${r.id}(${r.name})`)
|
||||
.join(', ');
|
||||
problems.push(
|
||||
`${stalledRows.length} stalled-forever job(s): ${sample}. ` +
|
||||
`Fix: gbrain jobs get <id> to inspect; gbrain jobs cancel <id> to force-kill.`
|
||||
);
|
||||
}
|
||||
if (depthRows.length > 0) {
|
||||
const sample = depthRows
|
||||
.map(r => `${r.name}@${r.queue}=${r.depth}`)
|
||||
.join(', ');
|
||||
problems.push(
|
||||
`waiting-queue depth exceeds ${threshold} for: ${sample}. ` +
|
||||
`Fix: set maxWaiting on the submitter (or raise GBRAIN_QUEUE_WAITING_THRESHOLD).`
|
||||
);
|
||||
}
|
||||
if (rssKillCount > 0) {
|
||||
problems.push(
|
||||
`${rssKillCount} job(s) dead-lettered for RSS-watchdog memory-limit kills in last 24h. ` +
|
||||
`Fix: raise the limit (e.g. \`gbrain jobs work --max-rss 4096\`) or opt out (\`--max-rss 0\`). ` +
|
||||
`→ see worker_oom_loop for the cap + fix (the authoritative OOM-loop signal).`
|
||||
);
|
||||
}
|
||||
if (promptTooLongCount > 0) {
|
||||
problems.push(
|
||||
`${promptTooLongCount} subagent job(s) dead-lettered with prompt_too_long in last 24h. ` +
|
||||
`Dream/synthesize transcripts exceeded the model's input context. ` +
|
||||
`Fix: \`gbrain dream --phase synthesize --dry-run --json\` to identify fat transcripts; ` +
|
||||
`set \`dream.synthesize.max_prompt_tokens\` to bound the per-chunk budget, or use a ` +
|
||||
`larger-context model (Opus 4.7 = 1M tokens vs Sonnet 4.6 = 200K).`
|
||||
);
|
||||
}
|
||||
|
||||
if (problems.length === 0) {
|
||||
checks.push({
|
||||
name: 'queue_health',
|
||||
status: 'ok',
|
||||
message: `No stalled-forever jobs; no queue over depth ${threshold}.`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: 'queue_health',
|
||||
status: 'warn',
|
||||
message: problems.join(' '),
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
checks.push({
|
||||
name: 'queue_health',
|
||||
status: 'warn',
|
||||
message: `queue_health scan skipped: ${e instanceof Error ? e.message : String(e)}`,
|
||||
});
|
||||
} finally {
|
||||
queueHealthHb();
|
||||
}
|
||||
}
|
||||
|
||||
// 11.4 subagent_capability (v0.38 — D7; was subagent_provider in v0.31.12). Surfaces a
|
||||
@@ -7324,17 +7178,9 @@ export async function buildChecks(
|
||||
let vanished = 0;
|
||||
const vanishedPaths: string[] = [];
|
||||
const fs = await import('node:fs');
|
||||
const nodePath = await import('node:path');
|
||||
// storage_path is repo-relative for sync-ingested assets. Resolving
|
||||
// against cwd made this check a false-positive WARN whenever doctor
|
||||
// ran outside the brain repo.
|
||||
const repoRoot = (await engine.getConfig('sync.repo_path')) ?? process.cwd();
|
||||
for (const r of rows) {
|
||||
const abs = nodePath.isAbsolute(r.storage_path)
|
||||
? r.storage_path
|
||||
: nodePath.join(repoRoot, r.storage_path);
|
||||
try {
|
||||
fs.statSync(abs);
|
||||
fs.statSync(r.storage_path);
|
||||
} catch {
|
||||
vanished++;
|
||||
if (vanishedPaths.length < 5) vanishedPaths.push(r.storage_path);
|
||||
@@ -7578,13 +7424,15 @@ function printAutoFixReport(report: AutoFixReport, dryRun: boolean, jsonOutput:
|
||||
|
||||
|
||||
/** Quick skill conformance check — frontmatter + required sections */
|
||||
export function skillConformanceCheck(skillsDir: string): Check {
|
||||
function checkSkillConformance(skillsDir: string): Check {
|
||||
const manifestPath = join(skillsDir, 'manifest.json');
|
||||
if (!existsSync(manifestPath)) {
|
||||
return { name: 'skill_conformance', status: 'warn', message: 'manifest.json not found' };
|
||||
}
|
||||
|
||||
try {
|
||||
// Host workspaces are allowed to omit a gbrain-specific manifest. Keep
|
||||
// conformance aligned with resolver_health and skill_brain_first by using
|
||||
// the canonical fallback that derives entries from direct SKILL.md files.
|
||||
const manifest = loadOrDeriveManifest(skillsDir);
|
||||
const skills = manifest.skills;
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8'));
|
||||
const skills = manifest.skills || [];
|
||||
let passing = 0;
|
||||
const failing: string[] = [];
|
||||
|
||||
@@ -7604,8 +7452,7 @@ export function skillConformanceCheck(skillsDir: string): Check {
|
||||
}
|
||||
|
||||
if (failing.length === 0) {
|
||||
const derivedNote = manifest.derived ? ' (derived from SKILL.md files)' : '';
|
||||
return { name: 'skill_conformance', status: 'ok', message: `${passing}/${skills.length} skills pass${derivedNote}` };
|
||||
return { name: 'skill_conformance', status: 'ok', message: `${passing}/${skills.length} skills pass` };
|
||||
}
|
||||
return {
|
||||
name: 'skill_conformance',
|
||||
@@ -7613,7 +7460,7 @@ export function skillConformanceCheck(skillsDir: string): Check {
|
||||
message: `${passing}/${skills.length} pass. Failing: ${failing.join(', ')}`,
|
||||
};
|
||||
} catch {
|
||||
return { name: 'skill_conformance', status: 'warn', message: 'Could not load or derive skills manifest' };
|
||||
return { name: 'skill_conformance', status: 'warn', message: 'Could not parse manifest.json' };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7924,71 +7771,27 @@ export async function runRemediationPlan(
|
||||
return;
|
||||
}
|
||||
|
||||
for (const line of renderRemediationPlanLines(plan, targetScore)) {
|
||||
console.log(line);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Human-render the remediation plan into a sequence of console lines.
|
||||
* Exported for unit-test access — `runRemediationPlan` consumes it
|
||||
* verbatim and only adds the JSON-mode short-circuit.
|
||||
*
|
||||
* Gating the "at target" line on `brain_score_current >= targetScore`
|
||||
* is load-bearing: when the plan is empty AND the target is unreachable,
|
||||
* the prior shape printed both "Target unreachable: …" and "Brain is at
|
||||
* target" back-to-back, which contradicted itself and hid the real next
|
||||
* step (manual prereq config to lift `max_reachable_score`).
|
||||
*/
|
||||
export function renderRemediationPlanLines(
|
||||
plan: RemediationPlanShape,
|
||||
targetScore: number,
|
||||
): string[] {
|
||||
const lines: string[] = [];
|
||||
lines.push(`Brain score: ${plan.brain_score_current}/100 → target ${targetScore}`);
|
||||
// Human output
|
||||
console.log(`Brain score: ${plan.brain_score_current}/100 → target ${targetScore}`);
|
||||
if (plan.target_unreachable) {
|
||||
lines.push(`Target unreachable: max with autonomous remediation is ${plan.max_reachable_score}/100.`);
|
||||
console.log(`Target unreachable: max with autonomous remediation is ${plan.max_reachable_score}/100.`);
|
||||
}
|
||||
if (plan.plan.length === 0) {
|
||||
if (plan.brain_score_current >= targetScore) {
|
||||
lines.push('No remediations needed. Brain is at target.');
|
||||
}
|
||||
// When brain_score < targetScore and plan is empty, the unreachable
|
||||
// line (if applicable) is the user-facing explanation; the blocked-
|
||||
// checks block below surfaces the manual gap. Don't follow with a
|
||||
// misleading "at target" claim.
|
||||
console.log('No remediations needed. Brain is at target.');
|
||||
} else {
|
||||
lines.push(`Plan: ${plan.plan.length} step(s), est ${plan.est_total_seconds}s, est $${plan.est_total_usd_cost.toFixed(2)}`);
|
||||
console.log(`Plan: ${plan.plan.length} step(s), est ${plan.est_total_seconds}s, est $${plan.est_total_usd_cost.toFixed(2)}`);
|
||||
for (const step of plan.plan) {
|
||||
const protectedMark = step.protected ? ' [PROTECTED]' : '';
|
||||
const costMark = step.est_usd_cost ? ` ($${step.est_usd_cost.toFixed(2)})` : '';
|
||||
lines.push(` ${step.step}. [${step.severity}] ${step.job}${protectedMark} — ${step.rationale}${costMark}`);
|
||||
console.log(` ${step.step}. [${step.severity}] ${step.job}${protectedMark} — ${step.rationale}${costMark}`);
|
||||
}
|
||||
}
|
||||
if (plan.blocked.length > 0) {
|
||||
lines.push(`\nBlocked checks (prereq missing):`);
|
||||
console.log(`\nBlocked checks (prereq missing):`);
|
||||
for (const b of plan.blocked) {
|
||||
lines.push(` - ${b.check}: ${b.reason}`);
|
||||
console.log(` - ${b.check}: ${b.reason}`);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
interface RemediationPlanShape {
|
||||
brain_score_current: number;
|
||||
target_unreachable: boolean;
|
||||
max_reachable_score: number;
|
||||
plan: Array<{
|
||||
step: number;
|
||||
severity: string;
|
||||
job: string;
|
||||
protected?: boolean;
|
||||
est_usd_cost?: number;
|
||||
rationale: string;
|
||||
}>;
|
||||
est_total_seconds: number;
|
||||
est_total_usd_cost: number;
|
||||
blocked: Array<{ check: string; reason: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+3
-88
@@ -26,7 +26,6 @@
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import {
|
||||
runCycle,
|
||||
resolveSourceForDir,
|
||||
ALL_PHASES,
|
||||
type CyclePhase,
|
||||
type CycleReport,
|
||||
@@ -77,18 +76,6 @@ interface DreamArgs {
|
||||
drain: boolean;
|
||||
/** Drain wallclock budget in seconds. Default 300 (5 min). */
|
||||
windowSeconds: number;
|
||||
/**
|
||||
* issue #2860 — `--once`. One-shot bypass of the named `--phase`'s own
|
||||
* `dream.<phase>.enabled` / `cycle.<phase>.enabled` config gate, for this
|
||||
* invocation only. Never reads or writes config — unlike the old
|
||||
* "toggle enabled true, run, toggle back to false" workaround, a crash
|
||||
* mid-run can't leave any global state stuck. Requires an explicit
|
||||
* `--phase <name>`; bare `--once` is a usage error (there'd be no single
|
||||
* phase to target). Applies only to phases with a config `.enabled` gate
|
||||
* (patterns, synthesize, conversation_facts_backfill, enrich_thin,
|
||||
* skillopt, drift) — a no-op for phases that always run when named directly.
|
||||
*/
|
||||
once: boolean;
|
||||
}
|
||||
|
||||
const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
|
||||
@@ -118,14 +105,6 @@ function collectFlagValues(args: string[], flag: string): string[] | null {
|
||||
|
||||
function parseArgs(args: string[]): DreamArgs {
|
||||
const phaseIdx = args.indexOf('--phase');
|
||||
// issue #2860 (Codex P3): captured BEFORE --input/--drain get a chance to
|
||||
// implicitly default `phase` below, so --once's validation can require
|
||||
// the user actually TYPED --phase, not merely that some phase ended up
|
||||
// resolved. Without this, `--input <f> --once` and `--drain --once`
|
||||
// slip past the "explicit --phase required" contract (the derived
|
||||
// `phase` value is already non-null by the time that check runs) and
|
||||
// --once becomes silently ineffective for both.
|
||||
const phaseWasExplicit = phaseIdx !== -1;
|
||||
const rawPhase = phaseIdx !== -1 ? args[phaseIdx + 1] : null;
|
||||
let phase = rawPhase && (ALL_PHASES as string[]).includes(rawPhase)
|
||||
? (rawPhase as CyclePhase)
|
||||
@@ -235,35 +214,6 @@ function parseArgs(args: string[]): DreamArgs {
|
||||
}
|
||||
}
|
||||
|
||||
// issue #2860: --once requires an EXPLICIT single --phase target (typed
|
||||
// by the user, not merely implied by --input/--drain — see
|
||||
// `phaseWasExplicit` above). Bare `--once` (full/default cycle) has no
|
||||
// single phase to bypass the gate for, and force-enabling EVERY
|
||||
// currently-disabled phase at once would be exactly the kind of
|
||||
// surprise-spend risk the flag exists to prevent. An implicit phase
|
||||
// (from --input or --drain) is rejected too: --drain returns before
|
||||
// onceForPhase is ever read, and --input already bypasses the
|
||||
// synthesize gate on its own, so --once would silently do nothing in
|
||||
// either case — reject loudly instead of pretending it worked (Codex
|
||||
// review finding).
|
||||
//
|
||||
// Codex review finding: `--help` must short-circuit BEFORE this exits(2),
|
||||
// matching the "IRON RULE" pinned by test/dream.test.ts's
|
||||
// "--help --source whatever prints help and exits 0" case — `gbrain
|
||||
// dream --help --once` (no --phase) must show help, not a usage error.
|
||||
const once = args.includes('--once');
|
||||
const wantsHelp = args.includes('--help') || args.includes('-h');
|
||||
if (once && !phaseWasExplicit && !wantsHelp) {
|
||||
console.error(
|
||||
'--once requires an explicit --phase <name> (bypasses that one ' +
|
||||
'phase\'s dream.<phase>.enabled / cycle.<phase>.enabled gate for ' +
|
||||
'this run only; never touches config). A phase implied by --input ' +
|
||||
'or --drain does not count — --once would silently do nothing for ' +
|
||||
'those. Usage: gbrain dream --phase <name> --once',
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
return {
|
||||
json: args.includes('--json'),
|
||||
dryRun: args.includes('--dry-run'),
|
||||
@@ -279,7 +229,6 @@ function parseArgs(args: string[]): DreamArgs {
|
||||
source,
|
||||
drain,
|
||||
windowSeconds,
|
||||
once,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -361,17 +310,6 @@ Options:
|
||||
"--dry-run" does NOT mean "zero LLM calls."
|
||||
--json Emit the CycleReport as JSON (agent-readable)
|
||||
--phase <name> Run a single phase: ${ALL_PHASES.join(' | ')}
|
||||
--once With --phase <name>: run that phase once even if its
|
||||
own dream.<phase>.enabled / cycle.<phase>.enabled
|
||||
config gate is false. Never reads or writes config —
|
||||
unlike toggling the flag on/off around the run, a
|
||||
crash mid-invocation can't leave it stuck. Applies to
|
||||
patterns, synthesize, conversation_facts_backfill,
|
||||
enrich_thin, skillopt, drift; no-op on phases with no such
|
||||
gate. Requires an EXPLICIT --phase <name> — a phase
|
||||
implied by --input or --drain does not count (bare
|
||||
--once, or --once with --input/--drain and no
|
||||
explicit --phase, is a usage error).
|
||||
--pull git pull the brain repo before syncing (default: no pull)
|
||||
--dir <path> Brain directory (default: configured brain). On a
|
||||
postgres/remote brain with no local checkout, the
|
||||
@@ -381,9 +319,9 @@ Options:
|
||||
|
||||
--source <id> Scope the cycle to one source so doctor's
|
||||
cycle_freshness check sees a fresh stamp on
|
||||
completion. When omitted, gbrain derives the
|
||||
source from --dir / the configured checkout
|
||||
when it matches a source's local_path (#1869).
|
||||
completion. Without this, gbrain dream's
|
||||
timestamp never lands and federated brains
|
||||
see "stale cycle" forever.
|
||||
--source-id <id> Alias for --source. Matches the v0.37.7.0+
|
||||
naming used by import/extract/graph-query.
|
||||
|
||||
@@ -415,7 +353,6 @@ Examples:
|
||||
gbrain dream
|
||||
gbrain dream --dry-run --json
|
||||
gbrain dream --phase lint
|
||||
gbrain dream --phase patterns --once # run once, ignore dream.patterns.enabled=false
|
||||
gbrain dream --phase synthesize --input ~/transcripts/2026-04-25.txt
|
||||
gbrain dream --phase synthesize --from 2026-04-01 --to 2026-04-25
|
||||
0 2 * * * gbrain dream --json # nightly via cron
|
||||
@@ -635,25 +572,6 @@ export async function runDream(engine: BrainEngine | null, args: string[]): Prom
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// #1869: a path-scoped run (--dir, or the configured sync.repo_path) whose
|
||||
// directory matches a registered source's local_path IS that source's cycle
|
||||
// — derive the source id so runCycle writes last_source_cycle_at /
|
||||
// last_full_cycle_at on success and doctor's cycle_freshness check stops
|
||||
// reading perpetually stale. Explicit --source still wins (resolved above).
|
||||
// Fixed here at the command level, NOT in runCycle's stamp gate, so legacy
|
||||
// global callers (autopilot-global-maintenance runs GLOBAL_PHASES with a
|
||||
// brainDir and no sourceId) can't falsely stamp per-source freshness.
|
||||
// A derived match on an archived source is skipped silently (falls back to
|
||||
// legacy unscoped behavior) — stamping it would mask staleness on restore,
|
||||
// mirroring the explicit --source archived guard above.
|
||||
if (resolvedSourceId === undefined && engine !== null && brainDir !== null) {
|
||||
const derived = await resolveSourceForDir(engine, brainDir);
|
||||
if (derived !== undefined) {
|
||||
const src = await fetchSource(engine, derived);
|
||||
if (src?.archived !== true) resolvedSourceId = derived;
|
||||
}
|
||||
}
|
||||
// ─── issue #1678: bounded single-hold extract_atoms drain ──────────
|
||||
if (opts.drain) {
|
||||
if (engine === null) {
|
||||
@@ -676,9 +594,6 @@ export async function runDream(engine: BrainEngine | null, args: string[]): Prom
|
||||
synthFrom: opts.from ?? undefined,
|
||||
synthTo: opts.to ?? undefined,
|
||||
synthBypassDreamGuard: opts.bypassDreamGuard,
|
||||
// issue #2860: opts.phase is guaranteed non-null here when opts.once is
|
||||
// set (parseArgs enforces --once requires --phase).
|
||||
onceForPhase: opts.once ? opts.phase! : undefined,
|
||||
});
|
||||
|
||||
if (opts.json) {
|
||||
|
||||
+19
-67
@@ -107,14 +107,6 @@ export interface EmbedOpts {
|
||||
* runs lock every source in sorted order. dryRun skips it.
|
||||
*/
|
||||
singleFlight?: boolean;
|
||||
/**
|
||||
* #394: suppress human stdout summaries (the `[dry-run] Would embed ...` /
|
||||
* `Embedded N chunks ...` slog lines). Set by structured-output callers —
|
||||
* the cycle's embed phase (dream --json must keep stdout JSON-clean per
|
||||
* docs/progress-events.md) reports counts via its own PhaseResult instead.
|
||||
* Errors/warnings still go to stderr regardless.
|
||||
*/
|
||||
quiet?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -261,7 +253,7 @@ export async function runEmbedCore(engine: BrainEngine, opts: EmbedOpts): Promis
|
||||
for (const s of opts.slugs) {
|
||||
if (isAborted(opts.signal)) break; // #1737: stop the per-slug loop on abort
|
||||
try {
|
||||
await embedPage(engine, s, !!opts.dryRun, result, opts.sourceId, opts.signal, opts.quiet);
|
||||
await embedPage(engine, s, !!opts.dryRun, result, opts.sourceId, opts.signal);
|
||||
} catch (e: unknown) {
|
||||
serr(` Error embedding ${s}: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
@@ -355,7 +347,6 @@ export async function runEmbedCore(engine: BrainEngine, opts: EmbedOpts): Promis
|
||||
catchUp: opts.catchUp,
|
||||
pacer,
|
||||
paceMaxConcurrency,
|
||||
quiet: opts.quiet,
|
||||
}, opts.signal);
|
||||
} finally {
|
||||
// E1: surface pacing telemetry (human + structured) when pacing was on.
|
||||
@@ -385,7 +376,7 @@ export async function runEmbedCore(engine: BrainEngine, opts: EmbedOpts): Promis
|
||||
return result;
|
||||
}
|
||||
if (opts.slug) {
|
||||
await embedPage(engine, opts.slug, !!opts.dryRun, result, opts.sourceId, opts.signal, opts.quiet);
|
||||
await embedPage(engine, opts.slug, !!opts.dryRun, result, opts.sourceId, opts.signal);
|
||||
return result;
|
||||
}
|
||||
throw new Error('No embed target specified. Pass { slug }, { slugs }, { all }, or { stale }.');
|
||||
@@ -530,7 +521,6 @@ async function embedPage(
|
||||
result: EmbedResult,
|
||||
sourceId?: string,
|
||||
signal?: AbortSignal,
|
||||
quiet?: boolean,
|
||||
) {
|
||||
const opts = sourceId ? { sourceId } : undefined;
|
||||
const page = await engine.getPage(slug, opts);
|
||||
@@ -575,7 +565,7 @@ async function embedPage(
|
||||
result.skipped += chunks.length - toEmbed.length;
|
||||
|
||||
if (toEmbed.length === 0) {
|
||||
if (!quiet) slog(`${slug}: all ${chunks.length} chunks already embedded`);
|
||||
slog(`${slug}: all ${chunks.length} chunks already embedded`);
|
||||
result.pages_processed++;
|
||||
return;
|
||||
}
|
||||
@@ -591,7 +581,7 @@ async function embedPage(
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
embeddingMap.set(toEmbed[j].chunk_index, embeddings[j]);
|
||||
}
|
||||
const updated: ChunkInput[] = chunks.map(c => preserveCodeMetadata(c, {
|
||||
const updated: ChunkInput[] = chunks.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
@@ -612,32 +602,7 @@ async function embedPage(
|
||||
}
|
||||
result.embedded += toEmbed.length;
|
||||
result.pages_processed++;
|
||||
if (!quiet) slog(`${slug}: embedded ${toEmbed.length} chunks`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Carry code-chunk metadata (language, symbol_name, symbol_type, line range,
|
||||
* parent scope, doc comment, qualified name) from a loaded Chunk back into a
|
||||
* ChunkInput destined for upsertChunks.
|
||||
*
|
||||
* Issue #769: every re-embed used to strip these fields, and upsertChunks
|
||||
* overwrites (does not COALESCE) the metadata columns from EXCLUDED, so
|
||||
* each pass clobbered code-def's primary index to NULL. Pulling the
|
||||
* preservation into one helper keeps the three re-embed call sites
|
||||
* (embedPage, embedAll non-stale, embedAllStale) in lock-step.
|
||||
*/
|
||||
function preserveCodeMetadata(loaded: any, base: ChunkInput): ChunkInput {
|
||||
return {
|
||||
...base,
|
||||
language: loaded.language ?? undefined,
|
||||
symbol_name: loaded.symbol_name ?? undefined,
|
||||
symbol_type: loaded.symbol_type ?? undefined,
|
||||
start_line: loaded.start_line ?? undefined,
|
||||
end_line: loaded.end_line ?? undefined,
|
||||
parent_symbol_path: loaded.parent_symbol_path ?? undefined,
|
||||
doc_comment: loaded.doc_comment ?? undefined,
|
||||
symbol_name_qualified: loaded.symbol_name_qualified ?? undefined,
|
||||
};
|
||||
slog(`${slug}: embedded ${toEmbed.length} chunks`);
|
||||
}
|
||||
|
||||
async function embedAll(
|
||||
@@ -655,8 +620,6 @@ async function embedAll(
|
||||
pacer?: DbPacer;
|
||||
/** Resolved concurrency cap (E-1: the worker count, no separate permit). */
|
||||
paceMaxConcurrency?: number;
|
||||
/** #394: suppress human stdout summaries (structured-output callers). */
|
||||
quiet?: boolean;
|
||||
},
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
@@ -754,10 +717,8 @@ async function embedAll(
|
||||
for (let j = 0; j < toEmbed.length; j++) {
|
||||
embeddingMap.set(toEmbed[j].chunk_index, embeddings[j]);
|
||||
}
|
||||
// Preserve ALL chunks, only update embeddings for stale ones.
|
||||
// preserveCodeMetadata threads code-chunk metadata (#769) so re-embed
|
||||
// doesn't clobber language/symbol_name/symbol_type to NULL.
|
||||
const updated: ChunkInput[] = chunks.map(c => preserveCodeMetadata(c, {
|
||||
// Preserve ALL chunks, only update embeddings for stale ones
|
||||
const updated: ChunkInput[] = chunks.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
@@ -802,12 +763,10 @@ async function embedAll(
|
||||
});
|
||||
|
||||
// Stdout summary preserved for scripts/tests that grep for counts.
|
||||
if (!staleOpts?.quiet) {
|
||||
if (dryRun) {
|
||||
slog(`[dry-run] Would embed ${result.would_embed} chunks across ${pages.length} pages`);
|
||||
} else {
|
||||
slog(`Embedded ${result.embedded} chunks across ${pages.length} pages`);
|
||||
}
|
||||
if (dryRun) {
|
||||
slog(`[dry-run] Would embed ${result.would_embed} chunks across ${pages.length} pages`);
|
||||
} else {
|
||||
slog(`Embedded ${result.embedded} chunks across ${pages.length} pages`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -843,8 +802,6 @@ async function embedAllStale(
|
||||
pacer?: DbPacer;
|
||||
/** Resolved concurrency cap (E-1: the worker count, no separate permit). */
|
||||
paceMaxConcurrency?: number;
|
||||
/** #394: suppress human stdout summaries (structured-output callers). */
|
||||
quiet?: boolean;
|
||||
},
|
||||
signature?: string,
|
||||
externalSignal?: AbortSignal,
|
||||
@@ -862,7 +819,7 @@ async function embedAllStale(
|
||||
signature,
|
||||
...(sourceId && { sourceId }),
|
||||
});
|
||||
if (invalidated > 0 && !staleOpts?.quiet) {
|
||||
if (invalidated > 0) {
|
||||
slog(`[embed] invalidated ${invalidated} chunk(s) embedded under a prior model signature`);
|
||||
}
|
||||
}
|
||||
@@ -873,12 +830,10 @@ async function embedAllStale(
|
||||
dryRun && signature ? { ...sourceOpt, signature } : sourceOpt,
|
||||
);
|
||||
if (staleCount === 0) {
|
||||
if (!staleOpts?.quiet) {
|
||||
if (dryRun) {
|
||||
slog('[dry-run] Would embed 0 chunks (0 stale found)');
|
||||
} else {
|
||||
slog('Embedded 0 chunks (0 stale found)');
|
||||
}
|
||||
if (dryRun) {
|
||||
slog('[dry-run] Would embed 0 chunks (0 stale found)');
|
||||
} else {
|
||||
slog('Embedded 0 chunks (0 stale found)');
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -887,7 +842,7 @@ async function embedAllStale(
|
||||
result.would_embed += staleCount;
|
||||
result.total_chunks += staleCount;
|
||||
if (onProgress) onProgress(1, 1, 0);
|
||||
if (!staleOpts?.quiet) slog(`[dry-run] Would embed ${staleCount} stale chunks`);
|
||||
slog(`[dry-run] Would embed ${staleCount} stale chunks`);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1057,10 +1012,7 @@ async function embedAllStale(
|
||||
for (let j = 0; j < stale.length; j++) {
|
||||
staleIdxToEmbedding.set(stale[j].chunk_index, embeddings[j]);
|
||||
}
|
||||
// preserveCodeMetadata threads code-chunk metadata (#769) so the
|
||||
// autopilot --stale path doesn't clobber language/symbol_name/etc
|
||||
// to NULL on every cycle.
|
||||
const merged: ChunkInput[] = existing.map(c => preserveCodeMetadata(c, {
|
||||
const merged: ChunkInput[] = existing.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
@@ -1130,7 +1082,7 @@ async function embedAllStale(
|
||||
if (budgetTimer) clearTimeout(budgetTimer);
|
||||
}
|
||||
|
||||
if (!staleOpts?.quiet) slog(`Embedded ${result.embedded} chunks across ${totalProcessedPages} pages`);
|
||||
slog(`Embedded ${result.embedded} chunks across ${totalProcessedPages} pages`);
|
||||
|
||||
// #1946 (OV2a): a catch-up pass that completed without being aborted but left
|
||||
// chunks unembedded means those chunks are stuck (a non-transient embed
|
||||
|
||||
@@ -33,7 +33,7 @@ import type { BrainEngine } from '../core/engine.ts';
|
||||
import type { EnrichCandidate, PageType } from '../core/types.ts';
|
||||
import { operations } from '../core/operations.ts';
|
||||
import type { OperationContext } from '../core/operations.ts';
|
||||
import { configureGatewayIfUninitialized, isAvailable, chat, getChatModel, withBudgetTracker } from '../core/ai/gateway.ts';
|
||||
import { isAvailable, chat, getChatModel, withBudgetTracker } from '../core/ai/gateway.ts';
|
||||
import { BudgetTracker, BudgetExhausted } from '../core/budget/budget-tracker.ts';
|
||||
import { hybridSearch } from '../core/search/hybrid.ts';
|
||||
import { serializeMarkdown } from '../core/markdown.ts';
|
||||
@@ -807,9 +807,7 @@ export async function runEnrich(engine: BrainEngine, args: string[]): Promise<vo
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Chat gateway is required for non-dry-run. Recover a cold singleton before
|
||||
// reporting an availability error (#2590).
|
||||
if (!parsed.dryRun && !isAvailable('chat')) configureGatewayIfUninitialized();
|
||||
// Chat gateway required for non-dry-run.
|
||||
if (!parsed.dryRun && !isAvailable('chat')) {
|
||||
console.error('Chat gateway unavailable. Configure a chat model (e.g. `gbrain config set chat_model anthropic:claude-haiku-4-5`), or pass --dry-run to preview candidates.');
|
||||
process.exit(1);
|
||||
|
||||
@@ -76,7 +76,7 @@ FLAGS:
|
||||
dimensions (goal, depth, sourcing, specificity, useful).
|
||||
--cycles N 1-3. Default: 3 in TTY, 1 in non-TTY (T11). Each
|
||||
cycle is 3 model calls; verdict aggregates over them.
|
||||
--slot-a-model <id> Override default 'openai:gpt-5.2'.
|
||||
--slot-a-model <id> Override default 'openai:gpt-4o'.
|
||||
--slot-b-model <id> Override default 'anthropic:claude-opus-4-7'.
|
||||
--slot-c-model <id> Override default 'google:gemini-1.5-pro'.
|
||||
--receipt-dir <path> Default: gbrainPath('eval-receipts').
|
||||
@@ -468,14 +468,6 @@ interface BatchRow {
|
||||
question_id: string;
|
||||
question: string;
|
||||
hypothesis: string;
|
||||
/**
|
||||
* Gold answer from the benchmark dataset, when the upstream eval emits
|
||||
* it (eval-longmemeval does). Folded into the judge task so CORRECTNESS
|
||||
* is verifiable — without it a judge panel that sees only
|
||||
* {question, hypothesis} cannot validate a terse factual answer against
|
||||
* a haystack it never saw.
|
||||
*/
|
||||
answer?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -589,7 +581,6 @@ function readBatchRows(path: string): BatchReadResult {
|
||||
question_id: typeof obj.question_id === 'string' ? obj.question_id : `line-${lineNo}`,
|
||||
question: obj.question,
|
||||
hypothesis: obj.hypothesis,
|
||||
...(typeof obj.answer === 'string' && obj.answer.length > 0 ? { answer: obj.answer } : {}),
|
||||
});
|
||||
}
|
||||
if (summarySkipped > 0) {
|
||||
@@ -706,11 +697,7 @@ async function runBatchMode(parsed: ParsedArgs, opts: RunCrossModalOpts): Promis
|
||||
fn: async (row, idx) => {
|
||||
process.stderr.write(`[eval cross-modal batch] ${idx + 1}/${rows.length} ${row.question_id} starting...\n`);
|
||||
return await runEvalFn({
|
||||
// With a gold answer the judges can actually verify correctness;
|
||||
// without one they see only {question, hypothesis} and cannot.
|
||||
task: row.answer
|
||||
? `${row.question}\n\nExpected answer (gold label from the benchmark dataset): ${row.answer}`
|
||||
: row.question,
|
||||
task: row.question,
|
||||
output: row.hypothesis,
|
||||
slug: row.question_id,
|
||||
dimensions,
|
||||
|
||||
@@ -33,7 +33,6 @@ import {
|
||||
type AliasMap,
|
||||
} from '../eval/longmemeval/extract.ts';
|
||||
import { extractCandidateEntities } from '../core/think/entity-extract.ts';
|
||||
import { splitProviderModelId } from '../core/model-id.ts';
|
||||
import { resolveEntitySlugWithSource, type ResolutionSource } from '../core/entities/resolve.ts';
|
||||
import { formatTrajectoryBlock } from '../core/trajectory-format.ts';
|
||||
|
||||
@@ -470,22 +469,14 @@ export async function runEvalLongMemEval(args: string[], runOpts: RunOpts = {}):
|
||||
});
|
||||
|
||||
// Wrap Anthropic SDK so its `.messages.create` shape matches ThinkLLMClient.
|
||||
// Same pattern as src/core/think/index.ts:247-249 — EXCEPT think's default
|
||||
// client routes through the gateway, which parses `provider:model` recipe
|
||||
// ids. This eval's client is a raw SDK by design (hermetic, no gateway
|
||||
// dependency), and resolveModel returns RECIPE ids (`anthropic:claude-…`);
|
||||
// passing one through unstripped 404s every answer/extractor call, which
|
||||
// surfaces downstream as all-upstream_error batches in the nightly probe.
|
||||
const toSdkModel = (m: string): string => splitProviderModelId(m).model || m;
|
||||
// Same pattern as src/core/think/index.ts:247-249.
|
||||
const realClient = new Anthropic();
|
||||
const client: ThinkLLMClient = runOpts.client ?? {
|
||||
create: (params, callOpts) =>
|
||||
realClient.messages.create({ ...params, model: toSdkModel(params.model) }, callOpts),
|
||||
create: (params, callOpts) => realClient.messages.create(params, callOpts),
|
||||
};
|
||||
// v0.40.2.0 — separate extractor client (defaults to same SDK).
|
||||
const extractorClient: ThinkLLMClient = runOpts.extractorClient ?? {
|
||||
create: (params, callOpts) =>
|
||||
realClient.messages.create({ ...params, model: toSdkModel(params.model) }, callOpts),
|
||||
create: (params, callOpts) => realClient.messages.create(params, callOpts),
|
||||
};
|
||||
const trajectoryEnabled = !opts.noTrajectory;
|
||||
const extractorModel = trajectoryEnabled
|
||||
@@ -760,11 +751,6 @@ async function runOneQuestion(
|
||||
// v0.40.1.0 (Track D / T2) — copy question_type into the row so the
|
||||
// by_type_summary can be rebuilt from the file on resume runs.
|
||||
question_type: q.question_type,
|
||||
// Gold answer for downstream consumers that verify correctness (the
|
||||
// cross-modal --batch judge folds it into the task; evaluate_qa.py
|
||||
// ignores unknown fields). Without it a judge can't validate a terse
|
||||
// factual hypothesis against a haystack it never saw.
|
||||
...(q.answer !== undefined ? { answer: q.answer } : {}),
|
||||
hypothesis,
|
||||
retrieved_session_ids: retrievedSessionIds,
|
||||
...(recallHit !== undefined ? { recall_hit: recallHit } : {}),
|
||||
|
||||
@@ -71,7 +71,7 @@ import {
|
||||
extractFactsFromTurn,
|
||||
isFactsExtractionEnabled,
|
||||
} from '../core/facts/extract.ts';
|
||||
import { configureGatewayIfUninitialized, isAvailable, withBudgetTracker } from '../core/ai/gateway.ts';
|
||||
import { isAvailable, withBudgetTracker } from '../core/ai/gateway.ts';
|
||||
import { BudgetTracker, BudgetExhausted } from '../core/budget/budget-tracker.ts';
|
||||
import { listSources } from '../core/sources-ops.ts';
|
||||
import {
|
||||
@@ -81,6 +81,7 @@ import {
|
||||
} from '../core/op-checkpoint.ts';
|
||||
import { createProgress } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions, maybeBackground } from '../core/cli-options.ts';
|
||||
import { loadConfig } from '../core/config.ts';
|
||||
import { createHash } from 'crypto';
|
||||
// v0.41.15.0 (T5): worker-pool primitive + per-source-clamp wrapper +
|
||||
// per-page advisory lock + delete-orphans-first replay safety. See plan
|
||||
@@ -140,14 +141,7 @@ export const DEFAULT_MAX_COST_USD = 5.0;
|
||||
* `--types` flag is an explicit per-run override; cycle config is
|
||||
* the single source of truth.
|
||||
*/
|
||||
export const ALLOWED_TYPES = [
|
||||
'conversation',
|
||||
'meeting',
|
||||
'slack',
|
||||
'email',
|
||||
'imessage',
|
||||
'imessage-daily',
|
||||
] as const;
|
||||
export const ALLOWED_TYPES = ['conversation', 'meeting', 'slack', 'email'] as const;
|
||||
export type AllowedType = (typeof ALLOWED_TYPES)[number];
|
||||
|
||||
/**
|
||||
@@ -763,12 +757,6 @@ async function processPage(
|
||||
source_markdown_slug: page.slug,
|
||||
source: PER_SEGMENT_SOURCE_PREFIX,
|
||||
source_session: sessionId,
|
||||
// Preserve the conversation's valid time instead of defaulting every
|
||||
// extracted fact to extraction time. Epoch-anchored parses have no
|
||||
// trustworthy date, so they retain the existing now() fallback.
|
||||
...(seg.startIso && !seg.startIso.startsWith('1970-')
|
||||
? { valid_from: new Date(seg.startIso) }
|
||||
: {}),
|
||||
context:
|
||||
fact.context ?? `from ${page.slug} segment ${seg.startIso}..${seg.endIso}`,
|
||||
}));
|
||||
@@ -1081,8 +1069,7 @@ export async function runExtractConversationFactsCore(
|
||||
}
|
||||
// Fall through to receipt+rollup write so the partial run is
|
||||
// still observable in extract_health doctor + extracts/ pages.
|
||||
// ...but not under --dry-run: a preview must not persist cache state.
|
||||
if (!dryRun) await writeRunReceiptAndRollup(engine, sourceId, result, /* halted */ true);
|
||||
await writeRunReceiptAndRollup(engine, sourceId, result, /* halted */ true);
|
||||
// Return partial result — caller (CLI / Minion) decides how to
|
||||
// surface. NOT a thrown failure.
|
||||
return result;
|
||||
@@ -1094,9 +1081,7 @@ export async function runExtractConversationFactsCore(
|
||||
// (queryable + citable per D-EXTRACT-17/19) AND UPSERTs the per-day
|
||||
// rollup row (best-effort cache per F-OUT-19). Both are best-effort —
|
||||
// failures stderr-warn but never fail the parent operation.
|
||||
// --dry-run must not persist cache/knowledge state: skip the rollup UPSERT +
|
||||
// receipt-page write so a preview leaves no extract cache row behind.
|
||||
if (!dryRun) await writeRunReceiptAndRollup(engine, sourceId, result, /* halted */ false);
|
||||
await writeRunReceiptAndRollup(engine, sourceId, result, /* halted */ false);
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -1366,9 +1351,7 @@ export async function runExtractConversationFacts(
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Chat gateway is required for non-dry-run. Recover a cold singleton before
|
||||
// reporting an availability error (#2590).
|
||||
if (!parsed.dryRun && !isAvailable('chat')) configureGatewayIfUninitialized();
|
||||
// Chat gateway is required for non-dry-run.
|
||||
if (!parsed.dryRun && !isAvailable('chat')) {
|
||||
console.error('Chat gateway unavailable. Configure an Anthropic or compatible chat model, or pass --dry-run to preview segmentation.');
|
||||
process.exit(1);
|
||||
|
||||
+9
-60
@@ -433,10 +433,7 @@ export async function extractLinksFromFile(
|
||||
async resolve(name: string, dirHint?: string | string[]): Promise<string | null> {
|
||||
if (!name) return null;
|
||||
const trimmed = name.trim();
|
||||
// Same broadened slug-shape as makeResolver step 1: accepts
|
||||
// digit-leading folders (`90-people/nicolai`) and nested paths.
|
||||
// Exact Set membership guards it — no false positives.
|
||||
if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed) && allSlugs.has(trimmed)) {
|
||||
if (/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(trimmed) && allSlugs.has(trimmed)) {
|
||||
return trimmed;
|
||||
}
|
||||
const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []);
|
||||
@@ -585,17 +582,6 @@ export interface ExtractOpts {
|
||||
* before (single-'default'-source brains unaffected).
|
||||
*/
|
||||
sourceId?: string;
|
||||
/**
|
||||
* v0.42 — also extract frontmatter links on the incremental (slugs) path.
|
||||
* `extractForSlugs` extracts BODY links only by default; set this true to also
|
||||
* parse each changed page's frontmatter so `sources:`/`related:` edges stay fresh
|
||||
* when YAML is edited externally and synced in. Applied PER changed page, so the
|
||||
* incremental walk stays bounded (no switch to a full DB scan). Only honored on
|
||||
* the incremental path (`slugs` defined); the full-walk path already covers
|
||||
* frontmatter via its own dispatch. Gated upstream by the config key
|
||||
* `autopilot.incremental_extract_include_frontmatter` (default off).
|
||||
*/
|
||||
includeFrontmatter?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -634,7 +620,7 @@ export async function runExtractCore(engine: BrainEngine, opts: ExtractOpts): Pr
|
||||
// Nothing changed — skip entirely.
|
||||
return result;
|
||||
}
|
||||
const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId, opts.includeFrontmatter);
|
||||
const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId);
|
||||
result.links_created = r.links_created;
|
||||
result.timeline_entries_created = r.timeline_created;
|
||||
result.pages_processed = r.pages;
|
||||
@@ -1025,11 +1011,6 @@ async function extractForSlugs(
|
||||
signal?: AbortSignal,
|
||||
// #1747/#1503: stamp resolved brain source id on batch rows (see ExtractOpts.sourceId).
|
||||
sourceId?: string,
|
||||
// v0.42: when true, also extract frontmatter links per changed page so
|
||||
// externally-edited YAML (`sources:`/`related:`) stays fresh on the cycle.
|
||||
// Default false preserves the body-only incremental behavior. Gated upstream
|
||||
// by `autopilot.incremental_extract_include_frontmatter`.
|
||||
includeFrontmatter: boolean = false,
|
||||
): Promise<{ links_created: number; timeline_created: number; pages: number }> {
|
||||
// Build the full slug set for link resolution (fast: just readdir, no file reads)
|
||||
const allFiles = walkMarkdownFiles(brainDir);
|
||||
@@ -1044,10 +1025,6 @@ async function extractForSlugs(
|
||||
let linksCreated = 0;
|
||||
let timelineCreated = 0;
|
||||
let pagesProcessed = 0;
|
||||
// #2636: successfully processed pages get their extraction watermark
|
||||
// stamped after the final flush (mode 'all' only — a partial-mode run
|
||||
// hasn't done the full extraction the watermark asserts).
|
||||
const processedRefs: Array<{ slug: string; source_id: string }> = [];
|
||||
|
||||
// Issue #972: read the basename flag once per extract run.
|
||||
const globalBasename = await isGlobalBasenameEnabled(engine);
|
||||
@@ -1108,7 +1085,7 @@ async function extractForSlugs(
|
||||
const content = readFileSync(fullPath, 'utf-8');
|
||||
|
||||
if (doLinks) {
|
||||
const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename, includeFrontmatter });
|
||||
const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename });
|
||||
for (const link of links) {
|
||||
if (dryRun) {
|
||||
if (!jsonMode) console.log(` ${link.from_slug} → ${link.to_slug} (${link.link_type})`);
|
||||
@@ -1136,7 +1113,6 @@ async function extractForSlugs(
|
||||
}
|
||||
|
||||
pagesProcessed++;
|
||||
if (!dryRun) processedRefs.push({ slug, source_id: sourceId ?? 'default' });
|
||||
} catch { /* skip unreadable */ }
|
||||
progress.tick(1);
|
||||
},
|
||||
@@ -1144,13 +1120,6 @@ async function extractForSlugs(
|
||||
|
||||
await flushLinks();
|
||||
await flushTimeline();
|
||||
// #2636: the Dream cycle disables sync's inline extraction and routes
|
||||
// changed slugs through this incremental path — without a stamp here,
|
||||
// those pages never get links_extracted_at and stay permanently visible
|
||||
// to `extract --stale` / doctor. Stamp only after BOTH batches flushed.
|
||||
if (!dryRun && mode === 'all') {
|
||||
await stampExtracted(engine, processedRefs);
|
||||
}
|
||||
progress.finish();
|
||||
|
||||
if (!jsonMode) {
|
||||
@@ -1682,7 +1651,7 @@ async function extractTimelineFromDB(
|
||||
* make re-extraction idempotent). EVERY processed page is stamped, including
|
||||
* zero-link pages — they WERE processed.
|
||||
*/
|
||||
export async function extractStaleFromDB(
|
||||
async function extractStaleFromDB(
|
||||
engine: BrainEngine,
|
||||
opts: {
|
||||
dryRun: boolean;
|
||||
@@ -1715,17 +1684,9 @@ export async function extractStaleFromDB(
|
||||
// Batch mode = pg_trgm + exact only, NO per-name search fallback. The
|
||||
// resolution map sees ALL sources so qualified cross-source wikilinks resolve
|
||||
// even when --source-id scopes the stale SCAN.
|
||||
//
|
||||
// #2576 bug 1: ALWAYS the real resolver — extractPageLinks's opts gate which
|
||||
// pass runs (`skipFrontmatter` for the frontmatter pass, `globalBasename` for
|
||||
// the issue-#972 bare-wikilink pass). The former `includeFrontmatter ?
|
||||
// resolver : nullResolver` ternary predates #972; the synthetic resolver has
|
||||
// no `resolveBasenameMatches`, so the --stale sweep silently skipped basename
|
||||
// resolution even with `link_resolution.global_basename` enabled, stamping
|
||||
// pages as extracted with their bare wikilinks dropped. Mirrors
|
||||
// extractLinksFromDB (including the codex-[P1] `sourceId` scoping).
|
||||
const resolver = makeResolver(engine, { mode: 'batch', sourceId: sourceIdFilter });
|
||||
const globalBasename = await isGlobalBasenameEnabled(engine);
|
||||
const resolver = makeResolver(engine, { mode: 'batch' });
|
||||
const nullResolver = { resolve: async () => null as string | null };
|
||||
const activeResolver = includeFrontmatter ? resolver : nullResolver;
|
||||
const allRefs = await engine.listAllPageRefs();
|
||||
const allSlugs = new Set<string>();
|
||||
const slugToSources = new Map<string, string[]>();
|
||||
@@ -1757,8 +1718,7 @@ export async function extractStaleFromDB(
|
||||
for (const page of rows) {
|
||||
const fullContent = page.compiled_truth + '\n' + page.timeline;
|
||||
const extracted = await extractPageLinks(
|
||||
page.slug, fullContent, page.frontmatter, page.type, resolver,
|
||||
{ skipFrontmatter: !includeFrontmatter, globalBasename },
|
||||
page.slug, fullContent, page.frontmatter, page.type, activeResolver,
|
||||
);
|
||||
for (const c of extracted.candidates) {
|
||||
const r = resolveCandidateSources(c, page.slug, page.source_id, allSlugs, slugToSources);
|
||||
@@ -1783,18 +1743,7 @@ export async function extractStaleFromDB(
|
||||
// `page.updated_at.toISOString()` — the JS Date is ms-truncated, so the
|
||||
// µs-precision DB updated_at stayed strictly greater and the page never
|
||||
// cleared on Postgres. Stamping the exact value makes them equal.
|
||||
//
|
||||
// BUT the stamp must also clear the version-staleness clause
|
||||
// (`links_extracted_at < versionTs`). A page whose updated_at predates
|
||||
// versionTs would otherwise be stamped below the threshold and read as
|
||||
// stale forever — a permanent re-extract loop that never clears the lag.
|
||||
// GREATEST(updated_at, versionTs) preserves the race semantics (a real
|
||||
// future edit advances updated_at > versionTs >= stamp → re-extracts)
|
||||
// while lifting old pages to the threshold so they clear.
|
||||
const stampIso = page.updated_at.getTime() >= Date.parse(versionTs)
|
||||
? page.updated_at_iso
|
||||
: versionTs;
|
||||
processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: stampIso });
|
||||
processedRefs.push({ slug: page.slug, source_id: page.source_id, extractedAt: page.updated_at_iso });
|
||||
}
|
||||
|
||||
// Flush NON-swallowing (CDX-4): a throw here propagates out of the sweep so
|
||||
|
||||
+8
-67
@@ -11,7 +11,6 @@ import {
|
||||
isCodeFilePath,
|
||||
isMarkdownFilePath,
|
||||
isImageFilePath as isImageFilePathFromSync,
|
||||
matchesAnyGlob,
|
||||
pruneDir,
|
||||
SYNC_SKIP_FILES,
|
||||
type SyncStrategy,
|
||||
@@ -21,7 +20,6 @@ import {
|
||||
loadCheckpoint,
|
||||
saveCheckpoint,
|
||||
clearCheckpoint,
|
||||
resolveImportTargetDir,
|
||||
resumeFilter,
|
||||
} from '../core/import-checkpoint.ts';
|
||||
|
||||
@@ -48,25 +46,7 @@ export interface RunImportResult {
|
||||
export async function runImport(
|
||||
engine: BrainEngine,
|
||||
args: string[],
|
||||
opts: {
|
||||
commit?: string;
|
||||
strategy?: SyncStrategy;
|
||||
sourceId?: string;
|
||||
managedBookmark?: boolean;
|
||||
/**
|
||||
* #753/#774: glob patterns to exclude from the import (same semantics as
|
||||
* `isSyncable`'s `exclude` — matched against the dir-relative path).
|
||||
* Threaded by performFullSync for `gbrain sync --exclude`.
|
||||
*/
|
||||
exclude?: string[];
|
||||
/**
|
||||
* #753/#774 monorepo subdir-source support: when set, slugs and
|
||||
* `source_path` are computed relative to this root (the git repo root)
|
||||
* instead of `dir` (the sync scope), so `wiki/page1.md` lands as slug
|
||||
* `wiki/page1` consistently across full and incremental sync.
|
||||
*/
|
||||
slugRoot?: string;
|
||||
} = {},
|
||||
opts: { commit?: string; strategy?: SyncStrategy; sourceId?: string; managedBookmark?: boolean } = {},
|
||||
): Promise<RunImportResult> {
|
||||
const noEmbed = args.includes('--no-embed');
|
||||
const fresh = args.includes('--fresh');
|
||||
@@ -188,19 +168,7 @@ export async function runImport(
|
||||
console.error('Usage: gbrain import <dir> [--no-embed] [--workers N] [--fresh] [--source-id <id>] [--json]');
|
||||
process.exit(1);
|
||||
}
|
||||
// #1728: capture the import target ONCE as an absolute real path. Every
|
||||
// downstream consumer of `dir` (collection, checkpoint load/save, resume
|
||||
// filtering) sees the same canonical identity — never the caller's `.`/
|
||||
// relative spelling, which would make the persisted checkpoint `dir`
|
||||
// resolve against whatever CWD a later process happens to run from.
|
||||
let dir: string;
|
||||
try {
|
||||
dir = resolveImportTargetDir(dirArg);
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
console.error(`Import target is not readable: ${dirArg} (${msg})`);
|
||||
process.exit(1);
|
||||
}
|
||||
const dir: string = dirArg; // narrowed; survives closure capture
|
||||
|
||||
// v0.31.2: collect under the right strategy. Pre-fix this called
|
||||
// collectMarkdownFiles unconditionally — code-strategy first sync
|
||||
@@ -209,30 +177,13 @@ export async function runImport(
|
||||
const strategy: SyncStrategy = opts.strategy ?? 'markdown';
|
||||
const _walkT0 = Date.now();
|
||||
console.error(`[gbrain phase] import.collect_files start dir=${dir} strategy=${strategy}`);
|
||||
let allFiles = collectSyncableFiles(dir, { strategy });
|
||||
const allFiles = collectSyncableFiles(dir, { strategy });
|
||||
console.error(
|
||||
`[gbrain phase] import.collect_files done ${Date.now() - _walkT0}ms files=${allFiles.length}`,
|
||||
);
|
||||
const fileTypeLabel = strategy === 'code' ? 'code'
|
||||
: strategy === 'auto' ? 'syncable' : 'markdown';
|
||||
// #753/#774: apply --exclude glob patterns (threaded by performFullSync).
|
||||
if (opts.exclude && opts.exclude.length > 0) {
|
||||
const beforeExclude = allFiles.length;
|
||||
allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(dir, abs), opts.exclude));
|
||||
console.log(
|
||||
`Found ${allFiles.length} ${fileTypeLabel} files ` +
|
||||
`(${beforeExclude - allFiles.length} excluded by --exclude patterns)`,
|
||||
);
|
||||
// NAV-4: everything excluded is almost always a mistyped pattern — warn.
|
||||
if (beforeExclude > 0 && allFiles.length === 0) {
|
||||
console.warn(
|
||||
`[gbrain sync] No files matched after applying ${opts.exclude.length} --exclude pattern(s). ` +
|
||||
`Check your --exclude flags. Patterns: ${JSON.stringify(opts.exclude)}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
console.log(`Found ${allFiles.length} ${fileTypeLabel} files`);
|
||||
}
|
||||
console.log(`Found ${allFiles.length} ${fileTypeLabel} files`);
|
||||
|
||||
// Sort newest-first so date-prefixed brain paths get embedded before older ones.
|
||||
// See src/core/sort-newest-first.ts for the policy.
|
||||
@@ -278,11 +229,6 @@ export async function runImport(
|
||||
|
||||
async function processFile(eng: BrainEngine, filePath: string) {
|
||||
const relativePath = relative(dir, filePath);
|
||||
// #753/#774: slug + source_path base. When performFullSync syncs a
|
||||
// monorepo subdir, slugRoot is the git root so slugs stay git-root-
|
||||
// relative (matching the incremental path's git-diff paths). The
|
||||
// checkpoint (`completed`) stays dir-relative — resumeFilter's contract.
|
||||
const importRelPath = opts.slugRoot ? relative(opts.slugRoot, filePath) : relativePath;
|
||||
// v0.31.2 (D5): per-file slow-path log. Fires only when a single
|
||||
// file takes >5s. The user's hang surfaces as one file taking
|
||||
// forever — without this, the agent can't see which file.
|
||||
@@ -293,8 +239,8 @@ export async function runImport(
|
||||
// up images when GBRAIN_EMBEDDING_MULTIMODAL=true so this branch is
|
||||
// unreachable when the gate is off; defense-in-depth check anyway.
|
||||
const result = isImageFilePath(relativePath) && process.env.GBRAIN_EMBEDDING_MULTIMODAL === 'true'
|
||||
? await importImageFile(eng, filePath, importRelPath, { noEmbed, sourceId })
|
||||
: await importFile(eng, filePath, importRelPath, { noEmbed, sourceId, activePack: importActivePack });
|
||||
? await importImageFile(eng, filePath, relativePath, { noEmbed, sourceId })
|
||||
: await importFile(eng, filePath, relativePath, { noEmbed, sourceId, activePack: importActivePack });
|
||||
const _fileMs = Date.now() - _fileT0;
|
||||
if (_fileMs > 5000) {
|
||||
console.error(`[gbrain phase] import.process_file slow ${_fileMs}ms ${relativePath}`);
|
||||
@@ -310,9 +256,7 @@ export async function runImport(
|
||||
if (result.error && result.error !== 'unchanged') {
|
||||
console.error(` Skipped ${relativePath}: ${result.error}`);
|
||||
// Bug 9 — non-"unchanged" skips carry a real error reason.
|
||||
// #774: ledger paths use the slug base so an incremental sync's
|
||||
// success at the same (git-root-relative) path clears the row.
|
||||
failures.push({ path: importRelPath, error: result.error });
|
||||
failures.push({ path: relativePath, error: result.error });
|
||||
} else {
|
||||
// 'unchanged' or no-error skip: content_hash matched a prior
|
||||
// successful import, so this file IS done for checkpoint purposes.
|
||||
@@ -330,7 +274,7 @@ export async function runImport(
|
||||
}
|
||||
errors++;
|
||||
skipped++;
|
||||
failures.push({ path: importRelPath, error: msg });
|
||||
failures.push({ path: relativePath, error: msg });
|
||||
}
|
||||
processed++;
|
||||
tickProgress();
|
||||
@@ -344,9 +288,6 @@ export async function runImport(
|
||||
catch { /* non-fatal */ }
|
||||
}
|
||||
saveCheckpoint(checkpointPath, {
|
||||
schema_version: 1,
|
||||
owner: 'gbrain',
|
||||
kind: 'import',
|
||||
dir,
|
||||
completedPaths: Array.from(completed),
|
||||
timestamp: new Date().toISOString(),
|
||||
|
||||
+13
-110
@@ -26,8 +26,6 @@ export async function runInit(args: string[]) {
|
||||
return;
|
||||
}
|
||||
|
||||
validateInitFlags(args);
|
||||
|
||||
const isSupabase = args.includes('--supabase');
|
||||
const isPGLite = args.includes('--pglite');
|
||||
const isMcpOnly = args.includes('--mcp-only');
|
||||
@@ -153,65 +151,6 @@ export async function runInit(args: string[]) {
|
||||
return initPostgres({ databaseUrl, jsonOutput, apiKey, aiOpts, schemaPack, skipEmbedCheck });
|
||||
}
|
||||
|
||||
const INIT_BOOLEAN_FLAGS = new Set([
|
||||
'--pglite',
|
||||
'--supabase',
|
||||
'--mcp-only',
|
||||
'--force',
|
||||
'--non-interactive',
|
||||
'--migrate-only',
|
||||
'--json',
|
||||
'--no-embedding',
|
||||
'--skip-embed-check',
|
||||
]);
|
||||
|
||||
const INIT_VALUE_FLAGS = new Set([
|
||||
'--url',
|
||||
'--key',
|
||||
'--path',
|
||||
'--schema-pack',
|
||||
'--embedding-model',
|
||||
'--model',
|
||||
'--embedding-dimensions',
|
||||
'--expansion-model',
|
||||
'--chat-model',
|
||||
'--mcp-url',
|
||||
'--issuer-url',
|
||||
'--oauth-client-id',
|
||||
'--oauth-client-secret',
|
||||
]);
|
||||
|
||||
function validateInitFlags(args: string[]) {
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i];
|
||||
if (!arg.startsWith('-')) continue;
|
||||
|
||||
if (INIT_BOOLEAN_FLAGS.has(arg)) continue;
|
||||
|
||||
if (INIT_VALUE_FLAGS.has(arg)) {
|
||||
if (i + 1 >= args.length || args[i + 1].startsWith('-')) {
|
||||
failInitFlag(`gbrain init: ${arg} requires a value`, args.includes('--json'));
|
||||
}
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg.startsWith('--')) {
|
||||
failInitFlag(`gbrain init: unknown flag ${arg}`, args.includes('--json'));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function failInitFlag(message: string, jsonOutput: boolean): never {
|
||||
if (jsonOutput) {
|
||||
console.log(JSON.stringify({ status: 'error', reason: 'invalid_flag', message }));
|
||||
} else {
|
||||
console.error(message);
|
||||
console.error('Run `gbrain init --help` for supported flags.');
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
interface ResolveAIOptionsArgs {
|
||||
verbose: string | null; // --embedding-model
|
||||
shorthand: string | null; // --model
|
||||
@@ -222,7 +161,7 @@ interface ResolveAIOptionsArgs {
|
||||
nonInteractive: boolean; // --non-interactive (forces D3 fail-loud, no picker)
|
||||
}
|
||||
|
||||
export interface ResolvedAIOptions {
|
||||
interface ResolvedAIOptions {
|
||||
embedding_model?: string;
|
||||
embedding_dimensions?: number;
|
||||
expansion_model?: string;
|
||||
@@ -231,41 +170,6 @@ export interface ResolvedAIOptions {
|
||||
noEmbedding?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Seed init's AI options from persisted config, falling back to the raw env
|
||||
* vars when loadConfig() returned null (#1058). On a cold install (no
|
||||
* config.json AND no DATABASE_URL) loadConfig short-circuits BEFORE its env
|
||||
* merge, so GBRAIN_EMBEDDING_MODEL / GBRAIN_EMBEDDING_DIMENSIONS /
|
||||
* GBRAIN_EXPANSION_MODEL / GBRAIN_CHAT_MODEL were silently ignored by init
|
||||
* and Tier-3 detection auto-picked by API key instead. Exported for unit
|
||||
* tests (env injectable).
|
||||
*/
|
||||
export function seedAIOptionsFromConfig(
|
||||
cfg: GBrainConfig | null,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): ResolvedAIOptions {
|
||||
const envDims = env.GBRAIN_EMBEDDING_DIMENSIONS
|
||||
? parseInt(env.GBRAIN_EMBEDDING_DIMENSIONS, 10)
|
||||
: NaN;
|
||||
const seed = cfg ?? {
|
||||
embedding_disabled: undefined,
|
||||
embedding_model: env.GBRAIN_EMBEDDING_MODEL,
|
||||
embedding_dimensions: Number.isFinite(envDims) ? envDims : undefined,
|
||||
expansion_model: env.GBRAIN_EXPANSION_MODEL,
|
||||
chat_model: env.GBRAIN_CHAT_MODEL,
|
||||
};
|
||||
const out: ResolvedAIOptions = {};
|
||||
if (seed.embedding_disabled) {
|
||||
out.noEmbedding = true;
|
||||
} else if (seed.embedding_model) {
|
||||
out.embedding_model = seed.embedding_model;
|
||||
if (seed.embedding_dimensions) out.embedding_dimensions = seed.embedding_dimensions;
|
||||
}
|
||||
if (seed.expansion_model) out.expansion_model = seed.expansion_model;
|
||||
if (seed.chat_model) out.chat_model = seed.chat_model;
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve AI provider options for `gbrain init`.
|
||||
*
|
||||
@@ -299,13 +203,18 @@ async function resolveAIOptions(opts: ResolveAIOptionsArgs): Promise<ResolvedAIO
|
||||
// user already opted into deferred mode.
|
||||
try {
|
||||
const { loadConfig } = await import('../core/config.ts');
|
||||
// #1058: loadConfig() returns null on a cold install (no config.json AND
|
||||
// no DATABASE_URL) — before it ever reaches its env merge. The seed helper
|
||||
// falls back to the same GBRAIN_* env vars directly in that case.
|
||||
Object.assign(out, seedAIOptionsFromConfig(loadConfig()));
|
||||
const cfg = loadConfig();
|
||||
if (cfg?.embedding_disabled) {
|
||||
out.noEmbedding = true;
|
||||
} else if (cfg?.embedding_model) {
|
||||
out.embedding_model = cfg.embedding_model;
|
||||
if (cfg.embedding_dimensions) out.embedding_dimensions = cfg.embedding_dimensions;
|
||||
}
|
||||
if (cfg?.expansion_model) out.expansion_model = cfg.expansion_model;
|
||||
if (cfg?.chat_model) out.chat_model = cfg.chat_model;
|
||||
} catch {
|
||||
// loadConfig threw — treat as first-time install, fall through to env
|
||||
// detection.
|
||||
// loadConfig throws when no brain configured — first-time install, fall
|
||||
// through to env detection.
|
||||
}
|
||||
|
||||
// --- Tier 1+2: explicit flags ---------------------------------------------
|
||||
@@ -1169,9 +1078,6 @@ async function initPostgres(opts: {
|
||||
console.warn(' Direct connections are IPv6 only and fail in many environments.');
|
||||
console.warn(' Use the Transaction pooler connection string instead (port 6543):');
|
||||
console.warn(' Supabase Dashboard > Connect (top bar) > Connection String > Transaction pooler');
|
||||
console.warn(' (With a pooler URL, gbrain derives a direct connection for DDL and falls back');
|
||||
console.warn(' to the pooler automatically if that host is unreachable. Power users:');
|
||||
console.warn(' GBRAIN_DIRECT_DATABASE_URL overrides the derived URL; GBRAIN_DISABLE_DIRECT_POOL=1 disables it.)');
|
||||
console.warn('');
|
||||
}
|
||||
|
||||
@@ -1185,9 +1091,6 @@ async function initPostgres(opts: {
|
||||
if (databaseUrl.includes('supabase.co') && (msg.includes('ECONNREFUSED') || msg.includes('ETIMEDOUT'))) {
|
||||
console.error('Connection failed. Supabase direct connections (db.*.supabase.co:5432) are IPv6 only.');
|
||||
console.error('Use the Transaction pooler connection string instead (port 6543).');
|
||||
console.error('(gbrain derives its own direct connection from pooler URLs for DDL; if that host is');
|
||||
console.error('unreachable it falls back to the pooler. GBRAIN_DIRECT_DATABASE_URL overrides the');
|
||||
console.error('derived URL; GBRAIN_DISABLE_DIRECT_POOL=1 disables the direct pool entirely.)');
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
@@ -1552,7 +1455,7 @@ export function reportModStatus(): void {
|
||||
console.log(' cd ~/.claude/skills/gstack && ./setup');
|
||||
}
|
||||
console.log('Resolver: skills/RESOLVER.md');
|
||||
console.log('Soul audit: ask your agent to "run a soul audit" to customize its identity (see skills/soul-audit)');
|
||||
console.log('Soul audit: run `gbrain soul-audit` to customize agent identity');
|
||||
// Retrieval Reflex (#1981): the deterministic pointer layer is ON by default
|
||||
// (no action needed). The policy skill is installed into the HOST repo on
|
||||
// request — we PRINT the command rather than silently mutating the host repo.
|
||||
|
||||
@@ -98,17 +98,8 @@ export function findBareTweetHits(compiledTruth: string, slug: string): BareTwee
|
||||
}
|
||||
// If the line already contains a tweet URL, it's cited — skip
|
||||
if (URL_NEARBY_RE.test(line)) continue;
|
||||
// If the line carries an explicit source citation (e.g.
|
||||
// "[Source: X, @handle, 2026-05-28]"), it's already attributed — skip.
|
||||
// Catches instructional/example lines in recipe docs that demonstrate
|
||||
// the CORRECT citation format. (v0.42.x)
|
||||
if (/\[\s*source:/i.test(line)) continue;
|
||||
// Strip inline-code spans (`...`) before matching: phrases shown as
|
||||
// inline-code templates in docs are examples, not bare claims. The
|
||||
// fenced-code skip above only covers ``` blocks, not inline backticks.
|
||||
const lineForMatch = line.replace(/`[^`]*`/g, '');
|
||||
for (const re of BARE_TWEET_PHRASES) {
|
||||
const m = lineForMatch.match(re);
|
||||
const m = line.match(re);
|
||||
if (m) {
|
||||
hits.push({ slug, line: i + 1, rawLine: line.trim(), phrase: m[0] });
|
||||
break; // one finding per line is enough
|
||||
|
||||
+18
-102
@@ -7,7 +7,7 @@ import type { BrainEngine } from '../core/engine.ts';
|
||||
import { MinionQueue } from '../core/minions/queue.ts';
|
||||
import { MinionWorker } from '../core/minions/worker.ts';
|
||||
import { WORKER_EXIT_RSS_WATCHDOG } from '../core/minions/worker-exit-codes.ts';
|
||||
import type { MinionHandler, MinionJob, MinionJobStatus } from '../core/minions/types.ts';
|
||||
import type { MinionJob, MinionJobStatus } from '../core/minions/types.ts';
|
||||
import type { PaceKeyOverrides } from '../core/pace-mode.ts';
|
||||
import { loadConfig, isThinClient } from '../core/config.ts';
|
||||
import { callRemoteTool, unpackToolResult } from '../core/mcp-client.ts';
|
||||
@@ -22,49 +22,6 @@ function hasFlag(args: string[], flag: string): boolean {
|
||||
return args.includes(flag);
|
||||
}
|
||||
|
||||
/**
|
||||
* Long-lived workers outlive operator config changes. Re-stamp the AI gateway
|
||||
* from DB-backed model config immediately before queued jobs enter gateway-backed
|
||||
* paths, so a stale process-level default cannot route new work to the wrong
|
||||
* provider.
|
||||
*/
|
||||
async function refreshGatewayForJob(engine: BrainEngine): Promise<void> {
|
||||
const { reconfigureGatewayWithEngine } = await import('../core/ai/gateway.ts');
|
||||
await reconfigureGatewayWithEngine(engine);
|
||||
}
|
||||
|
||||
const GATEWAY_REFRESH_JOB_NAMES = new Set([
|
||||
'embed',
|
||||
'extract-conversation-facts',
|
||||
'enrich',
|
||||
'contextual_reindex_per_chunk',
|
||||
'autopilot-cycle',
|
||||
'synthesize',
|
||||
'patterns',
|
||||
'consolidate',
|
||||
'extract_facts',
|
||||
'extract-atoms-drain',
|
||||
'embed-backfill',
|
||||
'extract-takes-from-pages',
|
||||
'embed-catch-up',
|
||||
]);
|
||||
|
||||
function registerBuiltinJob(
|
||||
worker: MinionWorker,
|
||||
engine: BrainEngine,
|
||||
name: string,
|
||||
handler: MinionHandler,
|
||||
): void {
|
||||
if (!GATEWAY_REFRESH_JOB_NAMES.has(name)) {
|
||||
worker.register(name, handler);
|
||||
return;
|
||||
}
|
||||
worker.register(name, async (job) => {
|
||||
await refreshGatewayForJob(engine);
|
||||
return await handler(job);
|
||||
});
|
||||
}
|
||||
|
||||
/** Parse `--max-waiting N` from CLI args. Returns undefined if absent.
|
||||
* Throws on malformed input (caller should surface the error and exit).
|
||||
* Clamps to [1, 100] to match the queue-layer clamp in MinionQueue.add.
|
||||
@@ -175,23 +132,9 @@ function formatJobDetail(job: MinionJob): string {
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export async function runJobs(engineOrNull: BrainEngine | null, args: string[]): Promise<void> {
|
||||
export async function runJobs(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const sub = args[0];
|
||||
|
||||
// Thin-client dispatch (cli.ts) passes engine=null for the subcommands
|
||||
// with remote MCP routing (`list`, `get`) so no scratch local engine is
|
||||
// ever built. Any other subcommand arriving with a null engine is a
|
||||
// routing bug upstream of this function — refuse instead of crashing
|
||||
// inside MinionQueue.
|
||||
if (!engineOrNull && sub !== 'list' && sub !== 'get') {
|
||||
console.error(`\`gbrain jobs ${sub ?? ''}\` needs a local engine and cannot run on a thin client.`);
|
||||
process.exit(1);
|
||||
}
|
||||
// Null only ever reaches the MCP-routed `list`/`get` branches, which
|
||||
// never touch the engine — narrowed once here so the host-only cases
|
||||
// below typecheck unchanged.
|
||||
const engine = engineOrNull as BrainEngine;
|
||||
|
||||
if (!sub || sub === '--help' || sub === '-h') {
|
||||
console.log(`gbrain jobs — Minions job queue
|
||||
|
||||
@@ -274,8 +217,6 @@ HANDLER TYPES (built in)
|
||||
return;
|
||||
}
|
||||
|
||||
// The constructor just stores the reference; on the null (thin-client
|
||||
// list/get) paths no queue method is ever reached.
|
||||
const queue = new MinionQueue(engine);
|
||||
|
||||
switch (sub) {
|
||||
@@ -1482,7 +1423,7 @@ export async function registerBuiltinHandlers(
|
||||
return { ...result, embed_job_id: embedJobId, embed_skip_reason: embedSkipReason };
|
||||
});
|
||||
|
||||
registerBuiltinJob(worker, engine, 'embed', async (job) => {
|
||||
worker.register('embed', async (job) => {
|
||||
const { runEmbedCore } = await import('./embed.ts');
|
||||
// Primary Minion progress channel is job.updateProgress (DB-backed,
|
||||
// readable via `gbrain jobs get <id>`). Stderr from the worker daemon
|
||||
@@ -1529,7 +1470,7 @@ export async function registerBuiltinHandlers(
|
||||
// BudgetTracker inside its own process. BudgetExhausted is caught at
|
||||
// the core level and returned as `result.budget_exhausted: true` (NOT
|
||||
// a job failure) so the user can resume with a higher cap.
|
||||
registerBuiltinJob(worker, engine, 'extract-conversation-facts', async (job) => {
|
||||
worker.register('extract-conversation-facts', async (job) => {
|
||||
const { runExtractConversationFactsCore } = await import('./extract-conversation-facts.ts');
|
||||
const sourceId = typeof job.data.sourceId === 'string' ? job.data.sourceId : undefined;
|
||||
if (!sourceId) {
|
||||
@@ -1540,7 +1481,7 @@ export async function registerBuiltinHandlers(
|
||||
}
|
||||
const types = Array.isArray(job.data.types)
|
||||
? (job.data.types as string[]).filter((t) =>
|
||||
['conversation', 'meeting', 'slack', 'email', 'imessage', 'imessage-daily'].includes(t),
|
||||
['conversation', 'meeting', 'slack', 'email'].includes(t),
|
||||
)
|
||||
: undefined;
|
||||
const result = await runExtractConversationFactsCore(engine, {
|
||||
@@ -1588,7 +1529,7 @@ export async function registerBuiltinHandlers(
|
||||
// at the core level and returned as result.budget_exhausted (NOT a failure).
|
||||
// Strict per-source: the CLI fans out one job per source when --source is
|
||||
// omitted, so a job ALWAYS carries data.sourceId.
|
||||
registerBuiltinJob(worker, engine, 'enrich', async (job) => {
|
||||
worker.register('enrich', async (job) => {
|
||||
const { runEnrichCore } = await import('./enrich.ts');
|
||||
const sourceId = typeof job.data.sourceId === 'string' ? job.data.sourceId : undefined;
|
||||
if (!sourceId) {
|
||||
@@ -1664,13 +1605,7 @@ export async function registerBuiltinHandlers(
|
||||
|
||||
worker.register('backlinks', async (job) => {
|
||||
const { runBacklinksCore } = await import('./backlinks.ts');
|
||||
// Default to 'check', not 'fix': backlinks jobs submitted with an empty
|
||||
// payload (e.g. the sync→embed→backlinks chains enqueued after ingestion)
|
||||
// must never rewrite tracked brain pages with generated "Referenced in"
|
||||
// timeline bullets. Mirrors the documented intent in src/core/cycle.ts
|
||||
// (runPhaseBacklinks). The filesystem fixer stays available explicitly
|
||||
// via '{"action":"fix"}' or `gbrain check-backlinks fix`.
|
||||
const action: 'check' | 'fix' = job.data.action === 'fix' ? 'fix' : 'check';
|
||||
const action: 'check' | 'fix' = job.data.action === 'check' ? 'check' : 'fix';
|
||||
const dir = typeof job.data.dir === 'string'
|
||||
? job.data.dir
|
||||
: (await engine.getConfig('sync.repo_path')) ?? '.';
|
||||
@@ -1734,13 +1669,13 @@ export async function registerBuiltinHandlers(
|
||||
const { makeContextualReindexHandler } = await import(
|
||||
'../core/minions/handlers/contextual-reindex-per-chunk.ts'
|
||||
);
|
||||
registerBuiltinJob(worker, engine, 'contextual_reindex_per_chunk', makeContextualReindexHandler({ engine }));
|
||||
worker.register('contextual_reindex_per_chunk', makeContextualReindexHandler({ engine }));
|
||||
}
|
||||
|
||||
// derivation); the handler returns { partial, status, report } so
|
||||
// `gbrain jobs get <id>` shows the full structured report. Does NOT
|
||||
// throw on partial: a flaky phase must not block every future cycle.
|
||||
registerBuiltinJob(worker, engine, 'autopilot-cycle', async (job) => {
|
||||
worker.register('autopilot-cycle', async (job) => {
|
||||
const { runCycle } = await import('../core/cycle.ts');
|
||||
// v0.41.30 (T2): fall back to null (NOT cwd '.') when no repo is configured.
|
||||
// The queued cycle is the same primitive `gbrain dream` uses; a checkout-less
|
||||
@@ -1845,7 +1780,6 @@ export async function registerBuiltinHandlers(
|
||||
brainDir: effectiveBrainDir,
|
||||
pull,
|
||||
signal: job.signal, // propagate abort so cycle bails on timeout/cancel
|
||||
deadlineAtMs: job.deadlineAtMs, // #2781: phases budget sub-work from remaining time
|
||||
...(sourceId ? { sourceId } : {}),
|
||||
...(requestedPhases && requestedPhases.length > 0 ? { phases: requestedPhases as any } : {}),
|
||||
yieldBetweenPhases: async () => {
|
||||
@@ -1883,9 +1817,7 @@ export async function registerBuiltinHandlers(
|
||||
brainDir: repoPath,
|
||||
pull: false, // brain-wide DB/maintenance work never git-pulls
|
||||
signal: job.signal,
|
||||
deadlineAtMs: job.deadlineAtMs, // #2781: phases budget sub-work from remaining time
|
||||
phases,
|
||||
forceGlobalOrphans: true,
|
||||
yieldBetweenPhases: async () => { await new Promise<void>((r) => setImmediate(r)); },
|
||||
});
|
||||
|
||||
@@ -2030,18 +1962,17 @@ export async function registerBuiltinHandlers(
|
||||
brainDir: repoPath,
|
||||
phases: [phase as any],
|
||||
signal: job.signal,
|
||||
deadlineAtMs: job.deadlineAtMs, // #2781: phases budget sub-work from remaining time
|
||||
});
|
||||
return { phase, status: report.status, report };
|
||||
};
|
||||
|
||||
// PROTECTED — internally spawn subagent children
|
||||
registerBuiltinJob(worker, engine, 'synthesize', makePhaseHandler('synthesize'));
|
||||
registerBuiltinJob(worker, engine, 'patterns', makePhaseHandler('patterns'));
|
||||
registerBuiltinJob(worker, engine, 'consolidate', makePhaseHandler('consolidate'));
|
||||
worker.register('synthesize', makePhaseHandler('synthesize'));
|
||||
worker.register('patterns', makePhaseHandler('patterns'));
|
||||
worker.register('consolidate', makePhaseHandler('consolidate'));
|
||||
|
||||
// Open — DB writes only, no LLM spend
|
||||
registerBuiltinJob(worker, engine, 'extract_facts', makePhaseHandler('extract_facts'));
|
||||
worker.register('extract_facts', makePhaseHandler('extract_facts'));
|
||||
worker.register('resolve_symbol_edges', makePhaseHandler('resolve_symbol_edges'));
|
||||
worker.register('recompute_emotional_weight', makePhaseHandler('recompute_emotional_weight'));
|
||||
|
||||
@@ -2051,7 +1982,7 @@ export async function registerBuiltinHandlers(
|
||||
// window / defer behavior. On LockUnavailableError (the routine cycle holds
|
||||
// the per-source lock) the job completes `{ deferred: true }` and retries
|
||||
// next tick instead of failing — cooperative interleave (CODEX accepted).
|
||||
registerBuiltinJob(worker, engine, 'extract-atoms-drain', async (job) => {
|
||||
worker.register('extract-atoms-drain', async (job) => {
|
||||
const { runExtractAtomsDrainForSource } = await import('../core/cycle/extract-atoms-drain.ts');
|
||||
const { LockUnavailableError } = await import('../core/db-lock.ts');
|
||||
const sourceId = typeof job.data.sourceId === 'string' ? job.data.sourceId : undefined;
|
||||
@@ -2062,26 +1993,11 @@ export async function registerBuiltinHandlers(
|
||||
? job.data.repoPath
|
||||
: ((await engine.getConfig('sync.repo_path')) ?? undefined);
|
||||
try {
|
||||
const result = await runExtractAtomsDrainForSource(engine, {
|
||||
return await runExtractAtomsDrainForSource(engine, {
|
||||
sourceId,
|
||||
windowSeconds,
|
||||
brainDir: repoPath,
|
||||
});
|
||||
// issue #3218: every item the drain attempted failed (0 succeeded, >=1
|
||||
// provider error) — completing this job normally would mark the
|
||||
// durable job done while the backlog sits untouched, and no retry
|
||||
// policy would ever fire on it again. Throw so the worker's ordinary
|
||||
// failJob path (attempt+backoff, or dead-letter once exhausted) takes
|
||||
// over instead — matching the existing behavior for every other
|
||||
// handler failure. Partial success (>=1 item extracted) keeps
|
||||
// completing normally, unchanged.
|
||||
if (result.status === 'provider_failure') {
|
||||
throw new Error(
|
||||
`extract-atoms-drain: all provider calls failed this batch ` +
|
||||
`(batches=${result.batches}, remaining=${result.remaining ?? '?'}) — retrying`,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
} catch (e) {
|
||||
if (e instanceof LockUnavailableError) {
|
||||
return { phase: 'extract_atoms', status: 'skipped', deferred: true, reason: 'cycle_already_running' };
|
||||
@@ -2094,7 +2010,7 @@ export async function registerBuiltinHandlers(
|
||||
// Cost-bounded via D6 ($10/job BudgetTracker) + D19 (source-level cooldown
|
||||
// + 24h rolling cap, gated at submit time). NOT in PROTECTED_JOB_NAMES —
|
||||
// embedding-only spend, no API-by-the-minute risk like subagent.
|
||||
registerBuiltinJob(worker, engine, 'embed-backfill', async (job) => {
|
||||
worker.register('embed-backfill', async (job) => {
|
||||
const { makeEmbedBackfillHandler } = await import('../core/minions/handlers/embed-backfill.ts');
|
||||
return await makeEmbedBackfillHandler(engine)(job);
|
||||
});
|
||||
@@ -2115,7 +2031,7 @@ export async function registerBuiltinHandlers(
|
||||
// (LLM-bearing). Two-gate consent enforced at the handler boundary:
|
||||
// refuses to run unless takes.bootstrap_enabled config is true, even
|
||||
// when allowProtectedSubmit was set at queue.add time.
|
||||
registerBuiltinJob(worker, engine, 'extract-takes-from-pages', async (job) => {
|
||||
worker.register('extract-takes-from-pages', async (job) => {
|
||||
const { extractTakesFromPages } = await import('../core/extract-takes-from-pages.ts');
|
||||
const data = (job.data ?? {}) as { sourceId?: string; maxPages?: number };
|
||||
const bootstrapCfg = await engine.getConfig('takes.bootstrap_enabled');
|
||||
@@ -2142,7 +2058,7 @@ export async function registerBuiltinHandlers(
|
||||
// remediation pipeline. Wraps runEmbedCore with stale + catchUp + the
|
||||
// priority/batchSize the recommendation supplies. NOT in
|
||||
// PROTECTED_JOB_NAMES (embedding spend only).
|
||||
registerBuiltinJob(worker, engine, 'embed-catch-up', async (job) => {
|
||||
worker.register('embed-catch-up', async (job) => {
|
||||
const { runEmbedCore } = await import('./embed.ts');
|
||||
const data = (job.data ?? {}) as {
|
||||
sourceId?: string;
|
||||
|
||||
+9
-49
@@ -127,12 +127,7 @@ export function lintContent(content: string, filePath: string, opts: LintContent
|
||||
}
|
||||
|
||||
// Rule: Wrapping code fences (```markdown ... ```)
|
||||
// Detector intentionally has NO /m flag so ^/$ match start/end of the whole
|
||||
// file, not inner lines. Keeps detector in sync with fixContent() below,
|
||||
// which also has no /m flag. Without this, lint reports "fixable" false
|
||||
// positives on any page that simply contains a ```markdown code block, but
|
||||
// fixContent can never strip them (its regex only matches whole-file wrappers).
|
||||
if (content.match(/^```(?:markdown|md)\s*\n/) && content.match(/\n```\s*$/)) {
|
||||
if (content.match(/^```(?:markdown|md)\s*\n/m) && content.match(/\n```\s*$/m)) {
|
||||
issues.push({
|
||||
file: filePath, line: 1, rule: 'code-fence-wrap',
|
||||
message: 'Page wrapped in ```markdown code fences (LLM artifact)',
|
||||
@@ -383,30 +378,15 @@ async function resolveLintContentSanity(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Directories never containing knowledge pages, skipped by default.
|
||||
* Deliberately tiny: only vendored dependency trees qualify. Anything
|
||||
* more opinionated (README.md, CHANGELOG.md, test/) is repo policy —
|
||||
* callers opt in via `--exclude` / `LintOpts.exclude`. Dot- and
|
||||
* underscore-prefixed entries are already skipped by the walk.
|
||||
*/
|
||||
const DEFAULT_LINT_EXCLUDE_DIRS = new Set(['node_modules']);
|
||||
|
||||
/** Collect markdown files from a directory */
|
||||
function collectPages(dir: string, extraExcludes: string[] = []): string[] {
|
||||
const extra = new Set(extraExcludes);
|
||||
function collectPages(dir: string): string[] {
|
||||
const pages: string[] = [];
|
||||
function walk(d: string) {
|
||||
for (const entry of readdirSync(d)) {
|
||||
if (entry.startsWith('.') || entry.startsWith('_')) continue;
|
||||
const full = join(d, entry);
|
||||
if (lstatSync(full).isDirectory()) {
|
||||
if (DEFAULT_LINT_EXCLUDE_DIRS.has(entry) || extra.has(entry)) continue;
|
||||
walk(full);
|
||||
} else if (entry.endsWith('.md')) {
|
||||
if (extra.has(entry)) continue;
|
||||
pages.push(full);
|
||||
}
|
||||
if (lstatSync(full).isDirectory()) walk(full);
|
||||
else if (entry.endsWith('.md')) pages.push(full);
|
||||
}
|
||||
}
|
||||
walk(dir);
|
||||
@@ -434,13 +414,6 @@ export interface LintOpts {
|
||||
* yields + checks this every 200 pages.
|
||||
*/
|
||||
signal?: AbortSignal;
|
||||
/**
|
||||
* #2649: extra dir/file basenames to skip while collecting pages, in
|
||||
* addition to node_modules and dot/underscore entries. For mixed-content
|
||||
* repos (knowledge pages alongside software trees). Ignored for
|
||||
* single-file targets.
|
||||
*/
|
||||
exclude?: string[];
|
||||
}
|
||||
|
||||
export interface LintResult {
|
||||
@@ -467,7 +440,7 @@ export async function runLintCore(opts: LintOpts): Promise<LintResult> {
|
||||
}
|
||||
|
||||
const isSingleFile = statSync(opts.target).isFile();
|
||||
const pages = isSingleFile ? [opts.target] : collectPages(opts.target, opts.exclude ?? []);
|
||||
const pages = isSingleFile ? [opts.target] : collectPages(opts.target);
|
||||
|
||||
// Resolve content-sanity config once for this lint run (D1: lift DB
|
||||
// config when reachable). Caller can pre-pass via opts.contentSanity
|
||||
@@ -518,27 +491,14 @@ export async function runLintCore(opts: LintOpts): Promise<LintResult> {
|
||||
}
|
||||
|
||||
export async function runLint(args: string[]) {
|
||||
// #2649: --exclude=a,b or --exclude a,b — extra basenames to skip.
|
||||
const extraExcludes: string[] = [];
|
||||
const skipIdx = new Set<number>();
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const a = args[i];
|
||||
if (a.startsWith('--exclude=')) {
|
||||
extraExcludes.push(...a.slice('--exclude='.length).split(',').map(s => s.trim()).filter(Boolean));
|
||||
} else if (a === '--exclude' && i + 1 < args.length) {
|
||||
extraExcludes.push(...args[i + 1].split(',').map(s => s.trim()).filter(Boolean));
|
||||
skipIdx.add(i + 1);
|
||||
}
|
||||
}
|
||||
const target = args.find((a, i) => !a.startsWith('--') && !skipIdx.has(i));
|
||||
const target = args.find(a => !a.startsWith('--'));
|
||||
const doFix = args.includes('--fix');
|
||||
const dryRun = args.includes('--dry-run');
|
||||
|
||||
if (!target) {
|
||||
console.error('Usage: gbrain lint <dir|file.md> [--fix] [--dry-run] [--exclude a,b]');
|
||||
console.error('Usage: gbrain lint <dir|file.md> [--fix] [--dry-run]');
|
||||
console.error(' --fix Auto-fix fixable issues (LLM preambles, code fences)');
|
||||
console.error(' --dry-run Preview fixes without writing');
|
||||
console.error(' --exclude Comma-separated dir/file basenames to skip (in addition to node_modules)');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -550,7 +510,7 @@ export async function runLint(args: string[]) {
|
||||
// Single file or directory — print human detail as we go, then rely on
|
||||
// Core for the aggregate numbers at the end.
|
||||
const isSingleFile = statSync(target).isFile();
|
||||
const pages = isSingleFile ? [target] : collectPages(target, extraExcludes);
|
||||
const pages = isSingleFile ? [target] : collectPages(target);
|
||||
|
||||
// Progress on stderr. Stdout keeps the per-issue human output it always had.
|
||||
const { createProgress } = await import('../core/progress.ts');
|
||||
@@ -597,7 +557,7 @@ export async function runLint(args: string[]) {
|
||||
// produces canonical numbers for the summary line).
|
||||
// Pass contentSanity through so runLintCore skips its own resolve
|
||||
// (we already resolved once for the human-detail loop above).
|
||||
const result = await runLintCore({ target, fix: doFix, dryRun, contentSanity, exclude: extraExcludes });
|
||||
const result = await runLintCore({ target, fix: doFix, dryRun, contentSanity });
|
||||
console.log(`\n${result.pages_scanned} pages scanned. ${result.total_issues} issue(s) in ${result.pages_with_issues} page(s).`);
|
||||
if (doFix) {
|
||||
console.log(`${dryRun ? '(dry run) ' : ''}${result.total_fixed} auto-fixed.`);
|
||||
|
||||
@@ -1,224 +0,0 @@
|
||||
/**
|
||||
* gbrain maintain — conservative self-healing maintenance.
|
||||
*
|
||||
* This command automates the safe parts of the operator runbook:
|
||||
* - stale link/timeline extraction
|
||||
* - stale per-source dream cycles when doctor reports cycle_freshness
|
||||
*
|
||||
* It deliberately does NOT mutate source files, apply schema-pack upgrades, or
|
||||
* invent semantic hub links. Those need review or a separate command with an
|
||||
* auditable proposal surface.
|
||||
*/
|
||||
|
||||
import { existsSync } from 'fs';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import type { BrainHealth } from '../core/types.ts';
|
||||
import { buildChecks, computeDoctorReport, type DoctorReport, type Check } from './doctor.ts';
|
||||
import { extractStaleFromDB } from './extract.ts';
|
||||
import { runCycle, type CycleReport } from '../core/cycle.ts';
|
||||
|
||||
type ActionStatus = 'ok' | 'would_apply' | 'applied' | 'blocked' | 'skipped';
|
||||
|
||||
export interface MaintenanceAction {
|
||||
name: string;
|
||||
status: ActionStatus;
|
||||
message: string;
|
||||
details?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface MaintainOptions {
|
||||
json: boolean;
|
||||
safe: boolean;
|
||||
dryRun: boolean;
|
||||
help: boolean;
|
||||
}
|
||||
|
||||
export interface MaintainReport {
|
||||
mode: 'dry-run' | 'safe';
|
||||
before: {
|
||||
health: BrainHealth;
|
||||
doctor: DoctorReport;
|
||||
};
|
||||
actions: MaintenanceAction[];
|
||||
after: {
|
||||
health: BrainHealth;
|
||||
doctor: DoctorReport;
|
||||
};
|
||||
}
|
||||
|
||||
export function parseMaintainArgs(args: string[]): MaintainOptions {
|
||||
const safe = args.includes('--safe');
|
||||
return {
|
||||
json: args.includes('--json'),
|
||||
safe,
|
||||
dryRun: args.includes('--dry-run') || !safe,
|
||||
help: args.includes('--help') || args.includes('-h'),
|
||||
};
|
||||
}
|
||||
|
||||
export function extractCycleFreshnessSourceIds(checks: Check[]): string[] {
|
||||
const ids = new Set<string>();
|
||||
for (const check of checks) {
|
||||
if (check.name !== 'cycle_freshness' || check.status === 'ok') continue;
|
||||
const re = /Source '([^']+)' last cycled/g;
|
||||
for (const match of check.message.matchAll(re)) {
|
||||
const id = match[1]?.trim();
|
||||
if (id) ids.add(id);
|
||||
}
|
||||
}
|
||||
return [...ids].sort();
|
||||
}
|
||||
|
||||
async function buildDoctorReport(engine: BrainEngine): Promise<DoctorReport> {
|
||||
const checks = await buildChecks(engine, ['--json', '--scope=brain']);
|
||||
return computeDoctorReport(checks);
|
||||
}
|
||||
|
||||
async function runStaleExtraction(
|
||||
engine: BrainEngine,
|
||||
beforeHealth: BrainHealth,
|
||||
dryRun: boolean,
|
||||
): Promise<MaintenanceAction> {
|
||||
if (beforeHealth.stale_pages <= 0) {
|
||||
return { name: 'extract_stale', status: 'ok', message: 'No stale pages.' };
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
return {
|
||||
name: 'extract_stale',
|
||||
status: 'would_apply',
|
||||
message: `Would run DB-backed stale extraction for ${beforeHealth.stale_pages} page(s).`,
|
||||
details: { stale_pages: beforeHealth.stale_pages },
|
||||
};
|
||||
}
|
||||
|
||||
const result = await extractStaleFromDB(engine, {
|
||||
dryRun: false,
|
||||
jsonMode: false,
|
||||
includeFrontmatter: false,
|
||||
catchUp: false,
|
||||
});
|
||||
|
||||
return {
|
||||
name: 'extract_stale',
|
||||
status: 'applied',
|
||||
message: `Processed ${result.pagesProcessed} stale page(s); ${result.staleRemaining} remain.`,
|
||||
details: {
|
||||
links_created: result.linksCreated,
|
||||
timeline_created: result.timelineCreated,
|
||||
pages_processed: result.pagesProcessed,
|
||||
stale_remaining: result.staleRemaining,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function runCycleFreshnessMaintenance(
|
||||
engine: BrainEngine,
|
||||
beforeDoctor: DoctorReport,
|
||||
dryRun: boolean,
|
||||
): Promise<MaintenanceAction[]> {
|
||||
const sourceIds = extractCycleFreshnessSourceIds(beforeDoctor.checks);
|
||||
if (sourceIds.length === 0) {
|
||||
return [{ name: 'cycle_freshness', status: 'ok', message: 'All sources cycled recently.' }];
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
return sourceIds.map((sourceId) => ({
|
||||
name: 'cycle_freshness',
|
||||
status: 'would_apply',
|
||||
message: `Would run source-scoped dream cycle for ${sourceId}.`,
|
||||
details: { source_id: sourceId },
|
||||
}));
|
||||
}
|
||||
|
||||
const sources = await engine.listAllSources();
|
||||
const actions: MaintenanceAction[] = [];
|
||||
|
||||
for (const sourceId of sourceIds) {
|
||||
const source = sources.find((s) => s.id === sourceId);
|
||||
const localPath = source?.local_path ?? null;
|
||||
const brainDir = localPath && existsSync(localPath) ? localPath : null;
|
||||
const report: CycleReport = await runCycle(engine, {
|
||||
brainDir,
|
||||
dryRun: false,
|
||||
pull: false,
|
||||
sourceId,
|
||||
});
|
||||
actions.push({
|
||||
name: 'cycle_freshness',
|
||||
status: report.status === 'failed' ? 'blocked' : 'applied',
|
||||
message: `Ran source-scoped dream cycle for ${sourceId}: ${report.status}.`,
|
||||
details: {
|
||||
source_id: sourceId,
|
||||
brain_dir: brainDir,
|
||||
cycle_status: report.status,
|
||||
phases: report.phases.map((p) => ({ phase: p.phase, status: p.status })),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return actions;
|
||||
}
|
||||
|
||||
export async function runMaintain(engine: BrainEngine, args: string[]): Promise<MaintainReport | void> {
|
||||
const opts = parseMaintainArgs(args);
|
||||
if (opts.help) {
|
||||
console.log(`Usage: gbrain maintain [--safe] [--dry-run] [--json]
|
||||
|
||||
Conservative self-healing maintenance.
|
||||
|
||||
Modes:
|
||||
--dry-run Preview safe actions without writes. Default when --safe is absent.
|
||||
--safe Apply safe actions: stale extraction and source cycle freshness.
|
||||
--json Emit a structured before/action/after report.
|
||||
|
||||
Not auto-applied:
|
||||
source-file frontmatter fixes, schema-pack upgrades, atom-pack changes,
|
||||
semantic hub-link guesses, and destructive cleanup.
|
||||
`);
|
||||
return;
|
||||
}
|
||||
|
||||
const beforeHealth = await engine.getHealth();
|
||||
const beforeDoctor = await buildDoctorReport(engine);
|
||||
const actions: MaintenanceAction[] = [];
|
||||
|
||||
actions.push(await runStaleExtraction(engine, beforeHealth, opts.dryRun));
|
||||
actions.push(...await runCycleFreshnessMaintenance(engine, beforeDoctor, opts.dryRun));
|
||||
|
||||
const afterHealth = await engine.getHealth();
|
||||
const afterDoctor = await buildDoctorReport(engine);
|
||||
const report: MaintainReport = {
|
||||
mode: opts.dryRun ? 'dry-run' : 'safe',
|
||||
before: { health: beforeHealth, doctor: beforeDoctor },
|
||||
actions,
|
||||
after: { health: afterHealth, doctor: afterDoctor },
|
||||
};
|
||||
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
} else {
|
||||
printMaintainReport(report);
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
function printMaintainReport(report: MaintainReport): void {
|
||||
console.log(`GBrain maintain (${report.mode})`);
|
||||
console.log(
|
||||
`Before: brain_score=${Math.round(report.before.health.brain_score)}/100 ` +
|
||||
`stale=${report.before.health.stale_pages} islands=${report.before.health.orphan_pages} ` +
|
||||
`doctor=${report.before.doctor.status}`,
|
||||
);
|
||||
for (const action of report.actions) {
|
||||
console.log(` ${action.status}: ${action.name} — ${action.message}`);
|
||||
}
|
||||
console.log(
|
||||
`After: brain_score=${Math.round(report.after.health.brain_score)}/100 ` +
|
||||
`stale=${report.after.health.stale_pages} islands=${report.after.health.orphan_pages} ` +
|
||||
`doctor=${report.after.doctor.status}`,
|
||||
);
|
||||
if (report.mode === 'dry-run') {
|
||||
console.log('Run `gbrain maintain --safe` to apply safe actions.');
|
||||
}
|
||||
}
|
||||
+98
-216
@@ -10,13 +10,12 @@
|
||||
import { createEngine } from '../core/engine-factory.ts';
|
||||
import { loadConfig, saveConfig, toEngineConfig, gbrainPath, effectiveEnvDatabaseUrl, type GBrainConfig } from '../core/config.ts';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import type { EngineConfig, Page } from '../core/types.ts';
|
||||
import type { EngineConfig } from '../core/types.ts';
|
||||
import { writeFileSync, readFileSync, existsSync, unlinkSync } from 'fs';
|
||||
import { createHash } from 'crypto';
|
||||
import { resolve } from 'path';
|
||||
import { createProgress } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
import { setCliExitVerdict } from '../core/cli-force-exit.ts';
|
||||
|
||||
interface MigrateOpts {
|
||||
targetEngine: 'postgres' | 'pglite';
|
||||
@@ -144,99 +143,6 @@ export async function copyMigrationSources(source: BrainEngine, target: BrainEng
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* postgres.js's UNDEFINED_VALUE guard rejects any bound parameter that is JS
|
||||
* `undefined` — unlike PGLite, it will not silently treat it as SQL NULL.
|
||||
* A page read back from a PGLite source can carry `undefined` for a column
|
||||
* that is legitimately empty/NULL (a read-side driver-shape difference, not
|
||||
* a data problem), and passing that value straight into a Postgres
|
||||
* `putPage` throws mid-insert (#3194). Normalizing at this migrate-only
|
||||
* boundary — rather than inside `putPage` itself, which many non-migrate
|
||||
* callers also use — turns that driver-shape difference into an explicit
|
||||
* SQL NULL, so only a genuine NOT-NULL constraint violation (an actual data
|
||||
* problem) still surfaces as a page-copy failure.
|
||||
*/
|
||||
function nullifyUndefinedColumns<T extends Record<string, unknown>>(row: T): T {
|
||||
const normalized = { ...row };
|
||||
for (const key of Object.keys(normalized) as (keyof T)[]) {
|
||||
if (normalized[key] === undefined) normalized[key] = null as T[typeof key];
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy one page's full row (page body, chunks, tags, timeline, raw data)
|
||||
* from source to target. Throws on any failure — the caller (the per-page
|
||||
* loop in runMigrateEngine) decides how to account for that: track it as a
|
||||
* failed page and keep going, rather than letting one bad row silently
|
||||
* disappear from the progress count (#3194). Exported so unit tests can
|
||||
* inject fake engines and exercise the failure path without a live
|
||||
* DATABASE_URL.
|
||||
*/
|
||||
export async function copyPageToTarget(
|
||||
source: BrainEngine,
|
||||
target: BrainEngine,
|
||||
page: Page,
|
||||
): Promise<void> {
|
||||
const sourceOpts = { sourceId: page.source_id };
|
||||
|
||||
// Copy page (preserve source_id). v0.32.8 F8: thread source_id end-to-end
|
||||
// so multi-source pages migrate intact.
|
||||
await target.putPage(page.slug, nullifyUndefinedColumns({
|
||||
type: page.type,
|
||||
title: page.title,
|
||||
compiled_truth: page.compiled_truth,
|
||||
timeline: page.timeline,
|
||||
frontmatter: page.frontmatter,
|
||||
content_hash: page.content_hash,
|
||||
}), sourceOpts);
|
||||
|
||||
// Copy chunks with embeddings.
|
||||
const chunks = await source.getChunksWithEmbeddings(page.slug, sourceOpts);
|
||||
if (chunks.length > 0) {
|
||||
await target.upsertChunks(page.slug, chunks.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
embedding: c.embedding || undefined,
|
||||
model: c.model,
|
||||
token_count: c.token_count || undefined,
|
||||
})), sourceOpts);
|
||||
}
|
||||
|
||||
// Copy tags
|
||||
const tags = await source.getTags(page.slug, sourceOpts);
|
||||
for (const tag of tags) {
|
||||
await target.addTag(page.slug, tag, sourceOpts);
|
||||
}
|
||||
|
||||
// Copy timeline
|
||||
const timeline = await source.getTimeline(page.slug, sourceOpts);
|
||||
for (const entry of timeline) {
|
||||
await target.addTimelineEntry(page.slug, {
|
||||
date: entry.date,
|
||||
source: entry.source,
|
||||
summary: entry.summary,
|
||||
detail: entry.detail,
|
||||
}, sourceOpts);
|
||||
}
|
||||
|
||||
// Copy raw data
|
||||
const rawData = await source.getRawData(page.slug, undefined, sourceOpts);
|
||||
for (const rd of rawData) {
|
||||
await target.putRawData(page.slug, rd.source, rd.data, sourceOpts);
|
||||
}
|
||||
}
|
||||
|
||||
/** A page that failed to copy during migrate — tracked so the run's final
|
||||
* summary reports it honestly instead of letting the "N copied" counter
|
||||
* imply every page landed (#3194). */
|
||||
export interface MigratePageFailure {
|
||||
source_id: string;
|
||||
slug: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]): Promise<void> {
|
||||
const opts = parseArgs(args);
|
||||
const config = loadConfig();
|
||||
@@ -271,47 +177,32 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
|
||||
await targetEngine.connect(targetConfig);
|
||||
await targetEngine.initSchema();
|
||||
|
||||
// Load or create manifest for resume. Checked BEFORE the non-empty-target
|
||||
// guard below: a manifest matching this exact target means the target's
|
||||
// existing rows came from OUR OWN in-progress migration (#3194's per-page
|
||||
// failures now leave the target non-empty by design instead of crashing),
|
||||
// so a resume must not be treated as "attempting to migrate into a
|
||||
// foreign non-empty brain".
|
||||
// Check if target has data
|
||||
const targetStats = await targetEngine.getStats();
|
||||
if (targetStats.page_count > 0 && !opts.force) {
|
||||
console.error(`Target brain is not empty (${targetStats.page_count} pages).`);
|
||||
console.error('Run with --force to overwrite, or migrate to an empty brain.');
|
||||
await targetEngine.disconnect();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (targetStats.page_count > 0 && opts.force) {
|
||||
console.log('--force: wiping target brain...');
|
||||
// v0.18.0+ multi-source: deletePage(slug) is now source-scoped (defaults
|
||||
// to 'default'), so per-page iteration would skip non-default-source
|
||||
// rows. migrate-engine --force is a destructive wipe across the entire
|
||||
// brain — all sources, all pages — so we issue a raw DELETE that matches
|
||||
// the original semantic. Cascades through content_chunks / page_links /
|
||||
// tags / timeline_entries / page_versions via existing FKs.
|
||||
await targetEngine.executeRaw('DELETE FROM pages');
|
||||
}
|
||||
|
||||
// Load or create manifest for resume
|
||||
let manifest = loadManifest();
|
||||
if (manifest && !manifestMatchesTarget(manifest, targetId)) {
|
||||
console.log('Previous migration was to a different target. Starting fresh.');
|
||||
manifest = null;
|
||||
}
|
||||
const resumingMatchingManifest = manifest !== null;
|
||||
|
||||
// Check if target has data
|
||||
const targetStats = await targetEngine.getStats();
|
||||
if (opts.force) {
|
||||
if (targetStats.page_count > 0) {
|
||||
console.log('--force: wiping target brain...');
|
||||
// v0.18.0+ multi-source: deletePage(slug) is now source-scoped (defaults
|
||||
// to 'default'), so per-page iteration would skip non-default-source
|
||||
// rows. migrate-engine --force is a destructive wipe across the entire
|
||||
// brain — all sources, all pages — so we issue a raw DELETE that matches
|
||||
// the original semantic. Cascades through content_chunks / page_links /
|
||||
// tags / timeline_entries / page_versions via existing FKs.
|
||||
await targetEngine.executeRaw('DELETE FROM pages');
|
||||
}
|
||||
// --force always starts this exact migration fresh against this target:
|
||||
// a manifest tracking a previous attempt must not be trusted to skip
|
||||
// pages, regardless of whether the target LOOKED non-empty just now
|
||||
// (e.g. the target DB file was recreated out-of-band but
|
||||
// ~/.gbrain/migrate-manifest.json survived) — round 2 of #3194.
|
||||
manifest = null;
|
||||
} else if (targetStats.page_count > 0 && !resumingMatchingManifest) {
|
||||
console.error(`Target brain is not empty (${targetStats.page_count} pages).`);
|
||||
console.error('Run with --force to overwrite, or migrate to an empty brain.');
|
||||
await targetEngine.disconnect();
|
||||
process.exit(1);
|
||||
} else if (targetStats.page_count > 0 && resumingMatchingManifest) {
|
||||
console.log(`Resuming previous migration: ${manifest!.completed_slugs.length} page(s) already copied.`);
|
||||
}
|
||||
|
||||
// v0.32.8 F8: manifest keys are now `${source_id}::${slug}` so multi-source
|
||||
// migrations don't collide on same-slug-different-source pages. Pre-v0.32.8
|
||||
// entries were bare slugs; we keep treating those as default-source for
|
||||
@@ -328,13 +219,6 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
|
||||
started_at: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
// Persist immediately, before any page copy runs. Otherwise a run where
|
||||
// EVERY page fails after its putPage lands (but before completed_slugs
|
||||
// ever gets a successful entry) leaves the target non-empty with no
|
||||
// manifest file on disk at all — the next invocation can't tell this
|
||||
// was a resumable in-progress migration and hits the non-empty guard
|
||||
// above requiring --force (round 2 of #3194).
|
||||
saveManifest(manifest);
|
||||
|
||||
// Pages.source_id is a foreign key. Copy the complete source catalog first,
|
||||
// including archived rows and sync/routing metadata, so every page write has
|
||||
@@ -351,68 +235,82 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
|
||||
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
|
||||
progress.start('migrate.copy_pages', pagesToMigrate.length);
|
||||
|
||||
// v0.32.8 F8: thread source_id end-to-end so multi-source pages migrate
|
||||
// intact. Pre-fix: putPage / getTags / getTimeline / getRawData / getLinks
|
||||
// all silently defaulted to source_id='default', so non-default-source
|
||||
// tags / timeline / raw / links were either dropped or attached to the
|
||||
// wrong row.
|
||||
let migrated = 0;
|
||||
const failures: MigratePageFailure[] = [];
|
||||
for (const page of pagesToMigrate) {
|
||||
try {
|
||||
await copyPageToTarget(sourceEngine, targetEngine, page);
|
||||
// Track progress with composite key so multi-source resume is correct.
|
||||
manifest!.completed_slugs.push(makeManifestKey(page.source_id, page.slug));
|
||||
saveManifest(manifest!);
|
||||
migrated++;
|
||||
} catch (e) {
|
||||
// #3194: a per-page write failure must never be swallowed into the
|
||||
// success count. Leave it OUT of completed_slugs (a resume retries
|
||||
// it — putPage/upsertChunks/etc. are all upserts, so re-running the
|
||||
// whole page copy is safe) and surface it in the final summary below
|
||||
// instead of letting "N pages copied" imply everything landed.
|
||||
failures.push({
|
||||
source_id: page.source_id,
|
||||
slug: page.slug,
|
||||
reason: e instanceof Error ? e.message : String(e),
|
||||
});
|
||||
// v0.32.8 F8: thread source_id end-to-end so multi-source pages migrate
|
||||
// intact. Pre-fix: putPage / getTags / getTimeline / getRawData / getLinks
|
||||
// all silently defaulted to source_id='default', so non-default-source
|
||||
// tags / timeline / raw / links were either dropped or attached to the
|
||||
// wrong row.
|
||||
const sourceOpts = { sourceId: page.source_id };
|
||||
|
||||
// Copy page (preserve source_id)
|
||||
await targetEngine.putPage(page.slug, {
|
||||
type: page.type,
|
||||
title: page.title,
|
||||
compiled_truth: page.compiled_truth,
|
||||
timeline: page.timeline,
|
||||
frontmatter: page.frontmatter,
|
||||
content_hash: page.content_hash,
|
||||
}, sourceOpts);
|
||||
|
||||
// Copy chunks with embeddings.
|
||||
const chunks = await sourceEngine.getChunksWithEmbeddings(page.slug, sourceOpts);
|
||||
if (chunks.length > 0) {
|
||||
await targetEngine.upsertChunks(page.slug, chunks.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
embedding: c.embedding || undefined,
|
||||
model: c.model,
|
||||
token_count: c.token_count || undefined,
|
||||
})), sourceOpts);
|
||||
}
|
||||
|
||||
// Copy tags
|
||||
const tags = await sourceEngine.getTags(page.slug, sourceOpts);
|
||||
for (const tag of tags) {
|
||||
await targetEngine.addTag(page.slug, tag, sourceOpts);
|
||||
}
|
||||
|
||||
// Copy timeline
|
||||
const timeline = await sourceEngine.getTimeline(page.slug, sourceOpts);
|
||||
for (const entry of timeline) {
|
||||
await targetEngine.addTimelineEntry(page.slug, {
|
||||
date: entry.date,
|
||||
source: entry.source,
|
||||
summary: entry.summary,
|
||||
detail: entry.detail,
|
||||
}, sourceOpts);
|
||||
}
|
||||
|
||||
// Copy raw data
|
||||
const rawData = await sourceEngine.getRawData(page.slug, undefined, sourceOpts);
|
||||
for (const rd of rawData) {
|
||||
await targetEngine.putRawData(page.slug, rd.source, rd.data, sourceOpts);
|
||||
}
|
||||
|
||||
// Copy versions
|
||||
const versions = await sourceEngine.getVersions(page.slug, sourceOpts);
|
||||
// Versions are snapshots, we recreate them on the target
|
||||
// (createVersion takes a snapshot of current state, which we just set)
|
||||
|
||||
// Track progress with composite key so multi-source resume is correct.
|
||||
manifest!.completed_slugs.push(makeManifestKey(page.source_id, page.slug));
|
||||
saveManifest(manifest!);
|
||||
migrated++;
|
||||
progress.tick(1, page.slug);
|
||||
}
|
||||
progress.finish();
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(`\n${failures.length} of ${pagesToMigrate.length} page(s) FAILED to copy and were NOT migrated:`);
|
||||
for (const f of failures) {
|
||||
const key = f.source_id === 'default' ? f.slug : `${f.source_id}::${f.slug}`;
|
||||
console.error(` - ${key}: ${f.reason}`);
|
||||
}
|
||||
console.error('Re-run `gbrain migrate` to retry the failed pages (already-copied pages resume via the manifest).');
|
||||
// Non-fatal so the run still copies links + config for everything that
|
||||
// DID land, but the process must exit non-zero — a partial migration
|
||||
// must never look identical to a clean one.
|
||||
setCliExitVerdict(1);
|
||||
}
|
||||
|
||||
// Copy links (after all pages exist in target).
|
||||
// v0.32.8 F8: thread source_id so cross-source links migrate correctly.
|
||||
// #3194: a page that failed to copy above does NOT exist on the target,
|
||||
// so any link touching it would violate the target's FK and abort this
|
||||
// whole phase (the exact "addLink failed: page ... not found" crash from
|
||||
// the original report). Skip links on either end of a known-failed page —
|
||||
// a retry that successfully copies the page also re-copies its links.
|
||||
const failedKeys = new Set(failures.map(f => makeManifestKey(f.source_id, f.slug)));
|
||||
console.log('Copying links...');
|
||||
progress.start('migrate.copy_links', allPages.length);
|
||||
for (const page of allPages) {
|
||||
if (failedKeys.has(makeManifestKey(page.source_id, page.slug))) {
|
||||
progress.tick(1);
|
||||
continue;
|
||||
}
|
||||
const sourceOpts = { sourceId: page.source_id };
|
||||
const links = await sourceEngine.getLinks(page.slug, sourceOpts);
|
||||
for (const link of links) {
|
||||
if (failedKeys.has(makeManifestKey(page.source_id, link.to_slug))) continue;
|
||||
await targetEngine.addLink(
|
||||
link.from_slug, link.to_slug,
|
||||
link.context, link.link_type,
|
||||
@@ -444,38 +342,22 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
|
||||
// Update local config. v0.37 fix wave: preserve existing file-plane
|
||||
// embedding/expansion/chat config across the engine migration; only
|
||||
// the engine + connection target should change.
|
||||
//
|
||||
// #3194: only flip the ACTIVE config when the migration is fully clean.
|
||||
// A partial migration leaves the target's data incomplete; auto-switching
|
||||
// every subsequent `gbrain` invocation onto that incomplete target would
|
||||
// (a) make the failure invisible behind otherwise-normal usage and (b)
|
||||
// break the natural retry — `gbrain migrate --to X` again would hit the
|
||||
// "Already using X engine" guard even though the migration never actually
|
||||
// finished. Leaving the file-plane config untouched keeps the source the
|
||||
// active engine, so a retry (which resumes via the still-intact manifest)
|
||||
// is a same-shaped command, not a special case.
|
||||
if (failures.length === 0) {
|
||||
const existingFile = (await import('../core/config.ts')).loadConfigFileOnly() ?? ({} as GBrainConfig);
|
||||
const newConfig: GBrainConfig = {
|
||||
...existingFile,
|
||||
engine: opts.targetEngine,
|
||||
...(opts.targetEngine === 'postgres'
|
||||
? { database_url: targetConfig.database_url, database_path: undefined }
|
||||
: { database_path: targetConfig.database_path, database_url: undefined }),
|
||||
};
|
||||
saveConfig(newConfig);
|
||||
// Clean up the resume manifest — only safe once nothing is left pending.
|
||||
clearManifest();
|
||||
}
|
||||
const existingFile = (await import('../core/config.ts')).loadConfigFileOnly() ?? ({} as GBrainConfig);
|
||||
const newConfig: GBrainConfig = {
|
||||
...existingFile,
|
||||
engine: opts.targetEngine,
|
||||
...(opts.targetEngine === 'postgres'
|
||||
? { database_url: targetConfig.database_url, database_path: undefined }
|
||||
: { database_path: targetConfig.database_path, database_url: undefined }),
|
||||
};
|
||||
saveConfig(newConfig);
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.log(`\nMigration completed with errors. ${migrated} of ${pagesToMigrate.length} pages copied, ${failures.length} failed (${completedSet.size} already done from a prior run). See failure list above.`);
|
||||
console.log(`Config NOT switched — still using engine: ${config.engine}. Re-run \`gbrain migrate --to ${opts.targetEngine}\` to retry; already-copied pages resume via the manifest.`);
|
||||
} else {
|
||||
console.log(`\nMigration complete. ${migrated} pages transferred.`);
|
||||
console.log(`Config updated to engine: ${opts.targetEngine}`);
|
||||
}
|
||||
if (failures.length === 0 && config.engine === 'pglite' && config.database_path) {
|
||||
// Clean up
|
||||
clearManifest();
|
||||
|
||||
console.log(`\nMigration complete. ${migrated} pages transferred.`);
|
||||
console.log(`Config updated to engine: ${opts.targetEngine}`);
|
||||
if (config.engine === 'pglite' && config.database_path) {
|
||||
console.log(`Original PGLite brain preserved at ${config.database_path} (backup).`);
|
||||
}
|
||||
|
||||
|
||||
@@ -186,6 +186,17 @@ async function phaseBFenceFacts(
|
||||
const localPathById = new Map<string, string | null>();
|
||||
for (const s of sources) localPathById.set(s.id, s.local_path);
|
||||
|
||||
// Dirty-tree refusal: check every source's local_path before writing.
|
||||
for (const [id, localPath] of localPathById) {
|
||||
if (localPath && isLocalPathDirty(localPath)) {
|
||||
return {
|
||||
name: 'fence_facts',
|
||||
status: 'failed',
|
||||
detail: `source "${id}" has uncommitted changes in ${localPath}. Commit or stash, then re-run.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Walk legacy rows in (source_id, entity_slug) groups for per-page
|
||||
// atomic writes.
|
||||
const legacy = await engine.executeRaw<LegacyFactRow>(
|
||||
@@ -224,21 +235,6 @@ async function phaseBFenceFacts(
|
||||
groups.set(key, list);
|
||||
}
|
||||
|
||||
// Dirty-tree refusal: check ONLY the sources we are about to write
|
||||
// into. A dirty tree in an unrelated source (or zero fenceable rows
|
||||
// at all) must not block a no-op or a targeted backfill (#927).
|
||||
const targetSourceIds = new Set([...groups.keys()].map(k => k.split('\0')[0]));
|
||||
for (const id of targetSourceIds) {
|
||||
const localPath = localPathById.get(id);
|
||||
if (localPath && isLocalPathDirty(localPath)) {
|
||||
return {
|
||||
name: 'fence_facts',
|
||||
status: 'failed',
|
||||
detail: `source "${id}" has uncommitted changes in ${localPath}. Commit or stash, then re-run.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
for (const [key, group] of groups) {
|
||||
const [sourceId, entitySlug] = key.split('\0');
|
||||
const localPath = localPathById.get(sourceId)!;
|
||||
|
||||
+1
-14
@@ -536,20 +536,7 @@ function shouldSkipProvider(modelStr: string, skip: string[]): boolean {
|
||||
|
||||
export async function runModels(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
const json = args.includes('--json');
|
||||
// args is `subArgs` from cli.ts `handleCliOnly` — the leading 'models'
|
||||
// token has already been stripped. The subcommand is at args[0], NOT
|
||||
// args[1]. Pre-fix this check was `args[1]`, so `gbrain models doctor`
|
||||
// silently fell through to the read view. The doctor probe path was
|
||||
// unreachable from the CLI.
|
||||
//
|
||||
// --help honored FIRST so `gbrain models doctor --help` shows usage
|
||||
// instead of running network probes (which would spend tokens or
|
||||
// exit nonzero when the user only asked for help). Pre-fix the
|
||||
// args[1] ternary happened to dodge this by always falling through
|
||||
// to the args.includes('--help') branch; the args[0] rewrite needs
|
||||
// explicit ordering to preserve that behavior.
|
||||
const hasHelp = args.includes('--help') || args.includes('-h') || args[0] === 'help';
|
||||
const sub = hasHelp ? 'help' : args[0] === 'doctor' ? 'doctor' : 'read';
|
||||
const sub = args[1] === 'doctor' ? 'doctor' : args[1] === 'help' || args.includes('--help') || args.includes('-h') ? 'help' : 'read';
|
||||
|
||||
if (sub === 'help') {
|
||||
process.stdout.write(
|
||||
|
||||
@@ -142,16 +142,12 @@ export async function runOnboard(engine: BrainEngine, args: string[]): Promise<v
|
||||
|
||||
// --auto path: runs through the T2 library orchestrator. Hooks emit CLI
|
||||
// progress to stderr; the final result lands as JSON on stdout (or human
|
||||
// summary). extraRemediations (gathered above from runAllOnboardChecks)
|
||||
// is threaded into the runner so the onboard-check remediations
|
||||
// (extract-ner, extract-timeline-from-meetings, etc.) reach the planner
|
||||
// — the same wiring the --check path uses above.
|
||||
// summary).
|
||||
const result = await runRemediation(
|
||||
engine,
|
||||
{
|
||||
targetScore,
|
||||
maxUsd,
|
||||
extraRemediations,
|
||||
// --auto --yes opts into the prompt_required tier too; library
|
||||
// doesn't distinguish auto_apply vs prompt_required, it just runs
|
||||
// every remediation in the plan. The plan-building side (T12 render)
|
||||
|
||||
+55
-10
@@ -15,11 +15,6 @@
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { createProgress, startHeartbeat } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
import {
|
||||
shouldExcludeFromOrphanReporting,
|
||||
loadOrphanPolicyOverrides,
|
||||
type OrphanPolicyOverrides,
|
||||
} from '../core/orphan-policy.ts';
|
||||
|
||||
// --- Types ---
|
||||
|
||||
@@ -37,14 +32,65 @@ export interface OrphanResult {
|
||||
excluded: number;
|
||||
}
|
||||
|
||||
// --- Filter constants ---
|
||||
|
||||
/** Slug suffixes that are always auto-generated root files */
|
||||
const AUTO_SUFFIX_PATTERNS = ['/_index', '/log'];
|
||||
|
||||
/** Page slugs that are pseudo-pages by convention */
|
||||
const PSEUDO_SLUGS = new Set(['_atlas', '_index', '_stats', '_orphans', '_scratch', 'claude']);
|
||||
|
||||
/** Slug segment that marks raw sources */
|
||||
const RAW_SEGMENT = '/raw/';
|
||||
|
||||
/** Slug prefixes where no inbound links is expected */
|
||||
const DENY_PREFIXES = [
|
||||
'output/',
|
||||
'dashboards/',
|
||||
'scripts/',
|
||||
'templates/',
|
||||
'openclaw/config/',
|
||||
];
|
||||
|
||||
/** First slug segments where no inbound links is expected */
|
||||
const FIRST_SEGMENT_EXCLUSIONS = new Set([
|
||||
'scratch',
|
||||
'thoughts',
|
||||
'catalog',
|
||||
'entities',
|
||||
'raw',
|
||||
'atoms',
|
||||
'skills',
|
||||
]);
|
||||
|
||||
// --- Filter logic ---
|
||||
|
||||
/**
|
||||
* Returns true if a slug should be excluded from orphan reporting by default.
|
||||
* These are pages where having no inbound links is expected / not a content problem.
|
||||
*/
|
||||
export function shouldExclude(slug: string, overrides?: OrphanPolicyOverrides): boolean {
|
||||
return shouldExcludeFromOrphanReporting(slug, overrides);
|
||||
export function shouldExclude(slug: string): boolean {
|
||||
// Pseudo-pages (exact match)
|
||||
if (PSEUDO_SLUGS.has(slug)) return true;
|
||||
|
||||
// Auto-generated suffix patterns
|
||||
for (const suffix of AUTO_SUFFIX_PATTERNS) {
|
||||
if (slug.endsWith(suffix)) return true;
|
||||
}
|
||||
|
||||
// Raw source slugs
|
||||
if (slug.includes(RAW_SEGMENT)) return true;
|
||||
|
||||
// Deny-prefix slugs
|
||||
for (const prefix of DENY_PREFIXES) {
|
||||
if (slug.startsWith(prefix)) return true;
|
||||
}
|
||||
|
||||
// First-segment exclusions
|
||||
const firstSegment = slug.split('/')[0];
|
||||
if (FIRST_SEGMENT_EXCLUSIONS.has(firstSegment)) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -110,7 +156,6 @@ export async function findOrphans(
|
||||
let allOrphans: { slug: string; title: string; domain: string | null }[];
|
||||
let total: number;
|
||||
let excludedAll: number;
|
||||
const overrides = includePseudo ? undefined : await loadOrphanPolicyOverrides(engine);
|
||||
try {
|
||||
allOrphans = await engine.findOrphanPages(
|
||||
sourceIds ? { sourceIds } : sourceId ? { sourceId } : undefined,
|
||||
@@ -139,7 +184,7 @@ export async function findOrphans(
|
||||
total = liveRows.length;
|
||||
excludedAll = includePseudo
|
||||
? 0
|
||||
: liveRows.reduce((n, r) => n + (shouldExclude(r.slug, overrides) ? 1 : 0), 0);
|
||||
: liveRows.reduce((n, r) => n + (shouldExclude(r.slug) ? 1 : 0), 0);
|
||||
} finally {
|
||||
stopHb();
|
||||
progress.finish();
|
||||
@@ -147,7 +192,7 @@ export async function findOrphans(
|
||||
|
||||
const filtered = includePseudo
|
||||
? allOrphans
|
||||
: allOrphans.filter(row => !shouldExclude(row.slug, overrides));
|
||||
: allOrphans.filter(row => !shouldExclude(row.slug));
|
||||
|
||||
const orphans: OrphanPage[] = filtered.map(row => ({
|
||||
slug: row.slug,
|
||||
|
||||
+14
-43
@@ -7,7 +7,6 @@
|
||||
|
||||
import { listRecipes, getRecipe } from '../core/ai/recipes/index.ts';
|
||||
import { configureGateway, embedOne, isAvailable as gwIsAvailable, chat as gwChat } from '../core/ai/gateway.ts';
|
||||
import { buildGatewayConfig } from '../core/ai/build-gateway-config.ts';
|
||||
import { probeOllama, probeLMStudio } from '../core/ai/probes.ts';
|
||||
import { loadConfig } from '../core/config.ts';
|
||||
import { AIConfigError, AITransientError } from '../core/ai/errors.ts';
|
||||
@@ -34,19 +33,16 @@ interface ProviderOption {
|
||||
|
||||
function configureFromEnv(): void {
|
||||
const config = loadConfig();
|
||||
// Route through buildGatewayConfig — the single ownership seam that folds
|
||||
// file-plane API keys (openrouter_api_key, zeroentropy_api_key, ...) into
|
||||
// the gateway env — instead of hand-assembling AIGatewayConfig field by
|
||||
// field. Hand-building it here let this diagnostic report a provider as
|
||||
// missing env even when ~/.gbrain/config.json had it and the real gateway
|
||||
// path resolved it fine (#2728). Pre-init (no file-plane config yet) falls
|
||||
// back to a bare env passthrough so the command still works before
|
||||
// `gbrain init`.
|
||||
if (config) {
|
||||
configureGateway(buildGatewayConfig(config));
|
||||
return;
|
||||
}
|
||||
configureGateway({ env: { ...process.env } });
|
||||
configureGateway({
|
||||
embedding_model: config?.embedding_model,
|
||||
embedding_dimensions: config?.embedding_dimensions,
|
||||
expansion_model: config?.expansion_model,
|
||||
chat_model: config?.chat_model,
|
||||
chat_fallback_chain: config?.chat_fallback_chain,
|
||||
base_urls: config?.provider_base_urls,
|
||||
provider_chat_options: config?.provider_chat_options,
|
||||
env: { ...process.env },
|
||||
});
|
||||
}
|
||||
|
||||
export function envReady(recipe: Recipe, env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
@@ -141,12 +137,7 @@ EXAMPLES
|
||||
}
|
||||
|
||||
function runList(_args: string[]): void {
|
||||
// Same env the gateway actually sees (file-plane keys folded in), not bare
|
||||
// process.env — keeps this table's STATUS column honest with what
|
||||
// `providers test` (and the real init/gateway path) would report.
|
||||
const cfg = loadConfig();
|
||||
const env = cfg ? buildGatewayConfig(cfg).env : process.env;
|
||||
console.log(formatRecipeTable(listRecipes(), env));
|
||||
console.log(formatRecipeTable(listRecipes()));
|
||||
}
|
||||
|
||||
async function runTest(args: string[]): Promise<void> {
|
||||
@@ -173,18 +164,8 @@ async function runTest(args: string[]): Promise<void> {
|
||||
// the divergence at the top of the test so the recovery experience
|
||||
// doesn't repeat the bug-reporter's "providers test ✓ but import still
|
||||
// broken" trap.
|
||||
//
|
||||
// #2863: `cfg` is lifted out of the try block (not just used for the
|
||||
// warning) so the configureGateway calls below can reuse it. Before this
|
||||
// fix, the --model override only forwarded embedding_model/chat_model +
|
||||
// env, dropping config.provider_base_urls entirely — a probe against a
|
||||
// custom endpoint (e.g. a regional DashScope base URL) would silently
|
||||
// fall back to the recipe's hardcoded default endpoint and fail with a
|
||||
// misleading "Incorrect API key" error even though the key was valid for
|
||||
// the configured endpoint.
|
||||
let cfg: ReturnType<typeof loadConfig> | null = null;
|
||||
try {
|
||||
cfg = loadConfig();
|
||||
const cfg = loadConfig();
|
||||
const configuredModel = tpArg === 'embedding' ? cfg?.embedding_model : cfg?.chat_model;
|
||||
if (!configuredModel) {
|
||||
console.error(
|
||||
@@ -200,27 +181,17 @@ async function runTest(args: string[]): Promise<void> {
|
||||
}
|
||||
} catch { /* loadConfig throws when no brain configured — first-time install path; the no-config branch above handles it. */ }
|
||||
|
||||
// Reuse the SAME resolver the production path uses (buildGatewayConfig —
|
||||
// also used by cli.ts#connectEngine and init-embed-check.ts) so the probe
|
||||
// sees the identical base_urls / provider_chat_options / folded API keys
|
||||
// that a real `gbrain import`/`gbrain query` call would. Only the
|
||||
// touchpoint's model (+ embedding dims) is overridden on top, so an
|
||||
// isolated `--model` probe still targets exactly the requested model —
|
||||
// it just resolves that model's endpoint the way the brain actually
|
||||
// would. Falls back to bare env when no brain is configured yet (cfg is
|
||||
// null on first-time install, matching the old behavior for that case).
|
||||
const baseGatewayConfig = cfg ? buildGatewayConfig(cfg) : { env: { ...process.env } };
|
||||
if (tpArg === 'embedding') {
|
||||
const dims = recipe?.touchpoints.embedding?.default_dims ?? 1536;
|
||||
configureGateway({
|
||||
...baseGatewayConfig,
|
||||
embedding_model: modelArg,
|
||||
embedding_dimensions: dims,
|
||||
env: { ...process.env },
|
||||
});
|
||||
} else {
|
||||
configureGateway({
|
||||
...baseGatewayConfig,
|
||||
chat_model: modelArg,
|
||||
env: { ...process.env },
|
||||
});
|
||||
}
|
||||
void modelId; // intentionally unused but preserved for readability
|
||||
|
||||
@@ -1,282 +0,0 @@
|
||||
/**
|
||||
* `gbrain reindex-search-vector` — recreate FTS trigger functions and
|
||||
* backfill existing rows under the language configured via
|
||||
* GBRAIN_FTS_LANGUAGE.
|
||||
*
|
||||
* Why this command exists: schema migration v123 (configurable_fts_language)
|
||||
* stamps the trigger functions with the configured language at first apply.
|
||||
* After that, changing the env var has no effect on the write side because
|
||||
* v123 already shows as "applied" — the migrations runner will skip it.
|
||||
* This command is the documented escape hatch: it re-runs the same
|
||||
* recreate-and-backfill logic v123 uses, gated on an explicit user
|
||||
* action so the operation is intentional and visible (writes touch
|
||||
* every row in pages and content_chunks).
|
||||
*
|
||||
* Idempotent: running twice with the same GBRAIN_FTS_LANGUAGE produces
|
||||
* the same trigger function bodies and the same tokenized vectors.
|
||||
*
|
||||
* Flags:
|
||||
* --dry-run Show what would happen, exit 0 without touching DB.
|
||||
* --yes Skip interactive [y/N]. Required for non-TTY (including --json).
|
||||
* --json Machine-readable result envelope. Does NOT imply --yes.
|
||||
*
|
||||
* Backfill runs in id-keyset batches (BACKFILL_BATCH_SIZE rows per UPDATE)
|
||||
* so a large brain never holds one giant row lock, and streams progress
|
||||
* through the shared reporter (stderr; stdout stays clean for --json).
|
||||
*
|
||||
* Cost: trigger recreate is sub-millisecond. Backfill is one tsvector
|
||||
* rebuild per page + per chunk. On a 20K-page brain with 80K chunks,
|
||||
* expect ~5-15s depending on Postgres CPU and content size.
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { getFtsLanguage } from '../core/fts-language.ts';
|
||||
import { createInterface } from 'readline';
|
||||
import { createProgress } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
|
||||
export interface ReindexSearchVectorOpts {
|
||||
dryRun?: boolean;
|
||||
yes?: boolean;
|
||||
json?: boolean;
|
||||
}
|
||||
|
||||
export interface ReindexSearchVectorResult {
|
||||
status: 'ok' | 'dry_run' | 'cancelled';
|
||||
language: string;
|
||||
pagesUpdated: number;
|
||||
chunksUpdated: number;
|
||||
triggersRecreated: number;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
interface CountRow {
|
||||
pages: number;
|
||||
chunks: number;
|
||||
}
|
||||
|
||||
/** Rows per backfill UPDATE. Keyset-batched so one statement never locks the whole table. */
|
||||
export const BACKFILL_BATCH_SIZE = 5000;
|
||||
|
||||
/**
|
||||
* Keyset-batched UPDATE: applies `setClause` to `table` rows where
|
||||
* search_vector IS NOT NULL, BACKFILL_BATCH_SIZE ids at a time, ticking
|
||||
* the shared progress reporter after each batch. Terminates when a batch
|
||||
* returns fewer rows than the batch size (or none).
|
||||
*/
|
||||
async function batchedBackfill(
|
||||
engine: BrainEngine,
|
||||
table: 'pages' | 'content_chunks',
|
||||
setClause: string,
|
||||
tick: (n: number) => void
|
||||
): Promise<void> {
|
||||
let cursor = 0;
|
||||
for (;;) {
|
||||
const rows = await engine.executeRaw<{ id: number }>(`
|
||||
UPDATE ${table} SET ${setClause}
|
||||
WHERE id IN (
|
||||
SELECT id FROM ${table}
|
||||
WHERE search_vector IS NOT NULL AND id > ${cursor}
|
||||
ORDER BY id
|
||||
LIMIT ${BACKFILL_BATCH_SIZE}
|
||||
)
|
||||
RETURNING id
|
||||
`);
|
||||
if (rows.length === 0) break;
|
||||
tick(rows.length);
|
||||
cursor = rows.reduce((m, r) => Math.max(m, Number(r.id)), cursor);
|
||||
if (rows.length < BACKFILL_BATCH_SIZE) break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Programmatic entrypoint — takes a typed opts object. Used by tests and
|
||||
* future internal callers. The CLI wrapper is `runReindexSearchVectorCli`
|
||||
* defined at the bottom of this file.
|
||||
*/
|
||||
export async function runReindexSearchVector(
|
||||
engine: BrainEngine,
|
||||
opts: ReindexSearchVectorOpts
|
||||
): Promise<ReindexSearchVectorResult> {
|
||||
const lang = getFtsLanguage();
|
||||
const startedAt = Date.now();
|
||||
|
||||
// Inventory: how many rows will the backfill touch?
|
||||
const counts = await engine.executeRaw<CountRow>(
|
||||
`SELECT
|
||||
(SELECT COUNT(*)::int FROM pages WHERE search_vector IS NOT NULL) AS pages,
|
||||
(SELECT COUNT(*)::int FROM content_chunks WHERE search_vector IS NOT NULL) AS chunks`
|
||||
);
|
||||
const pagesCount = counts[0]?.pages ?? 0;
|
||||
const chunksCount = counts[0]?.chunks ?? 0;
|
||||
|
||||
if (opts.dryRun) {
|
||||
const result: ReindexSearchVectorResult = {
|
||||
status: 'dry_run',
|
||||
language: lang,
|
||||
pagesUpdated: pagesCount,
|
||||
chunksUpdated: chunksCount,
|
||||
triggersRecreated: 0,
|
||||
durationMs: Date.now() - startedAt,
|
||||
};
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
console.log(`[dry-run] Would recreate 2 trigger functions with language='${lang}'`);
|
||||
console.log(`[dry-run] Would backfill ${pagesCount} pages + ${chunksCount} chunks`);
|
||||
console.log(`[dry-run] Skipping all DB writes. Pass --yes to apply.`);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Confirm unless --yes. --json does NOT bypass the gate — a machine
|
||||
// caller must pass --yes explicitly (mirrors reindex-code, #1784).
|
||||
if (!opts.yes) {
|
||||
if (!process.stdin.isTTY) {
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify({
|
||||
error: {
|
||||
class: 'ConfirmationRequired',
|
||||
code: 'reindex_requires_yes',
|
||||
message: `Refusing to recreate FTS triggers + backfill ${pagesCount} pages + ${chunksCount} chunks without --yes in a non-TTY environment.`,
|
||||
hint: 'Pass --yes to proceed, or --dry-run to preview.',
|
||||
},
|
||||
language: lang,
|
||||
pages: pagesCount,
|
||||
chunks: chunksCount,
|
||||
}));
|
||||
} else {
|
||||
console.error('Refusing to run without --yes in non-TTY environment.');
|
||||
}
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
||||
const answer = await new Promise<string>(resolve => {
|
||||
rl.question(
|
||||
`Recreate FTS triggers with language='${lang}' and backfill ${pagesCount} pages + ${chunksCount} chunks? [y/N]: `,
|
||||
resolve
|
||||
);
|
||||
});
|
||||
rl.close();
|
||||
|
||||
if (!/^y(es)?$/i.test(answer.trim())) {
|
||||
const result: ReindexSearchVectorResult = {
|
||||
status: 'cancelled',
|
||||
language: lang,
|
||||
pagesUpdated: 0,
|
||||
chunksUpdated: 0,
|
||||
triggersRecreated: 0,
|
||||
durationMs: Date.now() - startedAt,
|
||||
};
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
console.log('Cancelled.');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
// Recreate trigger functions. The strings are intentionally identical to
|
||||
// the v124 migration body — keeping them in lockstep is the contract.
|
||||
// `SET search_path = pg_catalog, public` mirrors the v120/#1647 hardening:
|
||||
// CREATE OR REPLACE resets proconfig, so omitting it here would strip the
|
||||
// hardening from every brain that runs this command.
|
||||
//
|
||||
// #2704: compiled_truth (the unbounded whole-page body) is deliberately
|
||||
// NOT indexed here — it overflows Postgres's 1MB tsvector cap on large
|
||||
// pages, and content_chunks.search_vector (populated separately, chunk-
|
||||
// grain, well under the cap) is what searchKeyword() actually queries.
|
||||
// See migrate.ts's v124 for the full rationale; keep this copy in sync.
|
||||
const recreatePagesFn = `
|
||||
CREATE OR REPLACE FUNCTION update_page_search_vector() RETURNS trigger SET search_path = pg_catalog, public AS $fn$
|
||||
DECLARE
|
||||
timeline_text TEXT;
|
||||
BEGIN
|
||||
SELECT coalesce(string_agg(summary || ' ' || detail, ' '), '')
|
||||
INTO timeline_text
|
||||
FROM timeline_entries
|
||||
WHERE page_id = NEW.id;
|
||||
|
||||
NEW.search_vector :=
|
||||
setweight(to_tsvector('${lang}', coalesce(NEW.title, '')), 'A') ||
|
||||
setweight(to_tsvector('${lang}', coalesce(NEW.timeline, '')), 'C') ||
|
||||
setweight(to_tsvector('${lang}', coalesce(timeline_text, '')), 'C');
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$fn$ LANGUAGE plpgsql;
|
||||
`;
|
||||
|
||||
const recreateChunksFn = `
|
||||
CREATE OR REPLACE FUNCTION update_chunk_search_vector() RETURNS TRIGGER SET search_path = pg_catalog, public AS $fn$
|
||||
BEGIN
|
||||
NEW.search_vector :=
|
||||
setweight(to_tsvector('${lang}', COALESCE(NEW.doc_comment, '')), 'A') ||
|
||||
setweight(to_tsvector('${lang}', COALESCE(NEW.symbol_name_qualified, '')), 'A') ||
|
||||
setweight(to_tsvector('${lang}', COALESCE(NEW.chunk_text, '')), 'B');
|
||||
RETURN NEW;
|
||||
END;
|
||||
$fn$ LANGUAGE plpgsql;
|
||||
`;
|
||||
|
||||
await engine.executeRaw(recreatePagesFn);
|
||||
await engine.executeRaw(recreateChunksFn);
|
||||
|
||||
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
|
||||
|
||||
// Backfill: UPDATE-to-self forces the pages trigger to re-fire
|
||||
// (Postgres re-fires on UPDATE-to-same-value); content_chunks gets a
|
||||
// direct vector compute since the column itself is what we want.
|
||||
progress.start('reindex_search_vector.pages', pagesCount);
|
||||
await batchedBackfill(engine, 'pages', 'id = id', n => progress.tick(n));
|
||||
progress.finish();
|
||||
|
||||
progress.start('reindex_search_vector.chunks', chunksCount);
|
||||
await batchedBackfill(
|
||||
engine,
|
||||
'content_chunks',
|
||||
`search_vector =
|
||||
setweight(to_tsvector('${lang}', COALESCE(doc_comment, '')), 'A') ||
|
||||
setweight(to_tsvector('${lang}', COALESCE(symbol_name_qualified, '')), 'A') ||
|
||||
setweight(to_tsvector('${lang}', COALESCE(chunk_text, '')), 'B')`,
|
||||
n => progress.tick(n)
|
||||
);
|
||||
progress.finish();
|
||||
|
||||
const result: ReindexSearchVectorResult = {
|
||||
status: 'ok',
|
||||
language: lang,
|
||||
pagesUpdated: pagesCount,
|
||||
chunksUpdated: chunksCount,
|
||||
triggersRecreated: 2,
|
||||
durationMs: Date.now() - startedAt,
|
||||
};
|
||||
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
console.log(`✅ Recreated 2 trigger functions with language='${lang}'`);
|
||||
console.log(`✅ Backfilled ${pagesCount} pages + ${chunksCount} chunks (${result.durationMs}ms)`);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* CLI entrypoint. Parses argv flags and dispatches to runReindexSearchVector.
|
||||
* Matches the style of `reindex-code`: --dry-run, --yes/-y, --json.
|
||||
*
|
||||
* Exit codes: 0 success/dry-run/cancelled, 2 if non-TTY without --yes.
|
||||
*/
|
||||
export async function runReindexSearchVectorCli(
|
||||
engine: BrainEngine,
|
||||
args: string[]
|
||||
): Promise<void> {
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const yes = args.includes('--yes') || args.includes('-y');
|
||||
const json = args.includes('--json');
|
||||
|
||||
await runReindexSearchVector(engine, { dryRun, yes, json });
|
||||
}
|
||||
+12
-18
@@ -105,19 +105,13 @@ function printHelp(): void {
|
||||
async function runRemotePing(config: NonNullable<ReturnType<typeof loadConfig>>, args: string[]): Promise<void> {
|
||||
const { json, timeoutMs } = parseFlags(args);
|
||||
|
||||
// submit_job / get_job return the MinionJob row verbatim — the lifecycle
|
||||
// field is `status` (src/core/minions/types.ts), not `state`. Reading
|
||||
// `state` here made every poll see `undefined`, so the terminal check
|
||||
// never matched and ping always exhausted its timeout (exit 1) even when
|
||||
// the cycle completed. The ping's own JSON *output* keys (`state`,
|
||||
// `last_state`) are kept as-is for consumers.
|
||||
let submitted: { id: number; name: string; status: string };
|
||||
let submitted: { id: number; name: string; state: string };
|
||||
try {
|
||||
const res = await callRemoteTool(config, 'submit_job', {
|
||||
name: 'autopilot-cycle',
|
||||
data: { phases: ['sync', 'extract', 'embed'] },
|
||||
});
|
||||
submitted = unpackToolResult<{ id: number; name: string; status: string }>(res);
|
||||
submitted = unpackToolResult<{ id: number; name: string; state: string }>(res);
|
||||
} catch (e) {
|
||||
return failPing(e, json);
|
||||
}
|
||||
@@ -128,43 +122,43 @@ async function runRemotePing(config: NonNullable<ReturnType<typeof loadConfig>>,
|
||||
|
||||
const startMs = Date.now();
|
||||
let attempt = 0;
|
||||
let lastState = submitted.status;
|
||||
let lastState = submitted.state;
|
||||
while (Date.now() - startMs < timeoutMs) {
|
||||
const elapsed = Date.now() - startMs;
|
||||
const intervalMs = elapsed < 30_000 ? 1_000 : elapsed < 5 * 60_000 + 30_000 ? 5_000 : 10_000;
|
||||
await sleep(intervalMs);
|
||||
attempt++;
|
||||
|
||||
let job: { id: number; status: string; failed_reason?: string };
|
||||
let job: { id: number; state: string; failed_reason?: string };
|
||||
try {
|
||||
const res = await callRemoteTool(config, 'get_job', { id: submitted.id });
|
||||
job = unpackToolResult<{ id: number; status: string; failed_reason?: string }>(res);
|
||||
job = unpackToolResult<{ id: number; state: string; failed_reason?: string }>(res);
|
||||
} catch (e) {
|
||||
// Network blip mid-poll: log and keep going. Surface only if persistent.
|
||||
if (!json) console.error(` poll #${attempt} failed (${e instanceof Error ? e.message : String(e)}); continuing...`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (job.status !== lastState) {
|
||||
lastState = job.status;
|
||||
if (!json) console.error(` job #${submitted.id} → ${job.status}`);
|
||||
if (job.state !== lastState) {
|
||||
lastState = job.state;
|
||||
if (!json) console.error(` job #${submitted.id} → ${job.state}`);
|
||||
}
|
||||
|
||||
const terminal = ['completed', 'failed', 'dead', 'cancelled'];
|
||||
if (terminal.includes(job.status)) {
|
||||
const ok = job.status === 'completed';
|
||||
if (terminal.includes(job.state)) {
|
||||
const ok = job.state === 'completed';
|
||||
if (json) {
|
||||
console.log(JSON.stringify({
|
||||
status: ok ? 'success' : 'error',
|
||||
job_id: submitted.id,
|
||||
state: job.status,
|
||||
state: job.state,
|
||||
...(job.failed_reason ? { failed_reason: job.failed_reason } : {}),
|
||||
elapsed_ms: Date.now() - startMs,
|
||||
}));
|
||||
} else {
|
||||
console.log(ok
|
||||
? `\nautopilot-cycle complete (${Math.round((Date.now() - startMs) / 1000)}s).`
|
||||
: `\nautopilot-cycle ended ${job.status}${job.failed_reason ? `: ${job.failed_reason}` : ''}.`);
|
||||
: `\nautopilot-cycle ended ${job.state}${job.failed_reason ? `: ${job.failed_reason}` : ''}.`);
|
||||
}
|
||||
process.exit(ok ? 0 : 1);
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ import {
|
||||
} from '../core/schema-pack/index.ts';
|
||||
import type { SchemaPackManifest, PackPrimitive } from '../core/schema-pack/manifest-v1.ts';
|
||||
import { PACK_PRIMITIVES } from '../core/schema-pack/manifest-v1.ts';
|
||||
import { gbrainPath, loadConfig, configPath, toEngineConfig } from '../core/config.ts';
|
||||
import { gbrainPath, loadConfig, configPath } from '../core/config.ts';
|
||||
|
||||
export async function runSchema(args: string[]): Promise<void> {
|
||||
const sub = args[0];
|
||||
@@ -434,12 +434,16 @@ function parseFlags(args: string[]): ParsedFlags {
|
||||
|
||||
async function withConnectedEngine<T>(fn: (engine: import('../core/engine.ts').BrainEngine) => Promise<T>): Promise<T> {
|
||||
const { createEngine } = await import('../core/engine-factory.ts');
|
||||
const cfg = loadConfig() ?? { engine: 'pglite' as const };
|
||||
const cfg = loadConfig() ?? {};
|
||||
const engineKind = (cfg as { engine?: string }).engine === 'postgres' ? 'postgres' : 'pglite';
|
||||
// PR #1321 (closed) defensive fix retained: build the EngineConfig once and
|
||||
// pass it to BOTH createEngine and engine.connect. The factory captures
|
||||
// config at construction; explicit re-pass at connect() is defense in depth
|
||||
// against future engine implementations that read URL from connect-time.
|
||||
const connectConfig = toEngineConfig(cfg);
|
||||
const connectConfig: import('../core/types.ts').EngineConfig = {
|
||||
engine: engineKind,
|
||||
database_url: (cfg as { database_url?: string }).database_url,
|
||||
};
|
||||
const engine = await createEngine(connectConfig);
|
||||
await engine.connect(connectConfig);
|
||||
try {
|
||||
|
||||
+10
-215
@@ -22,7 +22,6 @@ import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/
|
||||
import { ListToolsRequestSchema, CallToolRequestSchema } from '@modelcontextprotocol/sdk/types.js';
|
||||
import { mcpAuthRouter } from '@modelcontextprotocol/sdk/server/auth/router.js';
|
||||
import { requireBearerAuth } from '@modelcontextprotocol/sdk/server/auth/middleware/bearerAuth.js';
|
||||
import { OAuthTokenRevocationRequestSchema } from '@modelcontextprotocol/sdk/shared/auth.js';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { operations, OperationError } from '../core/operations.ts';
|
||||
import type { OperationContext, AuthInfo } from '../core/operations.ts';
|
||||
@@ -38,14 +37,12 @@ import { VERSION } from '../version.ts';
|
||||
import * as db from '../core/db.ts';
|
||||
import { sqlQueryForEngine, executeRawJsonb } from '../core/sql-query.ts';
|
||||
import { MinionQueue } from '../core/minions/queue.ts';
|
||||
import { isRetryableError } from '../core/retry-matcher.ts';
|
||||
import {
|
||||
computeContentHash,
|
||||
validateIngestionEvent,
|
||||
type IngestionContentType,
|
||||
type IngestionEvent,
|
||||
} from '../core/ingestion/types.ts';
|
||||
import { resolveOwnerHolder } from '../core/owner-holder.ts';
|
||||
|
||||
/**
|
||||
* /health endpoint timeout. 3s rather than 5s: Fly.io's default
|
||||
@@ -113,24 +110,6 @@ export function shouldSuppressBootstrapPrint(opts: {
|
||||
return !opts.isTty;
|
||||
}
|
||||
|
||||
export type OAuthTokenRateLimitConfig = {
|
||||
windowMs: number;
|
||||
max: number;
|
||||
};
|
||||
|
||||
function parsePositiveIntEnv(value: string | undefined, fallback: number): number {
|
||||
if (value === undefined) return fallback;
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
export function resolveOAuthTokenRateLimit(env: NodeJS.ProcessEnv = process.env): OAuthTokenRateLimitConfig {
|
||||
return {
|
||||
windowMs: parsePositiveIntEnv(env.GBRAIN_OAUTH_TOKEN_RATE_LIMIT_WINDOW_MS, 15 * 60 * 1000),
|
||||
max: parsePositiveIntEnv(env.GBRAIN_OAUTH_TOKEN_RATE_LIMIT_MAX, 50),
|
||||
};
|
||||
}
|
||||
|
||||
export type ProbeHealthResult =
|
||||
| { ok: true; status: 200; body: { status: 'ok'; version: string; engine: string; [k: string]: unknown } }
|
||||
| { ok: false; status: 503; body: { error: 'service_unavailable'; error_description: string } };
|
||||
@@ -449,34 +428,6 @@ export function skillPublishStatus(publishSkills: boolean): { bannerValue: strin
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* #1196: startup embedding-width guard for stateless host deployments.
|
||||
*
|
||||
* `embedding_model` / `embedding_dimensions` are file/env-plane only, so a
|
||||
* container booted WITHOUT a config.json (stateless host) resolves the
|
||||
* compiled-in default embedding width. Against an existing brain whose
|
||||
* `content_chunks.embedding` is a different `vector(N)`, every write then
|
||||
* fails with an opaque dim mismatch. Run doctor's existing
|
||||
* embedding_width_consistency check at serve startup and return a loud
|
||||
* banner (with the paste-ready recipe) when it isn't ok. Fail-open: a check
|
||||
* error never blocks serving read traffic.
|
||||
*/
|
||||
export async function embeddingWidthStartupWarning(engine: BrainEngine): Promise<string | null> {
|
||||
try {
|
||||
const { checkEmbeddingWidthConsistency } = await import('./doctor.ts');
|
||||
const check = await checkEmbeddingWidthConsistency(engine);
|
||||
if (check.status === 'ok') return null;
|
||||
return (
|
||||
`[serve-http] WARNING: embedding width check failed — writes that embed will fail until fixed.\n` +
|
||||
`${check.message}\n` +
|
||||
`Stateless hosts: embedding_model/embedding_dimensions resolve from env/config.json only — ` +
|
||||
`set GBRAIN_EMBEDDING_MODEL / GBRAIN_EMBEDDING_DIMENSIONS (or mount config.json) to match the brain's schema.`
|
||||
);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function runServeHttp(engine: BrainEngine, options: ServeHttpOptions) {
|
||||
const { port, tokenTtl, enableDcr, enableDcrInsecure, publicUrl, logFullParams } = options;
|
||||
// v0.34.1 (#864, D11): default bind flipped from 0.0.0.0 to 127.0.0.1.
|
||||
@@ -501,14 +452,6 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
);
|
||||
}
|
||||
|
||||
// #1196: fail-loud at startup when the resolved embedding width diverges
|
||||
// from the brain's actual vector(N) column (stateless containers falling
|
||||
// through to the compiled-in default). Non-fatal: reads still work.
|
||||
{
|
||||
const widthWarn = await embeddingWidthStartupWarning(engine);
|
||||
if (widthWarn) console.error(widthWarn);
|
||||
}
|
||||
|
||||
// Skill-publishing status for the banner + nudge. Mirrors readMcpPublishSkills
|
||||
// (skill-catalog.ts): the DB plane (`gbrain config set`) wins over the file
|
||||
// plane. When OFF, a connected coding agent can't see the host's skill
|
||||
@@ -687,13 +630,12 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// Custom client_credentials handler (before mcpAuthRouter)
|
||||
// SDK's token handler only supports authorization_code and refresh_token
|
||||
// ---------------------------------------------------------------------------
|
||||
const oauthTokenRateLimit = resolveOAuthTokenRateLimit();
|
||||
const ccRateLimiter = rateLimit({
|
||||
windowMs: oauthTokenRateLimit.windowMs,
|
||||
max: oauthTokenRateLimit.max,
|
||||
windowMs: 15 * 60 * 1000,
|
||||
max: 50,
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
message: { error: 'too_many_requests', error_description: 'Rate limit exceeded. Try again later.' },
|
||||
message: { error: 'too_many_requests', error_description: 'Rate limit exceeded. Try again in 15 minutes.' },
|
||||
});
|
||||
|
||||
// Magic-link rate limiter: 10 requests/min/IP. The bootstrap token is
|
||||
@@ -803,93 +745,6 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
}
|
||||
});
|
||||
|
||||
// The SDK's /revoke handler compares the presented secret with
|
||||
// client.client_secret as plaintext. GBrain stores only a SHA-256 hash, so
|
||||
// confidential clients need the same hash-aware validation used above for
|
||||
// authorization_code and refresh_token exchanges. Public clients present no
|
||||
// secret and continue through to the SDK's PKCE-compatible handler.
|
||||
app.post('/revoke', ccRateLimiter, express.urlencoded({ extended: false }), async (req, res, next) => {
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
|
||||
const rawClientId: unknown = req.body?.client_id;
|
||||
const rawBodySecret: unknown = req.body?.client_secret;
|
||||
const authHeader = (req.headers.authorization ?? '').toString();
|
||||
|
||||
// RFC 6749 §2.3: one client-authentication method per request. Reject
|
||||
// duplicates/arrays from express.urlencoded rather than letting them reach
|
||||
// hashToken() as non-strings and become a misleading invalid_client error.
|
||||
const hasBasicAuth = /^Basic\b/i.test(authHeader);
|
||||
if (
|
||||
(rawClientId !== undefined && typeof rawClientId !== 'string') ||
|
||||
(rawBodySecret !== undefined && typeof rawBodySecret !== 'string') ||
|
||||
(hasBasicAuth && (rawClientId !== undefined || rawBodySecret !== undefined))
|
||||
) {
|
||||
res.status(400).json({ error: 'invalid_request', error_description: 'Malformed or mixed client authentication' });
|
||||
return;
|
||||
}
|
||||
|
||||
let clientId = typeof rawClientId === 'string' ? rawClientId : undefined;
|
||||
let presentedSecret = typeof rawBodySecret === 'string' && rawBodySecret.length > 0
|
||||
? rawBodySecret
|
||||
: undefined;
|
||||
if (hasBasicAuth) {
|
||||
try {
|
||||
const match = authHeader.match(/^Basic\s+([^\s]+)$/i);
|
||||
if (!match) throw new Error('Malformed Basic authentication');
|
||||
const decoded = Buffer.from(match[1], 'base64').toString('utf8');
|
||||
const idx = decoded.indexOf(':');
|
||||
if (idx < 1) throw new Error('Malformed Basic authentication');
|
||||
clientId = decodeURIComponent(decoded.slice(0, idx).replace(/\+/g, ' '));
|
||||
presentedSecret = decodeURIComponent(decoded.slice(idx + 1).replace(/\+/g, ' '));
|
||||
if (!presentedSecret) throw new Error('Malformed Basic authentication');
|
||||
} catch {
|
||||
res.setHeader('WWW-Authenticate', 'Basic realm="gbrain"');
|
||||
res.status(401).json({ error: 'invalid_client', error_description: 'Invalid client' });
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (!clientId || !presentedSecret) return next();
|
||||
|
||||
const parsedRequest = OAuthTokenRevocationRequestSchema.safeParse(req.body);
|
||||
if (!parsedRequest.success || parsedRequest.data.token.length === 0) {
|
||||
res.status(400).json({ error: 'invalid_request', error_description: 'Valid token required' });
|
||||
return;
|
||||
}
|
||||
|
||||
let client;
|
||||
try {
|
||||
client = await oauthProvider.verifyConfidentialClientSecret(clientId, presentedSecret);
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : '';
|
||||
if (msg === 'Invalid client' || msg === 'Client has been revoked') {
|
||||
if (hasBasicAuth) res.setHeader('WWW-Authenticate', 'Basic realm="gbrain"');
|
||||
res.status(401).json({ error: 'invalid_client', error_description: 'Invalid client' });
|
||||
return;
|
||||
}
|
||||
console.error('[serve-http] revoke client verification failed:', msg || 'Unknown error');
|
||||
const retryable = isRetryableError(e);
|
||||
res.status(retryable ? 503 : 500).json({
|
||||
error: retryable ? 'temporarily_unavailable' : 'server_error',
|
||||
error_description: retryable ? 'Token revocation temporarily unavailable' : 'Token revocation failed',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await oauthProvider.revokeToken(client, parsedRequest.data);
|
||||
// RFC 7009 §2.2: successful revocation, including an unknown token, is 200.
|
||||
res.status(200).end();
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : 'Unknown error';
|
||||
console.error('[serve-http] token revocation failed:', msg);
|
||||
const retryable = isRetryableError(e);
|
||||
res.status(retryable ? 503 : 500).json({
|
||||
error: retryable ? 'temporarily_unavailable' : 'server_error',
|
||||
error_description: retryable ? 'Token revocation temporarily unavailable' : 'Token revocation failed',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MCP SDK Auth Router (OAuth endpoints)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -899,21 +754,6 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// reverse proxies / tunnels; default to localhost for dev.
|
||||
const issuerUrl = new URL(publicUrl || `http://localhost:${port}`);
|
||||
|
||||
// MCP authorization spec (2025-06-18 draft §5.1) and RFC 9728 require the
|
||||
// protected resource server to return its discovery metadata URL in the
|
||||
// WWW-Authenticate header on 401 responses:
|
||||
//
|
||||
// WWW-Authenticate: Bearer resource_metadata="<URL>"
|
||||
//
|
||||
// Clients (claude.ai, Cursor, every other MCP-aware OAuth client) use that
|
||||
// URL to find the authorization-server discovery doc + token endpoint
|
||||
// without the user having to paste those URLs manually. Pre-fix the header
|
||||
// shipped `Bearer error="invalid_token", ...` with no resource_metadata
|
||||
// parameter, so MCP clients couldn't begin the OAuth flow from a fresh
|
||||
// 401 — they would silently fail to connect with a generic "couldn't
|
||||
// reach the MCP server" error.
|
||||
const resourceMetadataUrl = `${issuerUrl.toString().replace(/\/$/, '')}/.well-known/oauth-protected-resource`;
|
||||
|
||||
// F9: cookie `secure` flag honors both the request's TLS state (req.secure
|
||||
// is set when express trust-proxy lands an X-Forwarded-Proto: https) AND
|
||||
// the operator's declared issuer protocol (so a Cloudflare-tunnel deploy
|
||||
@@ -956,16 +796,6 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
if (body?.grant_types_supported && !body.grant_types_supported.includes('client_credentials')) {
|
||||
body.grant_types_supported.push('client_credentials');
|
||||
}
|
||||
if (body?.token_endpoint_auth_methods_supported) {
|
||||
for (const method of ['client_secret_basic', 'none']) {
|
||||
if (!body.token_endpoint_auth_methods_supported.includes(method)) {
|
||||
body.token_endpoint_auth_methods_supported.push(method);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (body?.revocation_endpoint_auth_methods_supported && !body.revocation_endpoint_auth_methods_supported.includes('client_secret_basic')) {
|
||||
body.revocation_endpoint_auth_methods_supported.push('client_secret_basic');
|
||||
}
|
||||
return origJson(body);
|
||||
};
|
||||
}
|
||||
@@ -1261,7 +1091,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
app.get('/admin/api/calibration/pattern/:id', requireAdmin, async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { getLatestProfile } = await import('./calibration.ts');
|
||||
const holder = resolveOwnerHolder({ override: (req.query.holder as string) || undefined, configValue: await engine.getConfig('emotional_weight.user_holder') });
|
||||
const holder = (req.query.holder as string) || 'garry';
|
||||
const profile = await getLatestProfile(engine, { holder });
|
||||
if (!profile) {
|
||||
res.status(404).json({ error: 'no_profile' });
|
||||
@@ -1311,7 +1141,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
app.get('/admin/api/calibration/profile', requireAdmin, async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { getLatestProfile } = await import('./calibration.ts');
|
||||
const holder = resolveOwnerHolder({ override: (req.query.holder as string) || undefined, configValue: await engine.getConfig('emotional_weight.user_holder') });
|
||||
const holder = (req.query.holder as string) || 'garry';
|
||||
const profile = await getLatestProfile(engine, { holder });
|
||||
res.json(profile);
|
||||
} catch (err) {
|
||||
@@ -1328,7 +1158,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
renderAbandonedThreadsCard,
|
||||
renderPatternStatementsCard,
|
||||
} = await import('../core/calibration/svg-renderer.ts');
|
||||
const holder = resolveOwnerHolder({ override: (req.query.holder as string) || undefined, configValue: await engine.getConfig('emotional_weight.user_holder') });
|
||||
const holder = (req.query.holder as string) || 'garry';
|
||||
const type = req.params.type;
|
||||
const profile = await getLatestProfile(engine, { holder });
|
||||
|
||||
@@ -1567,38 +1397,6 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
}
|
||||
});
|
||||
|
||||
// v0.42.x (#1914): rescope an OAuth client's write source / federated read
|
||||
// scope. Admin-gated on purpose — DCR clients must never self-widen their
|
||||
// scope (fail-closed trust); only the operator rescopes, here or via
|
||||
// `gbrain auth rescope-client`. Source ids are validated by the canonical
|
||||
// validator inside rescopeClient.
|
||||
app.post('/admin/api/rescope-client', requireAdmin, express.json(), async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { clientId, sourceId, federatedRead } = req.body ?? {};
|
||||
if (!clientId || typeof clientId !== 'string') {
|
||||
res.status(400).json({ error: 'clientId required' });
|
||||
return;
|
||||
}
|
||||
if (federatedRead !== undefined &&
|
||||
!(Array.isArray(federatedRead) && federatedRead.every((s: unknown) => typeof s === 'string'))) {
|
||||
res.status(400).json({ error: 'federatedRead must be an array of source id strings' });
|
||||
return;
|
||||
}
|
||||
if (sourceId !== undefined && typeof sourceId !== 'string') {
|
||||
res.status(400).json({ error: 'sourceId must be a string' });
|
||||
return;
|
||||
}
|
||||
const result = await oauthProvider.rescopeClient(clientId, { sourceId, federatedRead });
|
||||
res.json(result);
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : 'Rescope failed';
|
||||
const status = /No OAuth client found/.test(message) ? 404
|
||||
: /Invalid source_id|requires --source|cannot be empty|does not exist/.test(message) ? 400
|
||||
: 500;
|
||||
res.status(status).json({ error: message });
|
||||
}
|
||||
});
|
||||
|
||||
// Revoke OAuth client
|
||||
app.post('/admin/api/revoke-client', requireAdmin, express.json(), async (req: Request, res: Response) => {
|
||||
try {
|
||||
@@ -1704,7 +1502,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
res.status(405).json({ jsonrpc: '2.0', error: { code: -32000, message: 'Method not allowed' }, id: null });
|
||||
});
|
||||
|
||||
app.post('/mcp', requireBearerAuth({ verifier: oauthProvider, resourceMetadataUrl }), async (req: Request, res: Response) => {
|
||||
app.post('/mcp', requireBearerAuth({ verifier: oauthProvider }), async (req: Request, res: Response) => {
|
||||
const startTime = Date.now();
|
||||
const authInfo = (req as any).auth as AuthInfo;
|
||||
|
||||
@@ -2047,7 +1845,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
app.post(
|
||||
'/ingest',
|
||||
ingestRateLimiter,
|
||||
requireBearerAuth({ verifier: oauthProvider, requiredScopes: ['write'], resourceMetadataUrl }),
|
||||
requireBearerAuth({ verifier: oauthProvider, requiredScopes: ['write'] }),
|
||||
express.raw({ type: '*/*', limit: ingestMaxBytes }),
|
||||
async (req: Request, res: Response) => {
|
||||
const startTime = Date.now();
|
||||
@@ -2249,10 +2047,8 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// Other event types (ping, pull_request, etc.) return 202 'ignored'
|
||||
// so GitHub doesn't retry.
|
||||
// D15.5: HMAC compare uses the shared safeHexEqual helper.
|
||||
// D18: submits 'sync' job with extraction + auto_embed_backfill enabled and
|
||||
// priority -10 (above autopilot's 0). This opts normal incremental pushes
|
||||
// into sync's inline extraction while pagesAffected still identifies the
|
||||
// changed pages. The sync core can still defer large (>100) changes.
|
||||
// D18: submits 'sync' job with auto_embed_backfill=true and priority -10
|
||||
// (above autopilot's 0).
|
||||
// ---------------------------------------------------------------------------
|
||||
const githubWebhookLimiter = rateLimit({
|
||||
windowMs: 60_000,
|
||||
@@ -2372,7 +2168,6 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
'sync',
|
||||
{
|
||||
sourceId: source.id,
|
||||
noExtract: false,
|
||||
auto_embed_backfill: true,
|
||||
embed_reason: 'webhook',
|
||||
},
|
||||
|
||||
+1
-68
@@ -9,17 +9,6 @@ import { startMcpServer } from '../mcp/server.ts';
|
||||
// the dir, sees a dead PID, and removes it).
|
||||
const CLEANUP_DEADLINE_MS = 5_000;
|
||||
|
||||
// Boot-readiness deadline (#3273). A serve process that wedges mid-boot
|
||||
// (e.g. an MCP boot step that never completes because a configured
|
||||
// upstream is unreachable) holds the PGLite write lock indefinitely: the
|
||||
// post-#2348 lock discipline never steals from a live holder, so every
|
||||
// CLI consumer times out until someone hunts down and kills the PID. If
|
||||
// startMcpServer hasn't finished connecting the transport within this
|
||||
// window, we release the engine (dropping the lock) and exit non-zero so
|
||||
// a supervisor can restart with backoff. Env-tunable via
|
||||
// GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS; 0 disables.
|
||||
const DEFAULT_BOOT_TIMEOUT_SECONDS = 60;
|
||||
|
||||
// How often the parent-process watchdog polls the live kernel parent PID
|
||||
// (via `readLiveParentPid`, NOT the cached `process.ppid` — see that
|
||||
// helper's comment). We don't receive a signal when our parent dies (the
|
||||
@@ -78,10 +67,6 @@ export interface ServeOptions {
|
||||
// transport.onclose still cover legitimate shutdown.
|
||||
// Defaults to `process.env.MCP_STDIO === '1'` when omitted.
|
||||
mcpStdio?: boolean;
|
||||
// Test seam for the boot-readiness deadline (#3273). Milliseconds.
|
||||
// Defaults to GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS (seconds; 60 when
|
||||
// unset, 0 disables) when omitted.
|
||||
bootTimeoutMs?: number;
|
||||
}
|
||||
|
||||
export async function runServe(
|
||||
@@ -157,43 +142,7 @@ export async function runServe(
|
||||
installStdioLifecycle(engine, args, opts);
|
||||
|
||||
const start = opts.startMcpServer ?? startMcpServer;
|
||||
|
||||
// Boot-readiness deadline (#3273): never sit on the PGLite write lock
|
||||
// forever with a boot that never completes. On expiry: log, release the
|
||||
// engine (drops the lock), exit non-zero so supervisors restart with
|
||||
// backoff. The disconnect itself is raced against CLEANUP_DEADLINE_MS,
|
||||
// same as the graceful-shutdown path, so a wedged WASM close can't trap
|
||||
// us either.
|
||||
const bootTimeoutMs = opts.bootTimeoutMs ?? resolveBootTimeoutMs();
|
||||
let bootDeadline: ReturnType<typeof setTimeout> | null = null;
|
||||
if (bootTimeoutMs > 0) {
|
||||
const log = opts.log ?? ((msg: string) => console.error(msg));
|
||||
const exit = opts.exit ?? ((code?: number) => { process.exit(code); });
|
||||
bootDeadline = setTimeout(() => {
|
||||
log(
|
||||
`GBrain MCP server: boot did not complete within ${bootTimeoutMs}ms — releasing DB lock and exiting so other consumers unblock (check configured provider endpoints; tune via GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS, 0 disables)`,
|
||||
);
|
||||
const cleanup = setTimeout(() => { exit(1); }, CLEANUP_DEADLINE_MS);
|
||||
cleanup.unref?.();
|
||||
Promise.resolve()
|
||||
.then(() => engine.disconnect())
|
||||
.catch((err: unknown) => {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
log(`GBrain MCP server: boot-deadline cleanup error: ${msg}`);
|
||||
})
|
||||
.finally(() => {
|
||||
clearTimeout(cleanup);
|
||||
exit(1);
|
||||
});
|
||||
}, bootTimeoutMs);
|
||||
bootDeadline.unref?.();
|
||||
}
|
||||
|
||||
try {
|
||||
await start(engine);
|
||||
} finally {
|
||||
if (bootDeadline) clearTimeout(bootDeadline);
|
||||
}
|
||||
await start(engine);
|
||||
// startMcpServer's `await server.connect(transport)` resolves once the
|
||||
// SDK has wired up its stdin 'data' listener; that listener keeps the
|
||||
// event loop alive. We deliberately do NOT add `await new Promise(() =>
|
||||
@@ -201,22 +150,6 @@ export async function runServe(
|
||||
// hooks from being able to call process.exit() cleanly.
|
||||
}
|
||||
|
||||
// Env resolution for the boot deadline. Lenient (warn + default) rather
|
||||
// than throw: this is an incident-time escape hatch, and a typo'd env var
|
||||
// must not turn a boot-safety net into a boot failure of its own.
|
||||
function resolveBootTimeoutMs(): number {
|
||||
const raw = process.env.GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS;
|
||||
if (raw === undefined || raw.trim() === '') return DEFAULT_BOOT_TIMEOUT_SECONDS * 1000;
|
||||
const n = Number(raw);
|
||||
if (!Number.isFinite(n) || n < 0) {
|
||||
console.error(
|
||||
`[gbrain serve] ignoring invalid GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS=${JSON.stringify(raw)} — using default ${DEFAULT_BOOT_TIMEOUT_SECONDS}s`,
|
||||
);
|
||||
return DEFAULT_BOOT_TIMEOUT_SECONDS * 1000;
|
||||
}
|
||||
return n * 1000;
|
||||
}
|
||||
|
||||
interface StdioLifecycleDeps {
|
||||
stdin: NodeJS.ReadableStream & { isTTY?: boolean };
|
||||
signals: Pick<NodeJS.Process, 'on'>;
|
||||
|
||||
+5
-18
@@ -7,9 +7,7 @@
|
||||
* full story.
|
||||
*
|
||||
* Subcommands:
|
||||
* gbrain sources add <id> --path <path> [--name <display>] [--federated|--no-federated] [--force]
|
||||
* --path must be a git-initialized repo (files committed,
|
||||
* not just present) — #2707. --force skips the check.
|
||||
* gbrain sources add <id> --path <path> [--name <display>] [--federated|--no-federated]
|
||||
* gbrain sources list [--json]
|
||||
* gbrain sources remove <id> [--yes] [--dry-run] [--keep-storage]
|
||||
* gbrain sources rename <id> <new-name>
|
||||
@@ -122,7 +120,7 @@ async function runAdd(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
if (!id) {
|
||||
console.error(
|
||||
'Usage: gbrain sources add <id> [--path <path> | --url <https-url>] ' +
|
||||
'[--name <display>] [--federated|--no-federated] [--clone-dir <path>] [--force]',
|
||||
'[--name <display>] [--federated|--no-federated] [--clone-dir <path>]',
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
@@ -134,7 +132,6 @@ async function runAdd(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
let cloneDir: string | undefined;
|
||||
let patFile: string | undefined;
|
||||
let noHarden = false;
|
||||
let force = false;
|
||||
|
||||
for (let i = 1; i < args.length; i++) {
|
||||
const a = args[i];
|
||||
@@ -146,7 +143,6 @@ async function runAdd(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
if (a === '--clone-dir') { cloneDir = args[++i]; continue; }
|
||||
if (a === '--pat-file') { patFile = args[++i]; continue; }
|
||||
if (a === '--no-harden') { noHarden = true; continue; }
|
||||
if (a === '--force') { force = true; continue; }
|
||||
console.error(`Unknown flag: ${a}`);
|
||||
process.exit(2);
|
||||
}
|
||||
@@ -166,7 +162,6 @@ async function runAdd(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
remoteUrl,
|
||||
federated,
|
||||
cloneDir,
|
||||
force,
|
||||
});
|
||||
|
||||
// Topology A discovery: if the just-added source carries a brain-resident
|
||||
@@ -1183,14 +1178,7 @@ async function runAudit(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
// frontmatter.type and estimates per-page segment count from body
|
||||
// bytes. Estimated per-segment Sonnet cost is a rough heuristic
|
||||
// (~2000 in + 500 out tokens at $3/MTok in + $15/MTok out ≈ $0.013).
|
||||
const FACTS_BACKFILL_ALLOWED = [
|
||||
'conversation',
|
||||
'meeting',
|
||||
'slack',
|
||||
'email',
|
||||
'imessage',
|
||||
'imessage-daily',
|
||||
];
|
||||
const FACTS_BACKFILL_ALLOWED = ['conversation', 'meeting', 'slack', 'email'];
|
||||
const FACTS_BACKFILL_CHARS_PER_SEGMENT = 6500; // matches SEGMENT_TEXT_CHAR_LIMIT
|
||||
const FACTS_BACKFILL_USD_PER_SEGMENT = 0.013;
|
||||
let factsBackfillPages = 0;
|
||||
@@ -1380,9 +1368,8 @@ function printHelp(): void {
|
||||
console.log(`gbrain sources — manage multi-source brain configuration (v0.26.5)
|
||||
|
||||
Subcommands:
|
||||
add <id> --path <p> [--name <n>] [--federated|--no-federated] [--force]
|
||||
Register a new source. --path must be a git repo
|
||||
with committed files; --force skips that check.
|
||||
add <id> --path <p> [--name <n>] [--federated|--no-federated]
|
||||
Register a new source.
|
||||
list [--json] List registered sources with page counts.
|
||||
remove <id> [--confirm-destructive] [--dry-run]
|
||||
Permanently delete a source and all its data.
|
||||
|
||||
+65
-707
File diff suppressed because it is too large
Load Diff
+8
-15
@@ -29,7 +29,6 @@ import {
|
||||
} from '../core/takes-fence.ts';
|
||||
import { withPageLock } from '../core/page-lock.ts';
|
||||
import { resolveSourceId } from '../core/source-resolver.ts';
|
||||
import { resolveOwnerHolder } from '../core/owner-holder.ts';
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
@@ -102,18 +101,12 @@ async function getPageId(engine: BrainEngine, slug: string, sourceId?: string):
|
||||
return rows[0].id;
|
||||
}
|
||||
|
||||
// Fail-closed (#2698 residual, TODOS.md): `resolveSourceId` only ever
|
||||
// throws when a source WAS explicitly in play — an invalid or
|
||||
// unregistered `GBRAIN_SOURCE`, a `.gbrain-source` dotfile pointing at a
|
||||
// source that doesn't exist, or a genuine DB error — never for "nothing
|
||||
// configured" (that path resolves cleanly to the seeded `'default'`
|
||||
// source, tier 6 of resolveSourceId). Swallowing those errors here used
|
||||
// to fall back to the unscoped slug-only page lookup, silently
|
||||
// reintroducing the pre-#2698 cross-source write bug whenever resolution
|
||||
// merely errored instead of resolving cleanly. Let it propagate so the
|
||||
// write is blocked instead of silently unscoped.
|
||||
async function resolveTakesSourceId(engine: BrainEngine): Promise<string> {
|
||||
return resolveSourceId(engine, null);
|
||||
async function resolveTakesSourceId(engine: BrainEngine): Promise<string | undefined> {
|
||||
try {
|
||||
return await resolveSourceId(engine, null);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function readBodyOrEmpty(path: string): string {
|
||||
@@ -292,7 +285,7 @@ async function cmdSupersede(engine: BrainEngine, args: string[], sourceId?: stri
|
||||
const pageId = await getPageId(engine, slug, sourceId);
|
||||
|
||||
// Read existing row to inherit kind/holder unless overridden
|
||||
const existing = await engine.listTakes({ page_id: pageId, active: true, limit: 500 });
|
||||
const existing = await engine.listTakes({ page_id: pageId, active: false, limit: 500 });
|
||||
const target = existing.find(t => t.row_num === rowNum);
|
||||
if (!target) {
|
||||
console.error(`Row #${rowNum} not found on ${slug}.`);
|
||||
@@ -365,7 +358,7 @@ async function cmdResolve(engine: BrainEngine, args: string[], sourceId?: string
|
||||
// --evidence is the v0.30.0 alias for --source on the resolve subcommand
|
||||
// (semantic clarity: "what evidence resolved this bet?").
|
||||
const source = flagValue(args, '--evidence') ?? flagValue(args, '--source');
|
||||
const resolvedBy = flagValue(args, '--by') ?? resolveOwnerHolder({ configValue: await engine.getConfig('emotional_weight.user_holder') });
|
||||
const resolvedBy = flagValue(args, '--by') ?? 'garry';
|
||||
const dirArg = flagValue(args, '--dir');
|
||||
|
||||
const pageId = await getPageId(engine, slug, sourceId);
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* degrades to gather-only output with a warning if missing.
|
||||
*/
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { runThink, persistSynthesis, stripGapsSection } from '../core/think/index.ts';
|
||||
import { runThink, persistSynthesis } from '../core/think/index.ts';
|
||||
import { loadConfig, isThinClient } from '../core/config.ts';
|
||||
import { callRemoteTool, unpackToolResult } from '../core/mcp-client.ts';
|
||||
|
||||
@@ -157,7 +157,7 @@ prints what would have been the input (exit 0).
|
||||
|
||||
// Human-readable output
|
||||
console.log(`# ${question}\n`);
|
||||
console.log(stripGapsSection(result.answer));
|
||||
console.log(result.answer);
|
||||
console.log('');
|
||||
if (result.gaps.length > 0) {
|
||||
console.log('## Gaps');
|
||||
|
||||
@@ -18,22 +18,12 @@
|
||||
import { loadConfig } from '../config.ts';
|
||||
|
||||
export function hasAnthropicKey(): boolean {
|
||||
return resolveAnthropicKey() !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the actual key value: env first, then the gbrain config file.
|
||||
* Callers constructing an Anthropic client directly (e.g. the legacy
|
||||
* subagent path) must pass this as `apiKey` — a bare `new Anthropic()`
|
||||
* only sees env, so launchd/MCP workers with config-stored keys fail.
|
||||
*/
|
||||
export function resolveAnthropicKey(): string | undefined {
|
||||
if (process.env.ANTHROPIC_API_KEY) return process.env.ANTHROPIC_API_KEY;
|
||||
if (process.env.ANTHROPIC_API_KEY) return true;
|
||||
try {
|
||||
const cfg = loadConfig();
|
||||
if (cfg?.anthropic_api_key) return cfg.anthropic_api_key;
|
||||
if (cfg?.anthropic_api_key) return true;
|
||||
} catch {
|
||||
// loadConfig may throw on first-run installs; treat as no key available.
|
||||
}
|
||||
return undefined;
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
* import it from `../../src/cli.ts`.
|
||||
*
|
||||
* The single ownership site for: (a) folding file-plane API keys
|
||||
* (openai/anthropic/zeroentropy/openrouter/voyage) into the gateway env, and (b) threading
|
||||
* (openai/anthropic/zeroentropy) into the gateway env, and (b) threading
|
||||
* local-server `*_BASE_URL` env vars into base_urls. Both matter for the
|
||||
* init-time embedding-key probe — without (a) it would false-warn on
|
||||
* config.json-keyed users, and without (b) a live probe could hit the wrong
|
||||
@@ -38,12 +38,6 @@ export function buildGatewayConfig(c: GBrainConfig): AIGatewayConfig {
|
||||
// config.json) must reach the openrouter recipe's OPENROUTER_API_KEY.
|
||||
// process.env still wins via the later spread.
|
||||
if (c.openrouter_api_key) envFromConfig.OPENROUTER_API_KEY = c.openrouter_api_key;
|
||||
// #2662: same seam for Voyage. Before this, config.json's voyage_api_key
|
||||
// was accepted at the file plane but never threaded into the gateway env,
|
||||
// so launchd/daemon/MCP contexts (no process-env export) silently failed
|
||||
// multimodal/image embeds despite config.json looking complete. process.env
|
||||
// still wins via the later spread.
|
||||
if (c.voyage_api_key) envFromConfig.VOYAGE_API_KEY = c.voyage_api_key;
|
||||
|
||||
// v0.32 codex finding #4+#5 fix: thread local-server _BASE_URL env vars
|
||||
// into base_urls so the gateway hits the user's configured port. Without
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
*/
|
||||
|
||||
import { resolveRecipe } from './model-resolver.ts';
|
||||
import { listRecipes } from './recipes/index.ts';
|
||||
import { AIConfigError } from './errors.ts';
|
||||
|
||||
export interface ProviderCapabilities {
|
||||
@@ -78,10 +77,7 @@ export function getProviderCapabilities(modelString: string): ProviderCapabiliti
|
||||
if (!chat) {
|
||||
throw new AIConfigError(
|
||||
`Provider "${recipe.id}" does not offer a chat touchpoint.`,
|
||||
// Computed from the registry so the hint can't drift into listing
|
||||
// chat-less providers (the pre-fix list falsely included embedding-only
|
||||
// recipes, sending users in circles — #1157).
|
||||
`Known providers with chat: ${listRecipes().filter(r => r.touchpoints.chat).map(r => r.id).join(', ')}. Pick one for models.tier.subagent.`,
|
||||
`Known providers with chat: openai, anthropic, google, openrouter, litellm-proxy, deepseek, groq, together, azure-openai, dashscope, minimax, zhipu, ollama, llama-server. Pick one for models.tier.subagent.`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -92,13 +88,9 @@ export function getProviderCapabilities(modelString: string): ProviderCapabiliti
|
||||
// boundary; this function returns capabilities for whatever the user asked
|
||||
// for, on the assumption it'll be validated elsewhere.
|
||||
|
||||
const promptCache = chat.supports_prompt_cache;
|
||||
|
||||
return {
|
||||
supportsToolCalling: chat.supports_tools === true,
|
||||
supportsPromptCaching: typeof promptCache === 'function'
|
||||
? promptCache(parsed.modelId)
|
||||
: promptCache === true,
|
||||
supportsPromptCaching: chat.supports_prompt_cache === true,
|
||||
// No recipe exposes parallel-tools-specifically yet; gate on supports_tools.
|
||||
// Subsequent waves can split this into its own recipe field if a provider
|
||||
// ever supports tools without parallel dispatch.
|
||||
@@ -109,6 +101,11 @@ export function getProviderCapabilities(modelString: string): ProviderCapabiliti
|
||||
supportsThinking: false,
|
||||
maxContext: chat.max_context_tokens ?? 128_000,
|
||||
};
|
||||
|
||||
// The `parsed` binding is intentionally unused — `resolveRecipe` is called
|
||||
// here for its validation side-effects (throws on unknown provider). Keeping
|
||||
// the destructure makes future per-model capability overrides cheap.
|
||||
void parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+3
-97
@@ -90,62 +90,6 @@ export function isValidOpenAITextEmbedding3Dim(modelId: string, dims: number): b
|
||||
return Number.isInteger(dims) && dims >= 1 && dims <= max;
|
||||
}
|
||||
|
||||
// Perplexity hosted embeddings (#1046): Matryoshka-style flexible dims,
|
||||
// any integer from 128 up to the model's native size. `dimensions` is the
|
||||
// native wire field (no translation needed); output encoding divergence
|
||||
// (base64 int8) is handled by perplexityCompatFetch in gateway.ts.
|
||||
const PERPLEXITY_EMBEDDING_MAX_DIMS: Record<string, number> = {
|
||||
'pplx-embed-v1-0.6b': 1024,
|
||||
'pplx-embed-v1-4b': 2560,
|
||||
};
|
||||
export const PERPLEXITY_MIN_DIMS = 128;
|
||||
|
||||
export function isPerplexityEmbeddingModel(modelId: string): boolean {
|
||||
return modelId in PERPLEXITY_EMBEDDING_MAX_DIMS;
|
||||
}
|
||||
|
||||
export function maxPerplexityEmbeddingDim(modelId: string): number | undefined {
|
||||
return PERPLEXITY_EMBEDDING_MAX_DIMS[modelId];
|
||||
}
|
||||
|
||||
export function isValidPerplexityDim(modelId: string, dims: number): boolean {
|
||||
const max = PERPLEXITY_EMBEDDING_MAX_DIMS[modelId];
|
||||
if (max === undefined) return false;
|
||||
return Number.isInteger(dims) && dims >= PERPLEXITY_MIN_DIMS && dims <= max;
|
||||
}
|
||||
|
||||
// NVIDIA NIM hosted embedding models use asymmetric input_type values. Most
|
||||
// emit fixed natural dimensions, but llama-nemotron-embed-1b-v2 accepts
|
||||
// Matryoshka-style dimension overrides (e.g. matching an existing 1280d
|
||||
// brain column without re-embedding through another provider).
|
||||
const NVIDIA_EMBEDDING_DIMS: Record<string, number> = {
|
||||
'nvidia/nv-embedqa-e5-v5': 1024,
|
||||
'nvidia/llama-nemotron-embed-1b-v2': 2048,
|
||||
'nvidia/nv-embed-v1': 4096,
|
||||
'nvidia/nv-embedcode-7b-v1': 4096,
|
||||
};
|
||||
|
||||
const NVIDIA_EMBEDDING_DIM_OPTIONS: Record<string, number[]> = {
|
||||
'nvidia/llama-nemotron-embed-1b-v2': [1024, 1280, 1536, 2048],
|
||||
};
|
||||
|
||||
export function isNvidiaEmbeddingModel(modelId: string): boolean {
|
||||
return modelId in NVIDIA_EMBEDDING_DIMS;
|
||||
}
|
||||
|
||||
export function nvidiaEmbeddingDim(modelId: string): number | undefined {
|
||||
return NVIDIA_EMBEDDING_DIMS[modelId];
|
||||
}
|
||||
|
||||
export function nvidiaEmbeddingDimOptions(modelId: string): number[] | undefined {
|
||||
return NVIDIA_EMBEDDING_DIM_OPTIONS[modelId];
|
||||
}
|
||||
|
||||
export function supportsNvidiaEmbeddingDimension(modelId: string, dims: number): boolean {
|
||||
const options = nvidiaEmbeddingDimOptions(modelId);
|
||||
return !!options && options.includes(dims);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the providerOptions blob for embedMany() that pins output dimensions.
|
||||
*
|
||||
@@ -250,34 +194,6 @@ export function dimsProviderOptions(
|
||||
},
|
||||
};
|
||||
}
|
||||
// Perplexity pplx-embed-v1-* — flexible dims via the native
|
||||
// `dimensions` field. Fail-loud when the configured dim is outside
|
||||
// the model's range (same rationale as the Voyage/ZE guards: the
|
||||
// upstream HTTP 400 misroutes as a transient network error).
|
||||
// Symmetric retrieval — inputType is never emitted.
|
||||
if (isPerplexityEmbeddingModel(modelId)) {
|
||||
if (!isValidPerplexityDim(modelId, dims)) {
|
||||
const max = maxPerplexityEmbeddingDim(modelId)!;
|
||||
throw new AIConfigError(
|
||||
`Perplexity model "${modelId}" supports embedding_dimensions in ` +
|
||||
`${PERPLEXITY_MIN_DIMS}..${max}, got ${dims}.`,
|
||||
`Set \`embedding_dimensions\` to a value between ${PERPLEXITY_MIN_DIMS} and ${max} ` +
|
||||
`in your gbrain config.`,
|
||||
);
|
||||
}
|
||||
return { openaiCompatible: { dimensions: dims } };
|
||||
}
|
||||
// NVIDIA NIM hosted embeddings are OpenAI-compatible but require
|
||||
// asymmetric input_type. Use passage for indexing/document-side vectors
|
||||
// and query for search-side vectors. Only llama-nemotron-embed-1b-v2
|
||||
// supports a dimensions override; fixed-dim models reject it.
|
||||
if (isNvidiaEmbeddingModel(modelId)) {
|
||||
const opts: Record<string, any> = {
|
||||
input_type: inputType === 'query' ? 'query' : 'passage',
|
||||
};
|
||||
if (supportsNvidiaEmbeddingDimension(modelId, dims)) opts.dimensions = dims;
|
||||
return { openaiCompatible: opts };
|
||||
}
|
||||
// OpenAI text-embedding-3 family on the openai-compatible adapter
|
||||
// (Azure OpenAI hosts these via its OpenAI-compatible /embeddings
|
||||
// endpoint). The provider defaults to the model's native size (3072
|
||||
@@ -285,10 +201,9 @@ export function dimsProviderOptions(
|
||||
// configured for a smaller width (e.g. 1536) hard-fail at first embed.
|
||||
// Azure/OpenAI-compat embeddings are symmetric — inputType ignored.
|
||||
// v0.36.0.0 (D13): same range validation as native-openai path.
|
||||
const bareModelId = modelId.includes('/') ? modelId.split('/').pop()! : modelId;
|
||||
if (bareModelId.startsWith('text-embedding-3')) {
|
||||
if (isOpenAITextEmbedding3Model(bareModelId) && !isValidOpenAITextEmbedding3Dim(bareModelId, dims)) {
|
||||
const max = maxOpenAITextEmbedding3Dim(bareModelId)!;
|
||||
if (modelId.startsWith('text-embedding-3')) {
|
||||
if (isOpenAITextEmbedding3Model(modelId) && !isValidOpenAITextEmbedding3Dim(modelId, dims)) {
|
||||
const max = maxOpenAITextEmbedding3Dim(modelId)!;
|
||||
throw new AIConfigError(
|
||||
`OpenAI model "${modelId}" supports embedding_dimensions in 1..${max}, got ${dims}.`,
|
||||
`Set \`embedding_dimensions\` to a value between 1 and ${max} ` +
|
||||
@@ -305,15 +220,6 @@ export function dimsProviderOptions(
|
||||
if (modelId === 'text-embedding-v3' || modelId === 'embedding-3') {
|
||||
return { openaiCompatible: { dimensions: dims } };
|
||||
}
|
||||
// Qwen3-Embedding family on Ollama (and any other openai-compatible
|
||||
// provider serving it) supports Matryoshka truncation via `dimensions`.
|
||||
// Native sizes: 0.6B=1024, 4B=2560, 8B=4096. Without `dimensions`,
|
||||
// Ollama returns the native size and brains configured for narrower
|
||||
// widths hard-fail with a dim-mismatch error. Pattern match the bare
|
||||
// model name + any `:tag` (e.g. `qwen3-embedding:4b`, `qwen3-embedding:0.6b`).
|
||||
if (modelId === 'qwen3-embedding' || modelId.startsWith('qwen3-embedding:')) {
|
||||
return { openaiCompatible: { dimensions: dims } };
|
||||
}
|
||||
// MiniMax embo-01 takes a `type: 'db' | 'query'` field for asymmetric
|
||||
// retrieval. Today still hardcoded to 'db' for back-compat — opting
|
||||
// into the new inputType seam is a follow-up (see plan's deferred
|
||||
|
||||
+5
-357
@@ -23,7 +23,6 @@
|
||||
|
||||
import { embed as aiEmbed, embedMany, generateObject, generateText, jsonSchema } from 'ai';
|
||||
import { AsyncLocalStorage } from 'node:async_hooks';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { listRecipes } from './recipes/index.ts';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import { createGoogleGenerativeAI } from '@ai-sdk/google';
|
||||
@@ -47,18 +46,12 @@ import type {
|
||||
TouchpointKind,
|
||||
} from './types.ts';
|
||||
import { resolveRecipe, assertTouchpoint, parseModelId } from './model-resolver.ts';
|
||||
import {
|
||||
OPENROUTER_CACHE_HEADER,
|
||||
openrouterRequiresExplicitPromptCache,
|
||||
} from './recipes/openrouter.ts';
|
||||
import { resolveModel, TIER_DEFAULTS } from '../model-config.ts';
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
import { dimsProviderOptions } from './dims.ts';
|
||||
import { hasAnthropicKey } from './anthropic-key.ts';
|
||||
import { AIConfigError, AITransientError, normalizeAIError } from './errors.ts';
|
||||
import { runGuardrails, hasGuardrails, type GuardrailHook } from '../guardrails.ts';
|
||||
import { loadConfig } from '../config.ts';
|
||||
import { buildGatewayConfig } from './build-gateway-config.ts';
|
||||
|
||||
// ---- Gateway-wide AI-HTTP timeout (v0.42.20.0, #1762/#1775) ----
|
||||
//
|
||||
@@ -123,18 +116,6 @@ const DEFAULT_RERANKER_MODEL = 'zeroentropyai:zerank-2';
|
||||
let _config: AIGatewayConfig | null = null;
|
||||
const _modelCache = new Map<string, any>();
|
||||
|
||||
/**
|
||||
* Recover the process-global gateway for foreground command entrypoints that
|
||||
* were reached without cli.ts's normal engine-connect initialization (#2590).
|
||||
* Existing configured gateways, including their DB-resolved model overrides,
|
||||
* are deliberately left unchanged.
|
||||
*/
|
||||
export function configureGatewayIfUninitialized(): void {
|
||||
if (_config) return;
|
||||
const config = loadConfig();
|
||||
if (config) configureGateway(buildGatewayConfig(config));
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.31.12 recipe-models merge: per-gateway-instance set of model ids the
|
||||
* user opted into via config. Keyed by provider id (`anthropic`, `openai`,
|
||||
@@ -267,18 +248,6 @@ export class ZeroEntropyResponseTooLargeError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/** Perplexity twin of the Voyage/ZE OOM caps (#1046). Int8 components are
|
||||
* 1 byte each, so a real response (512 texts × 2560 dims) is ~1.3 MB —
|
||||
* anything near this cap is unambiguously not legitimate. */
|
||||
const MAX_PERPLEXITY_RESPONSE_BYTES = 256 * 1024 * 1024;
|
||||
|
||||
export class PerplexityResponseTooLargeError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'PerplexityResponseTooLargeError';
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Unified auth resolution (D12=A) ----
|
||||
//
|
||||
// Pre-v0.32, openai-compatible auth was duplicated across instantiateEmbedding,
|
||||
@@ -537,20 +506,6 @@ export async function reconfigureGatewayWithEngine(engine: BrainEngine): Promise
|
||||
const expansionFull = newExpansion.includes(':') ? newExpansion : prefixWithProviderFrom(cfg.expansion_model ?? DEFAULT_EXPANSION_MODEL, newExpansion);
|
||||
const chatFull = newChat.includes(':') ? newChat : prefixWithProviderFrom(cfg.chat_model ?? DEFAULT_CHAT_MODEL, newChat);
|
||||
|
||||
// ALSO resolve the four tier models and register them as extended models.
|
||||
// assertTouchpoint's contract (model-resolver.ts) says config-chosen models —
|
||||
// `models.default` and `models.tier.*` included — bypass the native recipe
|
||||
// allowlist, but pre-fix only chat/expansion/embedding/reranker were
|
||||
// registered. A model reachable ONLY through a tier (e.g. `models.tier.deep`
|
||||
// set to an Opus newer than the recipe list) failed `probeChatModel` at call
|
||||
// time and silently degraded think/auto_think to the gather-only stub.
|
||||
// Resolving per-tier also honors `models.default` (it sits above tiers in
|
||||
// the resolveModel chain).
|
||||
const tierModels: string[] = [];
|
||||
for (const tier of ['utility', 'reasoning', 'deep', 'subagent'] as const) {
|
||||
tierModels.push(await resolveModel(engine, { tier, fallback: TIER_DEFAULTS[tier] }));
|
||||
}
|
||||
|
||||
_config = { ...cfg, expansion_model: expansionFull, chat_model: chatFull };
|
||||
_modelCache.clear();
|
||||
_shrinkState.clear();
|
||||
@@ -562,7 +517,6 @@ export async function reconfigureGatewayWithEngine(engine: BrainEngine): Promise
|
||||
_config.chat_model,
|
||||
_config.reranker_model,
|
||||
...(_config.chat_fallback_chain ?? []),
|
||||
...tierModels,
|
||||
]) {
|
||||
if (m) registerExtendedModel(m);
|
||||
}
|
||||
@@ -615,8 +569,6 @@ function warnRecipesMissingBatchTokens(): void {
|
||||
// LiteLLM proxy, llama-server) — they ship without a static cap because
|
||||
// the cap depends on a user-launched server. Warning is noise for them.
|
||||
if (embedding.no_batch_cap === true) continue;
|
||||
// A declared item-count cap is a real batch cap — no warning needed.
|
||||
if (embedding.max_batch_items !== undefined) continue;
|
||||
if (_warnedRecipes.has(recipe.id)) continue;
|
||||
_warnedRecipes.add(recipe.id);
|
||||
// eslint-disable-next-line no-console
|
||||
@@ -1092,30 +1044,6 @@ const voyageCompatFetch = (async (input: RequestInfo | URL, init?: RequestInit)
|
||||
* float[] (not base64), so the Layer 2 cap compares against the JSON
|
||||
* payload size of each embedding rather than a base64 string length.
|
||||
*/
|
||||
/**
|
||||
* NVIDIA NIM compatibility shim. NVIDIA uses the OpenAI embeddings wire
|
||||
* shape but requires asymmetric input_type values: query for retrieval and
|
||||
* passage for indexed documents. The generic gateway store carries
|
||||
* query/document across the AI SDK boundary; map document to passage here.
|
||||
*/
|
||||
const nvidiaCompatFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
let baseInit: RequestInit = init ?? {};
|
||||
if (baseInit.body && typeof baseInit.body === 'string') {
|
||||
try {
|
||||
const parsed = JSON.parse(baseInit.body);
|
||||
if (parsed && typeof parsed === 'object' && parsed.input_type === undefined) {
|
||||
parsed.input_type = __embedInputTypeStore.getStore() === 'query' ? 'query' : 'passage';
|
||||
const headers = new Headers(baseInit.headers ?? {});
|
||||
headers.delete('content-length');
|
||||
baseInit = { ...baseInit, body: JSON.stringify(parsed), headers };
|
||||
}
|
||||
} catch {
|
||||
// Preserve the provider response when the SDK body is unexpectedly non-JSON.
|
||||
}
|
||||
}
|
||||
return fetch(input as any, baseInit);
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
const zeroEntropyCompatFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
// OUTBOUND: normalize URL, rewrite path /embeddings → /models/embed, then
|
||||
// rewrite body. fetch accepts RequestInfo (string | Request) | URL; we
|
||||
@@ -1310,103 +1238,6 @@ const openAICompatAsymmetricFetch = (async (input: RequestInfo | URL, init?: Req
|
||||
return fetch(typeof input === 'string' ? input : input.toString(), baseInit);
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
/**
|
||||
* Perplexity compatibility shim (#1046). Perplexity's `/v1/embeddings`
|
||||
* endpoint is OpenAI-shaped but diverges on two points that break the AI
|
||||
* SDK's openai-compatible adapter:
|
||||
* - `encoding_format` only accepts 'base64_int8' (default) or
|
||||
* 'base64_binary'; the SDK sends 'float', which Perplexity rejects.
|
||||
* Force 'base64_int8' on the wire.
|
||||
* - The response `embedding` is a base64 string encoding SIGNED INT8
|
||||
* components (natively quantized output). The SDK schema expects
|
||||
* `number[]` — decode Int8Array → number[] here. Cosine similarity is
|
||||
* scale-invariant, so the raw int8 components rank correctly.
|
||||
* `dimensions` is Perplexity's native field name — no translation needed
|
||||
* (dims.ts emits it directly). Layer 1/Layer 2 OOM caps mirror the Voyage
|
||||
* pattern.
|
||||
*
|
||||
* Exported for tests (behavioral coverage of the int8 decode); not part of
|
||||
* the public gateway API.
|
||||
*/
|
||||
export const perplexityCompatFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
// OUTBOUND: force the encoding Perplexity actually accepts.
|
||||
if (init?.body && typeof init.body === 'string') {
|
||||
try {
|
||||
const parsed = JSON.parse(init.body);
|
||||
if (parsed && typeof parsed === 'object' && parsed.encoding_format !== 'base64_int8') {
|
||||
parsed.encoding_format = 'base64_int8';
|
||||
// Drop Content-Length so fetch recomputes from the new body.
|
||||
const headers = new Headers(init.headers ?? {});
|
||||
headers.delete('content-length');
|
||||
init = { ...init, body: JSON.stringify(parsed), headers };
|
||||
}
|
||||
} catch {
|
||||
// Body wasn't JSON — pass through untouched.
|
||||
}
|
||||
}
|
||||
|
||||
const resp = await fetch(input as any, init);
|
||||
if (!resp.ok) return resp;
|
||||
const ct = resp.headers.get('content-type') ?? '';
|
||||
if (!ct.toLowerCase().includes('application/json')) return resp;
|
||||
|
||||
// Layer 1: Content-Length pre-check BEFORE the body is parsed.
|
||||
const contentLengthHeader = resp.headers.get('content-length');
|
||||
if (contentLengthHeader) {
|
||||
const len = parseInt(contentLengthHeader, 10);
|
||||
if (Number.isFinite(len) && len > MAX_PERPLEXITY_RESPONSE_BYTES) {
|
||||
throw new PerplexityResponseTooLargeError(
|
||||
`Perplexity response Content-Length=${len} exceeds ${MAX_PERPLEXITY_RESPONSE_BYTES} bytes — ` +
|
||||
`likely compromised endpoint or misconfiguration`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// INBOUND: decode base64 int8 embeddings to number[] so the SDK's Zod
|
||||
// schema validates.
|
||||
try {
|
||||
const json: any = await resp.clone().json();
|
||||
if (!json || typeof json !== 'object') return resp;
|
||||
let modified = false;
|
||||
if (Array.isArray(json.data)) {
|
||||
for (const item of json.data) {
|
||||
if (item && typeof item.embedding === 'string') {
|
||||
// Layer 2: per-embedding cap for chunked responses that skipped
|
||||
// Layer 1. base64 → bytes is the canonical 0.75 ratio.
|
||||
const estDecoded = Math.ceil(item.embedding.length * 0.75);
|
||||
if (estDecoded > MAX_PERPLEXITY_RESPONSE_BYTES) {
|
||||
throw new PerplexityResponseTooLargeError(
|
||||
`Perplexity embedding base64 exceeds ${MAX_PERPLEXITY_RESPONSE_BYTES} bytes ` +
|
||||
`(estimated ${estDecoded} bytes from ${item.embedding.length} base64 chars)`,
|
||||
);
|
||||
}
|
||||
// base64_int8: one signed int8 per component.
|
||||
const bytes = Buffer.from(item.embedding, 'base64');
|
||||
item.embedding = Array.from(new Int8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength));
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (json.usage && typeof json.usage === 'object' && json.usage.prompt_tokens === undefined) {
|
||||
json.usage.prompt_tokens = typeof json.usage.total_tokens === 'number'
|
||||
? json.usage.total_tokens
|
||||
: 0;
|
||||
modified = true;
|
||||
}
|
||||
if (!modified) return resp;
|
||||
return new Response(JSON.stringify(json), {
|
||||
status: resp.status,
|
||||
statusText: resp.statusText,
|
||||
headers: resp.headers,
|
||||
});
|
||||
} catch (err) {
|
||||
// OOM-cap throws MUST propagate; anything else falls back to the
|
||||
// original response (same contract as voyageCompatFetch).
|
||||
if (err instanceof PerplexityResponseTooLargeError) throw err;
|
||||
return resp;
|
||||
}
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
async function resolveEmbeddingProvider(modelStr: string): Promise<{ model: any; recipe: Recipe; modelId: string }> {
|
||||
const { parsed, recipe } = resolveRecipe(modelStr);
|
||||
assertTouchpoint(recipe, 'embedding', parsed.modelId, getExtendedModelsForProvider(parsed.providerId));
|
||||
@@ -1451,10 +1282,6 @@ function instantiateEmbedding(recipe: Recipe, modelId: string, cfg: AIGatewayCon
|
||||
throw new AIConfigError(
|
||||
`Anthropic has no embedding model. Use openai or google for embeddings.`,
|
||||
);
|
||||
case 'claude-cli':
|
||||
throw new AIConfigError(
|
||||
`claude-cli has no embedding model. Use openai or google for embeddings.`,
|
||||
);
|
||||
case 'openai-compatible': {
|
||||
// D12=A: unified auth via Recipe.resolveAuth (or default).
|
||||
const auth = applyResolveAuth(recipe, cfg, 'embedding');
|
||||
@@ -1477,10 +1304,6 @@ function instantiateEmbedding(recipe: Recipe, modelId: string, cfg: AIGatewayCon
|
||||
? voyageCompatFetch
|
||||
: recipe.id === 'zeroentropyai'
|
||||
? zeroEntropyCompatFetch
|
||||
: recipe.id === 'nvidia'
|
||||
? nvidiaCompatFetch
|
||||
: recipe.id === 'perplexity'
|
||||
? perplexityCompatFetch
|
||||
: openAICompatAsymmetricFetch);
|
||||
const client = createOpenAICompatible({
|
||||
name: recipe.id,
|
||||
@@ -1638,17 +1461,10 @@ export async function embed(texts: string[], opts?: EmbedOpts): Promise<Float32A
|
||||
|
||||
// Pre-split is gated on max_batch_tokens. Recipes without it (e.g. OpenAI)
|
||||
// ride the fast path: one embedMany call, no recursion safety net.
|
||||
const tokenBatches = maxBatchTokens
|
||||
const batches = maxBatchTokens
|
||||
? splitByTokenBudget(truncated, Math.floor(maxBatchTokens * effectiveSafetyFactor(recipe)), charsPerToken)
|
||||
: [truncated];
|
||||
|
||||
// Hard COUNT cap (e.g. llama-server's "maximum allowed batch size 32").
|
||||
// Token budget can't bound item count, so re-split any oversized batch.
|
||||
const maxBatchItems = embedding?.max_batch_items;
|
||||
const batches = maxBatchItems
|
||||
? tokenBatches.flatMap(b => capBatchItems(b, maxBatchItems))
|
||||
: tokenBatches;
|
||||
|
||||
const allEmbeddings: Float32Array[] = [];
|
||||
let _embedThrew = false;
|
||||
try {
|
||||
@@ -1724,23 +1540,6 @@ export function splitByTokenBudget(
|
||||
return batches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a batch into sub-batches of at most `maxItems` inputs. Enforces a
|
||||
* hard COUNT cap that the token-budget split can't (many tiny inputs fit
|
||||
* under any token budget). Used for endpoints like llama.cpp's llama-server
|
||||
* that reject requests exceeding their launch batch size.
|
||||
*
|
||||
* @internal exported for tests; not part of the public gateway API.
|
||||
*/
|
||||
export function capBatchItems(texts: string[], maxItems: number): string[][] {
|
||||
if (maxItems <= 0 || texts.length <= maxItems) return [texts];
|
||||
const batches: string[][] = [];
|
||||
for (let i = 0; i < texts.length; i += maxItems) {
|
||||
batches.push(texts.slice(i, i + maxItems));
|
||||
}
|
||||
return batches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the error looks like a provider batch-token-limit error.
|
||||
*
|
||||
@@ -2399,15 +2198,6 @@ function instantiateExpansion(recipe: Recipe, modelId: string, cfg: AIGatewayCon
|
||||
const baseURL = resolveNativeBaseUrl('anthropic', cfg);
|
||||
return createAnthropic({ apiKey, ...(baseURL ? { baseURL } : {}) }).languageModel(modelId);
|
||||
}
|
||||
case 'claude-cli': {
|
||||
// The CLI handles its own auth (OAuth session); spawn the subprocess
|
||||
// directly via the same LanguageModelV2 implementation chat uses. There
|
||||
// is no separate expansion path because claude-cli does not declare a
|
||||
// separate expansion touchpoint — but routing here keeps the switch
|
||||
// exhaustive and lets a future expansion touchpoint use the same code.
|
||||
const { ClaudeCliLanguageModel } = require('./providers/claude-cli-language-model.ts');
|
||||
return new ClaudeCliLanguageModel(modelId);
|
||||
}
|
||||
case 'openai-compatible': {
|
||||
// D12=A: unified auth via Recipe.resolveAuth (or default).
|
||||
const auth = applyResolveAuth(recipe, cfg, 'expansion');
|
||||
@@ -2863,17 +2653,6 @@ export function probeChatModel(modelStr: string): ChatModelProbe {
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-model prompt-cache capability: `supports_prompt_cache` may be a static
|
||||
* boolean (native providers) or a per-model-id predicate (OpenRouter's
|
||||
* family-scoped caching).
|
||||
*/
|
||||
function chatSupportsPromptCache(recipe: Recipe, modelId: string): boolean {
|
||||
const support = recipe.touchpoints.chat?.supports_prompt_cache;
|
||||
if (typeof support === 'function') return support(modelId);
|
||||
return support === true;
|
||||
}
|
||||
|
||||
async function resolveChatProvider(modelStr: string): Promise<{ model: any; recipe: Recipe; modelId: string }> {
|
||||
const { parsed, recipe } = resolveRecipe(modelStr);
|
||||
assertTouchpoint(recipe, 'chat', parsed.modelId, getExtendedModelsForProvider(parsed.providerId));
|
||||
@@ -2907,15 +2686,6 @@ function instantiateChat(recipe: Recipe, modelId: string, cfg: AIGatewayConfig):
|
||||
const baseURL = resolveNativeBaseUrl('anthropic', cfg);
|
||||
return createAnthropic({ apiKey, ...(baseURL ? { baseURL } : {}) }).languageModel(modelId);
|
||||
}
|
||||
case 'claude-cli': {
|
||||
// The CLI handles its own auth (OAuth session managed by `claude`
|
||||
// login). Subprocess-based LanguageModelV2 dispatches via the recipe
|
||||
// path so per-call routing works: `claude-cli:claude-sonnet-4-6` lands
|
||||
// here, while sibling `litellm:gpt-5.4` continues through the
|
||||
// openai-compatible path below. No env-var switch, no global flag.
|
||||
const { ClaudeCliLanguageModel } = require('./providers/claude-cli-language-model.ts');
|
||||
return new ClaudeCliLanguageModel(modelId);
|
||||
}
|
||||
case 'openai-compatible': {
|
||||
// D12=A: unified auth via Recipe.resolveAuth (or default).
|
||||
const auth = applyResolveAuth(recipe, cfg, 'chat');
|
||||
@@ -3079,30 +2849,6 @@ async function classifyGatewayGuardrail(input: {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive OpenAI's `prompt_cache_key` (the AI SDK's `providerOptions.openai.
|
||||
* promptCacheKey`). It's a ROUTING hint, not a cache breakpoint: OpenAI caches
|
||||
* prefixes automatically, and a stable key makes requests sharing a prefix
|
||||
* land on the same engine, raising the hit rate (OpenAI cites 60%→87%).
|
||||
*
|
||||
* Hash the system prompt + sorted tool names — that's the stable prefix
|
||||
* gbrain's repeated loops (enrich, page-summary, skillopt, subagent) actually
|
||||
* share. Returns undefined when there's no system prompt (nothing stable to
|
||||
* key on), so one-off requests don't get pinned to a single engine. An
|
||||
* explicit key can still be set per provider/model via
|
||||
* `provider_chat_options` config, which overrides the derived key.
|
||||
*
|
||||
* @internal exported for tests; not part of the public gateway API.
|
||||
*/
|
||||
export function openAIPromptCacheKey(args: {
|
||||
system?: string;
|
||||
toolNames?: string[];
|
||||
}): string | undefined {
|
||||
if (!args.system) return undefined;
|
||||
const basis = `${args.system} ${(args.toolNames ?? []).slice().sort().join(',')}`;
|
||||
return `gbrain:${createHash('sha256').update(basis).digest('hex').slice(0, 32)}`;
|
||||
}
|
||||
|
||||
export function toAISDKTools(tools: ChatToolDef[] | undefined): Record<string, any> | undefined {
|
||||
if (!tools || tools.length === 0) return undefined;
|
||||
return tools.reduce((acc, t) => {
|
||||
@@ -3204,85 +2950,17 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
|
||||
const { model, recipe, modelId } = await resolveChatProvider(modelStr);
|
||||
const cfg = requireConfig();
|
||||
|
||||
const supportsCache = chatSupportsPromptCache(recipe, modelId);
|
||||
const supportsCache = recipe.touchpoints.chat?.supports_prompt_cache === true;
|
||||
const useCache = !!opts.cacheSystem && supportsCache;
|
||||
|
||||
// OpenRouter Claude routes need an explicit `cache_control` on the system
|
||||
// content block, but the openai-compatible adapter drops anthropic-namespace
|
||||
// providerOptions before building the wire body. Signal intent via a private
|
||||
// header; the recipe's compat fetch shim rewrites the body and strips the
|
||||
// header before the request leaves the process. OpenAI routes through
|
||||
// OpenRouter cache automatically — no marker needed.
|
||||
const requestHeaders = useCache && recipe.id === 'openrouter' && openrouterRequiresExplicitPromptCache(modelId)
|
||||
? { [OPENROUTER_CACHE_HEADER]: '1' }
|
||||
: undefined;
|
||||
|
||||
const tools = toAISDKTools(opts.tools);
|
||||
|
||||
const providerOptions: Record<string, any> = {};
|
||||
if (useCache) {
|
||||
// Call-level `providerOptions.anthropic.cacheControl` is NOT a no-op:
|
||||
// @ai-sdk/anthropic 3.0.47+ passes it through as a top-level
|
||||
// `cache_control` field on the Anthropic request body, which the
|
||||
// Messages API resolves as its documented "auto-cache the last
|
||||
// cacheable block in the request" shorthand (see Anthropic's
|
||||
// prompt-caching docs — "top-level auto-caching ... is the simplest
|
||||
// option when you don't need fine-grained placement"). Keep it: it's
|
||||
// what gives a growing multi-turn conversation (toolLoop()) a rolling
|
||||
// cache breakpoint on each turn's tail for free, without us having to
|
||||
// hand-roll the marker-walking logic subagent.ts's raw-SDK path uses.
|
||||
//
|
||||
// But "last cacheable block" is the wrong block for gbrain#2490's
|
||||
// actual callers (page-summary, skillopt, enrich): those are
|
||||
// single-turn calls with a STABLE system prompt and a DIFFERENT user
|
||||
// message every time, so the auto-marker lands on the ever-varying
|
||||
// tail — every call WRITES a fresh cache entry and never READS a prior
|
||||
// one (cache_read_input_tokens stays 0 forever). Caching the stable
|
||||
// prefix needs an EXPLICIT breakpoint on the system block itself,
|
||||
// which is applied below via a `SystemModelMessage` (round-trips its
|
||||
// own `providerOptions`) instead of a bare string.
|
||||
providerOptions.anthropic = { cacheControl: { type: 'ephemeral' } };
|
||||
}
|
||||
// OpenAI prompt_cache_key (native-openai only): a stable per-prefix routing
|
||||
// hint that keeps requests sharing a system prompt + tool set on the same
|
||||
// inference engine, lifting OpenAI's automatic prefix-cache hit rate. The
|
||||
// openai-compatible path (litellm/azure/groq/...) ignores
|
||||
// providerOptions.openai, so it gets nothing. Applied BEFORE the configured
|
||||
// provider options so `provider_chat_options.openai.promptCacheKey` from
|
||||
// config still overrides the derived key.
|
||||
if (recipe.implementation === 'native-openai') {
|
||||
const promptCacheKey = openAIPromptCacheKey({
|
||||
system: opts.system,
|
||||
toolNames: (opts.tools ?? []).map(t => t.name),
|
||||
});
|
||||
if (promptCacheKey) providerOptions.openai = { promptCacheKey };
|
||||
}
|
||||
applyConfiguredChatProviderOptions(providerOptions, cfg, recipe.id, modelId);
|
||||
|
||||
// Derive ONE canonical cache-control value AFTER config merging and reuse
|
||||
// it for every breakpoint (system block, last tool def, call-level). If
|
||||
// `provider_chat_options.anthropic.cacheControl` overrides the TTL (e.g.
|
||||
// `{ type: 'ephemeral', ttl: '1h' }`), that override lands in
|
||||
// `providerOptions.anthropic.cacheControl` via the deep-merge above —
|
||||
// reusing it here (instead of hardcoding `{ type: 'ephemeral' }` per
|
||||
// breakpoint) keeps every marker in the request on the same TTL.
|
||||
const cacheControlValue: { type: 'ephemeral'; ttl?: '5m' | '1h' } | undefined = useCache
|
||||
? (providerOptions.anthropic?.cacheControl ?? { type: 'ephemeral' })
|
||||
: undefined;
|
||||
|
||||
// Anthropic-only secondary breakpoint: mark the LAST tool def too (mirrors
|
||||
// subagent.ts's raw-SDK path — Anthropic caches everything up to and
|
||||
// including the last `cache_control` block it sees in the request, so
|
||||
// marking the last tool extends the cached prefix through the whole tool
|
||||
// list). `tool.providerOptions.anthropic.cacheControl` is the shape
|
||||
// @ai-sdk/anthropic 3.x reads for tool-def breakpoints.
|
||||
if (cacheControlValue && opts.tools && opts.tools.length > 0 && tools) {
|
||||
const lastTool = tools[opts.tools[opts.tools.length - 1]!.name];
|
||||
if (lastTool) {
|
||||
lastTool.providerOptions = { anthropic: { cacheControl: cacheControlValue } };
|
||||
}
|
||||
}
|
||||
|
||||
let _budgetRecorded = false;
|
||||
const _recordBudget = (modelLabel: string, inputTokens: number, outputTokens: number): void => {
|
||||
if (!tracker || _budgetRecorded) return;
|
||||
@@ -3299,28 +2977,10 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
|
||||
}
|
||||
};
|
||||
|
||||
// The actual Anthropic system-prompt cache breakpoint. A bare string
|
||||
// `system` produces `{ role: 'system', content }` with no `providerOptions`
|
||||
// field (ai@6's convertToLanguageModelPrompt), so @ai-sdk/anthropic's
|
||||
// getCacheControl(providerOptions) on that block always resolves to
|
||||
// nothing. Passing a `SystemModelMessage` object instead — the shape `ai`
|
||||
// documents specifically for "additional provider options (e.g. for
|
||||
// caching)" — round-trips `providerOptions` onto that block. Byte-identical
|
||||
// to the old bare-string form when useCache is false. Reuses
|
||||
// `cacheControlValue` (the config-merged value) so this breakpoint's TTL
|
||||
// always matches the last-tool and call-level breakpoints.
|
||||
const systemParam = cacheControlValue && opts.system
|
||||
? {
|
||||
role: 'system' as const,
|
||||
content: opts.system,
|
||||
providerOptions: { anthropic: { cacheControl: cacheControlValue } },
|
||||
}
|
||||
: opts.system;
|
||||
|
||||
try {
|
||||
const result = await _generateTextTransport({
|
||||
model,
|
||||
system: systemParam,
|
||||
system: opts.system,
|
||||
messages: toModelMessages(repairToolPairing(opts.messages)) as any,
|
||||
tools: opts.tools && opts.tools.length > 0 ? tools : undefined,
|
||||
maxOutputTokens: opts.maxTokens ?? defaultMaxOutputTokens(modelStr),
|
||||
@@ -3328,7 +2988,6 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
|
||||
// shorter wins). Covers native-anthropic (the default provider + facts Haiku).
|
||||
abortSignal: withDefaultTimeout(opts.abortSignal, AI_CHAT_TIMEOUT_MS),
|
||||
providerOptions: Object.keys(providerOptions).length > 0 ? providerOptions : undefined,
|
||||
...(requestHeaders ? { headers: requestHeaders } : {}),
|
||||
});
|
||||
|
||||
// Normalize blocks. Vercel SDK gives us `result.content` (an array of typed
|
||||
@@ -3377,10 +3036,7 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
|
||||
usage: {
|
||||
input_tokens: inTok,
|
||||
output_tokens: outTok,
|
||||
// `usage.cachedInputTokens` is the AI SDK's provider-neutral cache-read
|
||||
// count — it's how OpenAI-compatible routes (OpenRouter's
|
||||
// prompt_tokens_details.cached_tokens) surface cache hits.
|
||||
cache_read_tokens: Number(anthropicCache.cacheReadInputTokens ?? anthropicCache.cache_read_input_tokens ?? usage.cachedInputTokens ?? 0),
|
||||
cache_read_tokens: Number(anthropicCache.cacheReadInputTokens ?? anthropicCache.cache_read_input_tokens ?? 0),
|
||||
cache_creation_tokens: Number(anthropicCache.cacheCreationInputTokens ?? anthropicCache.cache_creation_input_tokens ?? 0),
|
||||
},
|
||||
model: `${recipe.id}:${modelId}`,
|
||||
@@ -3844,15 +3500,7 @@ export async function rerank(input: RerankInput): Promise<RerankResult[]> {
|
||||
// whose request/response shape differs from ZE/llama.cpp (e.g. Voyage with
|
||||
// `top_k` / `data[]`) needs separate adapter hooks in a follow-up plan.
|
||||
const url = `${compat.baseURL.replace(/\/$/, '')}${tp.path ?? '/models/rerank'}`;
|
||||
let auth: { apiKey?: string; headers?: Record<string, string> };
|
||||
try {
|
||||
auth = applyResolveAuth(recipe, cfg, 'reranker');
|
||||
} catch (err) {
|
||||
if (err instanceof AIConfigError) {
|
||||
throw new RerankError(err.message, 'auth');
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const auth = applyResolveAuth(recipe, cfg, 'reranker');
|
||||
// applyResolveAuth returns { apiKey } for Bearer-style auth (SDK's native
|
||||
// path) or { headers } for custom-header providers (Azure). v0.37.6.0:
|
||||
// recipes can ALSO declare default_headers (attribution etc.) which flow
|
||||
|
||||
@@ -1,444 +0,0 @@
|
||||
/**
|
||||
* ai-sdk LanguageModelV2 implementation that dispatches via the `claude --print`
|
||||
* CLI subprocess. Used by the `claude-cli` recipe to route gateway.toolLoop /
|
||||
* gateway.chat calls through Claude Code's OAuth session instead of the
|
||||
* Anthropic SDK + ANTHROPIC_API_KEY.
|
||||
*
|
||||
* Per-call routing is the contract: the gateway resolves the model string
|
||||
* to this recipe based on the `claude-cli:` prefix, instantiates one of
|
||||
* these objects per modelId, and dispatches doGenerate. Sibling subagent
|
||||
* jobs with `litellm:gpt-5.4` continue routing through litellm-proxy in
|
||||
* the same worker; no env-var switch, no global state.
|
||||
*
|
||||
* Tool use is supported via system-prompt-instructed JSON emission:
|
||||
* The recipe injects a fenced instruction block into the system prompt
|
||||
* that teaches the model the `<use_tools>[{id,name,input}, ...]</use_tools>`
|
||||
* emission format. The adapter parses those blocks back into ai-sdk
|
||||
* `tool-call` content parts. Parallel tool calls (multiple entries in
|
||||
* the JSON array) round-trip cleanly — this is the case that breaks
|
||||
* on the codex-proxy / litellm GPT-5.x bridge today.
|
||||
*
|
||||
* Context isolation:
|
||||
* The subprocess is spawned from a dedicated tmpdir so claude-cli's
|
||||
* CLAUDE.md auto-discovery has no local files to find. `--system-prompt`
|
||||
* replaces the default system prompt; `--disable-slash-commands` skips
|
||||
* skill resolution. User-level ~/.claude/CLAUDE.md still loads because
|
||||
* the only way to skip it is `--bare`, which forces ANTHROPIC_API_KEY
|
||||
* auth and defeats the whole point of this provider. The ~42k cached
|
||||
* tokens from user-level instructions are accepted as a cost-trivial
|
||||
* trade-off on the subscription path.
|
||||
*
|
||||
* doStream is not yet implemented; the model declares no streaming. Callers
|
||||
* (gateway.toolLoop primarily) use doGenerate.
|
||||
*/
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdirSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type {
|
||||
LanguageModelV2,
|
||||
LanguageModelV2CallOptions,
|
||||
LanguageModelV2Content,
|
||||
LanguageModelV2FunctionTool,
|
||||
LanguageModelV2Prompt,
|
||||
LanguageModelV2Message,
|
||||
LanguageModelV2ProviderDefinedTool,
|
||||
} from '@ai-sdk/provider';
|
||||
|
||||
function claudeBin(): string {
|
||||
return process.env.GBRAIN_CLAUDE_CLI_BIN ?? 'claude';
|
||||
}
|
||||
const CLAUDE_CWD = join(tmpdir(), `gbrain-claude-cli-cwd-${process.pid}`);
|
||||
let cwdEnsured = false;
|
||||
function ensureCleanCwd(): string {
|
||||
if (!cwdEnsured) {
|
||||
mkdirSync(CLAUDE_CWD, { recursive: true });
|
||||
cwdEnsured = true;
|
||||
}
|
||||
return CLAUDE_CWD;
|
||||
}
|
||||
|
||||
/** Parsed shape of `claude --print --output-format json`. */
|
||||
interface ClaudeJsonResult {
|
||||
type: 'result';
|
||||
subtype: 'success' | string;
|
||||
is_error: boolean;
|
||||
result: string;
|
||||
stop_reason: string | null;
|
||||
session_id: string;
|
||||
num_turns: number;
|
||||
usage?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
cache_read_input_tokens?: number;
|
||||
cache_creation_input_tokens?: number;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the system-prompt addendum that teaches the model the
|
||||
* `<use_tools>...</use_tools>` emission format. Returns the empty string
|
||||
* when no tools are registered for this turn so the model gets a normal
|
||||
* text-completion prompt without protocol noise.
|
||||
*/
|
||||
function buildToolUseInstructions(
|
||||
tools: ReadonlyArray<LanguageModelV2FunctionTool | LanguageModelV2ProviderDefinedTool> | undefined,
|
||||
): string {
|
||||
if (!tools || tools.length === 0) return '';
|
||||
|
||||
const functionTools = tools.filter((t): t is LanguageModelV2FunctionTool => t.type === 'function');
|
||||
if (functionTools.length === 0) return '';
|
||||
|
||||
const toolSpecs = functionTools.map(t => ({
|
||||
name: t.name,
|
||||
description: t.description ?? '',
|
||||
input_schema: t.inputSchema ?? { type: 'object', properties: {} },
|
||||
}));
|
||||
|
||||
return [
|
||||
'',
|
||||
'## Tool Use Protocol',
|
||||
'',
|
||||
'You have access to these tools:',
|
||||
'',
|
||||
'```json',
|
||||
JSON.stringify(toolSpecs, null, 2),
|
||||
'```',
|
||||
'',
|
||||
'To call one or more tools in this turn, emit EXACTLY ONE block of this form, ' +
|
||||
'with no other text outside the block on its own lines:',
|
||||
'',
|
||||
'<use_tools>',
|
||||
'[',
|
||||
' {"id": "<unique tool call id, like toolu_01ABC>", "name": "<tool name>", "input": <input object matching the tool\'s input_schema>}',
|
||||
']',
|
||||
'</use_tools>',
|
||||
'',
|
||||
'Multiple tool calls go in the array. Tool results are returned to you on the ' +
|
||||
'next turn as [tool_result <text>] entries. You may then call more tools or emit a final response.',
|
||||
'',
|
||||
'When you are ready to give a final answer instead of calling tools, respond with prose text only — ' +
|
||||
'do not include a <use_tools> block in that case.',
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Render the ai-sdk message array into a single text prompt for `claude --print`
|
||||
* stdin. System messages are extracted up-front and concatenated into the
|
||||
* `--system-prompt` flag value. Tool calls and tool results are rendered as
|
||||
* placeholders so the model sees the conversation in a coherent shape even
|
||||
* though the adapter does not natively round-trip tool calls through claude-cli.
|
||||
*/
|
||||
function renderPrompt(prompt: LanguageModelV2Prompt): { systemText: string; userPrompt: string } {
|
||||
const systemParts: string[] = [];
|
||||
const convo: string[] = [];
|
||||
|
||||
for (const msg of prompt as ReadonlyArray<LanguageModelV2Message>) {
|
||||
if (msg.role === 'system') {
|
||||
systemParts.push(msg.content);
|
||||
continue;
|
||||
}
|
||||
if (msg.role === 'user') {
|
||||
const text = msg.content
|
||||
.map(p => {
|
||||
if (p.type === 'text') return p.text;
|
||||
// File parts get a stub — multimodal is not supported via subprocess yet.
|
||||
if (p.type === 'file') return `[file ${p.mediaType ?? 'unknown'}]`;
|
||||
return '';
|
||||
})
|
||||
.filter(s => s.length > 0)
|
||||
.join('\n');
|
||||
if (text) convo.push(`User: ${text}`);
|
||||
continue;
|
||||
}
|
||||
if (msg.role === 'assistant') {
|
||||
const rendered = msg.content
|
||||
.map(p => {
|
||||
if (p.type === 'text') return p.text;
|
||||
if (p.type === 'reasoning') return ''; // dropped on replay
|
||||
if (p.type === 'tool-call') {
|
||||
return `[tool_use ${p.toolName}(${p.input})]`;
|
||||
}
|
||||
if (p.type === 'tool-result') {
|
||||
const out = typeof p.output === 'string' ? p.output : JSON.stringify(p.output);
|
||||
return `[tool_result ${out}]`;
|
||||
}
|
||||
return '';
|
||||
})
|
||||
.filter(s => s.length > 0)
|
||||
.join('\n');
|
||||
if (rendered) convo.push(`Assistant: ${rendered}`);
|
||||
continue;
|
||||
}
|
||||
if (msg.role === 'tool') {
|
||||
const rendered = msg.content
|
||||
.map(p => {
|
||||
const out = typeof p.output === 'string' ? p.output : JSON.stringify(p.output);
|
||||
return `[tool_result ${out}]`;
|
||||
})
|
||||
.join('\n');
|
||||
if (rendered) convo.push(`User: ${rendered}`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return { systemText: systemParts.join('\n'), userPrompt: convo.join('\n\n') };
|
||||
}
|
||||
|
||||
/**
|
||||
* Spawn `claude --print` with the contamination-suppression flags and return
|
||||
* the parsed `--output-format json` envelope. Aborts propagate to SIGTERM on
|
||||
* the child.
|
||||
*/
|
||||
function runClaude(
|
||||
systemPrompt: string,
|
||||
userPrompt: string,
|
||||
model: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ClaudeJsonResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const args = [
|
||||
'--print',
|
||||
'--output-format', 'json',
|
||||
'--model', model,
|
||||
'--disable-slash-commands',
|
||||
// Agent isolation: this subprocess must behave like a raw LLM, not a
|
||||
// full Claude Code agent. `--tools ""` disables every built-in tool
|
||||
// (Bash/Read/WebSearch/...); `--strict-mcp-config` ignores all user-level
|
||||
// MCP servers (without it, each call would boot the user's MCP servers —
|
||||
// including gbrain's own MCP → recursion + PGLite single-writer lock
|
||||
// contention). Verified against claude CLI 2.1.145 --help.
|
||||
'--tools', '',
|
||||
'--strict-mcp-config',
|
||||
];
|
||||
if (systemPrompt) {
|
||||
args.push('--system-prompt', systemPrompt);
|
||||
}
|
||||
// Env scrub: guarantee the CLI authenticates via its own OAuth session
|
||||
// (subscription), never via an inherited API key. Without this, an
|
||||
// ANTHROPIC_API_KEY in gbrain's env (the exact setup this recipe is meant
|
||||
// to replace) silently flips billing to per-token API usage.
|
||||
const env = { ...process.env };
|
||||
delete env.ANTHROPIC_API_KEY;
|
||||
delete env.ANTHROPIC_AUTH_TOKEN;
|
||||
delete env.ANTHROPIC_BASE_URL;
|
||||
const child = spawn(claudeBin(), args, {
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
cwd: ensureCleanCwd(),
|
||||
env,
|
||||
});
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
child.stdout.on('data', chunk => { stdout += String(chunk); });
|
||||
child.stderr.on('data', chunk => { stderr += String(chunk); });
|
||||
|
||||
const onAbort = () => {
|
||||
child.kill('SIGTERM');
|
||||
reject(new Error('claude-cli adapter aborted'));
|
||||
};
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
}
|
||||
|
||||
child.on('error', err => {
|
||||
if (signal) signal.removeEventListener('abort', onAbort);
|
||||
reject(new Error(`claude-cli spawn failed: ${err instanceof Error ? err.message : String(err)}`));
|
||||
});
|
||||
|
||||
child.on('close', code => {
|
||||
if (signal) signal.removeEventListener('abort', onAbort);
|
||||
if (code !== 0) {
|
||||
reject(new Error(`claude-cli exited ${code}: ${stderr.trim() || stdout.trim()}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
let parsed = JSON.parse(stdout) as unknown;
|
||||
// Compat: when the user has `"verbose": true` in ~/.claude/settings.json,
|
||||
// `--print --output-format json` emits an ARRAY of events
|
||||
// ([{type:"system",subtype:"init",...}, ..., {type:"result",...}])
|
||||
// instead of the bare result object. There is no CLI flag to force it
|
||||
// off (no --no-verbose; --settings '{}' merges, does not replace), so
|
||||
// tolerate both shapes and pick the result event. Verified on CLI 2.1.145.
|
||||
if (Array.isArray(parsed)) {
|
||||
const resultEvent = parsed.find(
|
||||
(ev): ev is ClaudeJsonResult =>
|
||||
!!ev && typeof ev === 'object' && (ev as { type?: unknown }).type === 'result',
|
||||
);
|
||||
if (!resultEvent) {
|
||||
reject(new Error(`claude-cli JSON event array had no "result" event\n--- raw ---\n${stdout.slice(0, 500)}`));
|
||||
return;
|
||||
}
|
||||
parsed = resultEvent;
|
||||
}
|
||||
const envelope = parsed as ClaudeJsonResult;
|
||||
if (envelope.is_error) {
|
||||
reject(new Error(`claude-cli reported error: ${envelope.result || envelope.subtype}`));
|
||||
return;
|
||||
}
|
||||
resolve(envelope);
|
||||
} catch (e) {
|
||||
reject(new Error(`claude-cli output not JSON: ${e instanceof Error ? e.message : String(e)}\n--- raw ---\n${stdout.slice(0, 500)}`));
|
||||
}
|
||||
});
|
||||
|
||||
// stdin error handler: if the binary does not exist (ENOENT) or the child
|
||||
// dies before draining stdin, write/end can emit an unhandled 'error'
|
||||
// (EPIPE) that would crash the worker. The spawn-level 'error' / non-zero
|
||||
// 'close' handlers above already surface the real failure, so the stdin
|
||||
// error itself is safe to swallow.
|
||||
child.stdin.on('error', () => { /* surfaced via child 'error'/'close' */ });
|
||||
try {
|
||||
child.stdin.write(userPrompt);
|
||||
child.stdin.end();
|
||||
} catch (e) {
|
||||
if (signal) signal.removeEventListener('abort', onAbort);
|
||||
reject(new Error(`claude-cli stdin write failed (is the claude binary installed?): ${e instanceof Error ? e.message : String(e)}`));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
interface ParsedToolCall {
|
||||
id: string;
|
||||
name: string;
|
||||
/** Stringified JSON, matching the ai-sdk LanguageModelV2ToolCall.input contract. */
|
||||
input: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Locate and parse the `<use_tools>...</use_tools>` block in the assistant's
|
||||
* raw text response. Returns the parsed tool calls plus whatever prose
|
||||
* surrounded the block. Returns an empty `toolCalls` array when no block is
|
||||
* present, malformed, or unterminated — the caller then treats the full
|
||||
* raw text as a final text response.
|
||||
*/
|
||||
function extractToolCalls(raw: string): {
|
||||
toolCalls: ParsedToolCall[];
|
||||
beforeText: string;
|
||||
afterText: string;
|
||||
} {
|
||||
const openTag = '<use_tools>';
|
||||
const closeTag = '</use_tools>';
|
||||
const openIdx = raw.indexOf(openTag);
|
||||
if (openIdx === -1) {
|
||||
return { toolCalls: [], beforeText: raw.trim(), afterText: '' };
|
||||
}
|
||||
const closeIdx = raw.indexOf(closeTag, openIdx + openTag.length);
|
||||
if (closeIdx === -1) {
|
||||
// Unterminated block — recover gracefully.
|
||||
return { toolCalls: [], beforeText: raw.trim(), afterText: '' };
|
||||
}
|
||||
|
||||
const beforeText = raw.slice(0, openIdx).trim();
|
||||
const afterText = raw.slice(closeIdx + closeTag.length).trim();
|
||||
let inner = raw.slice(openIdx + openTag.length, closeIdx).trim();
|
||||
|
||||
if (inner.startsWith('```')) {
|
||||
inner = inner.replace(/^```(?:json|JSON)?\s*\n?/, '').replace(/\n?```$/, '').trim();
|
||||
}
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(inner);
|
||||
} catch {
|
||||
return { toolCalls: [], beforeText: raw.trim(), afterText: '' };
|
||||
}
|
||||
if (!Array.isArray(parsed)) {
|
||||
return { toolCalls: [], beforeText: raw.trim(), afterText: '' };
|
||||
}
|
||||
|
||||
const toolCalls: ParsedToolCall[] = [];
|
||||
for (const entry of parsed) {
|
||||
if (!entry || typeof entry !== 'object') continue;
|
||||
const e = entry as Record<string, unknown>;
|
||||
const name = typeof e.name === 'string' ? e.name : null;
|
||||
if (!name) continue;
|
||||
const id = typeof e.id === 'string' && e.id.length > 0
|
||||
? e.id
|
||||
: `toolu_claude_cli_${Math.random().toString(36).slice(2, 12)}`;
|
||||
const inputJson = JSON.stringify(e.input ?? {});
|
||||
toolCalls.push({ id, name, input: inputJson });
|
||||
}
|
||||
|
||||
return { toolCalls, beforeText, afterText };
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip provider prefixes (`anthropic:`, `litellm:`, `claude-cli:`) that the
|
||||
* underlying CLI does not understand. The gateway hands us a bare model id
|
||||
* via `recipe.aliases` resolution, but defensive normalization here keeps
|
||||
* direct LanguageModelV2 construction (in tests, for example) ergonomic.
|
||||
*/
|
||||
function normalizeModel(model: string): string {
|
||||
const idx = model.indexOf(':');
|
||||
return idx >= 0 ? model.slice(idx + 1) : model;
|
||||
}
|
||||
|
||||
export class ClaudeCliLanguageModel implements LanguageModelV2 {
|
||||
readonly specificationVersion = 'v2' as const;
|
||||
readonly provider = 'claude-cli';
|
||||
readonly modelId: string;
|
||||
readonly supportedUrls = {};
|
||||
|
||||
constructor(modelId: string) {
|
||||
this.modelId = normalizeModel(modelId);
|
||||
}
|
||||
|
||||
async doGenerate(options: LanguageModelV2CallOptions): Promise<{
|
||||
content: LanguageModelV2Content[];
|
||||
finishReason: 'stop' | 'length' | 'content-filter' | 'tool-calls' | 'error' | 'other' | 'unknown';
|
||||
usage: { inputTokens: number | undefined; outputTokens: number | undefined; totalTokens: number | undefined };
|
||||
warnings: never[];
|
||||
}> {
|
||||
const { systemText, userPrompt } = renderPrompt(options.prompt);
|
||||
const toolInstructions = buildToolUseInstructions(options.tools);
|
||||
const systemPrompt = [systemText, toolInstructions].filter(s => s.length > 0).join('\n');
|
||||
|
||||
const result = await runClaude(systemPrompt, userPrompt, this.modelId, options.abortSignal);
|
||||
const { toolCalls, beforeText, afterText } = extractToolCalls(result.result);
|
||||
|
||||
const content: LanguageModelV2Content[] = [];
|
||||
if (beforeText) content.push({ type: 'text', text: beforeText });
|
||||
for (const call of toolCalls) {
|
||||
content.push({
|
||||
type: 'tool-call',
|
||||
toolCallId: call.id,
|
||||
toolName: call.name,
|
||||
input: call.input,
|
||||
});
|
||||
}
|
||||
if (afterText) content.push({ type: 'text', text: afterText });
|
||||
if (content.length === 0) {
|
||||
// Empty response — still hand the caller a well-formed content array.
|
||||
content.push({ type: 'text', text: result.result ?? '' });
|
||||
}
|
||||
|
||||
const finishReason = toolCalls.length > 0 ? 'tool-calls' as const : 'stop' as const;
|
||||
const inputTokens = result.usage?.input_tokens;
|
||||
const outputTokens = result.usage?.output_tokens;
|
||||
const totalTokens = (inputTokens ?? 0) + (outputTokens ?? 0);
|
||||
|
||||
return {
|
||||
content,
|
||||
finishReason,
|
||||
usage: {
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens: inputTokens !== undefined && outputTokens !== undefined ? totalTokens : undefined,
|
||||
},
|
||||
warnings: [],
|
||||
};
|
||||
}
|
||||
|
||||
async doStream(): Promise<never> {
|
||||
throw new Error(
|
||||
'claude-cli LanguageModel does not support streaming. Use doGenerate or set ' +
|
||||
'the model on a non-streaming chat surface (gateway.toolLoop is non-streaming).',
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -17,16 +17,13 @@ export const anthropic: Recipe = {
|
||||
touchpoints: {
|
||||
// No embedding model available.
|
||||
expansion: {
|
||||
models: ['claude-haiku-4-5-20251001', 'claude-sonnet-5', 'claude-sonnet-4-6'],
|
||||
models: ['claude-haiku-4-5-20251001', 'claude-sonnet-4-6'],
|
||||
cost_per_1m_tokens_usd: 0.25,
|
||||
price_last_verified: '2026-05-10',
|
||||
},
|
||||
chat: {
|
||||
models: [
|
||||
'claude-fable-5',
|
||||
'claude-opus-4-8',
|
||||
'claude-opus-4-7',
|
||||
'claude-sonnet-5',
|
||||
'claude-sonnet-4-6',
|
||||
'claude-haiku-4-5-20251001',
|
||||
],
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Claude via the local `claude` CLI binary, using its built-in OAuth session
|
||||
* (Claude Code / Claude Max subscription). No ANTHROPIC_API_KEY needed — the
|
||||
* CLI manages its own auth state and the gateway dispatches via subprocess.
|
||||
*
|
||||
* Solves the #334 case where Max subscribers want Minions subagent dispatch
|
||||
* to run against their existing subscription instead of paying per-token API
|
||||
* charges. The recipe sits alongside the existing `anthropic` recipe so users
|
||||
* pick per call: `anthropic:claude-sonnet-4-6` (API key + per-token billing)
|
||||
* vs `claude-cli:claude-sonnet-4-6` (OAuth subscription, no API key).
|
||||
*
|
||||
* Chat-only. Claude has no first-party embedding model; users wanting an
|
||||
* Anthropic chat path with embeddings still combine this with openai/google/
|
||||
* voyage for embedding the way the existing `anthropic` recipe documents.
|
||||
*
|
||||
* Auth: `auth_env.required: []` because the CLI handles auth itself. The
|
||||
* `claude` binary on PATH (or `GBRAIN_CLAUDE_CLI_BIN`) IS the auth surface;
|
||||
* there is nothing for the gateway to forward.
|
||||
*
|
||||
* Setup expectation: `claude` CLI installed and logged in (Claude Code
|
||||
* onboarding does this), or `GBRAIN_CLAUDE_CLI_BIN` pointing at the binary.
|
||||
*/
|
||||
export const claudeCli: Recipe = {
|
||||
id: 'claude-cli',
|
||||
name: 'Claude (via CLI)',
|
||||
tier: 'native',
|
||||
implementation: 'claude-cli',
|
||||
// The CLI owns auth; no env vars are required from the gateway side.
|
||||
auth_env: {
|
||||
required: [],
|
||||
},
|
||||
touchpoints: {
|
||||
// No embedding or expansion touchpoints — chat-only.
|
||||
chat: {
|
||||
models: [
|
||||
'claude-opus-4-7',
|
||||
'claude-sonnet-4-6',
|
||||
'claude-haiku-4-5-20251001',
|
||||
],
|
||||
supports_tools: true,
|
||||
supports_subagent_loop: true,
|
||||
// The CLI handles caching internally and does not surface it via the
|
||||
// standard cache_control control plane. From the gateway's POV the
|
||||
// model does not support prompt caching.
|
||||
supports_prompt_cache: false,
|
||||
max_context_tokens: 200000,
|
||||
// Cost figures match the underlying Claude API tier, but the actual
|
||||
// bill is borne by the subscription. We report them for the budget
|
||||
// ledger's per-call accounting; operators on flat-rate subscriptions
|
||||
// can treat the numbers as nominal.
|
||||
cost_per_1m_input_usd: 3.0,
|
||||
cost_per_1m_output_usd: 15.0,
|
||||
price_last_verified: '2026-06-17',
|
||||
},
|
||||
},
|
||||
// Friendly aliases mirror the `anthropic` recipe so config strings stay
|
||||
// portable: switching `anthropic:claude-sonnet-4-6` to `claude-cli:claude-sonnet-4-6`
|
||||
// is a one-token edit. Reverse aliases rewrite legacy IDs back to canonical.
|
||||
aliases: {
|
||||
'claude-haiku-4-5': 'claude-haiku-4-5-20251001',
|
||||
'claude-sonnet-4-6-20250929': 'claude-sonnet-4-6',
|
||||
'sonnet': 'claude-sonnet-4-6',
|
||||
'haiku': 'claude-haiku-4-5-20251001',
|
||||
'opus': 'claude-opus-4-7',
|
||||
},
|
||||
setup_hint:
|
||||
'Install Claude Code (`claude` CLI) and run `claude` once to log in. ' +
|
||||
'Set GBRAIN_CLAUDE_CLI_BIN if the binary is not on PATH.',
|
||||
};
|
||||
@@ -1,61 +0,0 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Alibaba DashScope (灵积) reranker. DashScope's OpenAI-compatible surface
|
||||
* splits by capability: embeddings live under `/compatible-mode/v1` (see the
|
||||
* sibling `dashscope` recipe) while rerank lives under `/compatible-api/v1`
|
||||
* with a PLURAL leaf — `POST {base}/reranks`. Wire shape matches ZeroEntropy:
|
||||
* request `{model, query, documents, top_n?}`, response
|
||||
* `{results: [{index, relevance_score}]}` — so it rides gateway.rerank()'s
|
||||
* native path with only the recipe-pluggable `path` override (v0.40.6.1).
|
||||
*
|
||||
* This is a SEPARATE recipe rather than a reranker touchpoint on `dashscope`
|
||||
* because the two capabilities need different base URLs (`compatible-mode`
|
||||
* vs `compatible-api`) and `provider_base_urls` is keyed by recipe id — one
|
||||
* recipe can't point embeddings and rerank at different prefixes. Same
|
||||
* topology precedent as llama-server vs llama-server-reranker.
|
||||
*
|
||||
* Live-verified against the China endpoint (2026-07): `/reranks` with
|
||||
* `qwen3-rerank` → 200 `results[].relevance_score`; `/rerank` (singular)
|
||||
* → 404; `gte-rerank-v2` → 404 "Unsupported model for OpenAI compatibility
|
||||
* mode" (native-API only, so it is deliberately NOT listed here).
|
||||
*
|
||||
* Note: the international endpoint requires a region-aware DASHSCOPE_API_KEY.
|
||||
* China-region users point at https://dashscope.aliyuncs.com/compatible-api/v1
|
||||
* via `provider_base_urls['dashscope-rerank']`, mirroring the embedding
|
||||
* recipe's convention.
|
||||
*/
|
||||
export const dashscopeRerank: Recipe = {
|
||||
id: 'dashscope-rerank',
|
||||
name: 'Alibaba DashScope (灵积, reranker)',
|
||||
tier: 'openai-compat',
|
||||
implementation: 'openai-compatible',
|
||||
base_url_default: 'https://dashscope-intl.aliyuncs.com/compatible-api/v1',
|
||||
auth_env: {
|
||||
required: ['DASHSCOPE_API_KEY'],
|
||||
setup_url: 'https://help.aliyun.com/zh/model-studio/getting-started/',
|
||||
},
|
||||
touchpoints: {
|
||||
reranker: {
|
||||
// Only the model verified live on the OpenAI-compat /reranks surface.
|
||||
// gte-rerank-v2 exists on DashScope's native API but the compat path
|
||||
// rejects it ("Unsupported model for OpenAI compatibility mode").
|
||||
models: ['qwen3-rerank'],
|
||||
default_model: 'qwen3-rerank',
|
||||
// Mirror ZE's defensive per-request ceiling; gateway.rerank()
|
||||
// pre-flights body size and fails open.
|
||||
max_payload_bytes: 5_000_000,
|
||||
// PLURAL leaf under compatible-api — the whole reason this recipe
|
||||
// exists. `${base_url}${path}` → `…/compatible-api/v1/reranks`.
|
||||
path: '/reranks',
|
||||
// Hosted API: no local warmup, but cross-region latency can exceed
|
||||
// the 5s gateway default (same rationale as llama-server-reranker).
|
||||
default_timeout_ms: 30_000,
|
||||
},
|
||||
},
|
||||
setup_hint:
|
||||
'Get an API key at https://help.aliyun.com/zh/model-studio/getting-started/, then ' +
|
||||
'`export DASHSCOPE_API_KEY=...` and `gbrain config set search.reranker.model ' +
|
||||
'dashscope-rerank:qwen3-rerank`. China-region accounts: `gbrain config set ' +
|
||||
'provider_base_urls.dashscope-rerank https://dashscope.aliyuncs.com/compatible-api/v1`.',
|
||||
};
|
||||
@@ -76,15 +76,6 @@ export const deepseek: Recipe = {
|
||||
setup_url: 'https://platform.deepseek.com/api_keys',
|
||||
},
|
||||
touchpoints: {
|
||||
// Query expansion reuses the same OpenAI-compatible chat endpoint (the
|
||||
// gateway's expansion path is a plain languageModel call). Without this
|
||||
// declaration an explicit `expansion_model: deepseek:...` silently
|
||||
// yields no expansion (#1135).
|
||||
expansion: {
|
||||
models: ['deepseek-chat'],
|
||||
cost_per_1m_tokens_usd: 0.14,
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
chat: {
|
||||
models: ['deepseek-chat', 'deepseek-reasoner'],
|
||||
supports_tools: true,
|
||||
|
||||
@@ -16,14 +16,6 @@ export const groq: Recipe = {
|
||||
setup_url: 'https://console.groq.com/keys',
|
||||
},
|
||||
touchpoints: {
|
||||
// Same OpenAI-compatible endpoint as chat; declared so an explicit
|
||||
// `expansion_model: groq:...` resolves instead of silently dropping
|
||||
// expansion (#1135). 8b-instant is the natural expansion pick (cheap,
|
||||
// fast, no tool-calling needed for multi-query rewrites).
|
||||
expansion: {
|
||||
models: ['llama-3.1-8b-instant', 'llama-3.3-70b-versatile'],
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
chat: {
|
||||
models: [
|
||||
'llama-3.3-70b-versatile',
|
||||
|
||||
@@ -9,7 +9,6 @@ import type { Recipe } from '../types.ts';
|
||||
import { openai } from './openai.ts';
|
||||
import { google } from './google.ts';
|
||||
import { anthropic } from './anthropic.ts';
|
||||
import { claudeCli } from './claude-cli.ts';
|
||||
import { ollama } from './ollama.ts';
|
||||
import { openrouter } from './openrouter.ts';
|
||||
import { voyage } from './voyage.ts';
|
||||
@@ -20,21 +19,15 @@ import { together } from './together.ts';
|
||||
import { llamaServer } from './llama-server.ts';
|
||||
import { minimax } from './minimax.ts';
|
||||
import { dashscope } from './dashscope.ts';
|
||||
import { dashscopeRerank } from './dashscope-rerank.ts';
|
||||
import { zhipu } from './zhipu.ts';
|
||||
import { azureOpenAI } from './azure-openai.ts';
|
||||
import { zeroentropyai } from './zeroentropyai.ts';
|
||||
import { llamaServerReranker } from './llama-server-reranker.ts';
|
||||
import { moonshot } from './moonshot.ts';
|
||||
import { mistral } from './mistral.ts';
|
||||
import { nvidia } from './nvidia.ts';
|
||||
import { perplexity } from './perplexity.ts';
|
||||
|
||||
const ALL: Recipe[] = [
|
||||
openai,
|
||||
google,
|
||||
anthropic,
|
||||
claudeCli,
|
||||
ollama,
|
||||
openrouter,
|
||||
voyage,
|
||||
@@ -46,14 +39,9 @@ const ALL: Recipe[] = [
|
||||
llamaServerReranker,
|
||||
minimax,
|
||||
dashscope,
|
||||
dashscopeRerank,
|
||||
zhipu,
|
||||
azureOpenAI,
|
||||
zeroentropyai,
|
||||
moonshot,
|
||||
mistral,
|
||||
nvidia,
|
||||
perplexity,
|
||||
];
|
||||
|
||||
/** Map from `provider:id` key to recipe. */
|
||||
|
||||
@@ -35,12 +35,9 @@ export const llamaServer: Recipe = {
|
||||
trust_custom_dims: true, // #2271: user knows the launched model's native dim
|
||||
cost_per_1m_tokens_usd: 0,
|
||||
price_last_verified: '2026-05-10',
|
||||
// llama-server enforces a hard request-COUNT cap equal to its launch
|
||||
// batch size (`--batch-size`, default 32): it rejects requests with
|
||||
// more inputs with `batch size N > maximum allowed batch size 32`.
|
||||
// The token-budget split can't bound item count, so cap it here. A
|
||||
// server launched with a larger `-b` can raise this. v0.32 (#779).
|
||||
max_batch_items: 32,
|
||||
// llama-server's batch capacity is set by `--ctx-size` at launch
|
||||
// time; no static cap to declare. v0.32 (#779).
|
||||
no_batch_cap: true,
|
||||
},
|
||||
},
|
||||
/**
|
||||
|
||||
@@ -1,100 +1,8 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* MiniMax transport shim (#1977). MiniMax's `/v1/embeddings` endpoint is NOT
|
||||
* OpenAI-compatible at the wire level despite the recipe's
|
||||
* `implementation: 'openai-compatible'`:
|
||||
* - Request: requires `texts` (the AI SDK sends `input`) plus an optional
|
||||
* `type: 'db' | 'query'` asymmetric-retrieval field, and rejects OpenAI's
|
||||
* `encoding_format`.
|
||||
* - Response: returns `{vectors: number[][], total_tokens}` where the AI
|
||||
* SDK's Zod schema expects `{data: [{embedding, index}], usage}`.
|
||||
*
|
||||
* Chat (`/chat/completions`) IS OpenAI-compatible, and this same fetch is
|
||||
* applied to every openai-compatible touchpoint by `applyOpenAICompatConfig`,
|
||||
* so everything outside the embeddings path passes through untouched — and
|
||||
* the response rewrite parses via `resp.clone()` only (never consume the
|
||||
* body of a response we return as-is; the DeepSeek shim rule). Fail-open:
|
||||
* any rewrite error returns the original request/response.
|
||||
*
|
||||
* @internal exported for tests.
|
||||
*/
|
||||
// Cast through `unknown` because Bun's `typeof fetch` carries a `preconnect`
|
||||
// member the arrow function does not implement (matches deepseek.ts).
|
||||
export const minimaxCompatFetch = (async (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
): Promise<Response> => {
|
||||
const url =
|
||||
typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
|
||||
const isEmbeddings = url.includes('/embeddings');
|
||||
|
||||
// OUTBOUND (embeddings only): `input` → `texts`, default `type: 'db'`
|
||||
// (the recipe's documented symmetric default — the AI SDK adapter strips
|
||||
// the `type` threaded via providerOptions before it reaches the wire,
|
||||
// same class as #1400), and drop `encoding_format` (not a MiniMax param).
|
||||
if (isEmbeddings && init?.body && typeof init.body === 'string') {
|
||||
try {
|
||||
const parsed = JSON.parse(init.body);
|
||||
if (
|
||||
parsed && typeof parsed === 'object' &&
|
||||
parsed.input !== undefined && parsed.texts === undefined
|
||||
) {
|
||||
parsed.texts = Array.isArray(parsed.input) ? parsed.input : [parsed.input];
|
||||
delete parsed.input;
|
||||
delete parsed.encoding_format;
|
||||
if (parsed.type === undefined) parsed.type = 'db';
|
||||
// Drop Content-Length so fetch recomputes from the new body.
|
||||
const headers = new Headers(init.headers ?? {});
|
||||
headers.delete('content-length');
|
||||
init = { ...init, body: JSON.stringify(parsed), headers };
|
||||
}
|
||||
} catch {
|
||||
// Body wasn't JSON — pass through untouched.
|
||||
}
|
||||
}
|
||||
|
||||
const res = await fetch(input as any, init as any);
|
||||
|
||||
// INBOUND (embeddings only): `{vectors: [[...]]}` → `{data: [{embedding}]}`.
|
||||
// Anything else (chat completions, MiniMax base_resp errors, non-JSON)
|
||||
// returns the ORIGINAL response with its body unread.
|
||||
if (!isEmbeddings || !res.ok) return res;
|
||||
const ctype = res.headers.get('content-type') ?? '';
|
||||
if (!ctype.toLowerCase().includes('application/json')) return res;
|
||||
try {
|
||||
const json = await res.clone().json();
|
||||
if (!json || typeof json !== 'object' || !Array.isArray(json.vectors)) return res;
|
||||
const totalTokens = typeof json.total_tokens === 'number' ? json.total_tokens : 0;
|
||||
const rewritten = {
|
||||
object: 'list',
|
||||
data: (json.vectors as number[][]).map((embedding, index) => ({
|
||||
object: 'embedding',
|
||||
embedding,
|
||||
index,
|
||||
})),
|
||||
model: typeof json.model === 'string' ? json.model : 'embo-01',
|
||||
usage: { prompt_tokens: totalTokens, total_tokens: totalTokens },
|
||||
};
|
||||
// Fresh header set: the body changed, so upstream content-length /
|
||||
// content-encoding would now be wrong.
|
||||
const headers = new Headers(res.headers);
|
||||
headers.delete('content-length');
|
||||
headers.delete('content-encoding');
|
||||
return new Response(JSON.stringify(rewritten), {
|
||||
status: res.status,
|
||||
statusText: res.statusText,
|
||||
headers,
|
||||
});
|
||||
} catch {
|
||||
return res;
|
||||
}
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
/**
|
||||
* MiniMax (海螺AI). `/embeddings` endpoint at api.minimaxi.com (wire shape
|
||||
* normalized by `minimaxCompatFetch` above); OpenAI-compatible
|
||||
* `/chat/completions`. The flagship embedding model is `embo-01` (1536 dims).
|
||||
* MiniMax (海螺AI). OpenAI-compatible /embeddings endpoint at
|
||||
* api.minimax.chat. The flagship embedding model is `embo-01` (1536 dims).
|
||||
*
|
||||
* MiniMax's API takes an extra `type: 'db' | 'query'` field for asymmetric
|
||||
* retrieval. gbrain currently has no notion of "this is a document vs a
|
||||
@@ -130,25 +38,7 @@ export const minimax: Recipe = {
|
||||
// halving in the gateway catches token-limit errors at runtime.
|
||||
max_batch_tokens: 4096,
|
||||
},
|
||||
chat: {
|
||||
// Model list from MiniMax's /v1/models (#1977). Chat is genuinely
|
||||
// OpenAI-compatible — no wire rewrite needed (minimaxCompatFetch
|
||||
// passes non-embedding requests through untouched).
|
||||
models: [
|
||||
'MiniMax-M3',
|
||||
'MiniMax-M2.7',
|
||||
'MiniMax-M2.7-highspeed',
|
||||
'MiniMax-M2.5',
|
||||
'MiniMax-M2.5-highspeed',
|
||||
'MiniMax-M2.1',
|
||||
'MiniMax-M2.1-highspeed',
|
||||
'MiniMax-M2',
|
||||
],
|
||||
supports_tools: false,
|
||||
supports_subagent_loop: false,
|
||||
},
|
||||
},
|
||||
setup_hint:
|
||||
'Get an API key at https://www.minimaxi.com, then `export MINIMAX_API_KEY=...`',
|
||||
compat: { fetch: minimaxCompatFetch },
|
||||
};
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Mistral AI exposes an OpenAI-compatible API at https://api.mistral.ai/v1
|
||||
* (/embeddings + /chat/completions). EU-hosted — the reason this recipe
|
||||
* exists: a brain that must stay inside EU jurisdiction can run embed +
|
||||
* expansion + chat on a single provider without a US hop.
|
||||
*
|
||||
* Verified against the live API on 2026-07-19 (model catalog, embedding
|
||||
* dimensions, dimension-parameter rejection, and the batch ceiling — see
|
||||
* the notes on each field below).
|
||||
*
|
||||
* DIMENSIONS — mistral-embed is FIXED 1024 and accepts NO dimension
|
||||
* parameter at all. Both spellings are rejected upstream:
|
||||
* {"dimensions": 512} -> 400 extra_forbidden (not in the API schema)
|
||||
* {"output_dimension": 512} -> 400 "This model does not support output_dimension"
|
||||
* The generic `openai-compatible` branch of dims.ts:dimsProviderOptions()
|
||||
* already falls through to `return undefined` for these model ids, so no
|
||||
* dimension field is emitted. Do NOT add mistral-embed to any of the
|
||||
* flexible-dim allowlists there — it would 400 every embed call. Same
|
||||
* contract as voyage-4-nano, for the same reason.
|
||||
*
|
||||
* codestral-embed / codestral-embed-2505 are deliberately NOT listed: they
|
||||
* return 1536 dims, and a touchpoint carries a single `default_dims`.
|
||||
* Mixing them under a 1024 declaration is the mixed-dim footgun
|
||||
* embedding-dim-check.ts exists to catch. They are code-retrieval models
|
||||
* anyway; a prose brain wants mistral-embed.
|
||||
*/
|
||||
export const mistral: Recipe = {
|
||||
id: 'mistral',
|
||||
name: 'Mistral AI',
|
||||
tier: 'openai-compat',
|
||||
implementation: 'openai-compatible',
|
||||
base_url_default: 'https://api.mistral.ai/v1',
|
||||
auth_env: {
|
||||
required: ['MISTRAL_API_KEY'],
|
||||
setup_url: 'https://console.mistral.ai/api-keys',
|
||||
},
|
||||
touchpoints: {
|
||||
embedding: {
|
||||
models: ['mistral-embed', 'mistral-embed-2312'],
|
||||
default_dims: 1024,
|
||||
// Mistral's published list price. Advisory only — canonical embedding
|
||||
// spend accounting lives in src/core/embedding-pricing.ts.
|
||||
cost_per_1m_tokens_usd: 0.1,
|
||||
price_last_verified: '2026-07-19',
|
||||
// Measured ceiling, not a doc guess: the /embeddings endpoint accepts a
|
||||
// 65,286-token batch and rejects 66,960 with
|
||||
// 400 code 3210 "Too many tokens overall, split into more batches."
|
||||
// -> the real cap is 65,536 (64K) tokens per request.
|
||||
max_batch_tokens: 65_536,
|
||||
// chars_per_token is a DIVISOR in splitByTokenBudget()
|
||||
// (estTokens = text.length / charsPerToken), so a LOWER value is the
|
||||
// conservative direction. The module default of 4 is an English-prose
|
||||
// assumption; German prose measured 3.58 here, and code/JSON/CJK runs
|
||||
// denser still. 2 keeps the estimate above the real token count for
|
||||
// every content shape we see.
|
||||
chars_per_token: 2,
|
||||
// With safety_factor 0.5 the pre-split budget is 32,768 estimated
|
||||
// tokens = 65,536 chars. Worst realistic density (~1.5 chars/token)
|
||||
// puts that at ~43.7K real tokens — still clear of the 64K ceiling.
|
||||
safety_factor: 0.5,
|
||||
},
|
||||
expansion: {
|
||||
models: ['ministral-3b-latest', 'mistral-small-latest'],
|
||||
price_last_verified: '2026-07-19',
|
||||
},
|
||||
chat: {
|
||||
models: [
|
||||
'mistral-small-latest', 'mistral-medium-latest', 'mistral-large-latest',
|
||||
'ministral-3b-latest', 'ministral-8b-latest', 'magistral-small-latest',
|
||||
],
|
||||
supports_tools: true,
|
||||
// Same call as the Moonshot recipe: ordinary tool calls are fine, but
|
||||
// gbrain's subagent loop stays Anthropic-pinned for stable tool_use_id
|
||||
// behavior across crashes/replays.
|
||||
supports_subagent_loop: false,
|
||||
supports_prompt_cache: false,
|
||||
max_context_tokens: 262144,
|
||||
price_last_verified: '2026-07-19',
|
||||
},
|
||||
},
|
||||
setup_hint: 'Get an API key at https://console.mistral.ai/api-keys, then `export MISTRAL_API_KEY=...` and use `mistral:mistral-embed` (1024 dims) for embeddings.',
|
||||
};
|
||||
@@ -1,42 +0,0 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Moonshot AI / Kimi Open Platform. Kimi exposes an OpenAI-compatible
|
||||
* /v1/chat/completions API at https://api.moonshot.ai/v1.
|
||||
*
|
||||
* Verified against Kimi API docs and live /v1/models on 2026-06-23.
|
||||
* The recipe is local-production glue until upstream GBrain carries a native
|
||||
* Moonshot recipe; keep it registered in the local patch registry.
|
||||
*/
|
||||
export const moonshot: Recipe = {
|
||||
id: 'moonshot',
|
||||
name: 'Moonshot AI / Kimi',
|
||||
tier: 'openai-compat',
|
||||
implementation: 'openai-compatible',
|
||||
base_url_default: 'https://api.moonshot.ai/v1',
|
||||
auth_env: {
|
||||
required: ['MOONSHOT_API_KEY'],
|
||||
setup_url: 'https://platform.kimi.ai/console/api-keys',
|
||||
},
|
||||
touchpoints: {
|
||||
expansion: {
|
||||
models: ['kimi-k2.7-code', 'kimi-k2.7-code-highspeed', 'kimi-k2.6', 'kimi-k2.5'],
|
||||
// Kimi pricing varies by current promotional/account terms; do not use
|
||||
// this advisory field for budget enforcement. Canonical budget pricing
|
||||
// belongs in src/core/model-pricing.ts when verified for the account.
|
||||
price_last_verified: '2026-06-23',
|
||||
},
|
||||
chat: {
|
||||
models: ['kimi-k2.7-code', 'kimi-k2.7-code-highspeed', 'kimi-k2.6', 'kimi-k2.5'],
|
||||
supports_tools: true,
|
||||
// Kimi tool calling is enough for ordinary chat/tool calls. GBrain's
|
||||
// subagent loop remains Anthropic-pinned because upstream requires stable
|
||||
// Anthropic-style tool_use_id behavior across crashes/replays.
|
||||
supports_subagent_loop: false,
|
||||
supports_prompt_cache: false,
|
||||
max_context_tokens: 256000,
|
||||
price_last_verified: '2026-06-23',
|
||||
},
|
||||
},
|
||||
setup_hint: 'Get an API key at https://platform.kimi.ai/console/api-keys, then `export MOONSHOT_API_KEY=...` and use `moonshot:kimi-k2.7-code`.',
|
||||
};
|
||||
@@ -1,71 +0,0 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* NVIDIA NIM / API Catalog exposes OpenAI-compatible /v1/chat/completions
|
||||
* and /v1/embeddings APIs.
|
||||
*
|
||||
* Retrieval models use asymmetric encoding. The gateway maps gbrain's
|
||||
* document/query distinction to NVIDIA's wire values:
|
||||
* document -> input_type: passage
|
||||
* query -> input_type: query
|
||||
*
|
||||
* The model ids below intentionally keep NVIDIA's full catalog ids because
|
||||
* the hosted endpoint expects values like `nvidia/nv-embedqa-e5-v5` in the
|
||||
* request body. Short aliases are provided for CLI ergonomics.
|
||||
*/
|
||||
export const nvidia: Recipe = {
|
||||
id: 'nvidia',
|
||||
name: 'NVIDIA NIM',
|
||||
tier: 'openai-compat',
|
||||
implementation: 'openai-compatible',
|
||||
base_url_default: 'https://integrate.api.nvidia.com/v1',
|
||||
auth_env: {
|
||||
required: ['NVIDIA_API_KEY'],
|
||||
setup_url: 'https://build.nvidia.com',
|
||||
},
|
||||
aliases: {
|
||||
'nv-embedqa-e5-v5': 'nvidia/nv-embedqa-e5-v5',
|
||||
'llama-nemotron-embed-1b-v2': 'nvidia/llama-nemotron-embed-1b-v2',
|
||||
'nemotron-3-super': 'nvidia/nemotron-3-super-120b-a12b',
|
||||
'nemotron-3-super-120b-a12b': 'nvidia/nemotron-3-super-120b-a12b',
|
||||
'nv-embed-v1': 'nvidia/nv-embed-v1',
|
||||
'nv-embedcode-7b-v1': 'nvidia/nv-embedcode-7b-v1',
|
||||
},
|
||||
// No resolveAuth override: NVIDIA is plain `Authorization: Bearer <key>`,
|
||||
// which defaultResolveAuth derives from auth_env.required. IRON RULE
|
||||
// (test/ai/recipes-existing-regression.test.ts): only Azure overrides
|
||||
// resolveAuth.
|
||||
touchpoints: {
|
||||
chat: {
|
||||
models: [
|
||||
'nvidia/nemotron-3-super-120b-a12b',
|
||||
],
|
||||
supports_tools: false,
|
||||
supports_subagent_loop: false,
|
||||
// Do not treat Nemotron as a Minions subagent driver until tool-calling
|
||||
// and replay stability are proven through a separate adapter test.
|
||||
max_context_tokens: 128000,
|
||||
price_last_verified: '2026-05-24',
|
||||
},
|
||||
embedding: {
|
||||
models: [
|
||||
'nvidia/nv-embedqa-e5-v5',
|
||||
'nvidia/llama-nemotron-embed-1b-v2',
|
||||
'nvidia/nv-embed-v1',
|
||||
'nvidia/nv-embedcode-7b-v1',
|
||||
],
|
||||
// Default to the lightest tested hosted model. Larger NVIDIA models are
|
||||
// supported via explicit embedding_dimensions (2048 or 4096).
|
||||
default_dims: 1024,
|
||||
dims_options: [1024, 2048, 4096],
|
||||
// Conservative split; hosted NVIDIA embedding endpoints require
|
||||
// input_type and may reject large payloads before tokenizing.
|
||||
max_batch_tokens: 8192,
|
||||
chars_per_token: 4,
|
||||
safety_factor: 0.75,
|
||||
cost_per_1m_tokens_usd: undefined,
|
||||
price_last_verified: '2026-05-24',
|
||||
},
|
||||
},
|
||||
setup_hint: 'Get an API key at https://build.nvidia.com, then `export NVIDIA_API_KEY=...`.',
|
||||
};
|
||||
@@ -1,101 +1,5 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Private in-process marker header. `gateway.chat()` sets it when the caller
|
||||
* asked for prompt caching (`cacheSystem`) on an OpenRouter route that needs
|
||||
* an explicit `cache_control` (Anthropic Claude). The compat fetch shim below
|
||||
* strips it and rewrites the body; the header NEVER leaves the process.
|
||||
*
|
||||
* Why a header and not providerOptions: the AI SDK's openai-compatible
|
||||
* adapter validates providerOptions against a fixed schema and silently
|
||||
* drops anthropic-namespace fields before building the wire body (same class
|
||||
* of problem as the embedding `input_type` ALS in gateway.ts). Headers pass
|
||||
* through untouched.
|
||||
*/
|
||||
export const OPENROUTER_CACHE_HEADER = 'x-gbrain-anthropic-prompt-cache';
|
||||
|
||||
/**
|
||||
* Family-scoped prompt-cache capability (per OpenRouter docs):
|
||||
* - OpenAI chat routes cache automatically (no request mutation needed).
|
||||
* - Anthropic Claude routes cache when the request carries `cache_control`
|
||||
* on a content block (applied by the fetch shim below).
|
||||
* Everything else is not marked cacheable — deliberately narrow rather than
|
||||
* blessing every routed model family forever.
|
||||
*/
|
||||
export function openrouterSupportsPromptCache(modelId: string): boolean {
|
||||
const normalized = modelId.trim().toLowerCase();
|
||||
if (normalized.startsWith('openai/gpt-') || /^openai\/o\d/.test(normalized)) return true;
|
||||
if (normalized.startsWith('anthropic/claude-')) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Only Anthropic Claude routes need an explicit cache_control block. */
|
||||
export function openrouterRequiresExplicitPromptCache(modelId: string): boolean {
|
||||
return modelId.trim().toLowerCase().startsWith('anthropic/claude-');
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite the last system message's string content into OpenRouter's
|
||||
* documented Anthropic caching shape: a content-part array carrying
|
||||
* `cache_control: { type: 'ephemeral' }` on the text block. (A top-level
|
||||
* body `cache_control` is NOT the OpenRouter format — OR forwards per-block
|
||||
* markers only.) Returns the input unchanged when it doesn't apply.
|
||||
*/
|
||||
function withSystemCacheControl(body: unknown): unknown {
|
||||
if (!body || typeof body !== 'object' || Array.isArray(body)) return body;
|
||||
const record = body as Record<string, unknown>;
|
||||
const model = typeof record.model === 'string' ? record.model : '';
|
||||
if (!openrouterRequiresExplicitPromptCache(model)) return body;
|
||||
const messages = Array.isArray(record.messages) ? record.messages : undefined;
|
||||
if (!messages) return body;
|
||||
let idx = -1;
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
const m = messages[i];
|
||||
if (m && typeof m === 'object' && (m as Record<string, unknown>).role === 'system') idx = i;
|
||||
}
|
||||
if (idx === -1) return body;
|
||||
const sys = messages[idx] as Record<string, unknown>;
|
||||
if (typeof sys.content !== 'string' || sys.content.length === 0) return body;
|
||||
const next = messages.slice();
|
||||
next[idx] = {
|
||||
...sys,
|
||||
content: [{ type: 'text', text: sys.content, cache_control: { type: 'ephemeral' } }],
|
||||
};
|
||||
return { ...record, messages: next };
|
||||
}
|
||||
|
||||
/**
|
||||
* Compat fetch: honors the OPENROUTER_CACHE_HEADER marker by splicing an
|
||||
* Anthropic cache_control breakpoint onto the system block, then strips the
|
||||
* marker. Fail-open: any parse problem sends the original body unchanged.
|
||||
*
|
||||
* @internal exported for tests. Cast through `unknown` because TS's
|
||||
* `typeof fetch` includes a `preconnect` member (matches azure-openai.ts).
|
||||
*/
|
||||
export const openrouterCompatFetch = (async (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
): Promise<Response> => {
|
||||
if (!init?.headers) return fetch(input as any, init as any);
|
||||
const headers = new Headers(init.headers as any);
|
||||
if (!headers.has(OPENROUTER_CACHE_HEADER)) return fetch(input as any, init as any);
|
||||
headers.delete(OPENROUTER_CACHE_HEADER);
|
||||
let body = init.body;
|
||||
if (typeof body === 'string') {
|
||||
try {
|
||||
const parsed = JSON.parse(body);
|
||||
const rewritten = withSystemCacheControl(parsed);
|
||||
if (rewritten !== parsed) {
|
||||
body = JSON.stringify(rewritten);
|
||||
headers.delete('content-length');
|
||||
}
|
||||
} catch {
|
||||
// Non-JSON body: let the provider surface the original problem.
|
||||
}
|
||||
}
|
||||
return fetch(input as any, { ...init, headers, body } as any);
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
/**
|
||||
* OpenRouter — single-key fan-out to OpenAI, Anthropic, Google, DeepSeek, and
|
||||
* dozens of other providers via a single OpenAI-compatible endpoint at
|
||||
@@ -189,9 +93,7 @@ export const openrouter: Recipe = {
|
||||
supports_tools: true,
|
||||
// Informational only — real gate is isAnthropicProvider() upstream.
|
||||
supports_subagent_loop: false,
|
||||
// Family-scoped: OpenAI routes cache automatically; Anthropic routes
|
||||
// cache via the compat fetch shim's cache_control rewrite.
|
||||
supports_prompt_cache: openrouterSupportsPromptCache,
|
||||
supports_prompt_cache: false,
|
||||
// No max_context_tokens: catalog spans 128K to 1M+; a single recipe-wide
|
||||
// value is either unsafe for smaller models or wasteful for larger ones.
|
||||
// Let upstream errors surface per-model.
|
||||
@@ -200,5 +102,4 @@ export const openrouter: Recipe = {
|
||||
},
|
||||
setup_hint:
|
||||
'Get an API key at https://openrouter.ai/settings/keys, then `export OPENROUTER_API_KEY=...` and use `openrouter:<provider>/<model>`. Optional overrides: OPENROUTER_BASE_URL (proxy), OPENROUTER_REFERER (attribution URL), OPENROUTER_TITLE (attribution name).',
|
||||
compat: { fetch: openrouterCompatFetch },
|
||||
};
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Perplexity's hosted embeddings API (#1046). OpenAI-shaped at
|
||||
* `POST {base}/embeddings` but diverges on the wire:
|
||||
* - `encoding_format` only accepts 'base64_int8' (default) or
|
||||
* 'base64_binary' — the AI SDK's 'float' default is rejected.
|
||||
* - The response `embedding` is a base64 string encoding SIGNED INT8
|
||||
* components (natively quantized output), not a float array.
|
||||
* Both divergences are handled by perplexityCompatFetch in gateway.ts
|
||||
* (force 'base64_int8' outbound; decode Int8Array → number[] inbound).
|
||||
* Cosine similarity is scale-invariant, so the raw int8 components store
|
||||
* and rank correctly as floats.
|
||||
*
|
||||
* Models (per docs.perplexity.ai/api-reference/embeddings-post, 2026-07):
|
||||
* - pplx-embed-v1-0.6b: dims 128..1024 (default 1024)
|
||||
* - pplx-embed-v1-4b: dims 128..2560 (default 2560)
|
||||
* The flexible-dim range validation lives in src/core/ai/dims.ts
|
||||
* (PERPLEXITY_EMBEDDING_MAX_DIMS). default_dims is pinned at 1024 so both
|
||||
* models work out of the box on a plain vector(N) column; users who want
|
||||
* the 4b model's full 2560 width set `embedding_dimensions: 2560` and the
|
||||
* existing halfvec path (dims > 2000) covers storage + ANN.
|
||||
*
|
||||
* Auth is PERPLEXITY_API_KEY only — deliberately NO OPENAI_API_KEY
|
||||
* fallback (a Perplexity brain must never silently bill/route through
|
||||
* OpenAI). If your key lives in PPLX_API_KEY, re-export it.
|
||||
*/
|
||||
export const perplexity: Recipe = {
|
||||
id: 'perplexity',
|
||||
name: 'Perplexity',
|
||||
tier: 'openai-compat',
|
||||
implementation: 'openai-compatible',
|
||||
base_url_default: 'https://api.perplexity.ai/v1',
|
||||
auth_env: {
|
||||
required: ['PERPLEXITY_API_KEY'],
|
||||
setup_url: 'https://www.perplexity.ai/settings/api',
|
||||
},
|
||||
touchpoints: {
|
||||
embedding: {
|
||||
models: ['pplx-embed-v1-0.6b', 'pplx-embed-v1-4b'],
|
||||
default_dims: 1024,
|
||||
cost_per_1m_tokens_usd: 0.03, // pplx-embed-v1-4b; 0.6b is $0.004/M
|
||||
price_last_verified: '2026-07-21',
|
||||
// Perplexity enforces 120K combined tokens (and 512 texts) per
|
||||
// request. Same pre-split posture as Voyage: assume a dense
|
||||
// tokenizer (1 char ≈ 1 token) at 0.5 utilization; the gateway's
|
||||
// recursive halving is the runtime safety net.
|
||||
max_batch_tokens: 120_000,
|
||||
chars_per_token: 1,
|
||||
safety_factor: 0.5,
|
||||
},
|
||||
},
|
||||
setup_hint: 'Get an API key at https://www.perplexity.ai/settings/api, then `export PERPLEXITY_API_KEY=...` (re-export PPLX_API_KEY if that is where your key lives).',
|
||||
};
|
||||
@@ -16,13 +16,6 @@ export const together: Recipe = {
|
||||
setup_url: 'https://api.together.ai/settings/api-keys',
|
||||
},
|
||||
touchpoints: {
|
||||
// Same OpenAI-compatible endpoint as chat; declared so an explicit
|
||||
// `expansion_model: together:...` resolves instead of silently dropping
|
||||
// expansion (#1135).
|
||||
expansion: {
|
||||
models: ['meta-llama/Llama-3.3-70B-Instruct-Turbo', 'Qwen/Qwen2.5-72B-Instruct-Turbo'],
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
chat: {
|
||||
models: [
|
||||
'Qwen/Qwen2.5-72B-Instruct-Turbo',
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Zhipu AI (智谱AI) BigModel Open Platform. OpenAI-compatible /embeddings and
|
||||
* /chat/completions endpoints at open.bigmodel.cn. Hosts embedding-2 (1024d),
|
||||
* embedding-3 (Matryoshka up to 2048d), and the GLM chat family (glm-5.1 etc.)
|
||||
* with native tool calling — usable for models.tier.subagent (#1157).
|
||||
* Zhipu AI (智谱AI) BigModel Open Platform. OpenAI-compatible /embeddings
|
||||
* endpoint at open.bigmodel.cn. Hosts embedding-2 (1024d) and embedding-3
|
||||
* (Matryoshka up to 2048d).
|
||||
*
|
||||
* embedding-3 at 2048 dims exceeds pgvector's HNSW cap of 2000 — those
|
||||
* brains fall back to exact vector scans (see
|
||||
@@ -26,20 +25,6 @@ export const zhipu: Recipe = {
|
||||
setup_url: 'https://open.bigmodel.cn/',
|
||||
},
|
||||
touchpoints: {
|
||||
chat: {
|
||||
// Informational list (openai-compat tier: assertTouchpoint doesn't
|
||||
// enforce it), so newer GLM ids pass without a recipe edit.
|
||||
models: ['glm-5.1', 'glm-4.6', 'glm-4.5'],
|
||||
supports_tools: true,
|
||||
// gbrain-side stable tool ids (v0.38 D11) decoupled the loop from
|
||||
// Anthropic response formats; GLM tool calling is stable through the
|
||||
// OpenAI-compat path, same as deepseek/groq.
|
||||
supports_subagent_loop: true,
|
||||
// Anthropic-style cache_control markers are not honored on the
|
||||
// OpenAI-compat path — the loop runs hot (degraded:no_caching warn).
|
||||
supports_prompt_cache: false,
|
||||
max_context_tokens: 128000,
|
||||
},
|
||||
embedding: {
|
||||
models: ['embedding-3', 'embedding-2'],
|
||||
default_dims: 1024,
|
||||
@@ -51,5 +36,5 @@ export const zhipu: Recipe = {
|
||||
},
|
||||
},
|
||||
setup_hint:
|
||||
'Get an API key at https://open.bigmodel.cn/, then `export ZHIPUAI_API_KEY=...`. Chat/subagent: use `zhipu:glm-5.1`.',
|
||||
'Get an API key at https://open.bigmodel.cn/, then `export ZHIPUAI_API_KEY=...`',
|
||||
};
|
||||
|
||||
+3
-19
@@ -22,8 +22,7 @@ export type Implementation =
|
||||
| 'native-openai'
|
||||
| 'native-google'
|
||||
| 'native-anthropic'
|
||||
| 'openai-compatible'
|
||||
| 'claude-cli';
|
||||
| 'openai-compatible';
|
||||
|
||||
export interface EmbeddingTouchpoint {
|
||||
models: string[];
|
||||
@@ -55,16 +54,6 @@ export interface EmbeddingTouchpoint {
|
||||
* `max_batch_tokens` is also set.
|
||||
*/
|
||||
safety_factor?: number;
|
||||
/**
|
||||
* Maximum number of inputs per embedding request. Some endpoints enforce a
|
||||
* hard COUNT cap independent of token budget — notably llama.cpp's
|
||||
* `llama-server`, which rejects requests with more inputs than its launch
|
||||
* batch size (e.g. `batch size 100 > maximum allowed batch size 32`). The
|
||||
* token-budget pre-split cannot bound item count (many tiny chunks fit under
|
||||
* any token budget), so this is enforced as a separate hard re-split after
|
||||
* the token split. When unset, no count cap is applied.
|
||||
*/
|
||||
max_batch_items?: number;
|
||||
/**
|
||||
* v0.27.1: when true, at least one model in this recipe accepts image
|
||||
* inputs via a multimodal embedding endpoint (e.g. Voyage's
|
||||
@@ -233,13 +222,8 @@ export interface ChatTouchpoint {
|
||||
* Strictly stronger than supports_tools.
|
||||
*/
|
||||
supports_subagent_loop: boolean;
|
||||
/**
|
||||
* Prompt caching honored for this chat touchpoint. Static booleans cover
|
||||
* native providers; openai-compatible aggregators may decide per model id
|
||||
* (e.g. OpenRouter caches OpenAI and Anthropic routes but not every routed
|
||||
* model family).
|
||||
*/
|
||||
supports_prompt_cache?: boolean | ((modelId: string) => boolean);
|
||||
/** Anthropic-style ephemeral prompt cache markers honored. */
|
||||
supports_prompt_cache?: boolean;
|
||||
max_context_tokens?: number;
|
||||
cost_per_1m_input_usd?: number;
|
||||
cost_per_1m_output_usd?: number;
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
/**
|
||||
* Nightly conversation-parser probe audit trail.
|
||||
*
|
||||
* One event per REAL probe run lands in
|
||||
* `~/.gbrain/audit/parser-probe-YYYY-Www.jsonl` (ISO-week rotation via the
|
||||
* shared audit-writer primitive; honors `GBRAIN_AUDIT_DIR`).
|
||||
* Scheduler-cadence skips (`rate_limited`) are NOT logged — the autopilot
|
||||
* loop ticks every few minutes, so logging every skip would flood the
|
||||
* audit file with rows that carry no signal.
|
||||
*
|
||||
* Read by `gbrain doctor`'s `conversation_parser_probe_health` check and
|
||||
* by the autopilot wiring's 24h rate-limit gate (`parserProbeRanWithin`).
|
||||
*/
|
||||
|
||||
import { createAuditWriter } from './audit/audit-writer.ts';
|
||||
import type { NightlyProbeResult } from './conversation-parser/nightly-probe.ts';
|
||||
|
||||
export type ParserProbeAuditEvent = NightlyProbeResult;
|
||||
|
||||
const writer = createAuditWriter<ParserProbeAuditEvent>({
|
||||
featureName: 'parser-probe',
|
||||
errorLabel: 'gbrain',
|
||||
errorMessagePrefix: 'parser-probe audit ',
|
||||
errorTrailer: '; probe continues',
|
||||
});
|
||||
|
||||
/** Append one parser-probe event. Best-effort; never throws. */
|
||||
export function logParserProbeEvent(event: ParserProbeAuditEvent): void {
|
||||
writer.log(event);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read recent parser-probe events (current + previous ISO week, filtered
|
||||
* to the window). Missing files and corrupt rows are skipped silently.
|
||||
*/
|
||||
export function readRecentParserProbeEvents(
|
||||
days = 7,
|
||||
now: Date = new Date(),
|
||||
): ParserProbeAuditEvent[] {
|
||||
return writer.readRecent(days, now);
|
||||
}
|
||||
|
||||
/** Exposed for tests pinning the rotation edge cases. */
|
||||
export function computeParserProbeAuditFilename(now: Date = new Date()): string {
|
||||
return writer.computeFilename(now);
|
||||
}
|
||||
|
||||
/**
|
||||
* 24h rate-limit gate for the autopilot wiring: true when any audited run
|
||||
* happened within `windowMs` of `now`. Only REAL outcomes are audited (see
|
||||
* module header), so a pass/fail today blocks re-runs until tomorrow while
|
||||
* scheduler-cadence skips never extend the window.
|
||||
*/
|
||||
export function parserProbeRanWithin(
|
||||
windowMs: number,
|
||||
now: Date = new Date(),
|
||||
): boolean {
|
||||
const cutoff = now.getTime() - windowMs;
|
||||
return readRecentParserProbeEvents(2, now).some((ev) => {
|
||||
const ts = Date.parse(ev.ts);
|
||||
return Number.isFinite(ts) && ts >= cutoff;
|
||||
});
|
||||
}
|
||||
@@ -81,20 +81,6 @@ function getLoad(): number {
|
||||
|
||||
/** Get memory usage fraction (0-1) */
|
||||
function getMemoryUsage(): number {
|
||||
// Prefer /proc/meminfo MemAvailable on Linux — os.freemem() returns
|
||||
// MemFree which excludes page cache, falsely reading "high pressure"
|
||||
// in any container where the kernel caches files.
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const fs = require('fs');
|
||||
const meminfo: string = fs.readFileSync('/proc/meminfo', 'utf8');
|
||||
const totalKb = Number(meminfo.match(/MemTotal:\s+(\d+)/)?.[1]);
|
||||
const availKb = Number(meminfo.match(/MemAvailable:\s+(\d+)/)?.[1]);
|
||||
if (totalKb > 0 && availKb >= 0) return 1 - availKb / totalKb;
|
||||
} catch {
|
||||
/* fall through to os.freemem() (non-Linux or /proc unavailable) */
|
||||
}
|
||||
// Non-Linux fallback (macOS, Windows, or any host without /proc/meminfo)
|
||||
const total = totalmem();
|
||||
if (total === 0) return 0;
|
||||
return 1 - (freemem() / total);
|
||||
|
||||
@@ -100,15 +100,7 @@ function gbrainHome(): string {
|
||||
* core→commands import). which gbrain → process.execPath → argv[1] → "gbrain". */
|
||||
function resolveGbrainCliPath(): string {
|
||||
try {
|
||||
// #2747: `env: process.env` required under Bun — see the sibling copy
|
||||
// of this function in commands/autopilot.ts for the full explanation
|
||||
// (Bun snapshots process.env at its own startup; execSync without an
|
||||
// explicit env is blind to any PATH mutation since then).
|
||||
const which = execSync('which gbrain', {
|
||||
encoding: 'utf-8',
|
||||
stdio: ['ignore', 'pipe', 'ignore'],
|
||||
env: process.env,
|
||||
}).trim();
|
||||
const which = execSync('which gbrain', { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
||||
if (which) return which;
|
||||
} catch { /* not on PATH */ }
|
||||
const exec = process.execPath ?? '';
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user