mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-15 09:22:18 +00:00
Compare commits
200
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
14d2689ec8 | ||
|
|
b022b17484 | ||
|
|
2a5dd27d68 | ||
|
|
5dfd2696d1 | ||
|
|
569e431e80 | ||
|
|
3e8d1ea6f4 | ||
|
|
ae8753c872 | ||
|
|
705a93e490 | ||
|
|
b30f0aa7cb | ||
|
|
2c758e23e8 | ||
|
|
4320527785 | ||
|
|
3a28d2612a | ||
|
|
bf4cf8a6dd | ||
|
|
0ce4064d13 | ||
|
|
032af6e5f7 | ||
|
|
29dd67c8ae | ||
|
|
d2ac2aef49 | ||
|
|
10079efe40 | ||
|
|
16782aee7f | ||
|
|
3126b8fdfc | ||
|
|
b6c75d802f | ||
|
|
07901b1886 | ||
|
|
d7c9625395 | ||
|
|
7a65f182aa | ||
|
|
9690140bf3 | ||
|
|
d014707e3c | ||
|
|
dde1bd9353 | ||
|
|
f0a28eb276 | ||
|
|
5ecab70a21 | ||
|
|
70beb16b8b | ||
|
|
7efb1694cc | ||
|
|
4beafbae46 | ||
|
|
5f84fb8813 | ||
|
|
14f0674bcf | ||
|
|
4871ae0c05 | ||
|
|
d9ac24744c | ||
|
|
c19a8808b4 | ||
|
|
ea08effd02 | ||
|
|
3fafb69b07 | ||
|
|
c44cdb52b1 | ||
|
|
32d42454e9 | ||
|
|
54c0c93376 | ||
|
|
f30d789c3a | ||
|
|
b35c617252 | ||
|
|
8b432b15d8 | ||
|
|
ef7351247a | ||
|
|
540b86ff55 | ||
|
|
8612da14bf | ||
|
|
278823828d | ||
|
|
d9a49564bd | ||
|
|
3fcca330cd | ||
|
|
31dca6837a | ||
|
|
be7b4b14d0 | ||
|
|
95ba2c70d5 | ||
|
|
8cd87968d1 | ||
|
|
f1cf5f14db | ||
|
|
f64505b75f | ||
|
|
38cc7198b7 | ||
|
|
1f319e6d5a | ||
|
|
d15e2ab8cf | ||
|
|
e1919fab9f | ||
|
|
5e665c1c06 | ||
|
|
f5e5736f09 | ||
|
|
cd252b080b | ||
|
|
97bdf6acc1 | ||
|
|
900ee3c678 | ||
|
|
96465d8c35 | ||
|
|
7fdecd5c01 | ||
|
|
06001248ef | ||
|
|
9e28379038 | ||
|
|
48d83bd200 | ||
|
|
454c26ab56 | ||
|
|
b3891fa7fc | ||
|
|
ca04874c8f | ||
|
|
9b3f1c6786 | ||
|
|
2944d9b7ae | ||
|
|
eba9680775 | ||
|
|
b313938e86 | ||
|
|
26c6bad445 | ||
|
|
38b8b1e41e | ||
|
|
eb6cb4a16f | ||
|
|
aae1a5107e | ||
|
|
40d9b83d5c | ||
|
|
69bc37f745 | ||
|
|
4b38724aa2 | ||
|
|
cd18081f4a | ||
|
|
ef1133df3e | ||
|
|
26b938c37d | ||
|
|
5d7a8d7d4b | ||
|
|
be722ee5f7 | ||
|
|
ca4cf2a0c6 | ||
|
|
d89d6ea293 | ||
|
|
03cd52631b | ||
|
|
800108e014 | ||
|
|
5e8816e7d6 | ||
|
|
593ba16535 | ||
|
|
ea6cb025be | ||
|
|
ca47c054b8 | ||
|
|
97df1e78b7 | ||
|
|
1392243d3b | ||
|
|
8345abce42 | ||
|
|
df22c81996 | ||
|
|
d2fd1f297c | ||
|
|
0a4f062cac | ||
|
|
2f4ad2c0a4 | ||
|
|
526c597ccf | ||
|
|
2a0c51d093 | ||
|
|
920aea5eb8 | ||
|
|
4be9d112cb | ||
|
|
b82f520314 | ||
|
|
00bcd66c3e | ||
|
|
22fca8f891 | ||
|
|
178d3404a4 | ||
|
|
1cc17f014d | ||
|
|
2b00b7abeb | ||
|
|
18513c65be | ||
|
|
f70c3fe9d8 | ||
|
|
c7dd0fa64b | ||
|
|
9fb046a110 | ||
|
|
581a1eed29 | ||
|
|
19c6b6ef67 | ||
|
|
4213ac8da8 | ||
|
|
6aa055024c | ||
|
|
cce774c904 | ||
|
|
b78fc56e98 | ||
|
|
38f446bb6f | ||
|
|
8901dc0f45 | ||
|
|
04e6b3af14 | ||
|
|
080b64e052 | ||
|
|
7e4094b2cd | ||
|
|
b5675437c0 | ||
|
|
8160236ade | ||
|
|
69e7e79a1f | ||
|
|
3594c316b5 | ||
|
|
0853491eb2 | ||
|
|
2283269932 | ||
|
|
c571bf82de | ||
|
|
8dc3310483 | ||
|
|
58606cc924 | ||
|
|
66f4cb6d82 | ||
|
|
c852abfcb6 | ||
|
|
b91350d778 | ||
|
|
e1156a5642 | ||
|
|
872d4eebb5 | ||
|
|
fecd331f02 | ||
|
|
79f6d1bfee | ||
|
|
ef840d9561 | ||
|
|
b139602119 | ||
|
|
7421efc41e | ||
|
|
b0f74017d7 | ||
|
|
941e7746d4 | ||
|
|
d6fe486370 | ||
|
|
080692fa47 | ||
|
|
d574e843a8 | ||
|
|
22cb074943 | ||
|
|
d21f34e96d | ||
|
|
fa43907df4 | ||
|
|
e0a208d7b7 | ||
|
|
418357332f | ||
|
|
45f85df8f4 | ||
|
|
9a70945152 | ||
|
|
aea6df3da7 | ||
|
|
35edd0e2d5 | ||
|
|
6388be2088 | ||
|
|
323610ecd7 | ||
|
|
e9a4fee97f | ||
|
|
054badbe60 | ||
|
|
b98fae9b61 | ||
|
|
11659743a2 | ||
|
|
5aa4795c04 | ||
|
|
e36251c023 | ||
|
|
7bbd087cb7 | ||
|
|
e0d2cbf353 | ||
|
|
7a1f61a31a | ||
|
|
9b8b829ca5 | ||
|
|
c27b2e4b0f | ||
|
|
9bcfa67748 | ||
|
|
16eb8cd06c | ||
|
|
92a3202198 | ||
|
|
8b7e30afcd | ||
|
|
68e4cebd1a | ||
|
|
8bbb19102c | ||
|
|
9ae4e04d22 | ||
|
|
fe2f2f6b2a | ||
|
|
fc169d9770 | ||
|
|
50406fc212 | ||
|
|
0556dbdc2c | ||
|
|
220af4b2d0 | ||
|
|
fe6850b067 | ||
|
|
8fc93c8fac | ||
|
|
3454dca0b4 | ||
|
|
5dcf3e7b2f | ||
|
|
dbca701008 | ||
|
|
4b6cf32c9f | ||
|
|
0c66715f90 | ||
|
|
55af5fc091 | ||
|
|
10b5746053 | ||
|
|
5bee08c3c4 | ||
|
|
f02919c041 | ||
|
|
66fa5fba22 |
@@ -28,5 +28,5 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: rhysd/actionlint@393031adb9afb225ee52ae2ccd7a5af5525e03e8 # v1.7.11
|
||||
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -116,7 +116,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
id-token: write # for attest-build-provenance (Sigstore OIDC)
|
||||
attestations: write # for attest-build-provenance
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
with:
|
||||
path: artifacts
|
||||
- name: Create release
|
||||
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||
with:
|
||||
files: |
|
||||
artifacts/gbrain-darwin-arm64/gbrain-darwin-arm64
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
container:
|
||||
image: semgrep/semgrep:1.170.0@sha256:c98f8829eea377274ee4b10656458b078b88232469b2ff913f091c2317347c9d
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
# Non-blocking initially (continue-on-error): the first runs establish a
|
||||
# baseline without failing unrelated PRs. Graduation path: once the
|
||||
# baseline findings are triaged (fixed or `# nosemgrep`'d), remove
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
hit: ${{ steps.lookup.outputs.cache-hit }}
|
||||
hash: ${{ steps.compute.outputs.hash }}
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- name: Compute content hash
|
||||
id: compute
|
||||
run: |
|
||||
@@ -84,7 +84,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: gitleaks/gitleaks-action@dcedce43c6f43de0b836d1fe38946645c9c638dc # v2
|
||||
@@ -103,7 +103,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 12
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -124,7 +124,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -149,7 +149,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 12
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -172,7 +172,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 12
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
@@ -216,7 +216,7 @@ jobs:
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
|
||||
@@ -35,6 +35,11 @@ export/
|
||||
# .context/test-shards/. Workspace-local by design — never committed.
|
||||
.context/
|
||||
|
||||
# Local agent instruction overrides (CLAUDE.local.md / AGENTS.local.md) — personal,
|
||||
# per-clone, loaded after the committed CLAUDE.md/AGENTS.md. Never committed.
|
||||
CLAUDE.local.md
|
||||
AGENTS.local.md
|
||||
|
||||
# Tier 3 PGLite snapshot fixture (built on demand by build:pglite-snapshot)
|
||||
test/fixtures/pglite-snapshot.tar
|
||||
test/fixtures/pglite-snapshot.version
|
||||
|
||||
+251
@@ -2,6 +2,257 @@
|
||||
|
||||
All notable changes to GBrain will be documented in this file.
|
||||
|
||||
## [0.42.66.1] - 2026-07-27
|
||||
|
||||
### Fixed
|
||||
|
||||
- `gbrain doctor` now treats embedding columns wider than pgvector's HNSW limit as healthy exact-scan configurations instead of prescribing an index PostgreSQL cannot build.
|
||||
- Local CI now passes an empty Docker mount list correctly and compiles the embedded-WASM smoke binary from container-local storage on Docker Desktop.
|
||||
|
||||
## [0.42.66.0] - 2026-07-24
|
||||
|
||||
**54 verified fixes from the community backlog: background enrichment stops wasting money on dead pages, autopilot stops killing its own healthy runs, and search respects your settings.**
|
||||
|
||||
This release is the second big sweep through the open pull-request backlog, with every change reviewed and tested individually before merging. The theme is trust in the background machinery. The overnight "dream" cycle now remembers which pages produced nothing and stops re-reading them every night, meters its small-model calls against your spend caps, and keeps claim proposals from silently overwriting each other. Long consolidation runs get a 30-minute deadline instead of being killed at 10 minutes mid-work. A wedged server boot now releases its database lock instead of blocking every later command.
|
||||
|
||||
Search behaves the way you configured it: the recency-decay setting now actually applies to hybrid search, a local `list_pages` call returns as many rows as you asked for, and when a listing is cut short it says so instead of looking complete. Slack conversation exports parse cleanly, with an optional AI fallback for formats the parser does not know.
|
||||
|
||||
New provider recipes: DashScope reranking, OpenRouter reranking, and a claude-cli recipe for dispatching subagents through the gateway.
|
||||
|
||||
## To take advantage of v0.42.66.0
|
||||
|
||||
`gbrain upgrade` should do this automatically. One schema migration ships in this release (v125, take-proposal idempotency); it is idempotent and needs no manual action.
|
||||
|
||||
1. **Upgrade and verify:**
|
||||
```bash
|
||||
gbrain upgrade
|
||||
gbrain doctor
|
||||
gbrain stats
|
||||
```
|
||||
2. **If `gbrain doctor` warns about a partial migration**, run the orchestrator manually:
|
||||
```bash
|
||||
gbrain apply-migrations --yes
|
||||
```
|
||||
3. **If any step fails,** please file an issue at https://github.com/garrytan/gbrain/issues with the output of `gbrain doctor` and `~/.gbrain/upgrade-errors.jsonl` if it exists.
|
||||
|
||||
### Itemized changes
|
||||
|
||||
#### Dream cycle, takes, and spend control
|
||||
|
||||
- Pages whose extraction yields zero claims are memoized, so the cycle stops re-spending on them every night. (#2514, #3319, contributed by @ivandebot)
|
||||
- Zero-yield pages are tombstoned so `extract_atoms` stops rediscovering them. (#2144, #3304, contributed by @ChenyqThu)
|
||||
- `extract_atoms` Haiku calls are metered against the cost gate. (#2371, #3329, contributed by @TheRealMrSystem)
|
||||
- `extract_atoms` stamps concepts so `synthesize_concepts` has material to work with. (#2123, #3308, contributed by @ChenyqThu)
|
||||
- `extract_facts` requires a live backing page, not just a non-NULL entity slug. (#2497, #3321, contributed by @javieraldape)
|
||||
- Multi-claim pages keep every proposal instead of only the first (migration v125 makes the idempotency key per claim). (#3297, contributed by @rp-agent-bot)
|
||||
- Superseding a take now queries the active row first. (#3275, contributed by @arisgysel-design)
|
||||
- Takes keyword search matches words inside long claims via `word_similarity`. (#3267)
|
||||
- Dream-generated orphan pages stay scoped to their source. (#2368, #3344, contributed by @snvtac)
|
||||
- Drift detection is wired into the dream cycle, report-only for now. (#2653, #3317)
|
||||
|
||||
#### Autopilot, jobs, and serve
|
||||
|
||||
- Full consolidation cycles get a 30-minute timeout floor; lighter dispatches keep the interval-derived budget. (#2852, #3338, contributed by @sanchalr)
|
||||
- The cron wrapper exports `~/.bun/bin` onto PATH so autopilot survives minimal environments. (#2013, #3305, contributed by @klampatech)
|
||||
- Dead or cancelled jobs no longer block idempotent re-submission. (#2253, #3306, contributed by @rafaelreis-r)
|
||||
- Contextual reindex jobs get a default timeout. (#2611, #3323, contributed by @spiky02plateau)
|
||||
- Onboarding stops repeating the same auto-remediation within a single run. (#2854, #3342, contributed by @sanchalr)
|
||||
- A wedged `gbrain serve` boot hits a readiness deadline and releases the PGLite lock. (#3335)
|
||||
|
||||
#### Search, retrieval, and health
|
||||
|
||||
- The recency-decay config is honored on the hybrid search path. (#2386, #3312, contributed by @rwbaker)
|
||||
- `list_pages` honors explicit limits for local callers, warns on remote clamping, and threads `offset`. (#2591, #3322, contributed by @deacon-botdoctor)
|
||||
- Truncated `list_pages` results say so instead of silently capping. (#2865, #3341, contributed by @paul-0320)
|
||||
- Negative metrics no longer invert trajectory regression signals. (#2621, #3324, contributed by @morluto)
|
||||
- Per-chunk synopsis generation in contextual retrieval is concurrency-bounded. (#2628, #3326, contributed by @spiky02plateau)
|
||||
- Graph health metrics count `entity` pages. (#2639, #3330, contributed by @tylr-r)
|
||||
|
||||
#### Ingestion, extraction, and links
|
||||
|
||||
- Conversation parsing gains an opt-in LLM fallback for unknown formats. (#2247, #3371, contributed by @danwiggins)
|
||||
- Normalized Slack markdown parses into conversations. (#3289, #3372, contributed by @danwiggins)
|
||||
- Conversation backfill outcomes are durable, so completed pages skip on the next run. (#3293, #3373, contributed by @danwiggins)
|
||||
- Reference-style wikilinks are recognized during extraction. (#2071, #3303, contributed by @mzkarami)
|
||||
- `[[wikilink]]` frontmatter values resolve via global basename lookup. (#2406, #3313, contributed by @spiky02plateau)
|
||||
- Incremental push syncs extract links. (#2850, #3337, contributed by @patentsong)
|
||||
- `<think>` reasoning tags in extractor output are handled. (#2559, #3318, contributed by @qaz8545355)
|
||||
- Tiktoken special tokens no longer crash code-chunker token estimates. (#2453, #3315, contributed by @Jiglet)
|
||||
- Source config stops re-wrapping into a growing JSON string scalar. (#2829, #3334, contributed by @1alessio)
|
||||
|
||||
#### Providers and recipes
|
||||
|
||||
- DashScope reranking recipe (DashScope serves a plural `/reranks` endpoint under its compatible API). (#2644, #3328, contributed by @YiconZiwei)
|
||||
- OpenRouter reranking touchpoint. (#2164, #3302, contributed by @Hippityy)
|
||||
- claude-cli recipe for native gateway-based subagent dispatch. (#2277, #3310, contributed by @brettdavies)
|
||||
- Prefixed model IDs work on the openai-compatible embedding-dimensions path. (#2325, #3309, contributed by @noetherly)
|
||||
- Embeddings stamp the gateway-resolved model in `content_chunks.model`, not the compiled default. (#2846, #3343, contributed by @SailorJoe6)
|
||||
- Bun-on-Windows write-through EEXIST fixed, non-Anthropic `--max-cost` pricing works, dream pages excluded from enrich. (#2407, #3316, contributed by @nguyenchiviet)
|
||||
- Supabase signed URLs prepend `/storage/v1`. (#2565, #3320, contributed by @danwiggins)
|
||||
|
||||
#### Sources, auth, and multi-brain
|
||||
|
||||
- Federated-source pages are visible to `get_page`, `list_pages`, `resolve_slugs`, and no-grant MCP callers. (#3242, #3301)
|
||||
- Admin-gated rescope surface for DCR clients stuck on a default scope. (#3299)
|
||||
- `whoami` exposes OAuth source grants. (#3279, #3332, contributed by @boundless-forest)
|
||||
- Thin-client `--source` maps onto `source_id` for remote-routed operations. (#3086)
|
||||
|
||||
#### CLI, doctor, and init
|
||||
|
||||
- `gbrain doctor` stops claiming "Brain is at target" when the target is unreachable. (#2151, #3339, contributed by @brettdavies)
|
||||
- Doctor gains a raw-source persistence guarantee for synthesized pages, warn-only for now. (#3300)
|
||||
- Doctor timeline labels disambiguate entity coverage from the brain-score component. (#2298, #3073, contributed by @TurgutKural)
|
||||
- Unknown `gbrain init` flags are rejected before migrations run. (#2201, #3307, contributed by @caioribeiroclw-pixel)
|
||||
- The init soul-audit hint points at the conversational skill, not a nonexistent CLI verb. (#2486, #3314, contributed by @SeanGearin)
|
||||
- `--force` retry escapes completed migration-ledger entries. (#2616, #3325, contributed by @spiky02plateau)
|
||||
- PGLite data-dir lock contention gets a clear error message. (#2658, #3336, contributed by @zaycruz)
|
||||
- Frontmatter validation derives slugs from the brain root, not the absolute path. (#2340, #3311, contributed by @alessioalionco)
|
||||
|
||||
#### For contributors
|
||||
|
||||
- Docker network isolation guidance for co-located self-hosted Postgres. (#3270, #3331)
|
||||
- `CLAUDE.local.md` / `AGENTS.local.md` are gitignored. (#3290, contributed by @igbymyboy)
|
||||
- The hybrid-reranker integration test isolates `GBRAIN_HOME`. (#1527, #3327, contributed by @Willisbest)
|
||||
- Test-shard scripts capture the real exit code before watchdog teardown in the no-timeout fallback. (#2864, #3340, contributed by @paul-0320)
|
||||
|
||||
## [0.42.65.0] - 2026-07-23
|
||||
|
||||
**A large maintenance release: 93 verified fixes and small features merged since v0.42.64.0, most of them community contributions.**
|
||||
|
||||
If you use gbrain day to day, this release makes the boring parts trustworthy. Importing and syncing notes is safer: a failed pull no longer pretends everything is up to date, imported pages are read back after writing to confirm they landed, and a page with real content can no longer be silently overwritten by an empty one. Search answers get better inputs: the think command now picks excerpts that actually match your question, and results respect your federated source settings. Background enrichment (the "dream" cycle) wastes less money and retries properly when an AI provider is down. Spending caps now fail closed, so a billing hiccup can never turn into an uncapped spend. And `gbrain doctor` is quieter, with several false alarms removed and real problems (like an embedding backlog with no worker running) now flagged.
|
||||
|
||||
More AI providers work out of the box, including OpenRouter prompt caching, MiniMax and Zhipu GLM recipes, Ollama Matryoshka embedding dimensions, and llama-server batch limits.
|
||||
|
||||
## To take advantage of v0.42.65.0
|
||||
|
||||
`gbrain upgrade` should do this automatically. No new schema migrations ship in this release.
|
||||
|
||||
1. **Upgrade and verify:**
|
||||
```bash
|
||||
gbrain upgrade
|
||||
gbrain doctor
|
||||
gbrain stats
|
||||
```
|
||||
2. **If `gbrain doctor` reports new findings after upgrading,** that is the quieter, more accurate check set working as intended. Each finding names its fix.
|
||||
3. **If any step fails,** please file an issue at https://github.com/garrytan/gbrain/issues with the output of `gbrain doctor` and `~/.gbrain/upgrade-errors.jsonl` if it exists.
|
||||
|
||||
### Itemized changes
|
||||
|
||||
#### Security
|
||||
|
||||
- MCP source scoping for remote callers got a hardening pass, so agent-facing connections stay confined to the sources they were granted. (#2881, contributed by @spinsirr)
|
||||
- Paid MCP spend accounting is now atomic and fails closed, and resolver spend is recorded before a cap error is raised, so caps cannot be raced past or undercounted. (#3203, #3204, contributed by @caterpillarC15)
|
||||
- The OAuth token endpoint rate limit on the HTTP server is now configurable via env for deployments behind shared IPs. (#3114, contributed by @time-attack)
|
||||
- `WWW-Authenticate` responses now carry `resource_metadata` per the MCP spec and RFC 9728, so conforming clients can discover the auth server. (#1410, contributed by @rayers)
|
||||
|
||||
#### Search, retrieval, and think
|
||||
|
||||
- `think` selects query-relevant excerpts instead of generic ones. (#3197, contributed by @Y0lan)
|
||||
- Unqualified local CLI `search`/`query` now honors `sources.config.federated` read visibility. (#2561, #3141, contributed by @time-attack)
|
||||
- Email citation metadata is projected into search results. (#2873, contributed by @amtagrwl)
|
||||
- The `think` Gaps section renders once instead of twice. (#1662, contributed by @howwohmm)
|
||||
- Fuzzy entity lookup threads the caller's source scope and skips soft-deleted entities. (#1508, contributed by @tim404x)
|
||||
- `code-def` surfaces method, constructor, field, and struct definitions, not just top-level symbols. (#1628, contributed by @rayers)
|
||||
- Briefing pages are excluded from their own Brain Pulse salience. (#1202, contributed by @rwbaker)
|
||||
- Reranker calls with missing auth are classified as configuration errors before falling back. (#2059, #3139, contributed by @time-attack)
|
||||
|
||||
#### Import, sync, and ingestion
|
||||
|
||||
- A failed git pull with zero imports reports `partial (pull_failed)` instead of `up_to_date`. (#3068, #3253, contributed by @Masashi-Ono0611)
|
||||
- Imports run a post-write read-back verification with a durable ingest-log record. (#2869, contributed by @Andredsouza1984)
|
||||
- `put` refuses to overwrite a non-empty page with empty content. (#2708, contributed by @symmetric-matthew)
|
||||
- `putPage` restores soft-deleted rows instead of colliding with them. (#2779, contributed by @RerankerGuo)
|
||||
- Mixed-case slugs are normalized before chunk upsert, ending duplicate-chunk churn. (#430, #3143, contributed by @time-attack)
|
||||
- Imports fall back to the body H1 for the title when frontmatter lacks `title:`. (#2446, #3072, contributed by @time-attack)
|
||||
- YAML comments inside the frontmatter fence are no longer treated as markdown headings. (#3225, #3247, contributed by @Masashi-Ono0611)
|
||||
- Write-through guards case-insensitive filesystem collisions before the atomic write. (#2831, #3119, contributed by @time-attack)
|
||||
- Path-qualified wikilinks outside the known directory pattern resolve on the DB/put_page path. (#2866, contributed by @paul-0320)
|
||||
- CJK slugs are supported in the slug registry and dream-cycle summary slugs. (#782, #738, #3083, contributed by @time-attack)
|
||||
- Three ingest/sync/serve singleton fixes: page-type round-trip, deleted-slug embed noise, and a stateless width guard. (#3140, contributed by @time-attack)
|
||||
- Sync honors the `embedding_disabled` sentinel as an implicit `--no-embed`. (#2879, contributed by @gawievanblerk)
|
||||
- Verified sync head sentinels are cleared correctly. (#2734, contributed by @symmetric-matthew)
|
||||
- Resumed syncs report the pinned commit they actually landed on. (#3202, contributed by @caterpillarC15)
|
||||
- The expected `discover_git_root` probe failure stays off stderr. (#3232, contributed by @Masashi-Ono0611)
|
||||
- `extract --stale` runs the real resolver so basename resolution reaches stale pages, and clears pre-version-bump pages. (#2576, #2717, contributed by @paul-0320; #1791, contributed by @Nazim22)
|
||||
- Oversized code chunks are capped so they stay embeddable, and code-chunk metadata survives re-embeds. (#1675, contributed by @lubosxyz; #769, #1232, contributed by @rayers)
|
||||
|
||||
#### Background cycle, dream, and facts
|
||||
|
||||
- Path-derived dream sources are stamped, and the engine closes cleanly on autopilot shutdown. (#3178, contributed by @time-attack)
|
||||
- All-provider-failed atom drains propagate so durable jobs retry instead of silently dropping work. (#3218, #3248, contributed by @Masashi-Ono0611)
|
||||
- Atom extraction raises `maxTokens` and case-normalizes `atom_type` for Gemini models. (#3211, contributed by @alexey-metaengage)
|
||||
- The conversation extractor gates anonymous-speaker self-attribution instead of guessing. (#3228, contributed by @asenkovskiy)
|
||||
- Incremental dream extraction stamps its watermark so re-runs stop reprocessing. (#2636, #3115, contributed by @time-attack)
|
||||
- `dream --dry-run --json` keeps stdout clean of embed summaries. (#394, #3109, contributed by @time-attack)
|
||||
- Synthesized dream pages require a self-contained opening summary. (#2770, contributed by @Masashi-Ono0611)
|
||||
- PGLite inline synth subagent drains complete, and `lint` gains `--exclude`. (#2699, #2649, #3162, contributed by @time-attack)
|
||||
- Live context reads the documented "P1 Today" heading form with plain checkbox tasks, matching the daily-task-manager skill's output format. (#2186, #3124, contributed by @time-attack)
|
||||
- Queued AI jobs refresh gateway config at execution time instead of using a stale snapshot. (#2125, contributed by @maxpetrusenkoagent)
|
||||
- `brainstorm`/`propose_takes` honor configured models: cost preview uses the configured model, the judge reads its config key, provider probes are skipped when unneeded, and page projection is narrowed. (#3120, contributed by @time-attack)
|
||||
- Backlog hardening wave: x-to-brain health check, propose_takes deadlines, capture title truncation, extract_atoms backlog handling, and pooler direct-URL routing. (#3165, contributed by @time-attack)
|
||||
- `skillopt` emits `proposed.md` in no-mutate mode. (#2635, #3182, contributed by @time-attack)
|
||||
- Nightly quality probe enable path and conversation-parser probe are wired up. (#2629, #2630, #3094, contributed by @time-attack)
|
||||
|
||||
#### Doctor, health, and maintenance
|
||||
|
||||
- New safe maintenance automation with a shared orphan-exclusion policy, so routine cleanup runs without risking linked content. (#3015, #3023, contributed by @time-attack)
|
||||
- `orphan_ratio` excludes the chronicle volume under `life/events/`. (#2264, #3214, contributed by @asenkovskiy)
|
||||
- `brain_score` orphan/timeline components use the orphans-audit linkable scope. (#3155, contributed by @time-attack)
|
||||
- Entity timeline coverage is measured separately from whole-brain density. (#2761, contributed by @TurgutKural)
|
||||
- Doctor flags embed backfills queued with no worker running. (#2696, contributed by @javieraldape)
|
||||
- Two doctor false-positive/timeout fixes: the drift walk skips `node_modules`, and the bare-tweet check skips inline code and cited lines. (#1772, contributed by @sonlndv)
|
||||
- A dead `llm_fallback_enabled` recommendation is dropped from conversation format coverage. (#1903, contributed by @ElliotDrel)
|
||||
- Skill triggers with CRLF line endings parse on Windows. (#1149, contributed by @samporter-31)
|
||||
- Onboard check names are registered in doctor categories, ending unknown-check warnings, and onboard-check remediations survive the `--apply --auto` path. (#3075, #3097, contributed by @time-attack)
|
||||
- Dead slug prefixes are counted by slug. (#2697, contributed by @RerankerGuo)
|
||||
- The backlinks worker defaults to check, not fix, and `check-backlinks` honors its positional directory argument. (#1853, contributed by @choomz; #3076, contributed by @time-attack)
|
||||
- Calibration resolves the owner holder via config, defaulting to `self`. (#3077, contributed by @time-attack)
|
||||
- Memory throttling on Linux reads `/proc/meminfo` MemAvailable. (#556, contributed by @chengzehsu)
|
||||
|
||||
#### AI providers and gateway
|
||||
|
||||
- OpenRouter gets family-scoped prompt caching, and query expansion works on chat-capable openai-compat recipes. (#3152, contributed by @time-attack)
|
||||
- MiniMax recipe: embedding wire-shape compat fetch plus a chat touchpoint. (#1977, #3089, contributed by @time-attack)
|
||||
- The Zhipu recipe gains a chat touchpoint so GLM subagents work. (#1157, #3084, contributed by @time-attack)
|
||||
- Tier-configured models reach the recipe allowlist, Anthropic model lists are refreshed, tier resolutions are registered, and probe labels are honest. (#2800, contributed by @p3ob7o)
|
||||
- Provider base URL config merges from the DB. (#1676, contributed by @TheLordArgus)
|
||||
- The gateway falls back to the pooler when the derived direct host is unreachable. (#1641, #3088, contributed by @time-attack)
|
||||
- Config-plane `voyage_api_key` folds into `VOYAGE_API_KEY` like the other hosted keys. (#3236, contributed by @Masashi-Ono0611)
|
||||
- The `zeroentropyai:zerank-2` reranker has a pricing entry so the budget tracker can meter it. (#3223, #3233, contributed by @Masashi-Ono0611)
|
||||
- llama-server embedding batches are capped at its 32-input request limit. (#1281, contributed by @mmekkaoui)
|
||||
- Matryoshka dimensions thread through for Qwen3-Embedding on Ollama. (#1072, contributed by @mgandal)
|
||||
- `init` seeds AI options from env on cold install, and `whoami` reports the stdio transport. (#3091, contributed by @time-attack)
|
||||
- The `models` dispatch subcommand reads its first argument correctly. (#1428, contributed by @BenjaminDSmithy)
|
||||
- Synopsis generation tail-truncates document text for small-model chat handlers. (#1427, contributed by @BenjaminDSmithy)
|
||||
- The contradiction judge token cap is raised for thinking models. (#3210, contributed by @alexey-metaengage)
|
||||
|
||||
#### Schema, migrations, and storage engines
|
||||
|
||||
- Engine migration counts and surfaces per-page copy failures instead of silently advancing. (#3241, contributed by @Masashi-Ono0611)
|
||||
- Invalid `CONCURRENTLY`-build index remnants are dropped without a DO block. (#3191, contributed by @Masashi-Ono0611)
|
||||
- Unsupported large-dimension HNSW indexes are skipped instead of failing schema setup. (#1734, #3080, contributed by @time-attack)
|
||||
- The v0.32.2 migration dirty-check scopes to targeted sources and surfaces failed phase detail. (#3093, contributed by @time-attack)
|
||||
- Schema packs merge the full `extends` chain and `borrow_from` into the resolved manifest. (#1749, #3181, contributed by @time-attack)
|
||||
- The schema-pack stats catch-all is narrowed so masked errors surface instead of fake zero-page counts. (#2466, #3133, contributed by @time-attack)
|
||||
- Bundled schema-pack inspection reports the pack actually shipped in the binary, and minion subagent auth resolves through config. (#3110, contributed by @time-attack)
|
||||
- PGLite `putPage` guards against zero-row RETURNING. (#1649, contributed by @alexhawkins)
|
||||
|
||||
#### MCP server and CLI surface
|
||||
|
||||
- `list_pages` rows include `source_id`. (#3209, contributed by @alexey-metaengage)
|
||||
- Running CLI commands while `gbrain serve` (MCP) holds the brain now notifies about the conflict instead of failing confusingly. (#3243, contributed by @fdefitte)
|
||||
- The OpenClaw plugin manifest entry is declared so the plugin loads. (#2551, #3185, contributed by @time-attack)
|
||||
|
||||
#### For contributors
|
||||
|
||||
- CI scanner roots are normalized on macOS. (#3198, contributed by @caterpillarC15)
|
||||
- CI shard timeout raised to 22 minutes plus a delta-assert reporter leak test. (#3231, contributed by @time-attack)
|
||||
- E2E suite hardening: flaky tests, no-op assertions, and cross-test coupling removed. (#1704, contributed by @auroracapital)
|
||||
- `mechanical.test.ts` isolates `$HOME` so the E2E suite stops clobbering user config. (#434, contributed by @lloydarmbrust)
|
||||
- The lint code-fence-wrap detector and fixer regex now agree. (#1597, contributed by @chungty)
|
||||
- README project links for OpenClaw and Hermes are corrected. (#1961, #3179, contributed by @time-attack)
|
||||
- A completed TODOS entry is dropped. (#3229, contributed by @Masashi-Ono0611)
|
||||
|
||||
## [0.42.64.0] - 2026-07-20
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -163,6 +163,14 @@ host port with `GBRAIN_CI_PG_PORT=5435 bun run ci:local` if 5434 collides.
|
||||
Fail-closed selector: an unmapped `src/` change runs all 29 E2E files. Hand-tune
|
||||
narrower mappings via `scripts/e2e-test-map.ts`.
|
||||
|
||||
### PR-side security checks
|
||||
|
||||
Besides the test gate, PRs may trigger three security workflows: Semgrep CE
|
||||
SAST (every PR — **advisory/non-blocking** while the baseline is tuned, so a
|
||||
Semgrep finding won't fail your PR), OSV-Scanner (only when `package.json` or
|
||||
`bun.lock` change), and actionlint (only when `.github/workflows/**` change).
|
||||
See `SECURITY.md` → "Automated security scanning" for details.
|
||||
|
||||
## Building
|
||||
|
||||
```bash
|
||||
|
||||
+36
@@ -8,6 +8,30 @@ on GitHub.
|
||||
|
||||
Do not open a public issue for security vulnerabilities.
|
||||
|
||||
## Automated security scanning
|
||||
|
||||
CI runs three automated security checks alongside secret scanning (Gitleaks):
|
||||
|
||||
- **Dependency vulnerabilities** — OSV-Scanner
|
||||
(`.github/workflows/osv-scanner.yml`) runs weekly and on any PR that touches
|
||||
`package.json` or `bun.lock`.
|
||||
- **Static analysis (SAST)** — Semgrep CE (`.github/workflows/semgrep.yml`)
|
||||
runs on every PR and weekly. It is currently **advisory (non-blocking)**
|
||||
while the finding baseline is tuned; the graduation path to a blocking check
|
||||
is documented in the workflow file.
|
||||
- **Release binary provenance** — release builds
|
||||
(`.github/workflows/release.yml`) attest each compiled binary with
|
||||
[GitHub artifact attestations](https://docs.github.com/en/actions/security-for-github-actions/using-artifact-attestations).
|
||||
Verify a downloaded release binary with:
|
||||
|
||||
```bash
|
||||
gh attestation verify ./gbrain-darwin-arm64 -R garrytan/gbrain
|
||||
gh attestation verify ./gbrain-linux-x64 -R garrytan/gbrain
|
||||
```
|
||||
|
||||
All security workflows use SHA-pinned actions and least-privilege permissions,
|
||||
enforced structurally by actionlint on every workflow change.
|
||||
|
||||
## Remote MCP Security
|
||||
|
||||
### ⚠️ Do NOT use open OAuth client registration for remote MCP
|
||||
@@ -135,6 +159,18 @@ the PGLite schema. Local agents continue to use stdio (`gbrain serve`).
|
||||
Running `--http` against a PGLite-backed install fails fast with a clear
|
||||
error message at startup.
|
||||
|
||||
### Docker network isolation (self-hosted Postgres)
|
||||
|
||||
OAuth and source scoping enforce isolation on the `serve --http` path only.
|
||||
Raw Postgres reachability bypasses both: a container that shares Docker's
|
||||
default `bridge` network with the brain's Postgres can open a direct DB
|
||||
session without any token and read every source. Put the brain's Postgres on
|
||||
a user-defined Docker network with nothing untrusted on it, publish its port
|
||||
loopback-only (if at all), and never put `DATABASE_URL` or a Postgres
|
||||
password in untrusted agent containers — those should reach the brain
|
||||
exclusively via OAuth against `serve --http`. Full operator checklist:
|
||||
[docs/mcp/DEPLOY.md — Co-located Docker workloads](docs/mcp/DEPLOY.md#co-located-docker-workloads-self-hosted-postgres).
|
||||
|
||||
### CORS
|
||||
|
||||
Default-deny: no `Access-Control-Allow-Origin` header is sent unless an
|
||||
|
||||
@@ -2,17 +2,10 @@
|
||||
|
||||
## community fix-wave follow-ups (filed v0.42.60.0)
|
||||
|
||||
- [ ] **P1 — take-writes source scoping fails open when source resolution errors (#2684 residual).**
|
||||
`resolveTakesSourceId` (src/commands/takes.ts) swallows resolution errors and returns
|
||||
`undefined`, which falls back to the unscoped slug-only page lookup — so an invalid
|
||||
`GBRAIN_SOURCE` (or a broken dotfile chain) silently restores the pre-#2698 cross-source
|
||||
write behavior on multi-source brains. Decide fail-closed semantics: error out when a
|
||||
source was explicitly requested but doesn't resolve; keep the unscoped fallback only for
|
||||
brains with no source configuration at all. Add a regression test for the invalid-source
|
||||
path. Found by cross-model adversarial review during the v0.42.60.0 release ship.
|
||||
- [ ] **P2 — cherry-pick #2112's uncovered doctor.ts hunk.** Fix-wave A (#2820) superseded
|
||||
- [x] **P2 — cherry-pick #2112's uncovered doctor.ts hunk.** Fix-wave A (#2820) superseded
|
||||
most of #2112 but not its `checkSubagentCapability` fix (check explicit `models.subagent`
|
||||
before `models.tier.subagent`). Refile or cherry-pick; the rest of that PR is covered.
|
||||
before `models.tier.subagent`). Implemented: `checkSubagentCapability` now resolves
|
||||
`models.subagent` before tier/default fallbacks and has regression coverage.
|
||||
|
||||
## v0.42.59.0 follow-ups (five-fix rollup #2735–#2739)
|
||||
|
||||
@@ -2287,10 +2280,25 @@ at plan time and got carved out:
|
||||
via `buildPerSourceBindings`. Document workaround: register
|
||||
source-scoped OAuth clients.
|
||||
|
||||
- [ ] **v0.41+: T20 — extends-chain merging in registry.ts.**
|
||||
`registry.ts:167` documents the gap. Implementing full child-wins
|
||||
merge cascades through every consumer of `manifest.page_types`. ~1
|
||||
day CC.
|
||||
- [x] **v0.41+: T20 — extends-chain merging in registry.ts.** DONE (#1749).
|
||||
`resolvePack` now merges parent → child (child-wins) for the six
|
||||
ingest/query-shaping fields (`page_types`, `link_types`,
|
||||
`frontmatter_links`, `enrichable_types`, `filing_rules`, `takes_kinds`)
|
||||
plus `borrow_from` materialization, in `src/core/schema-pack/merge.ts`.
|
||||
The cascade was transparent (consumers already read `resolved.manifest`),
|
||||
not per-consumer. `phases`/`calibration_domains` deliberately excluded —
|
||||
see the P3 follow-up below.
|
||||
|
||||
- [ ] **P3: explicit opt-in to inherit `phases` / `calibration_domains`.**
|
||||
T20 excludes these two from the child-wins merge because they gate real
|
||||
cycle execution (`cycle.ts` `packDeclaresPhase`) and the manifest
|
||||
contract says each pack declares its own participation explicitly —
|
||||
auto-inheriting would silently make a child run cycle phases it never
|
||||
requested. Multi-level lens packs (`gbrain-everything`) therefore still
|
||||
re-declare them by hand. If that redeclaration becomes painful, add an
|
||||
explicit manifest flag (e.g. `inherit_phases: true`) so a pack author
|
||||
opts in consciously. Depends on: T20 (landed). Start in
|
||||
`src/core/schema-pack/merge.ts` (`mergeInheritedManifest`).
|
||||
|
||||
- [ ] **v0.41+: T21 — comment-preserving YAML emitter.**
|
||||
v0.40.7.0 emitter does NOT preserve comments. Authors who care
|
||||
|
||||
Vendored
-56
File diff suppressed because one or more lines are too long
Vendored
+56
File diff suppressed because one or more lines are too long
Vendored
+1
-1
@@ -7,7 +7,7 @@
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
|
||||
<script type="module" crossorigin src="/admin/assets/index-CoGEje3-.js"></script>
|
||||
<script type="module" crossorigin src="/admin/assets/index-CviJXT-1.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/admin/assets/index-GxkWX7v3.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
+12
-2
@@ -39,11 +39,21 @@ export const api = {
|
||||
stats: () => apiFetch('/admin/api/stats'),
|
||||
health: () => apiFetch('/admin/api/health-indicators'),
|
||||
agents: () => apiFetch('/admin/api/agents'),
|
||||
sources: () => apiFetch('/admin/api/sources'),
|
||||
requests: (page = 1, qs = '') => apiFetch(`/admin/api/requests?page=${page}${qs}`),
|
||||
apiKeys: () => apiFetch('/admin/api/api-keys'),
|
||||
createApiKey: (name: string) => apiFetch('/admin/api/api-keys', { method: 'POST', body: JSON.stringify({ name }) }),
|
||||
revokeApiKey: (name: string) => apiFetch('/admin/api/api-keys/revoke', { method: 'POST', body: JSON.stringify({ name }) }),
|
||||
createApiKey(keyName: string) {
|
||||
return apiFetch('/admin/api/api-keys', { method: 'POST', body: JSON.stringify({ name: keyName }) });
|
||||
},
|
||||
revokeApiKey(keyName: string) {
|
||||
return apiFetch('/admin/api/api-keys/revoke', { method: 'POST', body: JSON.stringify({ name: keyName }) });
|
||||
},
|
||||
updateClientTtl: (clientId: string, tokenTtl: number | null) => apiFetch('/admin/api/update-client-ttl', { method: 'POST', body: JSON.stringify({ clientId, tokenTtl }) }),
|
||||
rescopeClient: (clientId: string, sourceId: string, federatedRead: string[]) =>
|
||||
apiFetch('/admin/api/rescope-client', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ clientId, sourceId, federatedRead }),
|
||||
}),
|
||||
revokeClient: (clientId: string) => apiFetch('/admin/api/revoke-client', { method: 'POST', body: JSON.stringify({ clientId }) }),
|
||||
// v0.36.1.0 (T15 / E6) — calibration endpoints.
|
||||
calibrationProfile: (holder?: string) =>
|
||||
|
||||
+169
-4
@@ -18,6 +18,8 @@ interface Agent {
|
||||
client_name?: string; // compat
|
||||
grant_types: string[];
|
||||
scope: string;
|
||||
source_id: string | null;
|
||||
federated_read: string[];
|
||||
created_at: string;
|
||||
last_used_at: string | null;
|
||||
total_requests: number;
|
||||
@@ -26,6 +28,12 @@ interface Agent {
|
||||
status: 'active' | 'revoked';
|
||||
}
|
||||
|
||||
interface Source {
|
||||
id: string;
|
||||
name: string;
|
||||
federated: boolean;
|
||||
}
|
||||
|
||||
interface ApiKey {
|
||||
id: string;
|
||||
name: string;
|
||||
@@ -36,6 +44,7 @@ interface ApiKey {
|
||||
|
||||
export function AgentsPage() {
|
||||
const [agents, setAgents] = useState<Agent[]>([]);
|
||||
const [sources, setSources] = useState<Source[]>([]);
|
||||
const [hideRevoked, setHideRevoked] = useState(true);
|
||||
const [showRegister, setShowRegister] = useState(false);
|
||||
const [showCredentials, setShowCredentials] = useState<{ clientId: string; clientSecret: string; name: string } | null>(null);
|
||||
@@ -43,7 +52,10 @@ export function AgentsPage() {
|
||||
const [showApiKeyToken, setShowApiKeyToken] = useState<{ name: string; token: string } | null>(null);
|
||||
const [selectedAgent, setSelectedAgent] = useState<Agent | null>(null);
|
||||
|
||||
useEffect(() => { loadAgents(); }, []);
|
||||
useEffect(() => {
|
||||
loadAgents();
|
||||
api.sources().then(setSources).catch(() => {});
|
||||
}, []);
|
||||
|
||||
const loadAgents = () => { api.agents().then(setAgents).catch(() => {}); };
|
||||
|
||||
@@ -88,6 +100,7 @@ export function AgentsPage() {
|
||||
<th>Name</th>
|
||||
<th>Type</th>
|
||||
<th>Scopes</th>
|
||||
<th>Sources</th>
|
||||
<th>Status</th>
|
||||
<th>Requests</th>
|
||||
<th>Last Used</th>
|
||||
@@ -108,6 +121,11 @@ export function AgentsPage() {
|
||||
<span key={s} className={`badge badge-${s}`} style={{ marginRight: 4 }}>{s}</span>
|
||||
))}
|
||||
</td>
|
||||
<td style={{ color: 'var(--text-secondary)', fontSize: 12 }}>
|
||||
{a.auth_type === 'oauth'
|
||||
? `${a.source_id || 'none'} · ${(a.federated_read || []).length} readable`
|
||||
: 'Unscoped'}
|
||||
</td>
|
||||
<td>
|
||||
<span className={`badge ${a.status === 'active' ? 'badge-success' : 'badge-danger'}`}>{a.status}</span>
|
||||
</td>
|
||||
@@ -144,7 +162,21 @@ export function AgentsPage() {
|
||||
)}
|
||||
|
||||
{selectedAgent && (
|
||||
<AgentDrawer agent={selectedAgent} onClose={() => setSelectedAgent(null)} onRevoked={loadAgents} />
|
||||
<AgentDrawer
|
||||
key={selectedAgent.id}
|
||||
agent={selectedAgent}
|
||||
sources={sources}
|
||||
onClose={() => setSelectedAgent(null)}
|
||||
onRevoked={loadAgents}
|
||||
onRescoped={({ sourceId, federatedRead }) => {
|
||||
setSelectedAgent(current => current ? {
|
||||
...current,
|
||||
source_id: sourceId,
|
||||
federated_read: federatedRead,
|
||||
} : current);
|
||||
loadAgents();
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showApiKeyCreate && (
|
||||
@@ -381,7 +413,127 @@ function CredentialsModal({ credentials, onClose }: {
|
||||
);
|
||||
}
|
||||
|
||||
function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: () => void; onRevoked: () => void }) {
|
||||
function SourceAccessEditor({ clientId, agent, sources, onRescoped }: {
|
||||
clientId: string;
|
||||
agent: Agent;
|
||||
sources: Source[];
|
||||
onRescoped: (scope: { sourceId: string; federatedRead: string[] }) => void;
|
||||
}) {
|
||||
const [writeSource, setWriteSource] = useState(agent.source_id || 'default');
|
||||
const [readSources, setReadSources] = useState<string[]>(agent.federated_read || []);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [error, setError] = useState('');
|
||||
const [saved, setSaved] = useState(false);
|
||||
const readableSet = new Set(readSources);
|
||||
const activeSourceIds = new Set(sources.map(source => source.id));
|
||||
const unavailableReadSources = readSources.filter(sourceId => !activeSourceIds.has(sourceId));
|
||||
const primaryUnavailable = !activeSourceIds.has(writeSource);
|
||||
|
||||
const save = async () => {
|
||||
if (readSources.length === 0) {
|
||||
setError('Select at least one readable source.');
|
||||
return;
|
||||
}
|
||||
setSaving(true);
|
||||
setError('');
|
||||
setSaved(false);
|
||||
try {
|
||||
const result = await api.rescopeClient(clientId, writeSource, readSources) as {
|
||||
sourceId: string;
|
||||
federatedRead: string[];
|
||||
};
|
||||
setWriteSource(result.sourceId);
|
||||
setReadSources(result.federatedRead);
|
||||
setSaved(true);
|
||||
onRescoped(result);
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : 'Failed to save source access');
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="section-title">Source Access</div>
|
||||
<div style={{ color: 'var(--text-secondary)', fontSize: 12, lineHeight: 1.5, marginBottom: 12 }}>
|
||||
The primary source is the write destination. Read access is an explicit allowlist and does not widen automatically.
|
||||
</div>
|
||||
<div style={{ marginBottom: 14 }}>
|
||||
<label htmlFor="agent-write-source">Primary / write source</label>
|
||||
<select
|
||||
id="agent-write-source"
|
||||
value={writeSource}
|
||||
onChange={e => { setWriteSource(e.target.value); setSaved(false); }}
|
||||
style={{ width: '100%', background: 'var(--bg-secondary)', color: 'var(--text-primary)', border: '1px solid var(--border)', borderRadius: 6, padding: '6px 10px', fontSize: 14 }}
|
||||
>
|
||||
{primaryUnavailable && (
|
||||
<option value={writeSource} disabled>{writeSource} · unavailable</option>
|
||||
)}
|
||||
{sources.map(source => (
|
||||
<option key={source.id} value={source.id}>{source.name} ({source.id})</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
<fieldset style={{ border: 0, padding: 0, margin: '0 0 14px' }}>
|
||||
<legend>Readable sources</legend>
|
||||
<div className="checkbox-group" style={{ marginTop: 6 }}>
|
||||
{sources.map(source => (
|
||||
<label key={source.id} className="checkbox-label">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={readableSet.has(source.id)}
|
||||
onChange={e => {
|
||||
setSaved(false);
|
||||
setReadSources(current => e.target.checked
|
||||
? [...current, source.id]
|
||||
: current.filter(id => id !== source.id));
|
||||
}}
|
||||
/>
|
||||
{source.name} ({source.id}){source.federated ? ' · federated' : ' · private'}
|
||||
</label>
|
||||
))}
|
||||
{unavailableReadSources.map(sourceId => (
|
||||
<label key={sourceId} className="checkbox-label" style={{ color: 'var(--warning)' }}>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked
|
||||
onChange={() => {
|
||||
setSaved(false);
|
||||
setReadSources(current => current.filter(id => id !== sourceId));
|
||||
}}
|
||||
/>
|
||||
{sourceId} · unavailable (clear to remove grant)
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</fieldset>
|
||||
{(primaryUnavailable || unavailableReadSources.length > 0) && (
|
||||
<div style={{ color: 'var(--warning)', fontSize: 13, marginBottom: 10 }}>
|
||||
This client references unavailable or archived sources. Choose an active primary source and clear unavailable read grants before saving.
|
||||
</div>
|
||||
)}
|
||||
{error && <div style={{ color: 'var(--error)', fontSize: 13, marginBottom: 10 }}>{error}</div>}
|
||||
{saved && <div style={{ color: 'var(--success)', fontSize: 13, marginBottom: 10 }}>Source access saved.</div>}
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-primary"
|
||||
disabled={saving || readSources.length === 0 || sources.length === 0 || primaryUnavailable || unavailableReadSources.length > 0}
|
||||
onClick={save}
|
||||
>
|
||||
{saving ? 'Saving...' : 'Save Source Access'}
|
||||
</button>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function AgentDrawer({ agent, sources, onClose, onRevoked, onRescoped }: {
|
||||
agent: Agent;
|
||||
sources: Source[];
|
||||
onClose: () => void;
|
||||
onRevoked: () => void;
|
||||
onRescoped: (scope: { sourceId: string; federatedRead: string[] }) => void;
|
||||
}) {
|
||||
const [tab, setTab] = useState<'claude-code' | 'chatgpt' | 'claude-cowork' | 'perplexity' | 'cursor' | 'json'>('claude-code');
|
||||
const copy = (text: string) => navigator.clipboard.writeText(text);
|
||||
const serverUrl = window.location.origin;
|
||||
@@ -553,6 +705,15 @@ function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: ()
|
||||
<span>{agent.token_ttl ? (agent.token_ttl >= 31536000 ? 'No expiry' : agent.token_ttl >= 86400 ? `${Math.floor(agent.token_ttl / 86400)}d` : agent.token_ttl >= 3600 ? `${Math.floor(agent.token_ttl / 3600)}h` : `${agent.token_ttl}s`) : '1h (default)'}</span>
|
||||
</div>
|
||||
|
||||
{isOAuth && (
|
||||
<SourceAccessEditor
|
||||
clientId={cid}
|
||||
agent={agent}
|
||||
sources={sources}
|
||||
onRescoped={onRescoped}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/*
|
||||
Config Export visible for both auth_type=oauth AND auth_type=api_key.
|
||||
Claude Code + Cursor + JSON tabs render real snippets regardless
|
||||
@@ -579,7 +740,11 @@ function AgentDrawer({ agent, onClose, onRevoked }: { agent: Agent; onClose: ()
|
||||
{(() => {
|
||||
const oauthOnlyTabs = new Set(['chatgpt', 'claude-cowork', 'perplexity']);
|
||||
if (!isOAuth && oauthOnlyTabs.has(tab)) {
|
||||
const clientName = { chatgpt: 'ChatGPT', 'claude-cowork': 'Claude.ai', perplexity: 'Perplexity' }[tab] || tab;
|
||||
const clientName = tab === 'chatgpt'
|
||||
? 'ChatGPT'
|
||||
: tab === 'claude-cowork'
|
||||
? 'Claude.ai'
|
||||
: 'Perplexity';
|
||||
return (
|
||||
<div style={{
|
||||
background: 'rgba(255, 200, 100, 0.08)',
|
||||
|
||||
@@ -51,8 +51,9 @@
|
||||
"@electric-sql/pglite",
|
||||
],
|
||||
"overrides": {
|
||||
"@hono/node-server": "^1.19.13",
|
||||
"fast-uri": "^3.1.2",
|
||||
"@hono/node-server": "^2.0.5",
|
||||
"body-parser": "^2.3.0",
|
||||
"fast-uri": "^3.1.4",
|
||||
"fast-xml-builder": "^1.1.7",
|
||||
"fast-xml-parser": "^5.7.0",
|
||||
"form-data": "^4.0.6",
|
||||
@@ -162,7 +163,7 @@
|
||||
|
||||
"@electric-sql/pglite": ["@electric-sql/pglite@0.4.3", "", {}, "sha512-ichuWTgtd4mOM1G4SpyGJa5trT03lWbMypDV0fUXUCXg5hiHqVAz/bZyV68NqmkLB7WcYmj1RMJVSp8HV/v/ZQ=="],
|
||||
|
||||
"@hono/node-server": ["@hono/node-server@1.19.14", "", { "peerDependencies": { "hono": "^4" } }, "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw=="],
|
||||
"@hono/node-server": ["@hono/node-server@2.0.11", "", { "peerDependencies": { "hono": "^4" } }, "sha512-bjD221KPLoJTWUwso1J6fGKiTXEUFedG/s0visavY4zakFPkeGURMRNly+FhBHs7T8Dz4qHaZIMX9ZoJHSJtKA=="],
|
||||
|
||||
"@jsquash/avif": ["@jsquash/avif@2.1.1", "", { "dependencies": { "wasm-feature-detect": "^1.2.11" } }, "sha512-LMRxd0fMgfCLtobDh0/sFYJMMiRJTNYSEEWvRDKXlAeZ08t3gI5V+1thIT0XjXJ+SVG7Zug9B0XPyx0Ti5VRNA=="],
|
||||
|
||||
@@ -326,7 +327,7 @@
|
||||
|
||||
"asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
|
||||
|
||||
"body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="],
|
||||
"body-parser": ["body-parser@2.3.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^2.0.0", "debug": "^4.4.3", "http-errors": "^2.0.1", "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", "qs": "^6.15.2", "raw-body": "^3.0.2", "type-is": "^2.1.0" } }, "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw=="],
|
||||
|
||||
"bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="],
|
||||
|
||||
@@ -400,7 +401,7 @@
|
||||
|
||||
"fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="],
|
||||
|
||||
"fast-uri": ["fast-uri@3.1.3", "", {}, "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg=="],
|
||||
"fast-uri": ["fast-uri@3.1.4", "", {}, "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw=="],
|
||||
|
||||
"fast-xml-builder": ["fast-xml-builder@1.3.0", "", { "dependencies": { "path-expression-matcher": "^1.6.2", "xml-naming": "^0.3.0" } }, "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ=="],
|
||||
|
||||
@@ -614,6 +615,10 @@
|
||||
|
||||
"@types/bun/bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
|
||||
|
||||
"body-parser/content-type": ["content-type@2.0.0", "", {}, "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ=="],
|
||||
|
||||
"body-parser/type-is": ["type-is@2.1.0", "", { "dependencies": { "content-type": "^2.0.0", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA=="],
|
||||
|
||||
"es-set-tostringtag/hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
|
||||
|
||||
"eventsource/eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
|
||||
|
||||
+2
-1
@@ -39,10 +39,11 @@ gbrain migrate --to pglite # Postgres → PGLite (rare)
|
||||
|
||||
For shared / large / multi-machine deployments (a team or company brain with multiple users hitting one server over HTTP MCP with OAuth scoping per user), follow the dedicated walkthrough: **[Tutorial: set up GBrain as your company brain](tutorials/company-brain.md)**.
|
||||
|
||||
API keys live in `~/.gbrain/config.json` (file plane) or env vars (`OPENAI_API_KEY`, `ZEROENTROPY_API_KEY`, `VOYAGE_API_KEY`, `ANTHROPIC_API_KEY`). Set via CLI:
|
||||
API keys live in `~/.gbrain/config.json` (file plane) or env vars (`OPENAI_API_KEY`, `OPENROUTER_API_KEY`, `ZEROENTROPY_API_KEY`, `VOYAGE_API_KEY`, `ANTHROPIC_API_KEY`). Set via CLI:
|
||||
|
||||
```bash
|
||||
gbrain config set zeroentropy_api_key sk-...
|
||||
gbrain config set openrouter_api_key sk-or-...
|
||||
gbrain config set anthropic_api_key sk-ant-...
|
||||
```
|
||||
|
||||
|
||||
@@ -189,8 +189,10 @@ Unit tests and what they cover:
|
||||
- `test/orphans.test.ts` — orphans command: detection, pseudo filtering, text/json/count outputs, MCP op.
|
||||
- `test/postgres-engine.test.ts` — `statement_timeout` scoping: `sql.begin` + `SET LOCAL` shape, source-level grep guardrail against a reintroduced bare `SET statement_timeout`.
|
||||
- `test/sync.test.ts` — sync logic + regression guard asserting top-level `engine.transaction` is not called.
|
||||
- `test/sync-pull-failed-anchor.serial.test.ts` — #3068 regression: a failed internal `git pull` (local-path origin vs `protocol.file.allow=never`) with zero imports returns `partial`/`pull_failed` (not `up_to_date`), freezes `last_commit` + `last_sync_at`, recovers after a manual pull; fall-through import of local commits preserved. Serial: pins `GBRAIN_HOME` to a temp dir for the whole file.
|
||||
- `test/sync-concurrency.test.ts` — `autoConcurrency()` thresholds + PGLite-forces-serial + explicit-override clamping; `shouldRunParallel()` explicit-bypasses-floor contract; `parseWorkers()` validation rejecting `'0'`/`'-3'`/`'foo'`/`'1.5'`/trailing chars.
|
||||
- `test/sync-parallel.test.ts` — PGLite-routed coverage of the bookmark gate under concurrency, head-drift gate, vanished-file failure capture, PGLite-stays-serial, and the `gbrain-sync` writer-lock contract.
|
||||
- `test/sync-all-missing-path.test.ts` — `sync --all --missing-path <fail|skip>` pure helpers: `parseMissingPathMode` (default fail, explicit values, loud rejection of bad/dangling values, never swallows a following flag) and `partitionMissingPathSources` (classification driven only by the injected pathExists predicate — no fs; null `local_path` passes through runnable; order preserved).
|
||||
- `test/sync-failures.test.ts` — `classifyErrorCode` regex coverage for all 12 codes against literal production message strings from `markdown.ts` and `import-file.ts`; `summarizeFailuresByCode` sort + pre-classified-honor; `recordSyncFailures` code-field persistence; `acknowledgeSyncFailures` `AcknowledgeResult` shape + backfill on legacy entries.
|
||||
- `test/doctor.test.ts` — doctor command; assertions that `jsonb_integrity` scans the four JSONB write sites and `markdown_body_completeness` is present.
|
||||
- `test/utils.test.ts` — shared SQL utilities + `tryParseEmbedding` null-return and single-warn semantics.
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,146 @@
|
||||
# Conversation parser patterns
|
||||
|
||||
The conversation parser turns exported chat and meeting transcripts into a
|
||||
common message stream without requiring an LLM call for known formats. This
|
||||
document describes the built-in pattern contract and the checks required when
|
||||
adding or changing a format.
|
||||
|
||||
## Data flow
|
||||
|
||||
`parseConversation` uses this sequence:
|
||||
|
||||
1. Resolve the page date and timezone context.
|
||||
2. Score every enabled built-in and user pattern against the first ten
|
||||
non-blank lines.
|
||||
3. Re-score the full body when the head score is inconclusive, or when a broad
|
||||
pattern explicitly requires full-body scoring.
|
||||
4. Reject the winner when its acceptance score is below the false-positive
|
||||
floor.
|
||||
5. Apply the winning pattern to every line and attach continuation lines to the
|
||||
preceding message.
|
||||
6. Optionally run LLM polish or fallback when those features are enabled.
|
||||
|
||||
Pattern order is only a tie-breaker. A new regex must be structurally distinct
|
||||
from neighboring formats; moving it earlier in the registry is not a valid
|
||||
non-shadowing strategy.
|
||||
|
||||
## Built-in pattern contract
|
||||
|
||||
Every `PatternEntry` in `builtins.ts` declares:
|
||||
|
||||
- A stable, kebab-case `id`.
|
||||
- A hand-vetted line regex and explicit capture-group indexes.
|
||||
- Where the date comes from and how the time is represented.
|
||||
- A timezone policy.
|
||||
- Whether the format supports multi-line message bodies.
|
||||
- Positive and negative samples that run during module initialization.
|
||||
- A documentation pointer describing the source format.
|
||||
|
||||
The registry refuses to load when a positive sample stops matching, a negative
|
||||
sample starts matching, or a capture map becomes invalid. This catches local
|
||||
regex mistakes before extraction can silently produce empty conversations.
|
||||
|
||||
### Date and timezone rules
|
||||
|
||||
Formats with an inline date should capture it from each message. Time-only
|
||||
formats use an explicit caller fallback first, then the page frontmatter date,
|
||||
then the page effective date. If none is available, the parser uses
|
||||
`1970-01-01` so the missing date remains visible instead of inventing a current
|
||||
date.
|
||||
|
||||
Time-only formats normally use `utc_assumed_with_warn`. The parser constructs a
|
||||
UTC timestamp and returns a timezone warning when the page does not provide a
|
||||
timezone. A new pattern should not imply local-time precision that the source
|
||||
format does not contain.
|
||||
|
||||
### Multi-line messages
|
||||
|
||||
An anchor regex identifies the first line of a message. Subsequent non-anchor
|
||||
lines are appended to that message until another anchor appears. Set
|
||||
`multi_line: true` when continuation content is part of the documented format,
|
||||
such as Markdown bullets, blockquotes, or an exported message body on the next
|
||||
line.
|
||||
|
||||
Tests for a multi-line format should assert the complete message text, including
|
||||
newlines. A message-count assertion alone will not detect lost bullets or a
|
||||
continuation attached to the wrong speaker.
|
||||
|
||||
### Scoring and false positives
|
||||
|
||||
The score compares matched anchors with the pattern's relevant candidate lines.
|
||||
The first pass uses the head of the page for speed. Low-confidence pages are
|
||||
re-scored across the full body before the parser accepts a winner.
|
||||
|
||||
Multi-line formats may opt into `score_continuations_as_body` when their anchor
|
||||
grammar is distinctive. Candidate-only scoring activates only after two anchors
|
||||
match, or when the first non-blank line is an anchor. This evidence threshold
|
||||
lets a single long message keep its continuation body without turning one stray
|
||||
anchor in a prose page into a conversation. Candidate anchor lines that fail the
|
||||
full regex still lower the score. Other patterns continue to use all non-blank
|
||||
lines in their density score.
|
||||
|
||||
Use `score_full_body: true` for a broad grammar that also occurs in ordinary
|
||||
prose. For example, `**Label:** text` can be either a transcript line or a bold
|
||||
label in meeting notes. Narrow formats with a timestamp and a distinctive
|
||||
separator generally do not need this override.
|
||||
|
||||
`quick_reject` is a performance hint, not an acceptance rule. It should cheaply
|
||||
exclude obviously unrelated lines while admitting every string accepted by the
|
||||
main regex.
|
||||
|
||||
## Normalized Slack Markdown
|
||||
|
||||
The `bold-time-dash` pattern parses message anchors shaped like:
|
||||
|
||||
```text
|
||||
**Alice Example** 09:15 — first message
|
||||
- supporting detail
|
||||
**Bob Example** 09:18 — second message
|
||||
```
|
||||
|
||||
Its grammar is:
|
||||
|
||||
```text
|
||||
**speaker** H:MM <dash> text
|
||||
```
|
||||
|
||||
where:
|
||||
|
||||
- `H:MM` is a valid 24-hour time from `0:00` through `23:59`.
|
||||
- `<dash>` may be an em dash (`—`), en dash (`–`), or ASCII hyphen (`-`).
|
||||
- The date comes from the resolved page date context.
|
||||
- Continuation lines belong to the preceding message.
|
||||
- The captured clock value is emitted with `Z`. Timezone metadata suppresses
|
||||
the missing-timezone warning but is not currently used for IANA conversion.
|
||||
|
||||
The required time and dash distinguish it from all existing bold-speaker
|
||||
formats:
|
||||
|
||||
- `**Speaker** (09:15): text` uses `bold-paren-time`.
|
||||
- `**Speaker** (9:15 AM): text` uses `bold-paren-time-12h`.
|
||||
- `**Speaker:** text` uses `bold-name-no-time`.
|
||||
- `**Speaker** (2026-04-09 9:15 AM): text` uses `imessage-slack`.
|
||||
|
||||
Keeping these examples in both `test_negative` and parser regression tests makes
|
||||
the non-shadowing contract executable.
|
||||
|
||||
## Adding a built-in format
|
||||
|
||||
1. Collect multiple anonymized examples, including separator and timestamp
|
||||
variants that occur in the same export family.
|
||||
2. Choose the narrowest grammar that represents the format. Constrain numeric
|
||||
fields such as hours and minutes when possible.
|
||||
3. Add at least two positive module-load samples and negative samples for every
|
||||
neighboring pattern that could plausibly overlap.
|
||||
4. Add parser tests that verify speakers, timestamps, text, continuation
|
||||
handling, and non-shadowing behavior.
|
||||
5. Add a dedicated JSONL fixture and include the same cases in
|
||||
`test/fixtures/conversation-formats/all.jsonl`.
|
||||
6. Run the focused parser tests and the fixture evaluator.
|
||||
7. Run the repository verification and full test suites before submission.
|
||||
8. Update `docs/architecture/KEY_FILES.md` when the registry count or supported
|
||||
format inventory changes.
|
||||
|
||||
Use generic fixture identities such as `Alice Example`, `Bob Example`, and
|
||||
`Summary Bot`. Never copy real transcript names or private content into source,
|
||||
tests, documentation, commits, or pull-request descriptions.
|
||||
@@ -75,6 +75,15 @@ Meta-pack stacking creator + investor + engineer via the v0.38
|
||||
preserved — this IS the active pack; the registry walks extends +
|
||||
borrow to materialize the merged view.
|
||||
|
||||
**Merge contract (T20 / #1749).** `resolvePack` merges parent → child
|
||||
(child-wins) for the six ingest/query-shaping fields: `page_types`,
|
||||
`link_types`, `frontmatter_links`, `enrichable_types`, `filing_rules`,
|
||||
and `takes_kinds` (unioned — a child cannot narrow it). `phases` and
|
||||
`calibration_domains` are **NOT** inherited: they gate cycle execution,
|
||||
so each pack must declare its own participation explicitly. That is why
|
||||
`gbrain-everything` re-declares all its phases and all 7
|
||||
`calibration_domains` — inheritance does not carry them.
|
||||
|
||||
Activate via `gbrain config set schema_pack gbrain-everything` and
|
||||
calibration_profile produces all 7 domain scorecards in one JSONB.
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ api_version: gbrain-schema-pack-v1
|
||||
name: my-pack
|
||||
version: 0.0.1
|
||||
gbrain_min_version: 0.39.0
|
||||
extends: gbrain-base # inherits everything from base; add overrides below
|
||||
extends: gbrain-base # inherits base's TYPES (see Merge contract below); add overrides
|
||||
description: |
|
||||
My personal pack.
|
||||
|
||||
@@ -170,6 +170,34 @@ enrichable_types: []
|
||||
filing_rules: []
|
||||
```
|
||||
|
||||
## Merge contract (`extends` + `borrow_from`)
|
||||
|
||||
`resolvePack` composes a pack against its `extends` chain (and any
|
||||
`borrow_from` targets) into the `resolved.manifest` every consumer reads
|
||||
(T20 / #1749). The rules:
|
||||
|
||||
- **Six fields inherit, child-wins:** `page_types`, `link_types`,
|
||||
`frontmatter_links`, `enrichable_types`, `filing_rules`, and `takes_kinds`.
|
||||
A child value with the same key (type name, link name, etc.) overrides the
|
||||
parent's; keys the child doesn't declare come through from the parent.
|
||||
- **`page_types` ordering:** overrides of a base type keep the base's declared
|
||||
position (base's `inferType` prefix priority is authoritative); a genuinely
|
||||
new type — from the child, a `borrow_from`, or a middle pack in the chain —
|
||||
is prepended nearest-first, so a more-derived type's `path_prefix` wins
|
||||
regardless of how deep the chain is.
|
||||
- **`takes_kinds` is UNION, not replace** — it carries a Zod default, so an
|
||||
omitted field is indistinguishable from an explicit one. A child can ADD
|
||||
kinds but **cannot narrow** `takes_kinds` below base ∪ parent. If you need a
|
||||
smaller set, don't `extends` a pack that declares the larger one.
|
||||
- **`phases` and `calibration_domains` are NOT inherited** (child-only). They
|
||||
gate real cycle execution, so each pack must declare its own participation
|
||||
explicitly — inheriting them would silently make a child run phases it never
|
||||
requested. This is why `gbrain-everything` re-declares all its phases and
|
||||
calibration domains by hand. See `lens-packs.md` for the worked example.
|
||||
- **`borrow_from` is selective + non-transitive + fail-closed:** it pulls only
|
||||
the named `types`/`link_types` from the target's OWN declarations (omitting a
|
||||
category borrows none of it); a missing target throws `UnknownPackError`.
|
||||
|
||||
## Recovery + revert
|
||||
|
||||
The single-PR cathedral is hard to revert atomically. Per codex finding
|
||||
|
||||
@@ -159,7 +159,8 @@ proxy for worker env.
|
||||
If a brain DB ever traverses a trust boundary, secrets stay out.
|
||||
- **Free-form names.** `inherit:` accepts any snake_case config-key on your
|
||||
worker — `database_url`, `anthropic_api_key`, `openai_api_key`,
|
||||
`voyage_api_key`, `groq_api_key`, `zeroentropy_api_key`, or any custom
|
||||
`openrouter_api_key`, `voyage_api_key`, `groq_api_key`,
|
||||
`zeroentropy_api_key`, or any custom
|
||||
field you stuff into `~/.gbrain/config.json`. The agent picks what it
|
||||
needs.
|
||||
- **`env:` still works** for non-secret values, or for cases where you
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
# Embedding migration — moving a brain to another embedding provider
|
||||
|
||||
`gbrain migrate embeddings` re-embeds an entire brain onto a different
|
||||
embedding provider/model, safely and resumably. It is the forward path off a
|
||||
sunsetting provider (for example ZeroEntropy's hosted API, which shuts down
|
||||
2026-09-04 and is the shipped default for brains that never picked a model) —
|
||||
but it is provider-agnostic: any configured `provider:model` works as a
|
||||
target.
|
||||
|
||||
Also reachable as `gbrain retrieval-upgrade` (the name `doctor` and the
|
||||
README reference).
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
# Preview the work + cost. Changes nothing.
|
||||
gbrain migrate embeddings --to openai:text-embedding-3-small --dry-run
|
||||
|
||||
# Run it (interactive confirm shows chunk count + $ estimate first).
|
||||
gbrain migrate embeddings --to openai:text-embedding-3-small
|
||||
|
||||
# Non-interactive (cron / scripts): --yes is required, else exit 2.
|
||||
gbrain migrate embeddings --to voyage:voyage-3-large --yes
|
||||
```
|
||||
|
||||
`--dim <N>` overrides the target width; it defaults to the provider recipe's
|
||||
declared width and is required for recipes that don't declare one (litellm,
|
||||
llama-server, and other bring-your-own-model providers).
|
||||
|
||||
## What it does, in order
|
||||
|
||||
1. **Plan.** Counts every chunk not already in the target embedding space —
|
||||
including chunks on pages with **no recorded embedding signature**
|
||||
(pages embedded before the v108 provenance stamp). Prices the re-embed
|
||||
from the pricing table; unknown providers print "estimate unavailable"
|
||||
instead of a fabricated number.
|
||||
2. **Consent gate.** Prints the plan; requires an interactive `y` or `--yes`.
|
||||
Non-TTY without `--yes` refuses with exit 2 (mirrors the `reindex-code`
|
||||
gate in [spend-controls](../operations/spend-controls.md)). Unlike the pure
|
||||
cost gates there, `spend.posture=tokenmax` does **not** bypass this one:
|
||||
posture waives the spend *ceiling*, and this gate also guards a
|
||||
destructive schema rebuild. Under `tokenmax` the dollar figure is marked
|
||||
informational and the confirmation is still asked. `--yes` is the single
|
||||
scripted bypass.
|
||||
3. **Live probe.** One tiny embed against the TARGET provider before any
|
||||
mutation — validates the API key, model id, and dimension support in a
|
||||
single call. A bad key fails here, with nothing changed.
|
||||
4. **Env-override gate.** Refuses when `GBRAIN_EMBEDDING_MODEL` /
|
||||
`GBRAIN_EMBEDDING_DIMENSIONS` would silently defeat the switch at
|
||||
runtime (the same guard `ze-switch` uses). `--ignore-env-override` for
|
||||
people running deliberate experiments.
|
||||
5. **Apply.** When the target width differs from the actual column width,
|
||||
runs the same atomic schema transition `ze-switch` uses, in one
|
||||
transaction. It rebuilds **all three dim-pinned text-embedding-space
|
||||
columns** — `content_chunks.embedding`, `query_cache.embedding`, and
|
||||
`facts.embedding` — at the new width, preserving each column's type
|
||||
(`vector` vs `halfvec`) and recreating its HNSW index. Missing any of the
|
||||
three leaves it silently broken: a narrow `query_cache.embedding` makes
|
||||
every cache write and read fail *by design* (the cache swallows errors so
|
||||
it can never break search) for a permanent 0% hit rate, and a narrow
|
||||
`facts.embedding` fails every per-fact embed write. The image/multimodal
|
||||
columns ARE deliberately untouched — they use separate models whose
|
||||
dimensions are independent of the text embedding model.
|
||||
Writes `embedding_model` + `embedding_dimensions` to BOTH config planes
|
||||
(file plane for the runtime gateway, DB plane for doctor), invalidates
|
||||
every chunk still in the old space — **including NULL-signature pages** —
|
||||
and purges the semantic query cache so stale cached results can't be
|
||||
served across the swap.
|
||||
6. **Re-embed.** The standard embed pipeline (`embed --stale --catch-up`)
|
||||
with per-source single-flight locks, rate-limit backoff, stderr progress,
|
||||
and optional DB-contention pacing (`--pace[=mode]`).
|
||||
|
||||
## What the rebuild deletes
|
||||
|
||||
The dimension change **deletes every stored embedding vector** in the brain —
|
||||
they are in the old model's space and unusable. They are not recoverable:
|
||||
going back to the previous provider means paying for a second full re-embed.
|
||||
`content_chunks` vectors are rebuilt by the re-embed pass, the query cache
|
||||
refills on the next query, and fact embeddings are rewritten on their next
|
||||
write (or a `gbrain extract` pass).
|
||||
|
||||
## Resume after a kill
|
||||
|
||||
The NULL-embedding column is the checkpoint. If the run is killed (or some
|
||||
pages fail to embed), re-run the **same command**: chunks already embedded on
|
||||
the target are never re-embedded, the schema/config steps no-op, and the run
|
||||
continues where it stopped. An in-flight marker (`embedding_migration.state`
|
||||
in DB config) records the target; it is cleared only when the backlog drains
|
||||
to zero.
|
||||
|
||||
A page whose chunks straddle two stale batches is embedded correctly but not
|
||||
stamped by the embed loop (which only stamps all-or-nothing per batch), so the
|
||||
migration runs one reconcile pass after the drain that stamps every
|
||||
fully-embedded page. Without it a large brain would report "incomplete" and the
|
||||
re-run would pay again for those pages. `--batch-size N` tunes the batch
|
||||
(default 2000).
|
||||
|
||||
`--no-embed` applies schema + config + invalidation and stops, so you can run
|
||||
the (potentially long) re-embed later or in the background:
|
||||
|
||||
```bash
|
||||
gbrain migrate embeddings --to openai:text-embedding-3-small --yes --no-embed
|
||||
gbrain embed --stale --catch-up --include-null-signature --background
|
||||
```
|
||||
|
||||
## During the migration
|
||||
|
||||
While the re-embed runs, semantic search returns degraded (lexical-arm-only)
|
||||
results for not-yet-re-embedded content. Pick a quiet window for large
|
||||
brains, or use `--pace` to keep the DB responsive.
|
||||
|
||||
## Pages without an embedding signature (#3391)
|
||||
|
||||
Pages embedded before provenance stamping have `embedding_signature IS NULL`
|
||||
and are grandfathered by the routine stale sweep (so an upgrade never
|
||||
surprise-re-embeds a whole corpus). After a provider swap that grandfather
|
||||
clause would silently leave those pages in the OLD embedding space — mixed
|
||||
vector spaces in one index, degrading retrieval with nothing in the logs.
|
||||
|
||||
- `gbrain migrate embeddings` always includes them.
|
||||
- Plain `gbrain embed --stale` warns when a model swap leaves NULL-signature
|
||||
pages behind, and `gbrain embed --stale --include-null-signature` re-embeds
|
||||
them.
|
||||
|
||||
## Reranker
|
||||
|
||||
Migrating embeddings does not touch the reranker. If
|
||||
`search.reranker.model` points at the outgoing provider, the plan prints a
|
||||
warning; disable it (`gbrain config set search.reranker.enabled false`) or
|
||||
point it at another provider.
|
||||
|
||||
## Self-hosting instead of migrating
|
||||
|
||||
If the outgoing model's weights are available (zembed-1's are Apache-2.0),
|
||||
serving them locally via `llama-server` / `ollama` / a LiteLLM proxy
|
||||
preserves your existing vectors — no re-embed at all. Point
|
||||
`embedding_model` at the local recipe and keep the same dimensions. The
|
||||
migration command is for when you'd rather move to a hosted provider.
|
||||
@@ -21,14 +21,17 @@ GBrain is tuned for the Supabase **Transaction pooler** (port 6543): it
|
||||
auto-disables prepared statements there and routes `engine.transaction()`
|
||||
(migrations, DDL, sync imports) to a derived **direct** connection
|
||||
(`db.<ref>.supabase.co:5432`). That direct host is IPv6-only, so on an
|
||||
IPv4-only host, reads work but sync **silently skips most pages**. This is the
|
||||
number one cause of "sync ran but nothing happened."
|
||||
IPv4-only host it is unreachable. When that happens gbrain now falls back to
|
||||
the pooler automatically (one stderr warning, then single-pool mode for the
|
||||
rest of the process) — but the pooler's ~2-min statement timeout can truncate
|
||||
very long migrations or bulk imports.
|
||||
|
||||
Fix: make the direct connection reachable over IPv4. Either set
|
||||
`GBRAIN_DIRECT_DATABASE_URL` to the **Session pooler** string (port 5432 on the
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on. Verify by
|
||||
running `gbrain sync` and checking that the page count in `gbrain stats` matches
|
||||
the syncable file count in the repo.
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on.
|
||||
`GBRAIN_DISABLE_DIRECT_POOL=1` skips the direct pool (and the fallback warning)
|
||||
entirely. Verify by running `gbrain sync` and checking that the page count in
|
||||
`gbrain stats` matches the syncable file count in the repo.
|
||||
|
||||
### The Primitives
|
||||
|
||||
|
||||
@@ -155,6 +155,7 @@ child-spawn time:
|
||||
- `inherit: ["database_url"]` → child env `GBRAIN_DATABASE_URL`
|
||||
- `inherit: ["anthropic_api_key"]` → child env `ANTHROPIC_API_KEY`
|
||||
- `inherit: ["openai_api_key"]` → child env `OPENAI_API_KEY`
|
||||
- `inherit: ["openrouter_api_key"]` → child env `OPENROUTER_API_KEY`
|
||||
- `inherit: ["voyage_api_key"]` → child env `VOYAGE_API_KEY`
|
||||
- `inherit: ["groq_api_key", "zeroentropy_api_key"]` → both injected
|
||||
- Or any arbitrary config-key your worker has (`my_custom_field` →
|
||||
|
||||
@@ -103,7 +103,7 @@ For GCP service-account / Vertex AI auth (production deployments), see the v0.32
|
||||
|
||||
### OpenRouter
|
||||
|
||||
Single OpenAI-compatible API for fan-out to OpenAI, Anthropic, Google, DeepSeek, Meta Llama, Qwen, and dozens of other hosted providers. One key, many models. Set `OPENROUTER_API_KEY` and use `openrouter:<provider>/<model>` (e.g. `openrouter:openai/gpt-5.2`, `openrouter:anthropic/claude-sonnet-4.6`).
|
||||
Single OpenAI-compatible API for fan-out to OpenAI, Anthropic, Google, DeepSeek, Meta Llama, Qwen, and dozens of other hosted providers. One key, many models. Set `OPENROUTER_API_KEY` or `openrouter_api_key` in `~/.gbrain/config.json`, then use `openrouter:<provider>/<model>` (e.g. `openrouter:openai/gpt-5.2`, `openrouter:anthropic/claude-sonnet-4.6`).
|
||||
|
||||
**Embedding**: `openai/text-embedding-3-small` (1536d default, Matryoshka shrink to 512/768/1024). OR's embedding catalog also includes `text-embedding-3-large`, `google/gemini-embedding-2-preview`, `qwen/qwen3-embedding-8b`, `bge-m3` — opt in via `--embedding-model openrouter:<id>`. Pricing matches the upstream provider (OR adds a small markup).
|
||||
|
||||
|
||||
@@ -258,6 +258,43 @@ the user owns the machine.
|
||||
See [ALTERNATIVES.md](ALTERNATIVES.md) for a comparison of ngrok, Tailscale
|
||||
Funnel, and cloud hosts (Fly.io, Railway).
|
||||
|
||||
### Co-located Docker workloads (self-hosted Postgres)
|
||||
|
||||
OAuth scopes and source scoping guard the `gbrain serve --http` path. They do
|
||||
NOT guard raw Postgres. If the brain's Postgres runs as a container on the same
|
||||
Docker host as other workloads (agent runtimes, n8n, staging fixtures), any
|
||||
container sharing Docker's default `bridge` network can open a direct DB
|
||||
session — no OAuth token required — and read every source. That silently
|
||||
recreates a privileged path underneath the isolation you configured at the MCP
|
||||
layer.
|
||||
|
||||
Network-zone the host so untrusted containers can never reach Postgres:
|
||||
|
||||
```
|
||||
Docker host
|
||||
├── gbrain-net ← ONLY the brain's Postgres (+ gbrain serve, if containerized)
|
||||
├── agent-<id>-net ← each untrusted agent runtime, isolated
|
||||
└── default bridge ← no secret-bearing databases
|
||||
```
|
||||
|
||||
Operator checklist:
|
||||
|
||||
```text
|
||||
[ ] Postgres is on a user-defined Docker network, not the default bridge
|
||||
(or nothing else runs on that bridge)
|
||||
[ ] If Postgres publishes a host port at all, it binds loopback only
|
||||
(`-p 127.0.0.1:5432:5432`, never `0.0.0.0`)
|
||||
[ ] Untrusted agent containers have no DATABASE_URL or Postgres password
|
||||
[ ] Untrusted agents reach the brain via OAuth/Bearer against serve --http only
|
||||
(host loopback via host.docker.internal / host gateway — never gbrain-net)
|
||||
[ ] OAuth clients are least-privilege: scoped --source / --federated-read,
|
||||
pre-minted short-lived tokens preferred over long-lived client secrets
|
||||
[ ] Isolation verified: a team-scoped client cannot read internal-only sources
|
||||
```
|
||||
|
||||
Optional defense-in-depth: a dedicated Postgres role (or RLS) limited to the
|
||||
allowed `source_id`s, so even a leaked connection string can't read everything.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"missing_auth" error**
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
# Conversation backfill durable outcomes
|
||||
|
||||
`gbrain extract-conversation-facts` stores page-level outcomes in `facts` so
|
||||
bulk runs, autopilot, and `gbrain doctor` can distinguish finished work from
|
||||
retryable work without adding another state table.
|
||||
|
||||
This is completion authority, not ordinary extracted knowledge. The authority
|
||||
is deliberately narrow: a marker is valid only for the exact page or transcript
|
||||
snapshot that was parsed, and only after every required operation succeeded.
|
||||
|
||||
## Outcome protocol
|
||||
|
||||
The current protocol is v2. Its source names are versioned so rows written by
|
||||
older best-effort implementations cannot suppress a corrective replay.
|
||||
|
||||
| Outcome | `facts.source` | Meaning |
|
||||
|---|---|---|
|
||||
| Complete | `cli:extract-conversation-facts:terminal:v2` | Every eligible segment was extracted and inserted successfully, the input remained unchanged, and the terminal write succeeded. |
|
||||
| Scanned, not extractable | `cli:extract-conversation-facts:non-extractable:v2` | A recognized input was scanned successfully but contained no eligible multi-message segment. |
|
||||
| Unfinished | no matching v2 outcome | Work is pending, failed, was not recognized, changed during extraction, or has only a legacy marker. |
|
||||
|
||||
The non-extractable outcome is intentionally separate from completion. It does
|
||||
not claim that knowledge facts were extracted. CLI counters, cycle details, and
|
||||
doctor output preserve that distinction.
|
||||
|
||||
## Snapshot identity
|
||||
|
||||
Every v2 marker binds `source_session` to the parser input snapshot:
|
||||
|
||||
```text
|
||||
<outcome-source>:<page-slug>:<version-token>
|
||||
```
|
||||
|
||||
There are two token forms.
|
||||
|
||||
### Database-backed page body
|
||||
|
||||
For pages parsed from `compiled_truth` and `timeline`, the token is:
|
||||
|
||||
```text
|
||||
page-<pages.content_hash>-<effective-date>
|
||||
```
|
||||
|
||||
`content_hash` covers title, type, compiled truth, timeline, and frontmatter.
|
||||
The effective-date suffix covers the remaining date input used by parsing. This
|
||||
identity does not depend on JavaScript's millisecond timestamp precision, so two
|
||||
writes within one PostgreSQL millisecond still produce different tokens when
|
||||
parser input changes. A legacy page with a null content hash uses a computed
|
||||
SHA-256 fallback and is verified in-process by both extraction and doctor.
|
||||
|
||||
### Raw transcript sidecar
|
||||
|
||||
When frontmatter contains `raw_transcript`, the source text lives outside the
|
||||
page row and may change without changing `pages.updated_at`. Its token is:
|
||||
|
||||
```text
|
||||
sidecar-<SHA-256>
|
||||
```
|
||||
|
||||
The digest covers the exact body given to the parser plus parser-relevant page
|
||||
metadata: title, type, frontmatter, and effective date. Selection recomputes
|
||||
the digest before skipping work. A sidecar-only edit therefore reopens the page.
|
||||
|
||||
`gbrain doctor` cannot read sidecars in its SQL aggregate, so it enumerates those
|
||||
pages in bounded batches and calls the same canonical verifier used by
|
||||
extraction. Doctor and extraction therefore agree after sidecar-only edits.
|
||||
|
||||
## Selection and locking
|
||||
|
||||
Bulk extraction follows this sequence:
|
||||
|
||||
1. Enumerate candidate pages in bounded batches.
|
||||
2. Filter candidates with matching v2 outcomes.
|
||||
3. Apply `--limit` to the remaining pages that actually need work.
|
||||
4. Acquire the source-and-slug advisory lock.
|
||||
5. Re-fetch the page under that lock.
|
||||
6. Recompute and recheck the snapshot-bound outcome.
|
||||
7. Prepare one immutable parser snapshot and process it.
|
||||
8. Re-fetch and recompute the snapshot before writing an outcome.
|
||||
|
||||
The pre-lock check avoids parser, filesystem, and model work for ordinary
|
||||
completed pages. The under-lock refetch prevents a stale enumeration object
|
||||
from becoming the certified input. The final comparison prevents an edit that
|
||||
happens during model or insertion work from receiving a marker for old content.
|
||||
|
||||
An edit can occur after the final comparison and before marker insertion. That
|
||||
is still safe because the marker contains the old version token. Future
|
||||
selection compares the token, not marker creation time, and reopens the page.
|
||||
|
||||
Single-page `--slug` runs use the same under-lock path.
|
||||
|
||||
## Strict extraction success
|
||||
|
||||
The general `extractFactsFromTurn` API remains best-effort for interactive
|
||||
callers. It historically returns an empty array for both a legitimate zero-fact
|
||||
answer and several model failures.
|
||||
|
||||
Conversation backfill instead uses `extractFactsFromTurnWithOutcome`, whose
|
||||
result separates:
|
||||
|
||||
- `{ ok: true, facts: [] }`, a successful extraction with no durable facts;
|
||||
- `{ ok: true, facts: [...] }`, a successful extraction with facts; and
|
||||
- `{ ok: false, reason, error? }`, an unavailable provider, provider error,
|
||||
refusal, content filter, malformed output, or repeated truncation.
|
||||
|
||||
Any failed segment aborts the page attempt. Any `insertFacts` failure also
|
||||
aborts it. The page receives neither a checkpoint advancement nor a terminal
|
||||
outcome. Facts inserted by earlier segments may remain temporarily, but the
|
||||
next claim deletes this command's rows for the page and replays cleanly.
|
||||
|
||||
Bulk workers continue past an individual page failure, but they do not hide it.
|
||||
`pages_failed` counts failed claims, stderr names each page, the CLI exits 1,
|
||||
the autopilot phase reports `warn`, and receipts/rollups classify the run as
|
||||
incomplete. A tolerant pool is therefore observable without sacrificing the
|
||||
rest of a large backfill.
|
||||
|
||||
This distinction is load-bearing. Treating a provider outage as a successful
|
||||
zero-fact response would make a transient failure durable and permanently hide
|
||||
the page from later runs.
|
||||
|
||||
## Non-extractable authority
|
||||
|
||||
A non-extractable marker is written only when all of the following are true:
|
||||
|
||||
- a deterministic or accepted parser format recognized the input;
|
||||
- ordinary segmentation produced no eligible multi-message segment;
|
||||
- the parser phase was not `no_match`;
|
||||
- cleanup of prior command-owned rows succeeded; and
|
||||
- the input snapshot was still current immediately before cleanup and write.
|
||||
|
||||
A `no_match` result stays unfinished so a new parser pattern, optional fallback,
|
||||
or corrected input can recover it. Oversize pages, disappeared pages, lock
|
||||
contention, dry runs, aborts, cleanup errors, provider failures, extraction
|
||||
failures, insertion failures, and outcome-write failures also stay unfinished.
|
||||
|
||||
Cleanup errors are never interpreted as "zero rows deleted." Propagating them
|
||||
prevents a fresh non-extractable marker from coexisting with stale extracted
|
||||
facts that could not be removed.
|
||||
|
||||
## Checkpoints are not authority
|
||||
|
||||
Operation checkpoints are only progress hints. They do not prove which page
|
||||
snapshot was processed, and old checkpoint entries do not include a snapshot
|
||||
token. When a page lacks a matching v2 outcome, the command discards that
|
||||
page's checkpoint entry and performs a delete-first full replay.
|
||||
|
||||
This rule prevents two corruption classes:
|
||||
|
||||
- edited text with timestamps older than the old watermark being skipped; and
|
||||
- command-owned facts being deleted while the checkpoint skips the segments
|
||||
needed to recreate them.
|
||||
|
||||
Deleting `op_checkpoints` does not reopen pages with matching v2 outcomes.
|
||||
Deleting or editing an outcome does not make a checkpoint authoritative.
|
||||
|
||||
## `--limit` semantics
|
||||
|
||||
`--limit N` caps pages that require processing, not completed pages inspected
|
||||
while finding them. Durable filtering happens before clipping a batch. With a
|
||||
completed page first and a pending page second, `--limit 1` processes the
|
||||
pending page rather than consuming the limit on the completed page.
|
||||
|
||||
`pages_considered` may therefore exceed `--limit` because it includes durable
|
||||
outcomes observed during selection. Model-bearing page work does not exceed the
|
||||
limit.
|
||||
|
||||
## `--force`
|
||||
|
||||
`--force` bypasses durable outcome selection and clears the page checkpoint.
|
||||
It still uses delete-first replay, strict extraction outcomes, advisory locks,
|
||||
and snapshot verification. Force means "recompute" rather than "relax safety."
|
||||
|
||||
## Operator signals
|
||||
|
||||
The result exposes separate counters:
|
||||
|
||||
- `pages_skipped_completed`
|
||||
- `pages_skipped_non_extractable`
|
||||
- `pages_marked_non_extractable`
|
||||
- `pages_failed`
|
||||
|
||||
The CLI aggregates these across sources. The autopilot backfill phase includes
|
||||
them in phase details. `gbrain doctor` reports `completed`,
|
||||
`scanned_not_extractable`, and `backlog` independently.
|
||||
|
||||
Run a small canary twice:
|
||||
|
||||
```bash
|
||||
gbrain extract-conversation-facts --source-id default --limit 10 --workers 1 --max-cost-usd 0.25 --yes
|
||||
gbrain extract-conversation-facts --source-id default --limit 10 --workers 1 --max-cost-usd 0.25 --yes
|
||||
gbrain doctor
|
||||
```
|
||||
|
||||
On the second run, unchanged pages should move through durable skip counters.
|
||||
Edit one page or raw transcript sidecar and rerun; that page should process
|
||||
again and receive a marker with a new token.
|
||||
|
||||
## Maintainer contracts
|
||||
|
||||
- Version completion protocols when their success guarantees change.
|
||||
- Require an exact `source`, page slug, and snapshot-bound `source_session`.
|
||||
- Keep completion and non-extractable as different sources and counters.
|
||||
- Re-fetch after acquiring the lock; never certify the enumeration object.
|
||||
- Revalidate the snapshot before writing either durable outcome.
|
||||
- Keep sidecar content in the version identity.
|
||||
- Keep regular-page content hash and effective date in the version identity.
|
||||
- Never turn model, insertion, cleanup, cancellation, or parser failures into
|
||||
successful empty extraction.
|
||||
- Never classify `no_match` or dry-run output as a durable negative.
|
||||
- Do not make operation checkpoints completion authority.
|
||||
- Apply work limits after durable filtering.
|
||||
- Keep doctor source-scoped by both page and fact `source_id`.
|
||||
- Give terminal completion precedence if both current outcome rows exist.
|
||||
- Update CLI and cycle aggregation whenever a result counter changes.
|
||||
|
||||
## Focused verification
|
||||
|
||||
```bash
|
||||
bun test test/extract-conversation-facts.test.ts
|
||||
bun test test/doctor-conversation-facts-backlog.test.ts
|
||||
bun x tsc --noEmit
|
||||
```
|
||||
|
||||
The focused suite covers checkpoint garbage collection, same-timestamp edits,
|
||||
edits during extraction, sidecar-only edits, legacy marker replay, provider and
|
||||
insert failures, cleanup failure, recognized non-extractable scans, retryable
|
||||
parser misses, post-filter limits, force replay, and doctor accounting.
|
||||
@@ -0,0 +1,240 @@
|
||||
# Conversation parser LLM fallback
|
||||
|
||||
The conversation parser has two stages:
|
||||
|
||||
1. A deterministic registry recognizes known transcript formats.
|
||||
2. An optional LLM fallback parses pages that every built-in pattern rejects.
|
||||
|
||||
The second stage is disabled by default. Enabling it is a privacy decision
|
||||
because unmatched transcript text can be sent to the configured utility-tier
|
||||
model provider.
|
||||
|
||||
## Enable or disable the fallback
|
||||
|
||||
Enable it for the current brain:
|
||||
|
||||
```bash
|
||||
gbrain config set conversation_parser.llm_fallback_enabled true
|
||||
```
|
||||
|
||||
Disable it:
|
||||
|
||||
```bash
|
||||
gbrain config set conversation_parser.llm_fallback_enabled false
|
||||
```
|
||||
|
||||
The key is registered explicitly, so neither command needs `--force`.
|
||||
Values other than the exact string `true` leave the fallback disabled.
|
||||
|
||||
The setting affects conversation fact extraction. It does not make the
|
||||
synchronous `conversation-parser scan` command call a model, and it does not
|
||||
enable the separate LLM polish scaffold.
|
||||
|
||||
## Select the utility model and run a canary
|
||||
|
||||
Inspect the model routing before enabling a production run:
|
||||
|
||||
```bash
|
||||
gbrain models
|
||||
```
|
||||
|
||||
The fallback uses the resolved `utility` tier. Override that tier when the
|
||||
brain should use a different configured provider or model:
|
||||
|
||||
```bash
|
||||
gbrain config set models.tier.utility <provider:model>
|
||||
```
|
||||
|
||||
Start with one known unmatched page and an explicit cost cap:
|
||||
|
||||
```bash
|
||||
gbrain extract-conversation-facts \
|
||||
--source-id <source-id> \
|
||||
--slug <conversation-slug> \
|
||||
--max-cost-usd 1
|
||||
```
|
||||
|
||||
Do not add `--dry-run` to this canary. Dry runs deliberately stop before the
|
||||
fallback boundary, so they cannot prove provider routing or model output.
|
||||
Success emits the per-page fallback log described under
|
||||
[Operator visibility](#operator-visibility). After the canary, remove `--slug`
|
||||
to process the source normally.
|
||||
|
||||
## When the fallback runs
|
||||
|
||||
For each eligible conversation page, extraction:
|
||||
|
||||
1. Reads the same body used by the deterministic parser, including a configured
|
||||
raw transcript sidecar for meeting pages.
|
||||
2. Calls `parseConversation(body, { page })`.
|
||||
3. Uses the deterministic messages when any built-in pattern succeeds.
|
||||
4. Calls the LLM fallback only when the parse phase is exactly `no_match`, the
|
||||
message list is empty, the opt-in key is `true`, and this is not a dry run.
|
||||
5. Splits accepted fallback messages into the normal extraction segments.
|
||||
|
||||
The fallback never replaces, edits, or polishes a successful deterministic
|
||||
parse. Adding a built-in pattern therefore removes model use for that format
|
||||
without changing configuration.
|
||||
|
||||
Dry runs remain local and cost-free. They report deterministic segmentation
|
||||
only and never send unmatched content to a provider.
|
||||
|
||||
## Data sent to the model
|
||||
|
||||
The full unmatched body is processed in overlapping windows of at most 100
|
||||
non-empty lines, with up to 20 lines of preceding context. Blank lines are
|
||||
omitted. Every model request receives:
|
||||
|
||||
- an instruction to treat the transcript as untrusted data;
|
||||
- an authoritative page date when one can be derived;
|
||||
- the sampled transcript inside an explicit chat-log envelope.
|
||||
|
||||
The system prompt tells the model not to follow commands or instructions found
|
||||
inside transcript content. It asks for message extraction only.
|
||||
|
||||
Each window is cached independently. Overlap results with the same normalized
|
||||
speaker and timestamp are deduplicated; when one body contains the other, the
|
||||
longer body wins. This preserves common multi-line messages that straddle a
|
||||
window boundary. If any later window has an ordinary provider or parse failure,
|
||||
the fallback returns no page result and extraction does not advance the
|
||||
checkpoint. Successful earlier windows stay cached for the retry.
|
||||
|
||||
Fallback calls allow up to 8,000 output tokens. Any non-terminal model stop,
|
||||
including length truncation, refusal, content filtering, tool use, or an
|
||||
unrecognized provider stop, is rejected before parsing and caching. A
|
||||
syntactically valid partial JSON array therefore cannot advance a checkpoint.
|
||||
|
||||
The utility model is resolved once per source run through the normal model
|
||||
configuration chain. The default fallback is the utility-tier Anthropic model.
|
||||
|
||||
## Date and timestamp behavior
|
||||
|
||||
The fallback uses the deterministic parser's date precedence:
|
||||
|
||||
1. an explicit caller date;
|
||||
2. `frontmatter.date`;
|
||||
3. the page effective date;
|
||||
4. `1970-01-01` when no date is known.
|
||||
|
||||
A real page date is included in both the prompt and the content-hash cache key.
|
||||
Two pages with identical time-only transcript text but different dates cannot
|
||||
share a cached parse.
|
||||
|
||||
Returned timestamps must be strict RFC3339 date-times with seconds and an
|
||||
explicit `Z` or numeric timezone offset. Calendar fields are validated before
|
||||
parsing. Accepted timestamps are normalized to whole-second UTC form:
|
||||
|
||||
```text
|
||||
YYYY-MM-DDTHH:MM:SSZ
|
||||
```
|
||||
|
||||
Date-only values, timezone-less values, impossible calendar dates, timestamps
|
||||
more than 24 hours in the future, blank speakers, and blank message bodies are
|
||||
discarded. Valid messages are stable-sorted by timestamp before segmentation.
|
||||
Canonical chronological UTC output keeps segment filtering and durable
|
||||
checkpoint comparisons stable and prevents future checkpoint poisoning.
|
||||
|
||||
If no page date is known, the prompt retains the historical epoch fallback.
|
||||
Full timestamps present in the transcript can still be extracted normally.
|
||||
|
||||
## Non-chat and failure behavior
|
||||
|
||||
The model is instructed to return an empty JSON array for non-chat content.
|
||||
An empty response, malformed JSON, unavailable provider, or transport failure
|
||||
leaves the page with no messages. Extraction skips that page and continues.
|
||||
|
||||
The fallback is fail-open with respect to parser availability. It does not turn
|
||||
a model outage into a deterministic-parser outage.
|
||||
|
||||
Cancellation and `BudgetExhausted` are control-flow signals, not provider
|
||||
failures. The extraction caller explicitly propagates them through the
|
||||
fail-open boundary so aborts stay prompt and hard cost caps remain effective.
|
||||
An `AbortError` from a provider timeout still fails open while the caller's own
|
||||
abort signal remains live.
|
||||
|
||||
The gateway can discover an underestimated budget overage only after the final
|
||||
provider result. Extraction checks tracker spend against its cap after the run,
|
||||
so an overage remains visible even when there is no next model reservation.
|
||||
|
||||
## Cache and repeat runs
|
||||
|
||||
Successful fallback results use the shared conversation-parser cache:
|
||||
|
||||
- an in-process map for repeat calls during one process;
|
||||
- the `conversation_parser_llm_cache` table for repeat calls across processes.
|
||||
|
||||
Each chunk's cache key includes the call shape, resolved model, page date
|
||||
metadata, and chunk content hash. A cached response is still validated before
|
||||
it originally enters the cache.
|
||||
|
||||
Once fallback messages produce extractable segments, the ordinary per-page
|
||||
checkpoint advances to the newest segment timestamp. A later run can read the
|
||||
cached parse, apply the checkpoint watermark, and skip already completed
|
||||
segments without another provider call.
|
||||
|
||||
## Operator visibility
|
||||
|
||||
`ExtractConversationFactsResult.pages_llm_fallback` counts pages for which the
|
||||
fallback returned at least one valid message. The command also logs:
|
||||
|
||||
```text
|
||||
[extract-conversation-facts] LLM fallback parsed N message(s) for <slug>
|
||||
```
|
||||
|
||||
The multi-source CLI summary reports the total number of fallback-parsed pages.
|
||||
A zero count means either the fallback was disabled, deterministic patterns
|
||||
handled every page, or fallback attempts returned no valid messages.
|
||||
|
||||
## Maintainer contracts
|
||||
|
||||
Keep these boundaries intact when changing the fallback:
|
||||
|
||||
- Default off. Page text must not reach the fallback without the exact opt-in.
|
||||
- Never call the provider during `--dry-run`.
|
||||
- Deterministic first. Invoke it only for phase `no_match`.
|
||||
- One model resolution per source run, not per page.
|
||||
- Use `deriveDateContext({ page })` so regex and LLM timestamps share metadata.
|
||||
- Put date metadata in the hashed request content to prevent cross-date cache
|
||||
collisions.
|
||||
- Process every non-empty line in bounded cached overlapping windows. Preserve
|
||||
common cross-boundary continuations through overlap and deterministic
|
||||
deduplication. Never checkpoint a partial page after a later window fails or
|
||||
returns a non-terminal stop reason.
|
||||
- Validate and canonicalize all model-produced fields before segmentation.
|
||||
- Stable-sort accepted messages before segmenting or checkpointing them.
|
||||
- Keep the exact config key in `KNOWN_CONFIG_KEYS`. Do not register the whole
|
||||
`conversation_parser.*` namespace while other scaffolded keys remain unwired.
|
||||
- Preserve `[]` and `null` as skip-page outcomes.
|
||||
- Propagate cancellation and budget-stop errors selected by the extraction
|
||||
caller; fail open only for ordinary provider and parse failures.
|
||||
- Never persist inferred regexes or promote model guesses into the built-in
|
||||
registry.
|
||||
|
||||
## Test coverage
|
||||
|
||||
The focused tests cover:
|
||||
|
||||
- default-off behavior with zero fallback calls;
|
||||
- enabled dry-run behavior with zero provider calls;
|
||||
- exact config-key registration;
|
||||
- a successful production-path fallback;
|
||||
- page-date prompt and cache-key separation;
|
||||
- durable checkpoint advancement and cache reuse;
|
||||
- complete processing beyond the first 100 non-empty lines;
|
||||
- cross-boundary continuation preservation and overlap deduplication;
|
||||
- rejection of truncated, refused, and content-filtered model results;
|
||||
- all-or-nothing page results when a later chunk fails;
|
||||
- non-chat empty arrays and malformed output;
|
||||
- strict timestamp normalization, ordering, and invalid-item filtering;
|
||||
- provider-unavailable and transport-failure behavior;
|
||||
- provider-timeout versus caller-cancellation behavior;
|
||||
- thrown and post-record budget-stop reporting.
|
||||
|
||||
Run the focused surface with:
|
||||
|
||||
```bash
|
||||
bun test test/conversation-parser/llm-base.test.ts \
|
||||
test/conversation-parser/llm-fallback.test.ts \
|
||||
test/extract-conversation-facts.test.ts \
|
||||
test/config-set.test.ts
|
||||
```
|
||||
@@ -49,6 +49,7 @@ The USD-limit knobs accept `off`, `unlimited`, or `none` (case-insensitive) to m
|
||||
| Backfill per-job budget | `embed.backfill_max_usd` | `10` | caps the job's tracker | `off` (`0` → default) | uncapped (still ledgered) |
|
||||
| Backfill cooldown | `embed.backfill_cooldown_min` | `10` | skips re-submission inside window | — (latency knob, not spend) | **not** bypassed |
|
||||
| `reindex-code` cost gate | — (preview before re-embed) | — | TTY prompt / non-TTY refuse + exit 2 | `--max-cost off` | informational |
|
||||
| `migrate embeddings` consent gate | — (plan + estimate before provider migration) | — | TTY y/N prompt / non-TTY refuse + exit 2 | `--yes` | estimate marked informational, but **still prompts** (guards a destructive schema rebuild, not just spend) |
|
||||
| `enrich` / `onboard --auto` | `--max-usd` (per-call) | — | refuse without a cap (non-TTY) | `--max-usd off` | runs uncapped (still ledgered) |
|
||||
|
||||
### Sync inline-embed cost gate
|
||||
|
||||
@@ -140,6 +140,9 @@ Stable phase names shipped in v0.15.2:
|
||||
- `import.files`
|
||||
- `sync.deletes`, `sync.renames`, `sync.imports`
|
||||
- `migrate.copy_pages`, `migrate.copy_links`
|
||||
- `migrate.reembed` (the re-embed pass of `gbrain migrate embeddings`; total is the
|
||||
stale-chunk backlog at the start of the pass, so it can grow slightly if a
|
||||
writer adds chunks mid-run)
|
||||
- `repair_jsonb.run`, `repair_jsonb.<table>.<column>`
|
||||
- `backlinks.scan`
|
||||
- `lint.pages`
|
||||
|
||||
@@ -91,3 +91,18 @@ First full takes extraction run on a ~100K-page brain:
|
||||
4. **Self-reported ≠ verified.** "Reports 7 figures" → holder=person, weight=0.75, NOT world/1.0
|
||||
5. **No false precision.** Use 0.05 increments (0.35, 0.55, 0.75), not 0.74 or 0.82
|
||||
6. **"So what" test.** Skip Twitter handles, follower counts, obvious metadata
|
||||
|
||||
## Owner-holder canonicalization
|
||||
|
||||
"The brain owner" is, by convention, the holder string **`self`** — the value the
|
||||
dream `consolidate` phase stamps when it promotes the owner's hot facts into cold
|
||||
takes. Calibration, `think`, and the `doctor` calibration check resolve the owner
|
||||
holder through `resolveOwnerHolder` (`src/core/owner-holder.ts`): explicit override
|
||||
> `emotional_weight.user_holder` config > `self`.
|
||||
|
||||
Known limitation (tracked in garrytan/gbrain#2465): the owner can also
|
||||
appear under `brain` (a take the owner asserts, via `propose_takes`) and
|
||||
`people/<owner>` (extraction that names the owner). The resolver selects the
|
||||
*default* canonical owner string for reads; it does not merge those other
|
||||
strings. Per-take attribution for other people (e.g. `people/george`) is
|
||||
unaffected and correct.
|
||||
|
||||
@@ -484,6 +484,10 @@ Returns a per-source dashboard: when each source last synced, how many pages, ho
|
||||
|
||||
The admin dashboard at `https://brain.acme-co.com/admin` shows live request volume, registered OAuth clients, recent activity, and brain stats. Use the admin bootstrap token from Part 4 to log in the first time, then register additional admin users from inside the dashboard.
|
||||
|
||||
### If agents run as containers on the same Docker host
|
||||
|
||||
OAuth source scoping only guards the HTTP MCP path. If the brain's Postgres and your teammates' agent runtimes are containers on the same Docker host, make sure the agents can't reach Postgres directly over Docker's default bridge network — a direct DB session skips OAuth entirely. Put Postgres on its own user-defined network, publish it loopback-only if at all, and never hand agent containers a `DATABASE_URL`. The copy-paste operator checklist lives in [docs/mcp/DEPLOY.md — Co-located Docker workloads](../mcp/DEPLOY.md#co-located-docker-workloads-self-hosted-postgres).
|
||||
|
||||
---
|
||||
|
||||
## Part 13: Cost and speed expectations
|
||||
|
||||
+45
-5
@@ -2720,14 +2720,17 @@ GBrain is tuned for the Supabase **Transaction pooler** (port 6543): it
|
||||
auto-disables prepared statements there and routes `engine.transaction()`
|
||||
(migrations, DDL, sync imports) to a derived **direct** connection
|
||||
(`db.<ref>.supabase.co:5432`). That direct host is IPv6-only, so on an
|
||||
IPv4-only host, reads work but sync **silently skips most pages**. This is the
|
||||
number one cause of "sync ran but nothing happened."
|
||||
IPv4-only host it is unreachable. When that happens gbrain now falls back to
|
||||
the pooler automatically (one stderr warning, then single-pool mode for the
|
||||
rest of the process) — but the pooler's ~2-min statement timeout can truncate
|
||||
very long migrations or bulk imports.
|
||||
|
||||
Fix: make the direct connection reachable over IPv4. Either set
|
||||
`GBRAIN_DIRECT_DATABASE_URL` to the **Session pooler** string (port 5432 on the
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on. Verify by
|
||||
running `gbrain sync` and checking that the page count in `gbrain stats` matches
|
||||
the syncable file count in the repo.
|
||||
`pooler.supabase.com` host, IPv4), or enable Supabase's IPv4 add-on.
|
||||
`GBRAIN_DISABLE_DIRECT_POOL=1` skips the direct pool (and the fallback warning)
|
||||
entirely. Verify by running `gbrain sync` and checking that the page count in
|
||||
`gbrain stats` matches the syncable file count in the repo.
|
||||
|
||||
### The Primitives
|
||||
|
||||
@@ -3902,6 +3905,43 @@ the user owns the machine.
|
||||
See [ALTERNATIVES.md](ALTERNATIVES.md) for a comparison of ngrok, Tailscale
|
||||
Funnel, and cloud hosts (Fly.io, Railway).
|
||||
|
||||
### Co-located Docker workloads (self-hosted Postgres)
|
||||
|
||||
OAuth scopes and source scoping guard the `gbrain serve --http` path. They do
|
||||
NOT guard raw Postgres. If the brain's Postgres runs as a container on the same
|
||||
Docker host as other workloads (agent runtimes, n8n, staging fixtures), any
|
||||
container sharing Docker's default `bridge` network can open a direct DB
|
||||
session — no OAuth token required — and read every source. That silently
|
||||
recreates a privileged path underneath the isolation you configured at the MCP
|
||||
layer.
|
||||
|
||||
Network-zone the host so untrusted containers can never reach Postgres:
|
||||
|
||||
```
|
||||
Docker host
|
||||
├── gbrain-net ← ONLY the brain's Postgres (+ gbrain serve, if containerized)
|
||||
├── agent-<id>-net ← each untrusted agent runtime, isolated
|
||||
└── default bridge ← no secret-bearing databases
|
||||
```
|
||||
|
||||
Operator checklist:
|
||||
|
||||
```text
|
||||
[ ] Postgres is on a user-defined Docker network, not the default bridge
|
||||
(or nothing else runs on that bridge)
|
||||
[ ] If Postgres publishes a host port at all, it binds loopback only
|
||||
(`-p 127.0.0.1:5432:5432`, never `0.0.0.0`)
|
||||
[ ] Untrusted agent containers have no DATABASE_URL or Postgres password
|
||||
[ ] Untrusted agents reach the brain via OAuth/Bearer against serve --http only
|
||||
(host loopback via host.docker.internal / host gateway — never gbrain-net)
|
||||
[ ] OAuth clients are least-privilege: scoped --source / --federated-read,
|
||||
pre-minted short-lived tokens preferred over long-lived client secrets
|
||||
[ ] Isolation verified: a team-scoped client cannot read internal-only sources
|
||||
```
|
||||
|
||||
Optional defense-in-depth: a dedicated Postgres role (or RLS) limited to the
|
||||
allowed `source_id`s, so even a leaked connection string can't read everything.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"missing_auth" error**
|
||||
|
||||
+5
-3
@@ -23,6 +23,7 @@
|
||||
"./backoff": "./src/core/backoff.ts",
|
||||
"./search/hybrid": "./src/core/search/hybrid.ts",
|
||||
"./search/expansion": "./src/core/search/expansion.ts",
|
||||
"./think": "./src/core/think/index.ts",
|
||||
"./ai/gateway": "./src/core/ai/gateway.ts",
|
||||
"./extract": "./src/commands/extract.ts",
|
||||
"./ingestion": "./src/core/ingestion/index.ts",
|
||||
@@ -144,10 +145,11 @@
|
||||
"bun": ">=1.3.10"
|
||||
},
|
||||
"license": "MIT",
|
||||
"version": "0.42.64.0",
|
||||
"version": "0.42.66.1",
|
||||
"overrides": {
|
||||
"@hono/node-server": "^1.19.13",
|
||||
"fast-uri": "^3.1.2",
|
||||
"@hono/node-server": "^2.0.5",
|
||||
"fast-uri": "^3.1.4",
|
||||
"body-parser": "^2.3.0",
|
||||
"fast-xml-builder": "^1.1.7",
|
||||
"fast-xml-parser": "^5.7.0",
|
||||
"form-data": "^4.0.6",
|
||||
|
||||
+13
-8
@@ -1,7 +1,7 @@
|
||||
---
|
||||
id: x-to-brain
|
||||
name: X-to-Brain
|
||||
version: 0.8.1
|
||||
version: 0.8.2
|
||||
description: Twitter timeline, mentions, and keyword monitoring flow into brain pages. Tracks deletions, engagement velocity, OCR on images, and real-time alerts.
|
||||
category: sense
|
||||
requires: []
|
||||
@@ -9,9 +9,12 @@ secrets:
|
||||
- name: X_BEARER_TOKEN
|
||||
description: X API v2 Bearer token (Basic tier minimum, $200/mo for full archive search)
|
||||
where: https://developer.x.com/en/portal/dashboard — create a project + app, copy the Bearer Token from "Keys and tokens"
|
||||
- name: X_HANDLE
|
||||
description: Your X username without the @ (used for the app-only health check — /users/me requires user-context OAuth, which app-only bearer tokens don't have)
|
||||
where: Your X profile — the handle in your profile URL, e.g. x.com/yourhandle → yourhandle
|
||||
health_checks:
|
||||
- type: http
|
||||
url: "https://api.x.com/2/users/me"
|
||||
url: "https://api.x.com/2/users/by/username/$X_HANDLE"
|
||||
auth: bearer
|
||||
auth_token: "$X_BEARER_TOKEN"
|
||||
label: "X API"
|
||||
@@ -110,15 +113,17 @@ Tell the user:
|
||||
4. Inside the project, create a new App
|
||||
5. Go to the app's 'Keys and tokens' tab
|
||||
6. Under 'Bearer Token', click 'Generate' (or 'Regenerate')
|
||||
7. Copy the Bearer Token and paste it to me
|
||||
7. Copy the Bearer Token and paste it to me, along with your X handle (without the @)
|
||||
|
||||
Note: Free tier gives read-only access with low limits. Basic tier ($200/mo)
|
||||
gives search/recent endpoint and higher limits. Pro tier gets full archive search."
|
||||
|
||||
Validate immediately:
|
||||
Set both `X_BEARER_TOKEN` and `X_HANDLE` in the environment. Validate immediately
|
||||
(app-only bearer tokens cannot call `/users/me` — that endpoint requires
|
||||
user-context OAuth — so validation uses the by-username lookup):
|
||||
```bash
|
||||
curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \
|
||||
"https://api.x.com/2/users/me" \
|
||||
"https://api.x.com/2/users/by/username/$X_HANDLE" \
|
||||
&& echo "PASS: X API connected" \
|
||||
|| echo "FAIL: X API token invalid"
|
||||
```
|
||||
@@ -134,10 +139,10 @@ starting with 'AAA...', (3) if you just created the app, the token is valid imme
|
||||
```bash
|
||||
# Look up the user's X user ID from their handle
|
||||
curl -sf -H "Authorization: Bearer $X_BEARER_TOKEN" \
|
||||
"https://api.x.com/2/users/by/username/USERNAME" | grep -o '"id":"[^"]*"'
|
||||
"https://api.x.com/2/users/by/username/$X_HANDLE" | grep -o '"id":"[^"]*"'
|
||||
```
|
||||
|
||||
Ask the user for their X handle (e.g., @yourhandle). Look up their user ID.
|
||||
Look up the user ID from the handle collected in Step 1.
|
||||
Save it — the collector needs the numeric ID, not the handle.
|
||||
|
||||
### Step 3: Configure the Collector
|
||||
@@ -205,7 +210,7 @@ The agent should review collected data 2-3x daily and run enrichment.
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.gbrain/integrations/x-to-brain
|
||||
echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.1","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl
|
||||
echo '{"ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","event":"setup_complete","source_version":"0.8.2","status":"ok","details":{"user_id":"X_USER_ID"}}' >> ~/.gbrain/integrations/x-to-brain/heartbeat.jsonl
|
||||
```
|
||||
|
||||
## Production Patterns (v0.8.1)
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
EXPECTED_COUNT=20
|
||||
EXPECTED_COUNT=21
|
||||
|
||||
# Count top-level keys in the exports object. `node -e` parses JSON
|
||||
# reliably without needing jq (which isn't in every CI environment).
|
||||
|
||||
@@ -70,7 +70,7 @@ PATTERN='import[[:space:]]+(\*[[:space:]]+as[[:space:]]+[a-zA-Z_$][a-zA-Z0-9_$]*
|
||||
FOUND_FILES=""
|
||||
while IFS= read -r f; do
|
||||
[ -n "$f" ] && FOUND_FILES="$FOUND_FILES$f"$'\n'
|
||||
done < <(grep -rlE --include='*.ts' "$PATTERN" src/ 2>/dev/null | sort -u || true)
|
||||
done < <(grep -rlE --include='*.ts' "$PATTERN" src 2>/dev/null | sort -u || true)
|
||||
|
||||
FAIL=0
|
||||
|
||||
|
||||
@@ -100,9 +100,9 @@ IFS='|' eval 'PATTERN="${PATTERN_PARTS[*]}"'
|
||||
|
||||
# Find tool.
|
||||
if command -v rg >/dev/null 2>&1; then
|
||||
matches="$(rg -niH --no-heading -t ts "$PATTERN" test/ 2>/dev/null || true)"
|
||||
matches="$(rg -niH --no-heading -t ts "$PATTERN" test 2>/dev/null || true)"
|
||||
elif command -v grep >/dev/null 2>&1; then
|
||||
matches="$(grep -rniE --include='*.test.ts' "$PATTERN" test/ 2>/dev/null || true)"
|
||||
matches="$(grep -rniE --include='*.test.ts' "$PATTERN" test 2>/dev/null || true)"
|
||||
else
|
||||
echo "check-test-real-names: ERROR: neither rg nor grep available." >&2
|
||||
exit 2
|
||||
|
||||
@@ -19,13 +19,25 @@ set -euo pipefail
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
OUT_BIN="$(mktemp /tmp/gbrain-wasm-check.XXXXXX)"
|
||||
trap 'rm -f "$OUT_BIN"' EXIT
|
||||
# Build from a container-local copy. On Docker Desktop, Bun canonicalizes a
|
||||
# bind-mounted input to /run/host_virtiofs but keeps /app as the output path;
|
||||
# its final atomic rename then fails with ENOENT even though both names refer
|
||||
# to the same mount. Keeping inputs and output under /tmp avoids that alias.
|
||||
BUILD_DIR="$(mktemp -d /tmp/gbrain-wasm-check.XXXXXX)"
|
||||
OUT_BIN="$BUILD_DIR/chunker-smoketest"
|
||||
trap 'rm -rf "$BUILD_DIR"' EXIT
|
||||
mkdir -p "$BUILD_DIR/scripts"
|
||||
cp -R "$REPO_ROOT/src" "$BUILD_DIR/src"
|
||||
cp "$REPO_ROOT/scripts/chunker-smoketest.ts" "$BUILD_DIR/scripts/chunker-smoketest.ts"
|
||||
ln -s "$REPO_ROOT/node_modules" "$BUILD_DIR/node_modules"
|
||||
|
||||
# Build a minimal smoketest binary that imports the chunker. We compile this
|
||||
# instead of the full gbrain CLI so the failure mode is laser-focused on
|
||||
# chunker + WASM path resolution, not unrelated CLI wiring.
|
||||
bun build --compile --outfile "$OUT_BIN" scripts/chunker-smoketest.ts >/dev/null 2>&1
|
||||
if ! (cd "$BUILD_DIR" && bun build --compile --outfile "$OUT_BIN" scripts/chunker-smoketest.ts >/dev/null); then
|
||||
echo "[check-wasm-embedded] FAIL: bun could not compile the smoketest binary." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Run it and capture JSON output.
|
||||
OUTPUT="$("$OUT_BIN" 2>&1)"
|
||||
|
||||
+1
-1
@@ -350,7 +350,7 @@ if [ -f .git ]; then
|
||||
fi
|
||||
|
||||
echo "[ci-local] Running checks inside runner container..."
|
||||
docker compose -f "$COMPOSE_FILE" run --rm "${EXTRA_MOUNTS[@]:-}" runner bash -c "$INNER_CMD"
|
||||
docker compose -f "$COMPOSE_FILE" run --rm "${EXTRA_MOUNTS[@]}" runner bash -c "$INNER_CMD"
|
||||
|
||||
echo ""
|
||||
echo "[ci-local] All checks passed."
|
||||
|
||||
+15
-2
@@ -42,8 +42,19 @@ export const E2E_TEST_MAP: Record<string, string[]> = {
|
||||
// phase, extract, integrity, embed, or migrate-engine change.
|
||||
"src/core/cycle/extract-takes.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/core/cycle/patterns.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/core/cycle/synthesize.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/commands/embed.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/core/cycle/synthesize.ts": [
|
||||
"test/e2e/multi-source-bug-class.test.ts",
|
||||
"test/e2e/synthesize-bigint-job-id-postgres.test.ts",
|
||||
],
|
||||
"src/commands/embed.ts": [
|
||||
"test/e2e/multi-source-bug-class.test.ts",
|
||||
// #3391: the NULL-signature stale predicates differ per engine.
|
||||
"test/e2e/migrate-embeddings-postgres.test.ts",
|
||||
],
|
||||
// #3390: runSchemaTransition's DDL path + the stale predicates behave
|
||||
// differently on real pgvector than on PGLite.
|
||||
"src/core/embedding-migration.ts": ["test/e2e/migrate-embeddings-postgres.test.ts"],
|
||||
"src/core/retrieval-upgrade-planner.ts": ["test/e2e/migrate-embeddings-postgres.test.ts"],
|
||||
"src/commands/extract.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
"src/commands/migrate-engine.ts": ["test/e2e/multi-source-bug-class.test.ts"],
|
||||
// Any minions queue/worker/handler change exercises all minion E2E.
|
||||
@@ -61,6 +72,8 @@ export const E2E_TEST_MAP: Record<string, string[]> = {
|
||||
"test/e2e/jsonb-roundtrip.test.ts",
|
||||
"test/e2e/engine-parity.test.ts",
|
||||
"test/e2e/schema-drift.test.ts",
|
||||
// #3391: includeNullSignature stale predicates (engine parity).
|
||||
"test/e2e/migrate-embeddings-postgres.test.ts",
|
||||
],
|
||||
// PGLite bootstrap path + parity guard.
|
||||
"src/core/pglite-engine.ts": [
|
||||
|
||||
@@ -133,6 +133,7 @@ for i in $(seq 1 "$N"); do
|
||||
env SHARD="$i/$N" \
|
||||
bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \
|
||||
> "$SHARD_LOG" 2>&1
|
||||
rc=$?
|
||||
else
|
||||
env SHARD="$i/$N" \
|
||||
bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \
|
||||
@@ -142,10 +143,20 @@ for i in $(seq 1 "$N"); do
|
||||
sleep 5 && kill -KILL "$pid" 2>/dev/null ) &
|
||||
cap_pid=$!
|
||||
wait "$pid" 2>/dev/null
|
||||
# Capture the shard's exit code from ITS `wait`, before any watchdog
|
||||
# teardown runs. The teardown commands below overwrite $? — the killed
|
||||
# watchdog reports 143 — which used to get stamped into every shard's
|
||||
# sentinel on machines with no gtimeout/timeout: every run "failed"
|
||||
# with rc=143 summaries even when all tests passed.
|
||||
rc=$?
|
||||
# Reap the watchdog's `sleep` child too (pkill -P), then the watchdog.
|
||||
# Killing only the subshell leaves the sleep orphaned until
|
||||
# $SHARD_TIMEOUT elapses — same quirk the heartbeat cleanup below works
|
||||
# around; CI's orphan-process sweep flags those.
|
||||
pkill -P "$cap_pid" 2>/dev/null
|
||||
kill "$cap_pid" 2>/dev/null
|
||||
wait "$cap_pid" 2>/dev/null
|
||||
fi
|
||||
rc=$?
|
||||
echo "$rc" > "$LOG_DIR/shard-$i.exit"
|
||||
[ "$rc" = "124" ] && echo "WEDGED" > "$LOG_DIR/shard-$i.wedged"
|
||||
) &
|
||||
|
||||
@@ -126,6 +126,7 @@ for c in "${CHECKS[@]}"; do
|
||||
(
|
||||
if [ -n "$TIMEOUT_BIN" ]; then
|
||||
"$TIMEOUT_BIN" "${TIMEOUT}s" bun run "$c" > "$LOG_FILE" 2>&1
|
||||
rc=$?
|
||||
else
|
||||
bun run "$c" > "$LOG_FILE" 2>&1 &
|
||||
pid=$!
|
||||
@@ -133,10 +134,20 @@ for c in "${CHECKS[@]}"; do
|
||||
sleep 5 && kill -KILL "$pid" 2>/dev/null ) &
|
||||
cap_pid=$!
|
||||
wait "$pid" 2>/dev/null
|
||||
# Capture the check's exit code from ITS `wait`, before any watchdog
|
||||
# teardown runs. The teardown commands below overwrite $? — the killed
|
||||
# watchdog reports 143 — which used to get stamped into every sentinel
|
||||
# on machines with no gtimeout/timeout: verify reported pass=0
|
||||
# fail=<all> while every per-check log said OK.
|
||||
rc=$?
|
||||
# Reap the watchdog's `sleep` child too (pkill -P), then the watchdog.
|
||||
# Killing only the subshell leaves the sleep orphaned until $TIMEOUT
|
||||
# elapses — same quirk the heartbeat cleanup in run-unit-parallel.sh
|
||||
# works around; CI's orphan-process sweep flags those.
|
||||
pkill -P "$cap_pid" 2>/dev/null
|
||||
kill "$cap_pid" 2>/dev/null
|
||||
wait "$cap_pid" 2>/dev/null
|
||||
fi
|
||||
rc=$?
|
||||
echo "$rc" > "$EXIT_FILE"
|
||||
) &
|
||||
PIDS+=($!)
|
||||
|
||||
@@ -62,7 +62,7 @@ gbrain capture "..." --json # structured output for agents
|
||||
- **Slug:** `inbox/YYYY-MM-DD-<hash8>` (stable for same content; the daemon's 24h dedup catches re-captures).
|
||||
- **Type:** `note` (override with `--type idea` etc.).
|
||||
- **Frontmatter stamps:** `captured_via: capture-cli`, `captured_at: <ISO>`.
|
||||
- **Title:** first non-empty line of the body, capped at 80 chars.
|
||||
- **Title:** first non-empty line of the body, capped at 80 chars (truncation appends `…`).
|
||||
|
||||
## Output Format
|
||||
|
||||
|
||||
@@ -60,7 +60,14 @@ Before skillifying, check:
|
||||
- Is there >20 lines of logic? (Trivial helpers don't need full infrastructure)
|
||||
- Does it have a clear trigger phrase a user would actually say?
|
||||
|
||||
If no to all three, it's a script, not a skill. Move on.
|
||||
If ANY answer is no, it's a script, not a skill — stop here. Do not scaffold, write a SKILL.md, run evals, or write tests for it. Tell the user why and move on.
|
||||
|
||||
Scope check (upper bound): one skill = one capability = one coherent trigger
|
||||
family. If the target spans multiple distinct intents users would invoke
|
||||
separately ("run the build" / "roll back the deploy" / "notify the team" are
|
||||
three intents, not one), do NOT build one skill covering them all. Stop,
|
||||
propose splitting into separate skillify targets, and ask the user which one
|
||||
to skillify first.
|
||||
|
||||
## Phase 1: Audit
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
// AUTO-GENERATED — do not edit by hand.
|
||||
// Run `bun run scripts/build-admin-embedded.ts` to regenerate.
|
||||
// Source: admin/dist/ at 2026-05-27.
|
||||
// Source: admin/dist/ at 2026-07-24.
|
||||
//
|
||||
// Bun resolves the file: imports to a path that works at runtime even
|
||||
// inside a compiled binary (`bun build --compile`). The manifest maps
|
||||
// the request path the express handler sees to (resolved-path, mime).
|
||||
|
||||
// @ts-ignore — type: 'file' is Bun ESM, not in lib.d.ts
|
||||
import A_0_assets_index_CoGEje3__js from '../admin/dist/assets/index-CoGEje3-.js' with { type: 'file' };
|
||||
import A_0_assets_index_CviJXT_1_js from '../admin/dist/assets/index-CviJXT-1.js' with { type: 'file' };
|
||||
// @ts-ignore — type: 'file' is Bun ESM, not in lib.d.ts
|
||||
import A_1_assets_index_GxkWX7v3_css from '../admin/dist/assets/index-GxkWX7v3.css' with { type: 'file' };
|
||||
// @ts-ignore — type: 'file' is Bun ESM, not in lib.d.ts
|
||||
@@ -19,7 +19,7 @@ export interface AdminAsset {
|
||||
}
|
||||
|
||||
export const ADMIN_ASSETS: Record<string, AdminAsset> = {
|
||||
"/admin/assets/index-CoGEje3-.js": { path: A_0_assets_index_CoGEje3__js as unknown as string, mime: "application/javascript; charset=utf-8" },
|
||||
"/admin/assets/index-CviJXT-1.js": { path: A_0_assets_index_CviJXT_1_js as unknown as string, mime: "application/javascript; charset=utf-8" },
|
||||
"/admin/assets/index-GxkWX7v3.css": { path: A_1_assets_index_GxkWX7v3_css as unknown as string, mime: "text/css; charset=utf-8" },
|
||||
"/admin/index.html": { path: A_2_index_html as unknown as string, mime: "text/html; charset=utf-8" },
|
||||
};
|
||||
|
||||
+119
-48
@@ -24,6 +24,7 @@ import type { GBrainConfig } from './core/config.ts';
|
||||
import type { AIGatewayConfig } from './core/ai/types.ts';
|
||||
import type { BrainEngine } from './core/engine.ts';
|
||||
import { operations, OperationError } from './core/operations.ts';
|
||||
import { resolveSourceIdEngineFree } from './core/source-resolver.ts';
|
||||
import { formatVolunteeredPage } from './core/context/volunteer.ts';
|
||||
import type { Operation, OperationContext } from './core/operations.ts';
|
||||
import { shouldForceExitAfterMain, finishCliTeardown, flushThenExit, currentExitCode, setCliExitVerdict } from './core/cli-force-exit.ts';
|
||||
@@ -54,7 +55,7 @@ export function bigintToStringReplacer(_key: string, value: unknown): unknown {
|
||||
}
|
||||
|
||||
// CLI-only commands that bypass the operation layer
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'maintain', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'reconcile-links', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'backfill', 'cache', 'ze-switch', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector']);
|
||||
export const CLI_ONLY = new Set(['init', 'reinit-pglite', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'extract-conversation-facts', 'enrich', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'maintain', 'sources', 'mounts', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'providers', 'storage', 'repos', 'code-def', 'code-refs', 'reindex', 'reindex-code', 'reindex-frontmatter', 'code-callers', 'code-callees', 'reconcile-links', 'frontmatter', 'auth', 'friction', 'claw-test', 'book-mirror', 'takes', 'think', 'salience', 'anomalies', 'calibration', 'transcripts', 'models', 'remote', 'recall', 'forget', 'edges-backfill', 'cache', 'ze-switch', 'retrieval-upgrade', 'founder', 'brainstorm', 'lsd', 'schema', 'capture', 'onboard', 'conversation-parser', 'status', 'connect', 'skillopt', 'quarantine', 'self-upgrade', 'advisor', 'watch', 'reindex-search-vector']);
|
||||
// CLI-only commands whose handlers print their own --help text. These are
|
||||
// excluded from the generic short-circuit so detailed per-command and
|
||||
// per-subcommand usage stays reachable.
|
||||
@@ -106,11 +107,10 @@ const CLI_ONLY_SELF_HELP = new Set([
|
||||
// `gbrain connect --help` prints its own usage (flags + examples) from
|
||||
// runConnect; route around the generic one-line short-circuit.
|
||||
'connect',
|
||||
// #3224 — `backfill` was missing from CLI_ONLY entirely (dispatch never
|
||||
// reached runBackfillCommand). Once added there, the generic short-circuit
|
||||
// still shadowed its own printHelp() (kind list + flags, same WARN-5 class
|
||||
// as capture); route around it so `gbrain backfill --help` reaches it.
|
||||
'backfill',
|
||||
// #3390 — `gbrain migrate embeddings --help` / `gbrain retrieval-upgrade
|
||||
// --help` print the migration flags from runMigrateEmbeddings. `migrate`
|
||||
// (engine transfer) keeps its own dispatch too.
|
||||
'migrate', 'retrieval-upgrade',
|
||||
]);
|
||||
|
||||
// v114 (#1941): alias -> operation lookup, kept separate from `cliOps` so
|
||||
@@ -389,6 +389,15 @@ async function main() {
|
||||
if (op.localOnly) {
|
||||
refuseThinClient(command, cfgPre!.remote_mcp!.mcp_url);
|
||||
}
|
||||
// #2098: the local path resolves --source / GBRAIN_SOURCE / .gbrain-source
|
||||
// inside makeContext (ctx.sourceId), which this route never reaches — so
|
||||
// scope must be mapped onto the op's source_id wire param before the call.
|
||||
try {
|
||||
applyThinClientSourceScope(op, params);
|
||||
} catch (e: unknown) {
|
||||
console.error(e instanceof Error ? e.message : String(e));
|
||||
process.exit(1);
|
||||
}
|
||||
await runThinClientRouted(op, params, cfgPre!, cliOpts);
|
||||
return;
|
||||
}
|
||||
@@ -809,18 +818,80 @@ export function parseOpArgs(op: Operation, args: string[]): Record<string, unkno
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* #2098: thin-client source scoping. Locally, --source / GBRAIN_SOURCE /
|
||||
* .gbrain-source resolve to ctx.sourceId in makeContext; the thin-client
|
||||
* route short-circuits before that, so `gbrain query --source X` against a
|
||||
* remote brain silently searched unscoped. This runs the engine-free tiers
|
||||
* (flag → env → dotfile; the DB-backed tiers can't run without an engine —
|
||||
* the server's grant scoping covers the rest) and maps the result onto the
|
||||
* op's `source_id` wire param.
|
||||
*
|
||||
* Ops that declare their OWN `source` param (facts add, etc.) are left
|
||||
* untouched — their --source is an op param, not scope. An explicit --source
|
||||
* on an op with no source_id wire param throws (loud beats silent drop);
|
||||
* ambient env/dotfile scope with nowhere to send it is ignored, matching the
|
||||
* pre-fix behavior for non-scopeable ops. Exported for tests.
|
||||
*/
|
||||
// Ops whose `source_id` wire param is NOT read-scope semantics: get_skill's
|
||||
// source_id flips the lookup from host catalog to brain-resident-pack
|
||||
// (getResidentSkillDetail). Ambient env/dotfile scope must never leak into
|
||||
// these; an explicit --source-id still passes through untouched above.
|
||||
const NON_SCOPE_SOURCE_ID_OPS = new Set(['get_skill']);
|
||||
|
||||
export function applyThinClientSourceScope(
|
||||
op: Operation,
|
||||
params: Record<string, unknown>,
|
||||
cwd?: string,
|
||||
): void {
|
||||
if ('source' in op.params) return; // the op owns --source; not a scope flag
|
||||
const explicit = typeof params.source === 'string' && params.source.length > 0
|
||||
? (params.source as string)
|
||||
: null;
|
||||
delete params.source; // never a wire param on these ops — don't leak it
|
||||
// Explicit per-call scope already on the wire wins over ambient tiers.
|
||||
if (params.source_id !== undefined || params.all_sources === true) {
|
||||
if (explicit) {
|
||||
throw new Error('Pass either --source or --source-id/--all-sources, not both.');
|
||||
}
|
||||
return;
|
||||
}
|
||||
const resolved = resolveSourceIdEngineFree(explicit, cwd);
|
||||
if (!resolved) return;
|
||||
if (!('source_id' in op.params) || NON_SCOPE_SOURCE_ID_OPS.has(op.name)) {
|
||||
if (explicit) {
|
||||
const hint = NON_SCOPE_SOURCE_ID_OPS.has(op.name)
|
||||
? `(its source_id parameter is not a scope filter; pass --source-id explicitly if you mean it)`
|
||||
: `(the remote op has no source_id parameter; the server scopes it to your grant)`;
|
||||
throw new Error(
|
||||
`gbrain ${op.cliHints?.name || op.name} does not accept --source on a thin-client install ${hint}.`,
|
||||
);
|
||||
}
|
||||
return; // ambient env/dotfile scope with nowhere to send it
|
||||
}
|
||||
params.source_id = resolved;
|
||||
}
|
||||
|
||||
async function makeContext(engine: BrainEngine, params: Record<string, unknown>): Promise<OperationContext> {
|
||||
// v0.31.8 (D11): resolve sourceId via the canonical 6-tier chain. Honors
|
||||
// --source / GBRAIN_SOURCE / .gbrain-source / path-match / brain default /
|
||||
// 'default'. Wrapped in try/catch so a doctor / single-source brain that
|
||||
// never set up sources still returns 'default' silently.
|
||||
let sourceId: string | undefined;
|
||||
// #2561: when the source resolved via a NON-explicit tier (path-match /
|
||||
// brain default / sole-non-default / seed default), unqualified search-shaped
|
||||
// reads span every `config.federated = true` source. Computed here (the
|
||||
// trusted local boundary) and consumed by federatedSearchScope in
|
||||
// operations.ts, which additionally gates on ctx.remote === false.
|
||||
let localFederated: string[] | undefined;
|
||||
try {
|
||||
const { resolveSourceId } = await import('./core/source-resolver.ts');
|
||||
const { resolveSourceWithTier, localFederatedSourceIds } = await import('./core/source-resolver.ts');
|
||||
// params.source is set when a CLI flag was parsed for the op (rare; most
|
||||
// CLI ops don't take --source). Falls through to env/dotfile/path-match.
|
||||
const explicit = (params.source as string | undefined) ?? null;
|
||||
sourceId = await resolveSourceId(engine, explicit);
|
||||
const resolved = await resolveSourceWithTier(engine, explicit);
|
||||
sourceId = resolved.source_id;
|
||||
localFederated = await localFederatedSourceIds(engine, resolved.source_id, resolved.tier);
|
||||
} catch {
|
||||
// Source resolution failed (e.g. sources table doesn't exist on a fresh
|
||||
// pre-init brain). Leave sourceId unset; engine read methods fall through
|
||||
@@ -841,6 +912,7 @@ async function makeContext(engine: BrainEngine, params: Record<string, unknown>)
|
||||
// table). Matches dispatch.ts's auto-fill so the contract holds across
|
||||
// every transport.
|
||||
sourceId: sourceId ?? 'default',
|
||||
...(localFederated ? { localFederatedSourceIds: localFederated } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -942,7 +1014,10 @@ export function formatResult(opName: string, result: unknown): string {
|
||||
lines.push(`Link coverage (entities): ${(h.link_coverage * 100).toFixed(1)}%`);
|
||||
}
|
||||
if (h.timeline_coverage !== undefined) {
|
||||
lines.push(`Timeline coverage (entities): ${(h.timeline_coverage * 100).toFixed(1)}%`);
|
||||
lines.push(`Timeline coverage (entity pages): ${(h.timeline_coverage * 100).toFixed(1)}%`);
|
||||
}
|
||||
if (h.timeline_coverage_score !== undefined) {
|
||||
lines.push(`Timeline density (all pages): ${h.timeline_coverage_score}/15 (whole-brain brain-score component)`);
|
||||
}
|
||||
if (Array.isArray(h.most_connected) && h.most_connected.length > 0) {
|
||||
lines.push('Most connected entities:');
|
||||
@@ -984,7 +1059,7 @@ export function formatResult(opName: string, result: unknown): string {
|
||||
* `runRemoteDoctor` for thin-client installs.
|
||||
*/
|
||||
const THIN_CLIENT_REFUSED_COMMANDS = new Set([
|
||||
'sync', 'embed', 'extract', 'extract-conversation-facts', 'enrich', 'migrate', 'apply-migrations',
|
||||
'sync', 'embed', 'extract', 'extract-conversation-facts', 'enrich', 'migrate', 'retrieval-upgrade', 'apply-migrations',
|
||||
'repair-jsonb', 'orphans', 'integrity', 'serve',
|
||||
// v0.43 (#2095): watch streams against a LOCAL engine; thin clients get
|
||||
// the volunteer_context MCP op instead.
|
||||
@@ -1012,17 +1087,6 @@ const THIN_CLIENT_REFUSED_COMMANDS = new Set([
|
||||
// it gets a partial dispatch (list/get route over MCP engine-free, the
|
||||
// rest refuse) in the main dispatch before connectEngine().
|
||||
'config',
|
||||
// #3224: `backfill` (like migrate/apply-migrations/repair-jsonb) opens
|
||||
// and mutates the local engine directly with no MCP equivalent op. On a
|
||||
// thin-client install it would otherwise fabricate the same kind of
|
||||
// ephemeral scratch-local PGLite `config` did before this audit —
|
||||
// refuse cleanly instead. This check runs before backfill's own
|
||||
// pre-connectEngine dispatch branch, so `--help` is ALSO refused on a
|
||||
// thin client rather than showing help — the same pre-existing tradeoff
|
||||
// `sync`/`enrich` already make (both are also THIN_CLIENT_REFUSED_COMMANDS
|
||||
// with their own pre-connectEngine `--help` branch further down); not a
|
||||
// new inconsistency introduced here.
|
||||
'backfill',
|
||||
]);
|
||||
|
||||
/**
|
||||
@@ -1042,6 +1106,7 @@ const THIN_CLIENT_REFUSE_HINTS: Record<string, string> = {
|
||||
'extract-conversation-facts': 'extract-conversation-facts runs on the host (requires local engine + chat gateway). Run on the host machine.',
|
||||
enrich: 'enrich runs on the host (requires local engine + chat gateway for grounded synthesis). Run on the host machine.',
|
||||
migrate: "migrate runs on the host's local engine. Run on the host machine.",
|
||||
'retrieval-upgrade': "retrieval-upgrade (embedding migration) rebuilds the host brain's schema + re-embeds. Run on the host machine.",
|
||||
'apply-migrations': 'schema migrations run on the host. SSH and run there.',
|
||||
'repair-jsonb': 'repair-jsonb operates on the local DB only.',
|
||||
integrity: 'integrity scans local files. Run on the host machine.',
|
||||
@@ -1063,8 +1128,6 @@ const THIN_CLIENT_REFUSE_HINTS: Record<string, string> = {
|
||||
// scratch-DB audit additions
|
||||
config: "config reads/writes the host brain's config plane. Edit the host's .gbrain/config.json (file-plane keys) or run on the host with GBRAIN_HOME set.",
|
||||
jobs: '`jobs list` and `jobs get <id>` are thin-client routable; this subcommand runs against the host queue. Use the submit_job / list_jobs / get_job MCP tools from your agent, or run on the host with GBRAIN_HOME set.',
|
||||
// #3224
|
||||
backfill: 'backfill mutates the host brain directly with no MCP equivalent. Run `gbrain backfill` on the host machine.',
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -1543,24 +1606,6 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
return;
|
||||
}
|
||||
|
||||
// #3224: `backfill` dispatches unconditionally here (not just --help).
|
||||
// runBackfillCommand manages its own engine end-to-end (createEngine +
|
||||
// connect at commands/backfill.ts, disconnect when done) — it takes no
|
||||
// engine parameter at all. The old `case 'backfill':` further down sits
|
||||
// behind this function's shared `const engine = await connectEngine()`;
|
||||
// reaching it there would open a SECOND PGLite connection to the same
|
||||
// database while the first is still held, and PGLite's single-writer
|
||||
// lock would make every real (non---help) backfill run hang for 30s and
|
||||
// fail. Dispatching before that shared connect — same as schema/init/
|
||||
// auth/remote above — avoids the double-connect entirely and, as a
|
||||
// bonus, makes `--help` reachable from a fresh tmpdir with no configured
|
||||
// brain (same motivation as the sync/capture/enrich branches below).
|
||||
if (command === 'backfill') {
|
||||
const { runBackfillCommand } = await import('./commands/backfill.ts');
|
||||
await runBackfillCommand(args);
|
||||
return;
|
||||
}
|
||||
|
||||
// v0.41.6.0 D3 (per outside-voice F1): connect-time + dispatch-time wallclock
|
||||
// timeouts for read-only commands whose hang would otherwise spin at 100% CPU
|
||||
// (the production "10-day zombie gbrain search ping" bug class). The wrap
|
||||
@@ -1712,10 +1757,33 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
}
|
||||
// doctor is handled before connectEngine() above
|
||||
case 'migrate': {
|
||||
// #3390: `gbrain migrate embeddings --to <provider:model>` — the
|
||||
// provider-agnostic embedding migration. Everything else stays the
|
||||
// engine-transfer path (`migrate --to <supabase|pglite>`).
|
||||
if (args[0] === 'embeddings') {
|
||||
const { runMigrateEmbeddings } = await import('./commands/migrate-embeddings.ts');
|
||||
await runMigrateEmbeddings(engine, args.slice(1));
|
||||
break;
|
||||
}
|
||||
if (args.includes('--help') || args.includes('-h')) {
|
||||
console.log('Usage: gbrain migrate --to <supabase|pglite> [--url <url>] [--path <path>] [--force]');
|
||||
console.log(' gbrain migrate embeddings --to <provider:model> [--dim N] [--dry-run] [--yes]');
|
||||
console.log('');
|
||||
console.log('The first form transfers the brain between engines; the second re-embeds');
|
||||
console.log('onto a different embedding provider (run `gbrain migrate embeddings --help`).');
|
||||
break;
|
||||
}
|
||||
const { runMigrateEngine } = await import('./commands/migrate-engine.ts');
|
||||
await runMigrateEngine(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'retrieval-upgrade': {
|
||||
// The command README.md + doctor.ts promised since v0.36 but never
|
||||
// dispatched. Alias for `migrate embeddings` (#3390).
|
||||
const { runMigrateEmbeddings } = await import('./commands/migrate-embeddings.ts');
|
||||
await runMigrateEmbeddings(engine, args);
|
||||
break;
|
||||
}
|
||||
case 'eval': {
|
||||
// v0.32 EXP-5: `eval takes-quality {run,trend,regress}` requires a
|
||||
// brain (samples takes from DB / reads runs table). `replay` was
|
||||
@@ -2097,13 +2165,13 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await reindexFrontmatterCli(args);
|
||||
return; // reindexFrontmatterCli handles its own engine lifecycle
|
||||
}
|
||||
// #3224: `backfill` (v0.30.1's first-class generic backfill command)
|
||||
// used to have its `case` right here, but runBackfillCommand manages
|
||||
// its own engine end-to-end and dispatching it AFTER this switch's
|
||||
// shared `connectEngine()` double-connects PGLite's single-writer
|
||||
// lock to itself (hangs 30s, then fails on every real run). Moved to
|
||||
// an unconditional pre-connectEngine branch above, next to schema/
|
||||
// init/auth/remote/sync/capture/enrich — see that branch's comment.
|
||||
case 'backfill': {
|
||||
// v0.30.1: first-class generic backfill command. Subcommand dispatch
|
||||
// is inside runBackfillCommand (kind | list | --help).
|
||||
const { runBackfillCommand } = await import('./commands/backfill.ts');
|
||||
await runBackfillCommand(args);
|
||||
return;
|
||||
}
|
||||
case 'code-callers': {
|
||||
// v0.20.0 Cathedral II Layer 10 (C4): "who calls <symbol>?"
|
||||
const { runCodeCallers } = await import('./commands/code-callers.ts');
|
||||
@@ -2312,6 +2380,7 @@ USAGE
|
||||
SETUP
|
||||
init [--pglite|--supabase|--url] Create brain (PGLite default, no server)
|
||||
migrate --to <supabase|pglite> Transfer brain between engines
|
||||
migrate embeddings --to <p:model> Re-embed onto another embedding provider
|
||||
upgrade Self-update
|
||||
check-update [--json] Check for new versions
|
||||
doctor [--json] [--fast] Health check (resolver, skills, pgvector, RLS, embeddings)
|
||||
@@ -2333,6 +2402,8 @@ IMPORT/EXPORT
|
||||
sync [--repo <path>] [flags] Git-to-brain incremental sync
|
||||
sync --watch [--interval N] Continuous sync (loops until stopped)
|
||||
See also: autopilot --install (continuous daemon).
|
||||
sync --all --missing-path skip Classify sources whose local_path is absent
|
||||
on this machine as skipped, not failed
|
||||
export [--dir ./out/] Export to markdown
|
||||
export --restore-only [--repo <p>] Restore missing supabase-only files
|
||||
[--type T] [--slug-prefix S] With optional filters
|
||||
|
||||
+19
-4
@@ -66,7 +66,9 @@ USAGE
|
||||
SUBMITTING
|
||||
gbrain agent run <prompt>
|
||||
--subagent-def <name> Named plugin subagent (from GBRAIN_PLUGIN_PATH)
|
||||
--model <id> Anthropic model id (defaults to sonnet)
|
||||
--model <id> Model id as provider:model (default: subagent tier model,
|
||||
anthropic:claude-sonnet-4-6). Non-Anthropic providers need
|
||||
agent.use_gateway_loop enabled — see NOTES below.
|
||||
--max-turns <n> Max assistant turns (default 20)
|
||||
--tools a,b,c Subset of registered tool names (comma list)
|
||||
--timeout-ms <n> Per-job wall-clock timeout
|
||||
@@ -87,9 +89,22 @@ VIEWING
|
||||
--since <spec> ISO-8601 timestamp OR relative ("5m","1h","2d")
|
||||
|
||||
NOTES
|
||||
Submitting subagent jobs is trusted-only; MCP submitters receive
|
||||
permission_denied. The worker needs ANTHROPIC_API_KEY set, or the
|
||||
first LLM turn of a claimed job fails.
|
||||
This CLI path is trusted-only. (Remote MCP callers reach subagents through
|
||||
the scoped submit_agent operation, not through this command.)
|
||||
|
||||
By default the worker runs the legacy Anthropic-direct path, which needs an
|
||||
Anthropic key — from ANTHROPIC_API_KEY or from anthropic_api_key in
|
||||
~/.gbrain/config.json — or the first LLM turn of a claimed job fails.
|
||||
|
||||
To run --model on a non-Anthropic provider, enable the provider-neutral
|
||||
gateway loop first, then supply whatever credential that provider needs
|
||||
(an API key for most; some recipes use OAuth or a local endpoint):
|
||||
gbrain config set agent.use_gateway_loop true
|
||||
Accepted values: true / 1 / yes / on.
|
||||
|
||||
The gateway loop needs a provider whose recipe supports chat WITH tool
|
||||
calling — not every recipe under src/core/ai/recipes/ qualifies. A model
|
||||
that cannot call tools is refused at job start with the reason named.
|
||||
`);
|
||||
}
|
||||
|
||||
|
||||
@@ -439,6 +439,13 @@ export async function runApplyMigrations(args: string[]): Promise<void> {
|
||||
const result = await m.orchestrator(orchestratorOptsFrom(cli));
|
||||
if (result.status === 'failed') {
|
||||
console.error(`Migration v${m.version} reported status=failed.`);
|
||||
// Surface each failed phase's detail — the ledger records it, but
|
||||
// the operator needs it on stderr to act (#921).
|
||||
for (const p of result.phases) {
|
||||
if (p.status === 'failed') {
|
||||
console.error(` phase ${p.name}: ${p.detail ?? '(no detail)'}`);
|
||||
}
|
||||
}
|
||||
// Record the attempt as 'partial' (not 'complete') so the cap counts
|
||||
// it. Don't let a failed orchestrator look like it never ran.
|
||||
try {
|
||||
|
||||
@@ -515,6 +515,60 @@ async function registerClient(name: string, args: string[]) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.42.x (#1914): rescope an existing OAuth client's write source and/or
|
||||
* federated read scope. This is the operator surface the DCR registration
|
||||
* comment promised ("rescope via the CLI later") — DCR clients land with
|
||||
* source_id='default' / federated_read=['default'] and must not self-widen,
|
||||
* so widening happens here (trusted local CLI) or via the requireAdmin
|
||||
* /admin/api/rescope-client endpoint.
|
||||
*/
|
||||
async function rescopeClient(clientId: string, args: string[]) {
|
||||
const usage = 'Usage: auth rescope-client <client_id> [--source SOURCE] [--federated-read SRC1,SRC2,...]';
|
||||
if (!clientId) {
|
||||
console.error(usage);
|
||||
process.exit(1);
|
||||
}
|
||||
let sourceId: string | undefined;
|
||||
let federatedRead: string[] | undefined;
|
||||
for (let i = 0; i < args.length; i += 2) {
|
||||
const flag = args[i];
|
||||
const value = args[i + 1];
|
||||
if (value === undefined || value.startsWith('--')) {
|
||||
console.error(`Error: ${flag} requires a value`);
|
||||
console.error(usage);
|
||||
process.exit(1);
|
||||
}
|
||||
if (flag === '--source') sourceId = value;
|
||||
else if (flag === '--federated-read') {
|
||||
federatedRead = value.split(',').map(s => s.trim()).filter(Boolean);
|
||||
} else {
|
||||
console.error(`Error: Unknown flag: ${flag}`);
|
||||
console.error(usage);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
if (sourceId === undefined && federatedRead === undefined) {
|
||||
console.error('Error: pass --source and/or --federated-read');
|
||||
console.error(usage);
|
||||
process.exit(1);
|
||||
}
|
||||
try {
|
||||
await withConfiguredSql(async (sql) => {
|
||||
const { GBrainOAuthProvider } = await import('../core/oauth-provider.ts');
|
||||
const provider = new GBrainOAuthProvider({ sql });
|
||||
const result = await provider.rescopeClient(clientId, { sourceId, federatedRead });
|
||||
console.log(`OAuth client rescoped: "${result.clientName}" (${result.clientId})\n`);
|
||||
console.log(` Write source: ${result.sourceId}`);
|
||||
console.log(` Federated reads: ${result.federatedRead.join(', ') || '<none>'}`);
|
||||
console.log('\nTakes effect on the client\'s next request (existing tokens included).');
|
||||
});
|
||||
} catch (e: any) {
|
||||
console.error('Error:', e.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Entry point for the `gbrain auth` CLI subcommand. Also reused by the
|
||||
* direct-script path (see bottom of file) so `bun run src/commands/auth.ts`
|
||||
@@ -556,6 +610,7 @@ export async function runAuth(args: string[]): Promise<void> {
|
||||
return;
|
||||
}
|
||||
case 'register-client': await registerClient(rest[0], rest.slice(1)); return;
|
||||
case 'rescope-client': await rescopeClient(rest[0], rest.slice(1)); return;
|
||||
case 'revoke-client': await revokeClient(rest[0]); return;
|
||||
case 'test': {
|
||||
const tokenIdx = rest.indexOf('--token');
|
||||
@@ -593,6 +648,11 @@ Usage:
|
||||
--bound-slug-prefixes <prefix1,prefix2> Bind submit_agent writes to slug prefixes
|
||||
--bound-max-concurrent <n> Bound submit_agent concurrency (default: 1)
|
||||
--budget-usd-per-day <usd> Bound submit_agent daily spend cap
|
||||
gbrain auth rescope-client <client_id> [options] Change an existing client's source scope (e.g. a DCR
|
||||
client stuck on the 'default' source). Only the flags
|
||||
you pass change; the other axis is left as-is.
|
||||
--source <id> New write source
|
||||
--federated-read <id1,id2,...> New read-scope source list
|
||||
gbrain auth revoke-client <client_id> Hard-delete an OAuth 2.1 client (cascades to tokens + codes)
|
||||
gbrain auth test <url> --token <token> Smoke-test a remote MCP server
|
||||
`);
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
export function resolveAutopilotDispatchTimeoutMs(
|
||||
baseIntervalSeconds: number,
|
||||
fullCycle: boolean,
|
||||
): number {
|
||||
const intervalDerivedTimeoutMs = Math.max(baseIntervalSeconds * 2 * 1000, 300_000);
|
||||
return fullCycle
|
||||
? Math.max(intervalDerivedTimeoutMs, 1_800_000)
|
||||
: intervalDerivedTimeoutMs;
|
||||
}
|
||||
+171
-12
@@ -19,7 +19,7 @@
|
||||
|
||||
import { existsSync, readFileSync, writeFileSync, mkdirSync, appendFileSync, utimesSync, unlinkSync, chmodSync } from 'fs';
|
||||
import { setCliExitVerdict } from '../core/cli-force-exit.ts';
|
||||
import { join } from 'path';
|
||||
import { join, dirname } from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { loadPreferences } from '../core/preferences.ts';
|
||||
@@ -38,6 +38,8 @@ import { logSelfUpgrade } from '../core/audit/self-upgrade-audit.ts';
|
||||
import { detectInstallMethod } from './upgrade.ts';
|
||||
import { evaluateQuietHours } from '../core/minions/quiet-hours.ts';
|
||||
import { inspectLock } from '../core/db-lock.ts';
|
||||
import { registerCleanup } from '../core/process-cleanup.ts';
|
||||
import { resolveAutopilotDispatchTimeoutMs } from './autopilot-timeout.ts';
|
||||
|
||||
/**
|
||||
* v0.37.7.0 #1162 — classify autopilot reconnect-loop errors.
|
||||
@@ -433,6 +435,37 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
let stopping = false;
|
||||
let childSupervisor: ChildWorkerSupervisor | null = null;
|
||||
|
||||
// #1872: graceful engine shutdown. On PGLite the cycle steps run INLINE in
|
||||
// this process, so a hard `process.exit` mid-write (systemctl stop →
|
||||
// SIGTERM) kills WASM Postgres with the WAL dirty and can corrupt the
|
||||
// brain. Two exit paths must both close the engine:
|
||||
// - autopilot's own shutdown() below (owns SIGINT + internal stops like
|
||||
// max_crashes / cycle-failure-cap), and
|
||||
// - process-cleanup's SIGTERM handler (installed at cli.ts module load;
|
||||
// it runs the cleanup registry with a 3s deadline and then exits) —
|
||||
// which is why closeEngine is ALSO registered there.
|
||||
// closeEngine aborts the in-flight inline cycle (runCycle checks the
|
||||
// signal between phases and threads it into phase sub-work), gives it a
|
||||
// short bounded window to wind down, then disconnects. PGLite's
|
||||
// disconnect() drains the pending query and checkpoints before closing;
|
||||
// a second call is a no-op (disconnect snapshots + nulls the handle), so
|
||||
// both paths firing is safe.
|
||||
const shutdownAbort = new AbortController();
|
||||
let inflightInlineCycle: Promise<unknown> | null = null;
|
||||
const closeEngine = async () => {
|
||||
shutdownAbort.abort(new Error('autopilot shutdown'));
|
||||
if (inflightInlineCycle) {
|
||||
// ponytail: 2s cap keeps us inside process-cleanup's 3s deadline; a
|
||||
// between-phase abort resolves instantly, a mid-phase one may not.
|
||||
await Promise.race([
|
||||
inflightInlineCycle.catch(() => { /* cycle errors already logged by the loop */ }),
|
||||
new Promise((r) => setTimeout(r, 2_000)),
|
||||
]);
|
||||
}
|
||||
try { await engine.disconnect(); } catch { /* best-effort */ }
|
||||
};
|
||||
const deregisterEngineClose = registerCleanup('autopilot-engine-close', closeEngine);
|
||||
|
||||
if (spawnManagedWorker) {
|
||||
const cliPath = resolveGbrainCliPath();
|
||||
// Cgroup-aware auto-sized RSS watchdog cap (issue #1678). The old flat
|
||||
@@ -520,6 +553,10 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
childSupervisor.killChild('SIGKILL');
|
||||
}
|
||||
}
|
||||
// #1872: abort the in-flight inline cycle and close the engine BEFORE
|
||||
// process.exit — a hard exit mid-write corrupts PGLite's WASM Postgres.
|
||||
await closeEngine();
|
||||
deregisterEngineClose();
|
||||
try { unlinkSync(lockPath); } catch { /* already gone */ }
|
||||
process.exit(0);
|
||||
};
|
||||
@@ -527,6 +564,9 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
process.on('SIGINT', () => { void shutdown('SIGINT'); });
|
||||
|
||||
let consecutiveErrors = 0;
|
||||
// Parser-probe fixture warning is once-per-process, not once-per-cycle
|
||||
// (compiled-binary installs have no source tree; don't spam the log).
|
||||
let parserProbeFixtureWarned = false;
|
||||
// v0.37.7.0 #1162 — counter for consecutive reconnect failures.
|
||||
// Reset on every successful health probe or reconnect. Threshold
|
||||
// controlled by GBRAIN_AUTOPILOT_MAX_RECONNECT_FAILS env (default 30).
|
||||
@@ -689,7 +729,7 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
const queue = new MinionQueue(engine);
|
||||
const slotMs = Math.floor(Date.now() / (baseInterval * 1000)) * baseInterval * 1000;
|
||||
const slot = new Date(slotMs).toISOString();
|
||||
const timeoutMs = Math.max(baseInterval * 2 * 1000, 300_000);
|
||||
const timeoutMs = resolveAutopilotDispatchTimeoutMs(baseInterval, false);
|
||||
|
||||
// ── v0.40 D17: per-source freshness check ────────────────────
|
||||
// Runs first; independent of score gate. Submits a 'sync' job per
|
||||
@@ -825,7 +865,10 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
{
|
||||
queue: 'default',
|
||||
idempotency_key: idemKey,
|
||||
max_attempts: 1,
|
||||
// issue #3218: the handler now throws on an
|
||||
// all-provider-failed batch, so give the queue's
|
||||
// backoff a chance (was 1 — dead-lettered instantly).
|
||||
max_attempts: 3,
|
||||
timeout_ms: timeoutMs,
|
||||
},
|
||||
{ allowProtectedSubmit: true },
|
||||
@@ -865,9 +908,19 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
} catch {
|
||||
embeddingModel = (await engine.getConfig('embedding_model')) ?? undefined;
|
||||
}
|
||||
const embedKeyCfg: Record<string, string | null> = {};
|
||||
// #2662 (codex round-3): HOSTED_EMBED_KEY_CONFIG entries are keys
|
||||
// buildGatewayConfig folds from the FILE plane only — `gbrain config
|
||||
// set <key> X` writes the DB plane, which never reaches the gateway
|
||||
// for these fields. Reading via engine.getConfig() here (DB plane)
|
||||
// would report a provider "configured" from a DB-only key that the
|
||||
// gateway can never actually use, dispatching a doomed embed job.
|
||||
// Read the same file-plane source context.ts (doctor) reads instead,
|
||||
// so autopilot and doctor agree with what the gateway can see.
|
||||
const { loadConfigFileOnly } = await import('../core/config.ts');
|
||||
const fileCfg = loadConfigFileOnly() as Record<string, unknown> | null;
|
||||
const embedKeyCfg: Record<string, unknown> = {};
|
||||
for (const field of Object.values(HOSTED_EMBED_KEY_CONFIG)) {
|
||||
embedKeyCfg[field] = await engine.getConfig(field);
|
||||
embedKeyCfg[field] = fileCfg?.[field];
|
||||
}
|
||||
const ctx = {
|
||||
repoPath,
|
||||
@@ -931,7 +984,9 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
const result = await dispatchPerSource(engine, queue, {
|
||||
repoPath,
|
||||
slot,
|
||||
timeoutMs,
|
||||
// Full cycles can outlive short daemon intervals. Keep lighter dispatches
|
||||
// interval-derived while giving per-source consolidation enough time.
|
||||
timeoutMs: resolveAutopilotDispatchTimeoutMs(baseInterval, true),
|
||||
fanoutMax,
|
||||
jsonMode,
|
||||
});
|
||||
@@ -1008,16 +1063,21 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
// path's phase set). Now both converge on the same primitive.
|
||||
try {
|
||||
const { runCycle } = await import('../core/cycle.ts');
|
||||
const report = await runCycle(engine, {
|
||||
// #1872: track the promise so closeEngine can drain it on shutdown,
|
||||
// and pass the abort signal so the cycle winds down between phases.
|
||||
const cyclePromise = runCycle(engine, {
|
||||
brainDir: repoPath,
|
||||
// Autopilot daemon path: pulls by default (matches
|
||||
// pre-v0.17 autopilot behavior). CLI dream defaults false
|
||||
// for cron safety; that choice is scoped to dream only.
|
||||
pull: true,
|
||||
signal: shutdownAbort.signal,
|
||||
yieldBetweenPhases: async () => {
|
||||
await new Promise(r => setImmediate(r));
|
||||
},
|
||||
});
|
||||
inflightInlineCycle = cyclePromise;
|
||||
const report = await cyclePromise.finally(() => { inflightInlineCycle = null; });
|
||||
// Only 'failed' (every attempted phase failed) trips the autopilot
|
||||
// circuit breaker. 'partial' means at least one phase warned or
|
||||
// failed while others ran — that's a soft signal, not a fatal
|
||||
@@ -1073,17 +1133,36 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
// loop. Probe runs even when cycleOk=false (probe may surface signal
|
||||
// explaining why the cycle is failing).
|
||||
try {
|
||||
const probeEnabled = cfg?.autopilot?.nightly_quality_probe?.enabled === true;
|
||||
const { resolveProbeEnabled, resolveProbeMaxUsd, runNightlyQualityProbe } = await import('../core/cycle/nightly-quality-probe.ts');
|
||||
// Dual-plane read: `gbrain config set` (what the doctor enable hint
|
||||
// prints) writes the DB plane; ~/.gbrain/config.json is the fallback.
|
||||
let dbEnabled: string | null = null;
|
||||
let dbMaxUsd: string | null = null;
|
||||
try {
|
||||
dbEnabled = await engine.getConfig('autopilot.nightly_quality_probe.enabled');
|
||||
dbMaxUsd = await engine.getConfig('autopilot.nightly_quality_probe.max_usd');
|
||||
} catch { /* DB unavailable → file plane only */ }
|
||||
const probeEnabled = resolveProbeEnabled(dbEnabled, cfg?.autopilot?.nightly_quality_probe?.enabled);
|
||||
if (probeEnabled) {
|
||||
const { runNightlyQualityProbe } = await import('../core/cycle/nightly-quality-probe.ts');
|
||||
const { runLongMemEvalForProbe, runCrossModalBatchForProbe } = await import('../core/cycle/nightly-probe-adapters.ts');
|
||||
const { isAvailable } = await import('../core/ai/gateway.ts');
|
||||
const maxUsd = Number(cfg?.autopilot?.nightly_quality_probe?.max_usd ?? 5);
|
||||
const { existsSync } = await import('node:fs');
|
||||
const { fileURLToPath } = await import('node:url');
|
||||
const { join } = await import('node:path');
|
||||
const maxUsd = resolveProbeMaxUsd(dbMaxUsd, cfg?.autopilot?.nightly_quality_probe?.max_usd);
|
||||
// The committed fixture (test/fixtures/longmemeval-nightly.jsonl)
|
||||
// lives in the gbrain PACKAGE, not the brain repo — repoPath is
|
||||
// sync.repo_path (the user's brain), where the fixture never
|
||||
// exists, so the probe error'd on every real install. Resolve the
|
||||
// package root from the module location; keep repoPath as the
|
||||
// fallback for setups that vendor the fixture into the brain repo.
|
||||
const pkgRoot = fileURLToPath(new URL('../..', import.meta.url));
|
||||
const fixtureAtPkgRoot = existsSync(join(pkgRoot, 'test', 'fixtures', 'longmemeval-nightly.jsonl'));
|
||||
await runNightlyQualityProbe({
|
||||
isEnabled: () => true, // already gated above; phase re-checks for defense-in-depth
|
||||
hasEmbeddingProvider: () => isAvailable('embedding'),
|
||||
resolveMaxUsd: () => maxUsd,
|
||||
resolveRepoRoot: () => repoPath ?? gbrainHomePath('.'),
|
||||
resolveRepoRoot: () => (fixtureAtPkgRoot ? pkgRoot : repoPath ?? gbrainHomePath('.')),
|
||||
runLongMemEval: runLongMemEvalForProbe,
|
||||
runCrossModalBatch: runCrossModalBatchForProbe,
|
||||
now: () => new Date(),
|
||||
@@ -1095,6 +1174,62 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
|
||||
// informational; autopilot loop continues.
|
||||
}
|
||||
|
||||
// 4.6 — Nightly conversation-parser probe (v0.41.16.0 phase module;
|
||||
// the scheduler wire-up was deferred at ship and is added here). Same
|
||||
// posture as 4.5: the phase owns its gates (enabled/mode-gate, LLM
|
||||
// key), the wiring owns invocation + the audit row, and a probe
|
||||
// failure NEVER crashes the autopilot loop. Per D10 the probe is
|
||||
// default-ON for search.mode=tokenmax, opt-in otherwise.
|
||||
try {
|
||||
const { runConversationParserNightlyProbe } = await import('../core/conversation-parser/nightly-probe.ts');
|
||||
const { logParserProbeEvent, parserProbeRanWithin } = await import('../core/audit-parser-probe.ts');
|
||||
const { isAvailable } = await import('../core/ai/gateway.ts');
|
||||
const { existsSync } = await import('node:fs');
|
||||
const { fileURLToPath } = await import('node:url');
|
||||
const { join } = await import('node:path');
|
||||
// Flag reads dual-plane: the DB row (`gbrain config set …`) wins,
|
||||
// ~/.gbrain/config.json is the fallback. search.mode lives on the
|
||||
// DB plane only (mode.ts owns it).
|
||||
let parserDbEnabled: string | null = null;
|
||||
let dbSearchMode: string | null = null;
|
||||
try {
|
||||
parserDbEnabled = await engine.getConfig('autopilot.conversation_parser_probe.enabled');
|
||||
dbSearchMode = await engine.getConfig('search.mode');
|
||||
} catch { /* DB unavailable → file plane only */ }
|
||||
const parserEnabled = parserDbEnabled != null
|
||||
? parserDbEnabled === 'true'
|
||||
: cfg?.autopilot?.conversation_parser_probe?.enabled === true;
|
||||
const searchMode = dbSearchMode ?? '';
|
||||
// Fixtures are committed in the gbrain package (test/fixtures/…),
|
||||
// NOT the brain repo — resolve from the module location. Compiled
|
||||
// binaries carry no source tree: skip quietly instead of writing
|
||||
// failure rows that would flip doctor to WARN on every binary install.
|
||||
const pkgRoot = fileURLToPath(new URL('../..', import.meta.url));
|
||||
const fixturePath = join(pkgRoot, 'test', 'fixtures', 'conversation-formats', 'all.jsonl');
|
||||
const adversarialPath = join(pkgRoot, 'test', 'fixtures', 'conversation-formats', 'adversarial.jsonl');
|
||||
const shouldInvoke = parserEnabled || searchMode === 'tokenmax';
|
||||
if (shouldInvoke && existsSync(fixturePath) && existsSync(adversarialPath)) {
|
||||
const result = await runConversationParserNightlyProbe({
|
||||
isEnabled: () => parserEnabled,
|
||||
searchMode: () => searchMode,
|
||||
hasLlmKey: () => isAvailable('chat'),
|
||||
resolveFixturePath: () => fixturePath,
|
||||
resolveAdversarialPath: () => adversarialPath,
|
||||
now: () => new Date(),
|
||||
shouldSkipForRateLimit: () => parserProbeRanWithin(24 * 60 * 60 * 1000),
|
||||
});
|
||||
// rate_limited is a non-run: the loop ticks every few minutes, so
|
||||
// logging every skip would flood the audit file with no-signal rows.
|
||||
if (result.outcome !== 'rate_limited') logParserProbeEvent(result);
|
||||
} else if (shouldInvoke && !parserProbeFixtureWarned) {
|
||||
parserProbeFixtureWarned = true;
|
||||
console.error(`[parser-probe] fixtures not found under ${pkgRoot}; skipping (probe needs a source-checkout install)`);
|
||||
}
|
||||
} catch (e) {
|
||||
logError('autopilot.parser_probe', e);
|
||||
// Informational, like 4.5: do NOT bump consecutiveErrors.
|
||||
}
|
||||
|
||||
// Wait for next cycle
|
||||
await new Promise(r => setTimeout(r, interval * 1000));
|
||||
}
|
||||
@@ -1177,6 +1312,17 @@ function writeWrapperScript(repoPath: string): string {
|
||||
const gbrainPath = resolveGbrainCliPath();
|
||||
const safeRepoPath = repoPath.replace(/'/g, "'\\''");
|
||||
const safeGbrainPath = gbrainPath.replace(/'/g, "'\\''");
|
||||
// Bake the dir of the bun runtime actually executing this install onto PATH,
|
||||
// so the wrapper finds bun wherever it lives — Homebrew (/opt/homebrew/bin),
|
||||
// npm -g, Docker (/usr/local/bin), a custom BUN_INSTALL, or nix — not just
|
||||
// ~/.bun/bin (which #3305 hardcoded, covering only the default bun.sh installer).
|
||||
// dirname('') === '.', so guard the degenerate/empty case — otherwise a missing
|
||||
// execPath would prepend '.' (cwd) onto a cron PATH. Empty prefix falls back to
|
||||
// the #3305 behavior exactly.
|
||||
const runtimeDir = dirname(process.execPath || '');
|
||||
const runtimePathPrefix = runtimeDir && runtimeDir !== '.'
|
||||
? `'${runtimeDir.replace(/'/g, "'\\''")}':`
|
||||
: '';
|
||||
const wrapper = `#!/bin/bash
|
||||
# Auto-generated by gbrain autopilot --install
|
||||
# Sources shell profile for API keys, then runs autopilot.
|
||||
@@ -1186,6 +1332,16 @@ function writeWrapperScript(repoPath: string): string {
|
||||
# OPENAI/ANTHROPIC keys exported in zshenv reach autopilot.
|
||||
[ -f ~/.zshenv ] && source ~/.zshenv 2>/dev/null
|
||||
source ~/.zshrc 2>/dev/null || source ~/.bashrc 2>/dev/null || true
|
||||
# Belt-and-suspenders PATH fix. ~/.bashrc ships with a non-interactive guard
|
||||
# (\`case $- in *i*) ;; *) return;; esac\`) that exits early when launched from
|
||||
# cron/systemd/launchd — so its PATH exports never reach this subprocess.
|
||||
# Without bun on PATH, the exec'd gbrain (a \`#!/usr/bin/env bun\` script) fails
|
||||
# silently with "env: bun: No such file or directory" and leaves a stale
|
||||
# lockfile that blocks every subsequent tick. Prepending the running bun's own
|
||||
# dir (derived from process.execPath at install time), with ~/.bun/bin kept as a
|
||||
# fallback, keeps the wrapper self-contained regardless of where bun is installed
|
||||
# or which init file the OS loaded.
|
||||
export PATH=${runtimePathPrefix}"$HOME/.bun/bin:$PATH"
|
||||
exec '${safeGbrainPath}' autopilot --repo '${safeRepoPath}'
|
||||
`;
|
||||
writeFileSync(wrapperPath, wrapper, { mode: 0o755 });
|
||||
@@ -1612,7 +1768,10 @@ function showStatus(json: boolean) {
|
||||
} else {
|
||||
try {
|
||||
const crontab = execSync('crontab -l 2>/dev/null || true', { encoding: 'utf-8' });
|
||||
installed = crontab.includes('gbrain autopilot');
|
||||
// The installed cron line invokes the generated wrapper (…/autopilot-run.sh);
|
||||
// older installs called `gbrain autopilot` directly. Match either so status
|
||||
// isn't a false negative after the wrapper indirection landed.
|
||||
installed = crontab.includes('autopilot-run.sh') || crontab.includes('gbrain autopilot');
|
||||
} catch { /* no crontab */ }
|
||||
}
|
||||
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
* checks if back-links exist, and optionally creates them.
|
||||
*
|
||||
* Usage:
|
||||
* gbrain check-backlinks check [--dir <brain-dir>] # report missing back-links
|
||||
* gbrain check-backlinks fix [--dir <brain-dir>] # create missing back-links
|
||||
* gbrain check-backlinks check [dir] [--dir <brain-dir>] # report missing back-links
|
||||
* gbrain check-backlinks fix [dir] [--dir <brain-dir>] # create missing back-links
|
||||
* gbrain check-backlinks fix --dry-run # preview fixes
|
||||
*/
|
||||
|
||||
@@ -201,6 +201,40 @@ export interface BacklinksResult {
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
export interface ParsedBacklinksArgs {
|
||||
subcommand: string | undefined;
|
||||
brainDir: string;
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
export function parseBacklinksArgs(args: string[]): ParsedBacklinksArgs {
|
||||
const subcommand = args[0];
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const dirIdx = args.indexOf('--dir');
|
||||
const flagDir = dirIdx >= 0 && args[dirIdx + 1] && !args[dirIdx + 1].startsWith('--')
|
||||
? args[dirIdx + 1]
|
||||
: undefined;
|
||||
|
||||
let positionalDir: string | undefined;
|
||||
for (let i = 1; i < args.length; i++) {
|
||||
const arg = args[i];
|
||||
if (arg === '--dir') {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (arg === '--dry-run') continue;
|
||||
if (arg.startsWith('--')) continue;
|
||||
positionalDir = arg;
|
||||
break;
|
||||
}
|
||||
|
||||
return {
|
||||
subcommand,
|
||||
brainDir: flagDir ?? positionalDir ?? '.',
|
||||
dryRun,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Library-level backlinks check/fix. Throws on validation errors; returns a
|
||||
* structured result so Minions handlers + autopilot-cycle can surface counts.
|
||||
@@ -236,16 +270,14 @@ export async function runBacklinksCore(opts: BacklinksOpts): Promise<BacklinksRe
|
||||
}
|
||||
|
||||
export async function runBacklinks(args: string[]) {
|
||||
const subcommand = args[0];
|
||||
const dirIdx = args.indexOf('--dir');
|
||||
const brainDir = dirIdx >= 0 ? args[dirIdx + 1] : '.';
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const { subcommand, brainDir, dryRun } = parseBacklinksArgs(args);
|
||||
|
||||
if (!subcommand || !['check', 'fix'].includes(subcommand)) {
|
||||
console.error('Usage: gbrain check-backlinks <check|fix> [--dir <brain-dir>] [--dry-run]');
|
||||
console.error('Usage: gbrain check-backlinks <check|fix> [dir] [--dir <brain-dir>] [--dry-run]');
|
||||
console.error(' check Report missing back-links');
|
||||
console.error(' fix Create missing back-links (appends to Timeline)');
|
||||
console.error(' --dir Brain directory (default: current directory)');
|
||||
console.error(' dir Brain directory (default: current directory)');
|
||||
console.error(' --dir Brain directory override');
|
||||
console.error(' --dry-run Preview fixes without writing');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import { runPhaseCalibrationProfile } from '../core/cycle/calibration-profile.ts
|
||||
import { sourceScopeOpts, type OperationContext } from '../core/operations.ts';
|
||||
import type { GBrainConfig } from '../core/config.ts';
|
||||
import { GBrainError } from '../core/types.ts';
|
||||
import { resolveOwnerHolder } from '../core/owner-holder.ts';
|
||||
|
||||
export interface CalibrationProfileRow {
|
||||
/** BIGSERIAL → string (postgres.js int8 wire shape; never Number() — int8
|
||||
@@ -167,7 +168,10 @@ export async function runCalibration(
|
||||
config: GBrainConfig,
|
||||
): Promise<void> {
|
||||
const { opts } = parseArgs(args);
|
||||
const holder = opts.holder ?? 'garry';
|
||||
const holder = resolveOwnerHolder({
|
||||
override: opts.holder,
|
||||
configValue: await engine.getConfig('emotional_weight.user_holder'),
|
||||
});
|
||||
// Resolve --source / GBRAIN_SOURCE / .gbrain-source so the (now reachable, #2035)
|
||||
// calibration command targets the right source in a multi-source brain instead
|
||||
// of always reading `default`. No signal → 'default' (prior behavior).
|
||||
@@ -253,12 +257,15 @@ export async function getCalibrationProfileOp(
|
||||
ctx: OperationContext,
|
||||
params: { holder?: string },
|
||||
): Promise<CalibrationProfileRow | null> {
|
||||
const holder = params.holder ?? 'garry';
|
||||
const holder = resolveOwnerHolder({
|
||||
override: params.holder,
|
||||
configValue: await ctx.engine.getConfig('emotional_weight.user_holder'),
|
||||
});
|
||||
if (typeof holder !== 'string' || holder.length === 0) {
|
||||
throw new GBrainError(
|
||||
'INVALID_HOLDER',
|
||||
'get_calibration_profile.holder must be a non-empty string',
|
||||
'pass holder="<slug>" or omit to default to "garry"',
|
||||
'pass holder="<slug>" or omit to default to the owner holder (config emotional_weight.user_holder, else "self")',
|
||||
);
|
||||
}
|
||||
const scope = sourceScopeOpts(ctx);
|
||||
|
||||
@@ -233,14 +233,18 @@ export function maybeRewriteSourceFkError(err: unknown, sourceId: string | undef
|
||||
|
||||
/**
|
||||
* Derive a title from the first non-empty, non-`---` line of the body,
|
||||
* stripping leading markdown heading marks, capped at 80 chars.
|
||||
* stripping leading markdown heading marks, capped at 80 chars. Truncation
|
||||
* is codepoint-aware (never splits an astral surrogate pair) and appends an
|
||||
* ellipsis so a cut title is visibly cut.
|
||||
* Falls back to 'Capture' when no usable line exists.
|
||||
*/
|
||||
function deriveTitle(rawBody: string): string {
|
||||
const firstLine = rawBody
|
||||
.split('\n')
|
||||
.find((l) => l.trim().length > 0 && l.trim() !== '---') ?? '';
|
||||
return firstLine.replace(/^#+\s*/, '').slice(0, 80) || 'Capture';
|
||||
const stripped = firstLine.replace(/^#+\s*/, '');
|
||||
const cps = [...stripped];
|
||||
return (cps.length > 80 ? cps.slice(0, 79).join('') + '…' : stripped) || 'Capture';
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -2,6 +2,7 @@ import { VERSION } from '../version.ts';
|
||||
import { detectInstallMethod } from './upgrade.ts';
|
||||
import {
|
||||
isMinorOrMajorBump,
|
||||
isNewerVersion,
|
||||
isValidVersionString,
|
||||
parseSemver,
|
||||
semverGt,
|
||||
@@ -21,7 +22,7 @@ function safeWriteCache(marker: UpdateMarker): void {
|
||||
// Back-compat re-exports: these used to live here; moved to ../core/semver.ts
|
||||
// so the self-upgrade decision module can depend on them without an import
|
||||
// cycle. Existing importers (`test/check-update.test.ts`, etc.) keep working.
|
||||
export { parseSemver, isMinorOrMajorBump };
|
||||
export { parseSemver, isMinorOrMajorBump, isNewerVersion };
|
||||
|
||||
interface CheckUpdateResult {
|
||||
current_version: string;
|
||||
@@ -131,7 +132,7 @@ export async function refreshUpdateCache(): Promise<void> {
|
||||
return;
|
||||
}
|
||||
const latestVersion = release.tag.replace(/^v/, '');
|
||||
if (!isValidVersionString(latestVersion) || !isMinorOrMajorBump(VERSION, latestVersion)) {
|
||||
if (!isValidVersionString(latestVersion) || !isNewerVersion(VERSION, latestVersion)) {
|
||||
safeWriteCache({ kind: 'up_to_date', current: VERSION });
|
||||
return;
|
||||
}
|
||||
@@ -140,7 +141,7 @@ export async function refreshUpdateCache(): Promise<void> {
|
||||
|
||||
export async function runCheckUpdate(args: string[]) {
|
||||
if (args.includes('--help') || args.includes('-h')) {
|
||||
console.log('Usage: gbrain check-update [--json] [--refresh-cache]\n\nCheck for new GBrain versions.\n\nOnly reports minor/major version bumps (v0.X.0), not patches.\nFails silently on network errors.\n\n--refresh-cache Fetch + update the self-upgrade cache, print nothing (used by\n the CLI startup hook\'s detached refresh).');
|
||||
console.log('Usage: gbrain check-update [--json] [--refresh-cache]\n\nCheck for new GBrain versions.\n\nReports any strictly newer release, including patch and micro updates.\nFails silently on network errors.\n\n--refresh-cache Fetch + update the self-upgrade cache, print nothing (used by\n the CLI startup hook\'s detached refresh).');
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -187,7 +188,7 @@ export async function runCheckUpdate(args: string[]) {
|
||||
}
|
||||
|
||||
const latestVersion = release.tag.replace(/^v/, '');
|
||||
const updateAvailable = isValidVersionString(latestVersion) && isMinorOrMajorBump(VERSION, latestVersion);
|
||||
const updateAvailable = isValidVersionString(latestVersion) && isNewerVersion(VERSION, latestVersion);
|
||||
|
||||
// Warm the self-upgrade cache so the next `gbrain <cmd>` startup hook can emit
|
||||
// the marker without a network call.
|
||||
|
||||
+592
-252
File diff suppressed because it is too large
Load Diff
+25
-5
@@ -26,6 +26,7 @@
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import {
|
||||
runCycle,
|
||||
resolveSourceForDir,
|
||||
ALL_PHASES,
|
||||
type CyclePhase,
|
||||
type CycleReport,
|
||||
@@ -85,7 +86,7 @@ interface DreamArgs {
|
||||
* `--phase <name>`; bare `--once` is a usage error (there'd be no single
|
||||
* phase to target). Applies only to phases with a config `.enabled` gate
|
||||
* (patterns, synthesize, conversation_facts_backfill, enrich_thin,
|
||||
* skillopt) — a no-op for phases that always run when named directly.
|
||||
* skillopt, drift) — a no-op for phases that always run when named directly.
|
||||
*/
|
||||
once: boolean;
|
||||
}
|
||||
@@ -366,7 +367,7 @@ Options:
|
||||
unlike toggling the flag on/off around the run, a
|
||||
crash mid-invocation can't leave it stuck. Applies to
|
||||
patterns, synthesize, conversation_facts_backfill,
|
||||
enrich_thin, skillopt; no-op on phases with no such
|
||||
enrich_thin, skillopt, drift; no-op on phases with no such
|
||||
gate. Requires an EXPLICIT --phase <name> — a phase
|
||||
implied by --input or --drain does not count (bare
|
||||
--once, or --once with --input/--drain and no
|
||||
@@ -380,9 +381,9 @@ Options:
|
||||
|
||||
--source <id> Scope the cycle to one source so doctor's
|
||||
cycle_freshness check sees a fresh stamp on
|
||||
completion. Without this, gbrain dream's
|
||||
timestamp never lands and federated brains
|
||||
see "stale cycle" forever.
|
||||
completion. When omitted, gbrain derives the
|
||||
source from --dir / the configured checkout
|
||||
when it matches a source's local_path (#1869).
|
||||
--source-id <id> Alias for --source. Matches the v0.37.7.0+
|
||||
naming used by import/extract/graph-query.
|
||||
|
||||
@@ -634,6 +635,25 @@ export async function runDream(engine: BrainEngine | null, args: string[]): Prom
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// #1869: a path-scoped run (--dir, or the configured sync.repo_path) whose
|
||||
// directory matches a registered source's local_path IS that source's cycle
|
||||
// — derive the source id so runCycle writes last_source_cycle_at /
|
||||
// last_full_cycle_at on success and doctor's cycle_freshness check stops
|
||||
// reading perpetually stale. Explicit --source still wins (resolved above).
|
||||
// Fixed here at the command level, NOT in runCycle's stamp gate, so legacy
|
||||
// global callers (autopilot-global-maintenance runs GLOBAL_PHASES with a
|
||||
// brainDir and no sourceId) can't falsely stamp per-source freshness.
|
||||
// A derived match on an archived source is skipped silently (falls back to
|
||||
// legacy unscoped behavior) — stamping it would mask staleness on restore,
|
||||
// mirroring the explicit --source archived guard above.
|
||||
if (resolvedSourceId === undefined && engine !== null && brainDir !== null) {
|
||||
const derived = await resolveSourceForDir(engine, brainDir);
|
||||
if (derived !== undefined) {
|
||||
const src = await fetchSource(engine, derived);
|
||||
if (src?.archived !== true) resolvedSourceId = derived;
|
||||
}
|
||||
}
|
||||
// ─── issue #1678: bounded single-hold extract_atoms drain ──────────
|
||||
if (opts.drain) {
|
||||
if (engine === null) {
|
||||
|
||||
+87
-19
@@ -107,6 +107,24 @@ export interface EmbedOpts {
|
||||
* runs lock every source in sorted order. dryRun skips it.
|
||||
*/
|
||||
singleFlight?: boolean;
|
||||
/**
|
||||
* #394: suppress human stdout summaries (the `[dry-run] Would embed ...` /
|
||||
* `Embedded N chunks ...` slog lines). Set by structured-output callers —
|
||||
* the cycle's embed phase (dream --json must keep stdout JSON-clean per
|
||||
* docs/progress-events.md) reports counts via its own PhaseResult instead.
|
||||
* Errors/warnings still go to stderr regardless.
|
||||
*/
|
||||
quiet?: boolean;
|
||||
/**
|
||||
* #3391: widen signature-drift invalidation to pages with NO recorded
|
||||
* embedding_signature (pre-v108). By default those are grandfathered
|
||||
* (never invalidated) so a routine upgrade doesn't surprise-re-embed a
|
||||
* whole corpus — but after a provider/model swap the grandfather clause
|
||||
* silently leaves them in the OLD embedding space, mixing two vector
|
||||
* spaces in one index. `gbrain migrate embeddings` and
|
||||
* `gbrain embed --stale --include-null-signature` set this.
|
||||
*/
|
||||
includeNullSignature?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -253,7 +271,7 @@ export async function runEmbedCore(engine: BrainEngine, opts: EmbedOpts): Promis
|
||||
for (const s of opts.slugs) {
|
||||
if (isAborted(opts.signal)) break; // #1737: stop the per-slug loop on abort
|
||||
try {
|
||||
await embedPage(engine, s, !!opts.dryRun, result, opts.sourceId, opts.signal);
|
||||
await embedPage(engine, s, !!opts.dryRun, result, opts.sourceId, opts.signal, opts.quiet);
|
||||
} catch (e: unknown) {
|
||||
serr(` Error embedding ${s}: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
@@ -347,6 +365,8 @@ export async function runEmbedCore(engine: BrainEngine, opts: EmbedOpts): Promis
|
||||
catchUp: opts.catchUp,
|
||||
pacer,
|
||||
paceMaxConcurrency,
|
||||
quiet: opts.quiet,
|
||||
includeNullSignature: opts.includeNullSignature,
|
||||
}, opts.signal);
|
||||
} finally {
|
||||
// E1: surface pacing telemetry (human + structured) when pacing was on.
|
||||
@@ -376,7 +396,7 @@ export async function runEmbedCore(engine: BrainEngine, opts: EmbedOpts): Promis
|
||||
return result;
|
||||
}
|
||||
if (opts.slug) {
|
||||
await embedPage(engine, opts.slug, !!opts.dryRun, result, opts.sourceId, opts.signal);
|
||||
await embedPage(engine, opts.slug, !!opts.dryRun, result, opts.sourceId, opts.signal, opts.quiet);
|
||||
return result;
|
||||
}
|
||||
throw new Error('No embed target specified. Pass { slug }, { slugs }, { all }, or { stale }.');
|
||||
@@ -460,6 +480,8 @@ export async function runEmbed(engine: BrainEngine, args: string[]): Promise<Emb
|
||||
const priorityRaw = priorityIdx >= 0 ? args[priorityIdx + 1] : undefined;
|
||||
const priority = priorityRaw === 'recent' ? 'recent' as const : undefined;
|
||||
const catchUp = args.includes('--catch-up');
|
||||
// #3391: re-embed pages that predate the embedding_signature stamp too.
|
||||
const includeNullSignature = args.includes('--include-null-signature');
|
||||
const pace = parsePaceArgs(args);
|
||||
|
||||
let opts: EmbedOpts;
|
||||
@@ -467,11 +489,11 @@ export async function runEmbed(engine: BrainEngine, args: string[]): Promise<Emb
|
||||
opts = { slugs: args.slice(slugsIdx + 1).filter(a => !a.startsWith('--')), dryRun, sourceId, batchSize, priority, catchUp };
|
||||
} else if (all || stale) {
|
||||
// E-2: CLI-only single-flight for stale runs (the minion path locks itself).
|
||||
opts = { all, stale, dryRun, sourceId, batchSize, priority, catchUp, ...(pace && { pace }), ...(stale && { singleFlight: true }) };
|
||||
opts = { all, stale, dryRun, sourceId, batchSize, priority, catchUp, ...(pace && { pace }), ...(stale && { singleFlight: true }), ...(includeNullSignature && { includeNullSignature: true }) };
|
||||
} else {
|
||||
const slug = args.find(a => !a.startsWith('--'));
|
||||
if (!slug) {
|
||||
serr('Usage: gbrain embed [<slug>|--all|--stale|--slugs s1 s2 ...] [--dry-run] [--batch-size N] [--priority recent] [--catch-up]');
|
||||
serr('Usage: gbrain embed [<slug>|--all|--stale|--slugs s1 s2 ...] [--dry-run] [--batch-size N] [--priority recent] [--catch-up] [--include-null-signature]');
|
||||
process.exit(1);
|
||||
}
|
||||
opts = { slug, dryRun, sourceId, batchSize, priority, catchUp };
|
||||
@@ -521,6 +543,7 @@ async function embedPage(
|
||||
result: EmbedResult,
|
||||
sourceId?: string,
|
||||
signal?: AbortSignal,
|
||||
quiet?: boolean,
|
||||
) {
|
||||
const opts = sourceId ? { sourceId } : undefined;
|
||||
const page = await engine.getPage(slug, opts);
|
||||
@@ -565,7 +588,7 @@ async function embedPage(
|
||||
result.skipped += chunks.length - toEmbed.length;
|
||||
|
||||
if (toEmbed.length === 0) {
|
||||
slog(`${slug}: all ${chunks.length} chunks already embedded`);
|
||||
if (!quiet) slog(`${slug}: all ${chunks.length} chunks already embedded`);
|
||||
result.pages_processed++;
|
||||
return;
|
||||
}
|
||||
@@ -602,7 +625,7 @@ async function embedPage(
|
||||
}
|
||||
result.embedded += toEmbed.length;
|
||||
result.pages_processed++;
|
||||
slog(`${slug}: embedded ${toEmbed.length} chunks`);
|
||||
if (!quiet) slog(`${slug}: embedded ${toEmbed.length} chunks`);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -645,6 +668,10 @@ async function embedAll(
|
||||
pacer?: DbPacer;
|
||||
/** Resolved concurrency cap (E-1: the worker count, no separate permit). */
|
||||
paceMaxConcurrency?: number;
|
||||
/** #394: suppress human stdout summaries (structured-output callers). */
|
||||
quiet?: boolean;
|
||||
/** #3391: lift the NULL-signature grandfather clause (see EmbedOpts). */
|
||||
includeNullSignature?: boolean;
|
||||
},
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
@@ -790,10 +817,12 @@ async function embedAll(
|
||||
});
|
||||
|
||||
// Stdout summary preserved for scripts/tests that grep for counts.
|
||||
if (dryRun) {
|
||||
slog(`[dry-run] Would embed ${result.would_embed} chunks across ${pages.length} pages`);
|
||||
} else {
|
||||
slog(`Embedded ${result.embedded} chunks across ${pages.length} pages`);
|
||||
if (!staleOpts?.quiet) {
|
||||
if (dryRun) {
|
||||
slog(`[dry-run] Would embed ${result.would_embed} chunks across ${pages.length} pages`);
|
||||
} else {
|
||||
slog(`Embedded ${result.embedded} chunks across ${pages.length} pages`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -829,6 +858,10 @@ async function embedAllStale(
|
||||
pacer?: DbPacer;
|
||||
/** Resolved concurrency cap (E-1: the worker count, no separate permit). */
|
||||
paceMaxConcurrency?: number;
|
||||
/** #394: suppress human stdout summaries (structured-output callers). */
|
||||
quiet?: boolean;
|
||||
/** #3391: lift the NULL-signature grandfather clause (see EmbedOpts). */
|
||||
includeNullSignature?: boolean;
|
||||
},
|
||||
signature?: string,
|
||||
externalSignal?: AbortSignal,
|
||||
@@ -836,6 +869,7 @@ async function embedAllStale(
|
||||
// D7: thread sourceId so source-scoped runs only count + visit
|
||||
// that source's NULL embeddings.
|
||||
const sourceOpt = sourceId ? { sourceId } : undefined;
|
||||
const includeNullSig = !!staleOpts?.includeNullSignature;
|
||||
|
||||
// v0.41.31: re-embed pages whose embedding_signature drifted (model/dims
|
||||
// swap). dry-run must NOT mutate, so it counts signature-stale via the
|
||||
@@ -845,22 +879,54 @@ async function embedAllStale(
|
||||
const invalidated = await engine.invalidateStaleSignatureEmbeddings({
|
||||
signature,
|
||||
...(sourceId && { sourceId }),
|
||||
...(includeNullSig && { includeNullSignature: true }),
|
||||
});
|
||||
if (invalidated > 0) {
|
||||
if (invalidated > 0 && !staleOpts?.quiet) {
|
||||
slog(`[embed] invalidated ${invalidated} chunk(s) embedded under a prior model signature`);
|
||||
}
|
||||
// #3391: the grandfather clause keeps NULL-signature pages on their OLD
|
||||
// vectors — two embedding spaces mixed in one index. Loud stderr warning
|
||||
// with the fix, instead of silent retrieval degradation.
|
||||
//
|
||||
// Deliberately NOT gated on `invalidated > 0`: the original bug report's
|
||||
// shape is a brain where EVERY embedded page predates the signature stamp,
|
||||
// so nothing drifts, nothing is invalidated — and pre-fix that brain got
|
||||
// no warning AND no work, the exact silent case #3391 is about. The probe
|
||||
// below computes the left-behind count directly, which is 0 on a healthy
|
||||
// brain, so an unaffected run stays quiet.
|
||||
if (!includeNullSig) {
|
||||
try {
|
||||
const wide = await engine.countStaleChunks({ ...sourceOpt, signature, includeNullSignature: true });
|
||||
const narrow = await engine.countStaleChunks({ ...sourceOpt, signature });
|
||||
const leftBehind = wide - narrow;
|
||||
if (leftBehind > 0) {
|
||||
serr(
|
||||
` [embed] WARNING: ${leftBehind} embedded chunk(s) sit on pages with no recorded ` +
|
||||
`embedding signature and were NOT invalidated — they remain in the previous model's ` +
|
||||
`embedding space. Re-run with --include-null-signature (or use ` +
|
||||
`\`gbrain migrate embeddings\`) to re-embed them.`,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// The warning probe is best-effort; never break the embed run.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-flight: 0 stale chunks → nothing to do, no further DB reads.
|
||||
// dry-run includes signature-drift in the count without mutating.
|
||||
const staleCount = await engine.countStaleChunks(
|
||||
dryRun && signature ? { ...sourceOpt, signature } : sourceOpt,
|
||||
dryRun && signature
|
||||
? { ...sourceOpt, signature, ...(includeNullSig && { includeNullSignature: true }) }
|
||||
: sourceOpt,
|
||||
);
|
||||
if (staleCount === 0) {
|
||||
if (dryRun) {
|
||||
slog('[dry-run] Would embed 0 chunks (0 stale found)');
|
||||
} else {
|
||||
slog('Embedded 0 chunks (0 stale found)');
|
||||
if (!staleOpts?.quiet) {
|
||||
if (dryRun) {
|
||||
slog('[dry-run] Would embed 0 chunks (0 stale found)');
|
||||
} else {
|
||||
slog('Embedded 0 chunks (0 stale found)');
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -869,7 +935,7 @@ async function embedAllStale(
|
||||
result.would_embed += staleCount;
|
||||
result.total_chunks += staleCount;
|
||||
if (onProgress) onProgress(1, 1, 0);
|
||||
slog(`[dry-run] Would embed ${staleCount} stale chunks`);
|
||||
if (!staleOpts?.quiet) slog(`[dry-run] Would embed ${staleCount} stale chunks`);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1112,7 +1178,7 @@ async function embedAllStale(
|
||||
if (budgetTimer) clearTimeout(budgetTimer);
|
||||
}
|
||||
|
||||
slog(`Embedded ${result.embedded} chunks across ${totalProcessedPages} pages`);
|
||||
if (!staleOpts?.quiet) slog(`Embedded ${result.embedded} chunks across ${totalProcessedPages} pages`);
|
||||
|
||||
// #1946 (OV2a): a catch-up pass that completed without being aborted but left
|
||||
// chunks unembedded means those chunks are stuck (a non-transient embed
|
||||
@@ -1120,7 +1186,9 @@ async function embedAllStale(
|
||||
// as a clean run — re-running won't help until the underlying failure is fixed.
|
||||
if (staleOpts?.catchUp && !effectiveSignal.aborted && embedFailures > 0) {
|
||||
const remaining = await engine.countStaleChunks(
|
||||
signature ? { signature, ...(sourceId ? { sourceId } : {}) } : (sourceId ? { sourceId } : undefined),
|
||||
signature
|
||||
? { signature, ...(sourceId ? { sourceId } : {}), ...(includeNullSig && { includeNullSignature: true }) }
|
||||
: (sourceId ? { sourceId } : undefined),
|
||||
);
|
||||
if (remaining > 0) {
|
||||
serr(`\n [embed] catch-up finished but ${remaining} chunk(s) remain stale after ${embedFailures} embed failure(s). These are not embeddable as-is; re-running won't clear them until the underlying error is resolved.`);
|
||||
|
||||
@@ -76,7 +76,7 @@ FLAGS:
|
||||
dimensions (goal, depth, sourcing, specificity, useful).
|
||||
--cycles N 1-3. Default: 3 in TTY, 1 in non-TTY (T11). Each
|
||||
cycle is 3 model calls; verdict aggregates over them.
|
||||
--slot-a-model <id> Override default 'openai:gpt-4o'.
|
||||
--slot-a-model <id> Override default 'openai:gpt-5.2'.
|
||||
--slot-b-model <id> Override default 'anthropic:claude-opus-4-7'.
|
||||
--slot-c-model <id> Override default 'google:gemini-1.5-pro'.
|
||||
--receipt-dir <path> Default: gbrainPath('eval-receipts').
|
||||
@@ -468,6 +468,14 @@ interface BatchRow {
|
||||
question_id: string;
|
||||
question: string;
|
||||
hypothesis: string;
|
||||
/**
|
||||
* Gold answer from the benchmark dataset, when the upstream eval emits
|
||||
* it (eval-longmemeval does). Folded into the judge task so CORRECTNESS
|
||||
* is verifiable — without it a judge panel that sees only
|
||||
* {question, hypothesis} cannot validate a terse factual answer against
|
||||
* a haystack it never saw.
|
||||
*/
|
||||
answer?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -581,6 +589,7 @@ function readBatchRows(path: string): BatchReadResult {
|
||||
question_id: typeof obj.question_id === 'string' ? obj.question_id : `line-${lineNo}`,
|
||||
question: obj.question,
|
||||
hypothesis: obj.hypothesis,
|
||||
...(typeof obj.answer === 'string' && obj.answer.length > 0 ? { answer: obj.answer } : {}),
|
||||
});
|
||||
}
|
||||
if (summarySkipped > 0) {
|
||||
@@ -697,7 +706,11 @@ async function runBatchMode(parsed: ParsedArgs, opts: RunCrossModalOpts): Promis
|
||||
fn: async (row, idx) => {
|
||||
process.stderr.write(`[eval cross-modal batch] ${idx + 1}/${rows.length} ${row.question_id} starting...\n`);
|
||||
return await runEvalFn({
|
||||
task: row.question,
|
||||
// With a gold answer the judges can actually verify correctness;
|
||||
// without one they see only {question, hypothesis} and cannot.
|
||||
task: row.answer
|
||||
? `${row.question}\n\nExpected answer (gold label from the benchmark dataset): ${row.answer}`
|
||||
: row.question,
|
||||
output: row.hypothesis,
|
||||
slug: row.question_id,
|
||||
dimensions,
|
||||
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
type AliasMap,
|
||||
} from '../eval/longmemeval/extract.ts';
|
||||
import { extractCandidateEntities } from '../core/think/entity-extract.ts';
|
||||
import { splitProviderModelId } from '../core/model-id.ts';
|
||||
import { resolveEntitySlugWithSource, type ResolutionSource } from '../core/entities/resolve.ts';
|
||||
import { formatTrajectoryBlock } from '../core/trajectory-format.ts';
|
||||
|
||||
@@ -469,14 +470,22 @@ export async function runEvalLongMemEval(args: string[], runOpts: RunOpts = {}):
|
||||
});
|
||||
|
||||
// Wrap Anthropic SDK so its `.messages.create` shape matches ThinkLLMClient.
|
||||
// Same pattern as src/core/think/index.ts:247-249.
|
||||
// Same pattern as src/core/think/index.ts:247-249 — EXCEPT think's default
|
||||
// client routes through the gateway, which parses `provider:model` recipe
|
||||
// ids. This eval's client is a raw SDK by design (hermetic, no gateway
|
||||
// dependency), and resolveModel returns RECIPE ids (`anthropic:claude-…`);
|
||||
// passing one through unstripped 404s every answer/extractor call, which
|
||||
// surfaces downstream as all-upstream_error batches in the nightly probe.
|
||||
const toSdkModel = (m: string): string => splitProviderModelId(m).model || m;
|
||||
const realClient = new Anthropic();
|
||||
const client: ThinkLLMClient = runOpts.client ?? {
|
||||
create: (params, callOpts) => realClient.messages.create(params, callOpts),
|
||||
create: (params, callOpts) =>
|
||||
realClient.messages.create({ ...params, model: toSdkModel(params.model) }, callOpts),
|
||||
};
|
||||
// v0.40.2.0 — separate extractor client (defaults to same SDK).
|
||||
const extractorClient: ThinkLLMClient = runOpts.extractorClient ?? {
|
||||
create: (params, callOpts) => realClient.messages.create(params, callOpts),
|
||||
create: (params, callOpts) =>
|
||||
realClient.messages.create({ ...params, model: toSdkModel(params.model) }, callOpts),
|
||||
};
|
||||
const trajectoryEnabled = !opts.noTrajectory;
|
||||
const extractorModel = trajectoryEnabled
|
||||
@@ -751,6 +760,11 @@ async function runOneQuestion(
|
||||
// v0.40.1.0 (Track D / T2) — copy question_type into the row so the
|
||||
// by_type_summary can be rebuilt from the file on resume runs.
|
||||
question_type: q.question_type,
|
||||
// Gold answer for downstream consumers that verify correctness (the
|
||||
// cross-modal --batch judge folds it into the task; evaluate_qa.py
|
||||
// ignores unknown fields). Without it a judge can't validate a terse
|
||||
// factual hypothesis against a haystack it never saw.
|
||||
...(q.answer !== undefined ? { answer: q.answer } : {}),
|
||||
hypothesis,
|
||||
retrieved_session_ids: retrievedSessionIds,
|
||||
...(recallHit !== undefined ? { recall_hit: recallHit } : {}),
|
||||
|
||||
@@ -43,11 +43,10 @@
|
||||
* (source_id, source_markdown_slug, row_num); per-segment row_num
|
||||
* would collide on segment 2. Per-page counter increments across
|
||||
* segments.
|
||||
* - Terminal audit row on completion. After all segments commit, one
|
||||
* extra fact row with source='cli:extract-conversation-facts:terminal'
|
||||
* marks the page complete. Doctor's backlog query checks for the
|
||||
* terminal row, NOT any fact — partial extraction → no terminal →
|
||||
* next run resumes.
|
||||
* - Snapshot-bound terminal audit row on completion. After all segments
|
||||
* commit, one v2 row binds completion to the exact page version or raw
|
||||
* transcript digest. Partial extraction has no matching terminal and the
|
||||
* next claim performs a delete-first full replay.
|
||||
* - Optional budgetTracker via opts. If a tracker is in opts, use it
|
||||
* as-is (NO `withBudgetTracker` wrap, which would REPLACE the active
|
||||
* tracker per gateway.ts AsyncLocalStorage semantics, defeating an
|
||||
@@ -68,7 +67,7 @@
|
||||
import type { BrainEngine, NewFact } from '../core/engine.ts';
|
||||
import type { Page } from '../core/types.ts';
|
||||
import {
|
||||
extractFactsFromTurn,
|
||||
extractFactsFromTurnWithOutcome,
|
||||
isFactsExtractionEnabled,
|
||||
} from '../core/facts/extract.ts';
|
||||
import { configureGatewayIfUninitialized, isAvailable, withBudgetTracker } from '../core/ai/gateway.ts';
|
||||
@@ -172,7 +171,15 @@ export const PER_SEGMENT_SOURCE_PREFIX = 'cli:extract-conversation-facts';
|
||||
* the per-segment source. Partial extraction = no terminal row = page
|
||||
* stays in backlog.
|
||||
*/
|
||||
export const TERMINAL_AUDIT_SOURCE = 'cli:extract-conversation-facts:terminal';
|
||||
export const TERMINAL_AUDIT_SOURCE = 'cli:extract-conversation-facts:terminal:v2';
|
||||
|
||||
/**
|
||||
* Durable outcome for a successfully scanned page that contains no eligible
|
||||
* multi-message segment. Kept distinct from successful extraction so operator
|
||||
* surfaces can report the truth without rescanning the page forever.
|
||||
*/
|
||||
export const NON_EXTRACTABLE_AUDIT_SOURCE =
|
||||
'cli:extract-conversation-facts:non-extractable:v2';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public types.
|
||||
@@ -253,6 +260,19 @@ export interface ExtractConversationFactsResult {
|
||||
pages_skipped: number;
|
||||
pages_skipped_too_large: number;
|
||||
pages_skipped_disappeared: number;
|
||||
/** Fresh terminal outcomes skipped before parsing or model work. */
|
||||
pages_skipped_completed: number;
|
||||
/** Fresh scanned-not-extractable outcomes skipped before parser work. */
|
||||
pages_skipped_non_extractable: number;
|
||||
/** Durable scanned-not-extractable outcomes written by this run. */
|
||||
pages_marked_non_extractable: number;
|
||||
/** Pages whose claim reached extraction but failed before durable outcome. */
|
||||
pages_failed: number;
|
||||
/**
|
||||
* Pages whose built-in parse returned `no_match` and whose messages were
|
||||
* recovered by the explicitly enabled LLM fallback.
|
||||
*/
|
||||
pages_llm_fallback: number;
|
||||
/**
|
||||
* v0.41.15.0 (D6): pages we attempted to claim but skipped because
|
||||
* another worker / parallel process held the advisory lock. The pages
|
||||
@@ -290,10 +310,13 @@ export interface ExtractConversationFactsResult {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
import {
|
||||
deriveDateContext,
|
||||
parseConversation,
|
||||
type ParseConversationOpts as OrchestratorParseOpts,
|
||||
} from '../core/conversation-parser/parse.ts';
|
||||
import { readConversationBodyForParsing } from '../core/conversation-parser/body.ts';
|
||||
import { runLlmFallback } from '../core/conversation-parser/llm-fallback.ts';
|
||||
import { resolveModel } from '../core/model-config.ts';
|
||||
|
||||
/**
|
||||
* v0.41.13.0 — back-compat shape for direct callers + the existing
|
||||
@@ -583,31 +606,21 @@ async function deleteOrphanFactsForPage(
|
||||
sourceId: string,
|
||||
slug: string,
|
||||
): Promise<number> {
|
||||
try {
|
||||
// The two write-source variants this command may have left behind:
|
||||
// - PER_SEGMENT_SOURCE_PREFIX ('cli:extract-conversation-facts')
|
||||
// - TERMINAL_AUDIT_SOURCE ('cli:extract-conversation-facts:terminal')
|
||||
// Using a LIKE prefix match covers both with one statement.
|
||||
const rows = await engine.executeRaw<{ count: string }>(
|
||||
`WITH del AS (
|
||||
DELETE FROM facts
|
||||
WHERE source_id = $1
|
||||
AND source_markdown_slug = $2
|
||||
AND source LIKE 'cli:extract-conversation-facts%'
|
||||
RETURNING 1
|
||||
)
|
||||
SELECT COUNT(*)::text AS count FROM del`,
|
||||
[sourceId, slug],
|
||||
);
|
||||
const n = parseInt(rows[0]?.count ?? '0', 10);
|
||||
return Number.isFinite(n) ? n : 0;
|
||||
} catch {
|
||||
// Best-effort: a missing source_markdown_slug column on pre-v0.32
|
||||
// brains (or other rare DDL drift) falls through to "no orphans
|
||||
// cleaned." The subsequent insertFacts call will surface any real
|
||||
// schema issues with a clearer error.
|
||||
return 0;
|
||||
}
|
||||
// A cleanup failure is authoritative: callers must not write a terminal or
|
||||
// non-extractable marker while facts from an older snapshot may remain.
|
||||
const rows = await engine.executeRaw<{ count: string }>(
|
||||
`WITH del AS (
|
||||
DELETE FROM facts
|
||||
WHERE source_id = $1
|
||||
AND source_markdown_slug = $2
|
||||
AND source LIKE 'cli:extract-conversation-facts%'
|
||||
RETURNING 1
|
||||
)
|
||||
SELECT COUNT(*)::text AS count FROM del`,
|
||||
[sourceId, slug],
|
||||
);
|
||||
const n = parseInt(rows[0]?.count ?? '0', 10);
|
||||
return Number.isFinite(n) ? n : 0;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -631,6 +644,12 @@ interface ExtractCoreState {
|
||||
* batch boundaries + final flush.
|
||||
*/
|
||||
cpMap: Map<string, string>;
|
||||
/**
|
||||
* Opt-in LLM parser state, resolved once per source run. A null model means
|
||||
* the fallback is disabled and no chat content leaves the deterministic
|
||||
* parser path.
|
||||
*/
|
||||
llmFallbackModel: string | null;
|
||||
}
|
||||
|
||||
function cpMapKey(sourceId: string, slug: string): string {
|
||||
@@ -663,11 +682,150 @@ function cpEntriesToMap(entries: string[]): Map<string, string> {
|
||||
return map;
|
||||
}
|
||||
|
||||
export type DurableExtractionOutcome = 'complete' | 'non_extractable';
|
||||
|
||||
interface ConversationPageSnapshot {
|
||||
page: Page;
|
||||
body: string;
|
||||
versionToken: string;
|
||||
}
|
||||
|
||||
function hasRawTranscriptSidecar(page: Page): boolean {
|
||||
const raw = page.frontmatter?.raw_transcript;
|
||||
return typeof raw === 'string' && raw.trim().length > 0;
|
||||
}
|
||||
|
||||
function regularPageVersionToken(page: Page): string {
|
||||
// content_hash covers title, type, compiled_truth, timeline, and frontmatter.
|
||||
// Unlike JavaScript Date, it cannot collapse distinct PostgreSQL updates that
|
||||
// happen within the same millisecond. effective_date is parser input too.
|
||||
const hash = page.content_hash ?? createHash('sha256')
|
||||
.update(JSON.stringify({
|
||||
title: page.title,
|
||||
type: page.type,
|
||||
compiled_truth: page.compiled_truth,
|
||||
timeline: page.timeline || '',
|
||||
frontmatter: page.frontmatter || {},
|
||||
}))
|
||||
.digest('hex');
|
||||
const effectiveDate = page.effective_date
|
||||
? new Date(page.effective_date).toISOString().slice(0, 10)
|
||||
: 'none';
|
||||
return `page-${hash}-${effectiveDate}`;
|
||||
}
|
||||
|
||||
function snapshotVersionToken(page: Page, body: string): string {
|
||||
if (!hasRawTranscriptSidecar(page)) return regularPageVersionToken(page);
|
||||
// Sidecar contents can change without touching pages.updated_at. Hash the
|
||||
// exact parser input plus parser-relevant page metadata so those edits reopen
|
||||
// the page without a schema migration.
|
||||
return `sidecar-${createHash('sha256')
|
||||
.update(
|
||||
JSON.stringify({
|
||||
body,
|
||||
title: page.title,
|
||||
type: page.type,
|
||||
frontmatter: page.frontmatter,
|
||||
effective_date: page.effective_date ?? null,
|
||||
}),
|
||||
)
|
||||
.digest('hex')}`;
|
||||
}
|
||||
|
||||
async function preparePageSnapshot(
|
||||
engine: BrainEngine,
|
||||
page: Page,
|
||||
): Promise<ConversationPageSnapshot> {
|
||||
const body = await readConversationBodyForParsing(engine, page);
|
||||
return { page, body, versionToken: snapshotVersionToken(page, body) };
|
||||
}
|
||||
|
||||
function outcomeSession(source: string, slug: string, versionToken: string): string {
|
||||
return `${source}:${slug}:${versionToken}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find v2 outcomes bound to the exact parser input snapshot. Legacy outcome
|
||||
* rows deliberately do not match and are replayed once under the strict v2
|
||||
* protocol. Sidecar files are hashed because pages.updated_at cannot see them.
|
||||
*/
|
||||
export async function findFreshExtractionOutcomes(
|
||||
engine: BrainEngine,
|
||||
sourceId: string,
|
||||
pages: readonly Page[],
|
||||
): Promise<Map<string, DurableExtractionOutcome>> {
|
||||
if (pages.length === 0) return new Map();
|
||||
const expected = new Map<string, string>();
|
||||
for (const page of pages) {
|
||||
// Batch enumeration can already be stale. Refresh before deciding to skip
|
||||
// so an edit between listPages and this check cannot match an old marker.
|
||||
const current = await engine.getPage(page.slug, { sourceId });
|
||||
if (!current) continue;
|
||||
const token = hasRawTranscriptSidecar(current)
|
||||
? (await preparePageSnapshot(engine, current)).versionToken
|
||||
: regularPageVersionToken(current);
|
||||
expected.set(current.slug, token);
|
||||
}
|
||||
const rows = await engine.executeRaw<{
|
||||
slug: string;
|
||||
source: string;
|
||||
source_session: string | null;
|
||||
}>(
|
||||
`SELECT source_markdown_slug AS slug, source, source_session
|
||||
FROM facts
|
||||
WHERE source_id = $1
|
||||
AND source_markdown_slug = ANY($2::text[])
|
||||
AND source = ANY($3::text[])
|
||||
ORDER BY source_markdown_slug,
|
||||
CASE WHEN source = $4 THEN 0 ELSE 1 END`,
|
||||
[
|
||||
sourceId,
|
||||
pages.map((page) => page.slug),
|
||||
[TERMINAL_AUDIT_SOURCE, NON_EXTRACTABLE_AUDIT_SOURCE],
|
||||
TERMINAL_AUDIT_SOURCE,
|
||||
],
|
||||
);
|
||||
const outcomes = new Map<string, DurableExtractionOutcome>();
|
||||
for (const row of rows) {
|
||||
if (outcomes.has(row.slug)) continue;
|
||||
const token = expected.get(row.slug);
|
||||
if (!token || row.source_session !== outcomeSession(row.source, row.slug, token)) {
|
||||
continue;
|
||||
}
|
||||
outcomes.set(
|
||||
row.slug,
|
||||
row.source === TERMINAL_AUDIT_SOURCE ? 'complete' : 'non_extractable',
|
||||
);
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
|
||||
function recordDurableOutcomeSkip(
|
||||
state: ExtractCoreState,
|
||||
outcome: DurableExtractionOutcome,
|
||||
): void {
|
||||
state.result.pages_considered++;
|
||||
if (outcome === 'complete') state.result.pages_skipped_completed++;
|
||||
else state.result.pages_skipped_non_extractable++;
|
||||
}
|
||||
|
||||
async function snapshotIsCurrent(
|
||||
engine: BrainEngine,
|
||||
sourceId: string,
|
||||
snapshot: ConversationPageSnapshot,
|
||||
): Promise<boolean> {
|
||||
const current = await engine.getPage(snapshot.page.slug, { sourceId });
|
||||
if (!current) return false;
|
||||
const currentSnapshot = await preparePageSnapshot(engine, current);
|
||||
return currentSnapshot.versionToken === snapshot.versionToken;
|
||||
}
|
||||
|
||||
async function processPage(
|
||||
state: ExtractCoreState,
|
||||
page: Page,
|
||||
snapshot: ConversationPageSnapshot,
|
||||
sinceIso: string | undefined,
|
||||
): Promise<{ newEndIso: string | null }> {
|
||||
const { page, body } = snapshot;
|
||||
state.result.pages_considered++;
|
||||
|
||||
// Body cap check first — pre-parse, pre-segment, pre-extraction.
|
||||
@@ -680,7 +838,6 @@ async function processPage(
|
||||
return { newEndIso: null };
|
||||
}
|
||||
|
||||
const body = await readConversationBodyForParsing(state.engine, page);
|
||||
// v0.41.13.0: thread the full Page through the orchestrator so D8
|
||||
// date-derivation chain (frontmatter.date > effective_date >
|
||||
// '1970-01-01') AND timezone_policy warnings apply. The historical
|
||||
@@ -688,13 +845,71 @@ async function processPage(
|
||||
// meant Telegram-bracket pages with frontmatter dates landed at
|
||||
// 1970-01-01. Now they pick up the correct date.
|
||||
const parseResult = parseConversation(body, { page });
|
||||
const messages = parseResult.messages;
|
||||
let messages = parseResult.messages;
|
||||
if (parseResult.timezone_warning) {
|
||||
process.stderr.write(parseResult.timezone_warning + '\n');
|
||||
}
|
||||
// The fallback runs only for a true built-in miss. It never replaces or
|
||||
// polishes a deterministic parse, and it remains unreachable unless the
|
||||
// operator explicitly enables conversation_parser.llm_fallback_enabled.
|
||||
if (
|
||||
!state.dryRun &&
|
||||
messages.length === 0 &&
|
||||
parseResult.phase === 'no_match' &&
|
||||
state.llmFallbackModel
|
||||
) {
|
||||
const fallbackMessages = await runLlmFallback({
|
||||
modelStr: state.llmFallbackModel,
|
||||
body,
|
||||
engine: state.engine,
|
||||
signal: state.signal,
|
||||
fallbackDate: deriveDateContext({ page }).fallbackDate,
|
||||
propagateError: (error) =>
|
||||
error instanceof BudgetExhausted ||
|
||||
(state.signal?.aborted === true && isAbortError(error)),
|
||||
});
|
||||
if (fallbackMessages && fallbackMessages.length > 0) {
|
||||
messages = fallbackMessages;
|
||||
state.result.pages_llm_fallback++;
|
||||
process.stderr.write(
|
||||
`[extract-conversation-facts] LLM fallback parsed ${fallbackMessages.length} message(s) for ${page.slug}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const allSegments = splitIntoSegments(messages);
|
||||
const segments = splitIntoSegments(messages, { sinceIso });
|
||||
if (segments.length === 0) {
|
||||
state.result.pages_skipped++;
|
||||
if (
|
||||
!state.dryRun &&
|
||||
parseResult.phase !== 'no_match' &&
|
||||
allSegments.length === 0
|
||||
) {
|
||||
if (await snapshotIsCurrent(state.engine, state.sourceId, snapshot)) {
|
||||
const cleaned = await deleteOrphanFactsForPage(
|
||||
state.engine,
|
||||
state.sourceId,
|
||||
page.slug,
|
||||
);
|
||||
state.result.orphan_facts_cleaned += cleaned;
|
||||
const rowNum = await peekRowNumStart(
|
||||
state.engine,
|
||||
state.sourceId,
|
||||
page.slug,
|
||||
);
|
||||
await writeNonExtractableAuditRow(
|
||||
state.engine,
|
||||
state.sourceId,
|
||||
page.slug,
|
||||
rowNum,
|
||||
snapshot.versionToken,
|
||||
messages.length === 0
|
||||
? 'no conversation messages found'
|
||||
: 'fewer than two eligible messages',
|
||||
);
|
||||
state.result.pages_marked_non_extractable++;
|
||||
}
|
||||
}
|
||||
return { newEndIso: null };
|
||||
}
|
||||
|
||||
@@ -730,24 +945,22 @@ async function processPage(
|
||||
const text = renderSegmentForExtraction(page.title || page.slug, seg);
|
||||
const sessionId = `${PER_SEGMENT_SOURCE_PREFIX}:${page.slug}`;
|
||||
|
||||
let extracted: Awaited<ReturnType<typeof extractFactsFromTurn>> = [];
|
||||
try {
|
||||
extracted = await extractFactsFromTurn({
|
||||
turnText: text,
|
||||
sessionId,
|
||||
source: PER_SEGMENT_SOURCE_PREFIX,
|
||||
engine: state.engine,
|
||||
abortSignal: state.signal,
|
||||
});
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
if (err instanceof BudgetExhausted) throw err;
|
||||
// Per-segment LLM failures are best-effort; loop continues.
|
||||
process.stderr.write(
|
||||
`[extract-conversation-facts] segment ${seg.startIso}..${seg.endIso} extractor failed: ${(err as Error).message}\n`,
|
||||
const extraction = await extractFactsFromTurnWithOutcome({
|
||||
turnText: text,
|
||||
sessionId,
|
||||
source: PER_SEGMENT_SOURCE_PREFIX,
|
||||
engine: state.engine,
|
||||
abortSignal: state.signal,
|
||||
});
|
||||
if (!extraction.ok) {
|
||||
const detail = extraction.error instanceof Error
|
||||
? `: ${extraction.error.message}`
|
||||
: '';
|
||||
throw new Error(
|
||||
`segment ${seg.startIso}..${seg.endIso} extraction failed (${extraction.reason})${detail}`,
|
||||
);
|
||||
extracted = [];
|
||||
}
|
||||
const extracted = extraction.facts;
|
||||
|
||||
state.result.segments_processed++;
|
||||
segmentsThisPage++;
|
||||
@@ -772,19 +985,9 @@ async function processPage(
|
||||
context:
|
||||
fact.context ?? `from ${page.slug} segment ${seg.startIso}..${seg.endIso}`,
|
||||
}));
|
||||
try {
|
||||
const ins = await state.engine.insertFacts(rows, { source_id: state.sourceId }); // gbrain-allow-direct-insert: canonical bulk extraction path for conversation pages — fences-as-system-of-record doesn't apply because conversations don't carry `## Facts` fences (the chat-log shape is the source-of-truth)
|
||||
pageInsertedTotal += ins.inserted;
|
||||
state.result.facts_inserted += ins.inserted;
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
// Batch failure is best-effort — segment is the transactional
|
||||
// boundary, so a duplicate-key or constraint error rolls back
|
||||
// this segment only. Loop continues.
|
||||
process.stderr.write(
|
||||
`[extract-conversation-facts] segment ${seg.startIso}..${seg.endIso} insertFacts failed: ${(err as Error).message}\n`,
|
||||
);
|
||||
}
|
||||
const ins = await state.engine.insertFacts(rows, { source_id: state.sourceId }); // gbrain-allow-direct-insert: canonical bulk extraction path for conversation pages — fences-as-system-of-record doesn't apply because conversations don't carry `## Facts` fences (the chat-log shape is the source-of-truth)
|
||||
pageInsertedTotal += ins.inserted;
|
||||
state.result.facts_inserted += ins.inserted;
|
||||
rowNum += extracted.length;
|
||||
} else {
|
||||
// dry-run: count for reporting, no DB write.
|
||||
@@ -800,20 +1003,28 @@ async function processPage(
|
||||
// segment (no break on segmentLimit; that's an explicit partial run).
|
||||
const fullyProcessed =
|
||||
state.segmentLimit === 0 || segmentsThisPage < state.segmentLimit;
|
||||
if (!state.dryRun && fullyProcessed && newestEnd !== null) {
|
||||
try {
|
||||
await writeTerminalAuditRow(state.engine, state.sourceId, page.slug, rowNum);
|
||||
rowNum++;
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
// Terminal-row write failure: page is NOT marked complete; next
|
||||
// run resumes. Loud stderr so users see partial-success state.
|
||||
process.stderr.write(
|
||||
`[extract-conversation-facts] ${page.slug} terminal audit write failed: ${(err as Error).message}\n`,
|
||||
);
|
||||
// Suppress the resume-state update so doctor still flags this page.
|
||||
newestEnd = null;
|
||||
}
|
||||
if (
|
||||
!state.dryRun &&
|
||||
fullyProcessed &&
|
||||
newestEnd !== null &&
|
||||
await snapshotIsCurrent(state.engine, state.sourceId, snapshot)
|
||||
) {
|
||||
// A terminal insert is part of the page transaction contract. Propagate
|
||||
// failure so bulk accounting, CLI exit status, cycle status, and rollups all
|
||||
// report the page as unfinished.
|
||||
await writeTerminalAuditRow(
|
||||
state.engine,
|
||||
state.sourceId,
|
||||
page.slug,
|
||||
rowNum,
|
||||
snapshot.versionToken,
|
||||
);
|
||||
rowNum++;
|
||||
} else if (!state.dryRun && fullyProcessed && newestEnd !== null) {
|
||||
process.stderr.write(
|
||||
`[extract-conversation-facts] ${page.slug} changed during extraction; leaving it unfinished for replay\n`,
|
||||
);
|
||||
newestEnd = null;
|
||||
}
|
||||
|
||||
if (!state.dryRun && newestEnd !== null) {
|
||||
@@ -838,13 +1049,14 @@ async function writeTerminalAuditRow(
|
||||
sourceId: string,
|
||||
slug: string,
|
||||
rowNum: number,
|
||||
versionToken: string,
|
||||
): Promise<void> {
|
||||
const fact: NewFact & { row_num: number; source_markdown_slug: string } = {
|
||||
fact: 'EXTRACTION_COMPLETE',
|
||||
kind: 'fact',
|
||||
entity_slug: null,
|
||||
source: TERMINAL_AUDIT_SOURCE,
|
||||
source_session: `${TERMINAL_AUDIT_SOURCE}:${slug}`,
|
||||
source_session: outcomeSession(TERMINAL_AUDIT_SOURCE, slug, versionToken),
|
||||
confidence: 1.0,
|
||||
notability: 'low',
|
||||
row_num: rowNum,
|
||||
@@ -863,6 +1075,33 @@ async function writeTerminalAuditRow(
|
||||
* - If absent: create a fresh tracker scoped to `opts.maxCostUsd`
|
||||
* and run the body inside `withBudgetTracker`.
|
||||
*/
|
||||
async function writeNonExtractableAuditRow(
|
||||
engine: BrainEngine,
|
||||
sourceId: string,
|
||||
slug: string,
|
||||
rowNum: number,
|
||||
versionToken: string,
|
||||
reason: string,
|
||||
): Promise<void> {
|
||||
const fact: NewFact & { row_num: number; source_markdown_slug: string } = {
|
||||
fact: 'EXTRACTION_NOT_APPLICABLE',
|
||||
kind: 'fact',
|
||||
entity_slug: null,
|
||||
source: NON_EXTRACTABLE_AUDIT_SOURCE,
|
||||
source_session: outcomeSession(
|
||||
NON_EXTRACTABLE_AUDIT_SOURCE,
|
||||
slug,
|
||||
versionToken,
|
||||
),
|
||||
confidence: 1.0,
|
||||
notability: 'low',
|
||||
context: `scanned, not extractable: ${reason}`,
|
||||
row_num: rowNum,
|
||||
source_markdown_slug: slug,
|
||||
};
|
||||
await engine.insertFacts([fact], { source_id: sourceId }); // gbrain-allow-direct-insert: durable non-extractable audit outcome prevents repeated scans while remaining distinct from successful extraction
|
||||
}
|
||||
|
||||
export async function runExtractConversationFactsCore(
|
||||
engine: BrainEngine,
|
||||
opts: ExtractConversationFactsCoreOpts,
|
||||
@@ -879,6 +1118,11 @@ export async function runExtractConversationFactsCore(
|
||||
pages_skipped: 0,
|
||||
pages_skipped_too_large: 0,
|
||||
pages_skipped_disappeared: 0,
|
||||
pages_skipped_completed: 0,
|
||||
pages_skipped_non_extractable: 0,
|
||||
pages_marked_non_extractable: 0,
|
||||
pages_failed: 0,
|
||||
pages_llm_fallback: 0,
|
||||
pages_lock_skipped: 0,
|
||||
orphan_facts_cleaned: 0,
|
||||
segments_processed: 0,
|
||||
@@ -924,6 +1168,18 @@ export async function runExtractConversationFactsCore(
|
||||
);
|
||||
const workers = workersResolved.workers;
|
||||
|
||||
// Privacy boundary: the parser never sends page content to an LLM unless
|
||||
// this exact DB-plane key is explicitly true. Resolve the model once rather
|
||||
// than probing configuration for every page.
|
||||
const llmFallbackEnabled =
|
||||
(await engine.getConfig('conversation_parser.llm_fallback_enabled')) === 'true';
|
||||
const llmFallbackModel = llmFallbackEnabled
|
||||
? await resolveModel(engine, {
|
||||
tier: 'utility',
|
||||
fallback: 'anthropic:claude-haiku-4-5-20251001',
|
||||
})
|
||||
: null;
|
||||
|
||||
const state: ExtractCoreState = {
|
||||
result,
|
||||
engine,
|
||||
@@ -934,6 +1190,7 @@ export async function runExtractConversationFactsCore(
|
||||
types,
|
||||
signal,
|
||||
cpMap: new Map(),
|
||||
llmFallbackModel,
|
||||
};
|
||||
|
||||
// Run body. Either inside the externally-provided tracker scope (no
|
||||
@@ -957,21 +1214,41 @@ export async function runExtractConversationFactsCore(
|
||||
*/
|
||||
const processPageWithLock = async (page: Page): Promise<void> => {
|
||||
const lockId = extractConversationFactsLockId(sourceId, page.slug);
|
||||
|
||||
let sinceIso: string | undefined;
|
||||
// Per-page resume: --force clears prior entries; normal path uses
|
||||
// the latest endIso for this (sourceId, slug) from the shared map.
|
||||
if (opts.force) {
|
||||
state.cpMap.delete(cpMapKey(sourceId, page.slug));
|
||||
}
|
||||
const checkpointed = state.cpMap.get(cpMapKey(sourceId, page.slug)) ?? null;
|
||||
sinceIso = pickLaterIso(checkpointed, opts.sinceIso);
|
||||
|
||||
try {
|
||||
await withRefreshingLock(
|
||||
engine,
|
||||
lockId,
|
||||
() => processPage(state, page, sinceIso),
|
||||
async () => {
|
||||
// Re-fetch under the advisory lock. Batch enumeration is only a
|
||||
// candidate list; it must never become the snapshot we certify.
|
||||
const currentPage = await engine.getPage(page.slug, { sourceId });
|
||||
if (!currentPage) {
|
||||
state.result.pages_skipped_disappeared++;
|
||||
return { newEndIso: null };
|
||||
}
|
||||
|
||||
// Close the race between batch selection and lock acquisition.
|
||||
if (!opts.force) {
|
||||
const outcome = (
|
||||
await findFreshExtractionOutcomes(engine, sourceId, [currentPage])
|
||||
).get(currentPage.slug);
|
||||
if (outcome) {
|
||||
recordDurableOutcomeSkip(state, outcome);
|
||||
return { newEndIso: null };
|
||||
}
|
||||
}
|
||||
|
||||
// A checkpoint without a matching durable v2 outcome cannot prove
|
||||
// which page snapshot it describes. Clear it and replay safely;
|
||||
// delete-orphans-first makes that replay deterministic.
|
||||
state.cpMap.delete(cpMapKey(sourceId, currentPage.slug));
|
||||
const snapshot = await preparePageSnapshot(engine, currentPage);
|
||||
return processPage(state, snapshot, opts.sinceIso);
|
||||
},
|
||||
{ ttlMinutes: PER_PAGE_LOCK_TTL_MINUTES },
|
||||
).then(() => undefined);
|
||||
} catch (err) {
|
||||
@@ -1022,21 +1299,59 @@ export async function runExtractConversationFactsCore(
|
||||
});
|
||||
if (batch.length === 0) break;
|
||||
|
||||
// Respect --limit at batch granularity: clip the batch so we
|
||||
// never overshoot the cap by `workers - 1` extra pages.
|
||||
let claimable = batch;
|
||||
if (opts.limit) {
|
||||
const remaining = opts.limit - processedPagesCount;
|
||||
if (remaining < batch.length) claimable = batch.slice(0, remaining);
|
||||
// Checkpoints are an intra-page cursor; fresh durable outcomes are
|
||||
// the page-level selection authority and survive checkpoint GC.
|
||||
if (!opts.force && claimable.length > 0) {
|
||||
const fresh = await findFreshExtractionOutcomes(
|
||||
engine,
|
||||
sourceId,
|
||||
claimable,
|
||||
);
|
||||
claimable = claimable.filter((page) => {
|
||||
const outcome = fresh.get(page.slug);
|
||||
if (!outcome) return true;
|
||||
recordDurableOutcomeSkip(state, outcome);
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
await runSlidingPool({
|
||||
// Apply --limit after durable filtering. The limit caps pages that
|
||||
// need work, not already-completed pages scanned to find that work.
|
||||
if (opts.limit) {
|
||||
const remaining = opts.limit - processedPagesCount;
|
||||
if (remaining < claimable.length) {
|
||||
claimable = claimable.slice(0, remaining);
|
||||
}
|
||||
}
|
||||
|
||||
const poolResult = await runSlidingPool({
|
||||
items: claimable,
|
||||
workers,
|
||||
signal,
|
||||
onItem: (page) => processPageWithLock(page),
|
||||
onError: (error) => (isAbortError(error) ? 'abort' : 'continue'),
|
||||
failureLabel: (page) => page.slug,
|
||||
});
|
||||
const cancellation = poolResult.failures.find((failure) =>
|
||||
isAbortError(failure.error),
|
||||
);
|
||||
if (cancellation) throw cancellation.error;
|
||||
if (signal?.aborted) {
|
||||
if (signal.reason instanceof Error) throw signal.reason;
|
||||
throw Object.assign(new Error('caller cancelled'), {
|
||||
name: 'AbortError',
|
||||
});
|
||||
}
|
||||
result.pages_failed += poolResult.errored;
|
||||
for (const failure of poolResult.failures) {
|
||||
const message = failure.error instanceof Error
|
||||
? failure.error.message
|
||||
: String(failure.error);
|
||||
process.stderr.write(
|
||||
`[extract-conversation-facts] ${failure.label} failed: ${message}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
processedPagesCount += claimable.length;
|
||||
offset += batch.length;
|
||||
@@ -1057,6 +1372,7 @@ export async function runExtractConversationFactsCore(
|
||||
}
|
||||
};
|
||||
|
||||
let ownedTracker: BudgetTracker | null = null;
|
||||
try {
|
||||
if (opts.budgetTracker) {
|
||||
// Caller-managed scope — use as-is, no wrap (nested wrap REPLACES
|
||||
@@ -1067,6 +1383,7 @@ export async function runExtractConversationFactsCore(
|
||||
maxCostUsd: opts.maxCostUsd ?? DEFAULT_MAX_COST_USD,
|
||||
label: `extract-conversation-facts:${sourceId}`,
|
||||
});
|
||||
ownedTracker = tracker;
|
||||
try {
|
||||
await withBudgetTracker(tracker, body);
|
||||
} finally {
|
||||
@@ -1090,13 +1407,34 @@ export async function runExtractConversationFactsCore(
|
||||
throw err;
|
||||
}
|
||||
|
||||
// gateway.chat preserves a successful provider result when the final
|
||||
// tracker.record() discovers an underestimated overage. Usually the next
|
||||
// reserve surfaces it, but a fallback that yields fewer than two messages
|
||||
// has no next call. Detect that terminal overage so the result and rollup
|
||||
// remain honest.
|
||||
const effectiveTracker = opts.budgetTracker ?? ownedTracker;
|
||||
if (
|
||||
effectiveTracker?.cap !== undefined &&
|
||||
effectiveTracker.totalSpent > effectiveTracker.cap
|
||||
) {
|
||||
result.budget_exhausted = true;
|
||||
result.spent_usd = effectiveTracker.totalSpent;
|
||||
}
|
||||
|
||||
// v0.42 — Wave B1: extract-conversation-facts writes a receipt page
|
||||
// (queryable + citable per D-EXTRACT-17/19) AND UPSERTs the per-day
|
||||
// rollup row (best-effort cache per F-OUT-19). Both are best-effort —
|
||||
// failures stderr-warn but never fail the parent operation.
|
||||
// --dry-run must not persist cache/knowledge state: skip the rollup UPSERT +
|
||||
// receipt-page write so a preview leaves no extract cache row behind.
|
||||
if (!dryRun) await writeRunReceiptAndRollup(engine, sourceId, result, /* halted */ false);
|
||||
if (!dryRun) {
|
||||
await writeRunReceiptAndRollup(
|
||||
engine,
|
||||
sourceId,
|
||||
result,
|
||||
/* halted */ result.budget_exhausted === true,
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -1134,7 +1472,12 @@ async function writeRunReceiptAndRollup(
|
||||
extracted_at: now,
|
||||
total_rows: result.facts_inserted,
|
||||
cost_usd: result.spent_usd ?? 0,
|
||||
summary: `Extracted ${result.facts_inserted} facts from ${result.pages_processed}/${result.pages_considered} eligible pages.`,
|
||||
summary:
|
||||
`Extracted ${result.facts_inserted} facts from ` +
|
||||
`${result.pages_processed}/${result.pages_considered} eligible pages` +
|
||||
(result.pages_failed > 0
|
||||
? `; ${result.pages_failed} page(s) failed and remain unfinished.`
|
||||
: '.'),
|
||||
});
|
||||
} catch (err) {
|
||||
// Best-effort: receipt write failure shouldn't kill the run.
|
||||
@@ -1148,12 +1491,13 @@ async function writeRunReceiptAndRollup(
|
||||
// Rollup UPSERT: ALWAYS fire so doctor's extract_health sees the
|
||||
// cycle ran (even no-op runs are signal — they prove the extractor
|
||||
// was alive). Best-effort per F-OUT-19.
|
||||
const incomplete = halted || result.pages_failed > 0;
|
||||
await upsertExtractRollup(engine, {
|
||||
kind: 'facts.conversation',
|
||||
source_id: sourceId,
|
||||
cost_delta: result.spent_usd ?? 0,
|
||||
round_completed_delta: halted ? 0 : 1,
|
||||
halt_delta: halted ? 1 : 0,
|
||||
round_completed_delta: incomplete ? 0 : 1,
|
||||
halt_delta: incomplete ? 1 : 0,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1381,6 +1725,11 @@ export async function runExtractConversationFacts(
|
||||
pages_skipped: 0,
|
||||
pages_skipped_too_large: 0,
|
||||
pages_skipped_disappeared: 0,
|
||||
pages_skipped_completed: 0,
|
||||
pages_skipped_non_extractable: 0,
|
||||
pages_marked_non_extractable: 0,
|
||||
pages_failed: 0,
|
||||
pages_llm_fallback: 0,
|
||||
pages_lock_skipped: 0,
|
||||
orphan_facts_cleaned: 0,
|
||||
segments_processed: 0,
|
||||
@@ -1421,6 +1770,11 @@ export async function runExtractConversationFacts(
|
||||
aggregate.pages_skipped += perSource.pages_skipped;
|
||||
aggregate.pages_skipped_too_large += perSource.pages_skipped_too_large;
|
||||
aggregate.pages_skipped_disappeared += perSource.pages_skipped_disappeared;
|
||||
aggregate.pages_skipped_completed += perSource.pages_skipped_completed;
|
||||
aggregate.pages_skipped_non_extractable += perSource.pages_skipped_non_extractable;
|
||||
aggregate.pages_marked_non_extractable += perSource.pages_marked_non_extractable;
|
||||
aggregate.pages_failed += perSource.pages_failed;
|
||||
aggregate.pages_llm_fallback += perSource.pages_llm_fallback;
|
||||
aggregate.pages_lock_skipped += perSource.pages_lock_skipped;
|
||||
aggregate.orphan_facts_cleaned += perSource.orphan_facts_cleaned;
|
||||
aggregate.segments_processed += perSource.segments_processed;
|
||||
@@ -1452,6 +1806,21 @@ export async function runExtractConversationFacts(
|
||||
if (aggregate.pages_skipped_disappeared > 0) {
|
||||
console.log(` Skipped ${aggregate.pages_skipped_disappeared} page(s) that disappeared between enumeration and fetch.`);
|
||||
}
|
||||
if (aggregate.pages_skipped_completed > 0) {
|
||||
console.log(` Skipped ${aggregate.pages_skipped_completed} page(s) with fresh durable completion outcomes.`);
|
||||
}
|
||||
if (aggregate.pages_skipped_non_extractable > 0) {
|
||||
console.log(` Skipped ${aggregate.pages_skipped_non_extractable} page(s) previously scanned as not extractable.`);
|
||||
}
|
||||
if (aggregate.pages_marked_non_extractable > 0) {
|
||||
console.log(` Marked ${aggregate.pages_marked_non_extractable} page(s) as scanned, not extractable.`);
|
||||
}
|
||||
if (aggregate.pages_failed > 0) {
|
||||
console.error(` Failed ${aggregate.pages_failed} page(s); they remain unfinished and will retry.`);
|
||||
}
|
||||
if (aggregate.pages_llm_fallback > 0) {
|
||||
console.log(` Parsed ${aggregate.pages_llm_fallback} page(s) with the opt-in LLM fallback.`);
|
||||
}
|
||||
if (aggregate.pages_lock_skipped > 0) {
|
||||
console.log(` Skipped ${aggregate.pages_lock_skipped} page(s) held by another worker / process (will retry next run).`);
|
||||
}
|
||||
@@ -1468,6 +1837,9 @@ export async function runExtractConversationFacts(
|
||||
// anyBudgetExhausted doesn't trigger exit 3; the budget message
|
||||
// above already tells the user what to do, and exit 0 is the right
|
||||
// signal for "ran to the cap intentionally."
|
||||
if (aggregate.pages_failed > 0) {
|
||||
process.exit(1);
|
||||
}
|
||||
if (aggregate.pages_lock_skipped > 0 && !anyBudgetExhausted) {
|
||||
process.exit(3);
|
||||
}
|
||||
|
||||
+49
-47
@@ -433,7 +433,10 @@ export async function extractLinksFromFile(
|
||||
async resolve(name: string, dirHint?: string | string[]): Promise<string | null> {
|
||||
if (!name) return null;
|
||||
const trimmed = name.trim();
|
||||
if (/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(trimmed) && allSlugs.has(trimmed)) {
|
||||
// Same broadened slug-shape as makeResolver step 1: accepts
|
||||
// digit-leading folders (`90-people/nicolai`) and nested paths.
|
||||
// Exact Set membership guards it — no false positives.
|
||||
if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed) && allSlugs.has(trimmed)) {
|
||||
return trimmed;
|
||||
}
|
||||
const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []);
|
||||
@@ -469,53 +472,15 @@ export async function extractLinksFromFile(
|
||||
|
||||
// --- Timeline extraction ---
|
||||
|
||||
/**
|
||||
* Index of the first dash (—, –, -) that can serve as the Source — Summary
|
||||
* delimiter: it must have whitespace on both sides and sit outside every
|
||||
* markdown-link span. Hyphens inside link targets
|
||||
* (`../people/alice-example.md`) and dashes inside link labels
|
||||
* (`[Deals — Q1 Review](...)`) are content, not delimiters — splitting on
|
||||
* them shatters one entry into two fragments whose halves re-insert on
|
||||
* every sync (the (page_id, date, summary, source) uniqueness sees each
|
||||
* fragment shape as a new row). Returns -1 when the line has no delimiter.
|
||||
*/
|
||||
function findDelimiterOutsideLinks(text: string): number {
|
||||
let depth = 0;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const c = text[i];
|
||||
if (c === '[' || c === '(') depth++;
|
||||
else if (c === ']' || c === ')') { if (depth > 0) depth--; }
|
||||
else if (
|
||||
depth === 0 &&
|
||||
(c === '—' || c === '–' || c === '-') &&
|
||||
i > 0 && /\s/.test(text[i - 1]) &&
|
||||
i + 1 < text.length && /\s/.test(text[i + 1])
|
||||
) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/** Extract timeline entries from markdown content */
|
||||
export function extractTimelineFromContent(content: string, slug: string): ExtractedTimelineEntry[] {
|
||||
const entries: ExtractedTimelineEntry[] = [];
|
||||
|
||||
// Format 1: Bullet — - **YYYY-MM-DD** | Source — Summary
|
||||
// The delimiter search is link-aware (see findDelimiterOutsideLinks); a
|
||||
// bullet with no delimiter (e.g. an auto-generated backlink line
|
||||
// `- **date** | Referenced in [X](y.md)`) is kept whole as the summary
|
||||
// rather than dropped or fragmented.
|
||||
const bulletPattern = /^-\s+\*\*(\d{4}-\d{2}-\d{2})\*\*\s*\|\s*(.+)$/gm;
|
||||
const bulletPattern = /^-\s+\*\*(\d{4}-\d{2}-\d{2})\*\*\s*\|\s*(.+?)\s*[—–-]\s*(.+)$/gm;
|
||||
let match;
|
||||
while ((match = bulletPattern.exec(content)) !== null) {
|
||||
const rest = match[2].trim();
|
||||
const at = findDelimiterOutsideLinks(rest);
|
||||
if (at >= 0) {
|
||||
entries.push({ slug, date: match[1], source: rest.slice(0, at).trim(), summary: rest.slice(at + 1).trim() });
|
||||
} else {
|
||||
entries.push({ slug, date: match[1], source: 'markdown', summary: rest });
|
||||
}
|
||||
entries.push({ slug, date: match[1], source: match[2].trim(), summary: match[3].trim() });
|
||||
}
|
||||
|
||||
// Format 2: Header — ### YYYY-MM-DD — Title
|
||||
@@ -620,6 +585,17 @@ export interface ExtractOpts {
|
||||
* before (single-'default'-source brains unaffected).
|
||||
*/
|
||||
sourceId?: string;
|
||||
/**
|
||||
* v0.42 — also extract frontmatter links on the incremental (slugs) path.
|
||||
* `extractForSlugs` extracts BODY links only by default; set this true to also
|
||||
* parse each changed page's frontmatter so `sources:`/`related:` edges stay fresh
|
||||
* when YAML is edited externally and synced in. Applied PER changed page, so the
|
||||
* incremental walk stays bounded (no switch to a full DB scan). Only honored on
|
||||
* the incremental path (`slugs` defined); the full-walk path already covers
|
||||
* frontmatter via its own dispatch. Gated upstream by the config key
|
||||
* `autopilot.incremental_extract_include_frontmatter` (default off).
|
||||
*/
|
||||
includeFrontmatter?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -658,7 +634,7 @@ export async function runExtractCore(engine: BrainEngine, opts: ExtractOpts): Pr
|
||||
// Nothing changed — skip entirely.
|
||||
return result;
|
||||
}
|
||||
const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId);
|
||||
const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId, opts.includeFrontmatter);
|
||||
result.links_created = r.links_created;
|
||||
result.timeline_entries_created = r.timeline_created;
|
||||
result.pages_processed = r.pages;
|
||||
@@ -1049,6 +1025,11 @@ async function extractForSlugs(
|
||||
signal?: AbortSignal,
|
||||
// #1747/#1503: stamp resolved brain source id on batch rows (see ExtractOpts.sourceId).
|
||||
sourceId?: string,
|
||||
// v0.42: when true, also extract frontmatter links per changed page so
|
||||
// externally-edited YAML (`sources:`/`related:`) stays fresh on the cycle.
|
||||
// Default false preserves the body-only incremental behavior. Gated upstream
|
||||
// by `autopilot.incremental_extract_include_frontmatter`.
|
||||
includeFrontmatter: boolean = false,
|
||||
): Promise<{ links_created: number; timeline_created: number; pages: number }> {
|
||||
// Build the full slug set for link resolution (fast: just readdir, no file reads)
|
||||
const allFiles = walkMarkdownFiles(brainDir);
|
||||
@@ -1063,6 +1044,10 @@ async function extractForSlugs(
|
||||
let linksCreated = 0;
|
||||
let timelineCreated = 0;
|
||||
let pagesProcessed = 0;
|
||||
// #2636: successfully processed pages get their extraction watermark
|
||||
// stamped after the final flush (mode 'all' only — a partial-mode run
|
||||
// hasn't done the full extraction the watermark asserts).
|
||||
const processedRefs: Array<{ slug: string; source_id: string }> = [];
|
||||
|
||||
// Issue #972: read the basename flag once per extract run.
|
||||
const globalBasename = await isGlobalBasenameEnabled(engine);
|
||||
@@ -1123,7 +1108,7 @@ async function extractForSlugs(
|
||||
const content = readFileSync(fullPath, 'utf-8');
|
||||
|
||||
if (doLinks) {
|
||||
const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename });
|
||||
const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename, includeFrontmatter });
|
||||
for (const link of links) {
|
||||
if (dryRun) {
|
||||
if (!jsonMode) console.log(` ${link.from_slug} → ${link.to_slug} (${link.link_type})`);
|
||||
@@ -1151,6 +1136,7 @@ async function extractForSlugs(
|
||||
}
|
||||
|
||||
pagesProcessed++;
|
||||
if (!dryRun) processedRefs.push({ slug, source_id: sourceId ?? 'default' });
|
||||
} catch { /* skip unreadable */ }
|
||||
progress.tick(1);
|
||||
},
|
||||
@@ -1158,6 +1144,13 @@ async function extractForSlugs(
|
||||
|
||||
await flushLinks();
|
||||
await flushTimeline();
|
||||
// #2636: the Dream cycle disables sync's inline extraction and routes
|
||||
// changed slugs through this incremental path — without a stamp here,
|
||||
// those pages never get links_extracted_at and stay permanently visible
|
||||
// to `extract --stale` / doctor. Stamp only after BOTH batches flushed.
|
||||
if (!dryRun && mode === 'all') {
|
||||
await stampExtracted(engine, processedRefs);
|
||||
}
|
||||
progress.finish();
|
||||
|
||||
if (!jsonMode) {
|
||||
@@ -1722,9 +1715,17 @@ export async function extractStaleFromDB(
|
||||
// Batch mode = pg_trgm + exact only, NO per-name search fallback. The
|
||||
// resolution map sees ALL sources so qualified cross-source wikilinks resolve
|
||||
// even when --source-id scopes the stale SCAN.
|
||||
const resolver = makeResolver(engine, { mode: 'batch' });
|
||||
const nullResolver = { resolve: async () => null as string | null };
|
||||
const activeResolver = includeFrontmatter ? resolver : nullResolver;
|
||||
//
|
||||
// #2576 bug 1: ALWAYS the real resolver — extractPageLinks's opts gate which
|
||||
// pass runs (`skipFrontmatter` for the frontmatter pass, `globalBasename` for
|
||||
// the issue-#972 bare-wikilink pass). The former `includeFrontmatter ?
|
||||
// resolver : nullResolver` ternary predates #972; the synthetic resolver has
|
||||
// no `resolveBasenameMatches`, so the --stale sweep silently skipped basename
|
||||
// resolution even with `link_resolution.global_basename` enabled, stamping
|
||||
// pages as extracted with their bare wikilinks dropped. Mirrors
|
||||
// extractLinksFromDB (including the codex-[P1] `sourceId` scoping).
|
||||
const resolver = makeResolver(engine, { mode: 'batch', sourceId: sourceIdFilter });
|
||||
const globalBasename = await isGlobalBasenameEnabled(engine);
|
||||
const allRefs = await engine.listAllPageRefs();
|
||||
const allSlugs = new Set<string>();
|
||||
const slugToSources = new Map<string, string[]>();
|
||||
@@ -1756,7 +1757,8 @@ export async function extractStaleFromDB(
|
||||
for (const page of rows) {
|
||||
const fullContent = page.compiled_truth + '\n' + page.timeline;
|
||||
const extracted = await extractPageLinks(
|
||||
page.slug, fullContent, page.frontmatter, page.type, activeResolver,
|
||||
page.slug, fullContent, page.frontmatter, page.type, resolver,
|
||||
{ skipFrontmatter: !includeFrontmatter, globalBasename },
|
||||
);
|
||||
for (const c of extracted.candidates) {
|
||||
const r = resolveCandidateSources(c, page.slug, page.source_id, allSlugs, slugToSources);
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
|
||||
import { readFileSync, writeFileSync, existsSync, lstatSync, readdirSync } from 'fs';
|
||||
import { setCliExitVerdict } from '../core/cli-force-exit.ts';
|
||||
import { join, relative, resolve } from 'path';
|
||||
import { join, relative, resolve, basename, dirname } from 'path';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { loadConfig, toEngineConfig } from '../core/config.ts';
|
||||
import { createEngine } from '../core/engine-factory.ts';
|
||||
@@ -155,6 +155,27 @@ interface FileValidation {
|
||||
backupPath?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk up from `start` (file or dir) to the brain root — the nearest ancestor
|
||||
* containing a `.git` marker — so slug derivation is brain-root-relative,
|
||||
* matching how sync/extract compute slugs. Falls back to the start's own
|
||||
* directory when no marker is found. Fixes #565: for a single-file target,
|
||||
* `relative(resolve(target), file)` was empty (target === file) and fell back
|
||||
* to the ABSOLUTE path, yielding bogus "root/brain/..." slugs and false
|
||||
* SLUG_MISMATCH — which the install-hook pre-commit hook hits on every commit.
|
||||
*/
|
||||
function findBrainRoot(start: string): string {
|
||||
const startDir = lstatSync(start).isDirectory() ? start : dirname(start);
|
||||
let candidate = startDir;
|
||||
for (let i = 0; i < 40; i++) {
|
||||
if (existsSync(join(candidate, '.git'))) return candidate;
|
||||
const parent = resolve(candidate, '..');
|
||||
if (parent === candidate) break;
|
||||
candidate = parent;
|
||||
}
|
||||
return startDir;
|
||||
}
|
||||
|
||||
async function runValidate(rest: string[]): Promise<void> {
|
||||
const flags: ValidateFlags = { json: false, fix: false, dryRun: false };
|
||||
let target: string | null = null;
|
||||
@@ -177,13 +198,17 @@ async function runValidate(rest: string[]): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
const brainRoot = findBrainRoot(resolved);
|
||||
const files = collectFiles(resolved);
|
||||
const results: FileValidation[] = [];
|
||||
const backupRunId = makeFrontmatterBackupRunId();
|
||||
|
||||
for (const file of files) {
|
||||
const content = readFileSync(file, 'utf8');
|
||||
const expectedSlug = slugifyPath(relative(resolve(target), file) || file);
|
||||
const rel = relative(brainRoot, file);
|
||||
// Files above/outside the brain root fall back to basename rather than
|
||||
// emitting a "../"-prefixed slug for non-brain files.
|
||||
const expectedSlug = slugifyPath(rel && !rel.startsWith('..') ? rel : basename(file));
|
||||
const parsed = parseMarkdown(content, file, { validate: true, expectedSlug });
|
||||
const errs = parsed.errors ?? [];
|
||||
const result: FileValidation = {
|
||||
|
||||
+13
-4
@@ -59,6 +59,11 @@ export async function runImport(
|
||||
* Threaded by performFullSync for `gbrain sync --exclude`.
|
||||
*/
|
||||
exclude?: string[];
|
||||
/**
|
||||
* Opt out of the git-visible fast path and walk the filesystem directly,
|
||||
* so markdown/code files matched by .gitignore can still be imported.
|
||||
*/
|
||||
includeGitignored?: boolean;
|
||||
/**
|
||||
* #753/#774 monorepo subdir-source support: when set, slugs and
|
||||
* `source_path` are computed relative to this root (the git repo root)
|
||||
@@ -71,6 +76,7 @@ export async function runImport(
|
||||
const noEmbed = args.includes('--no-embed');
|
||||
const fresh = args.includes('--fresh');
|
||||
const jsonOutput = args.includes('--json');
|
||||
const includeGitignored = args.includes('--include-gitignored') || opts.includeGitignored === true;
|
||||
|
||||
// T7 (D9): refuse cleanly when init persisted the deferred-setup sentinel,
|
||||
// unless the user is explicitly skipping embedding via `--no-embed` (in
|
||||
@@ -185,7 +191,7 @@ export async function runImport(
|
||||
const dirArg = args.find((a, i) => !a.startsWith('--') && !flagValues.has(i));
|
||||
|
||||
if (!dirArg) {
|
||||
console.error('Usage: gbrain import <dir> [--no-embed] [--workers N] [--fresh] [--source-id <id>] [--json]');
|
||||
console.error('Usage: gbrain import <dir> [--no-embed] [--workers N] [--fresh] [--source-id <id>] [--include-gitignored] [--json]');
|
||||
process.exit(1);
|
||||
}
|
||||
// #1728: capture the import target ONCE as an absolute real path. Every
|
||||
@@ -209,7 +215,7 @@ export async function runImport(
|
||||
const strategy: SyncStrategy = opts.strategy ?? 'markdown';
|
||||
const _walkT0 = Date.now();
|
||||
console.error(`[gbrain phase] import.collect_files start dir=${dir} strategy=${strategy}`);
|
||||
let allFiles = collectSyncableFiles(dir, { strategy });
|
||||
let allFiles = collectSyncableFiles(dir, { strategy, includeGitignored });
|
||||
console.error(
|
||||
`[gbrain phase] import.collect_files done ${Date.now() - _walkT0}ms files=${allFiles.length}`,
|
||||
);
|
||||
@@ -545,6 +551,7 @@ function resolveMaxWalkDepth(): number {
|
||||
|
||||
interface CollectOpts {
|
||||
strategy?: SyncStrategy;
|
||||
includeGitignored?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -675,8 +682,10 @@ export function collectSyncableFiles(dir: string, opts: CollectOpts = {}): strin
|
||||
// vendored data/fixtures). `--cached --others --exclude-standard` = tracked
|
||||
// PLUS untracked-not-ignored, so uncommitted source is still indexed. Non-git
|
||||
// dirs (or git unavailable) fall through to the FS walk below.
|
||||
const gitFiles = gitListSyncableFiles(dir, strategy, multimodalOn);
|
||||
if (gitFiles) return gitFiles;
|
||||
if (!opts.includeGitignored) {
|
||||
const gitFiles = gitListSyncableFiles(dir, strategy, multimodalOn);
|
||||
if (gitFiles) return gitFiles;
|
||||
}
|
||||
|
||||
const maxDepth = resolveMaxWalkDepth();
|
||||
const visitedInodes = new Map<string, true>();
|
||||
|
||||
+129
-23
@@ -26,6 +26,8 @@ export async function runInit(args: string[]) {
|
||||
return;
|
||||
}
|
||||
|
||||
validateInitFlags(args);
|
||||
|
||||
const isSupabase = args.includes('--supabase');
|
||||
const isPGLite = args.includes('--pglite');
|
||||
const isMcpOnly = args.includes('--mcp-only');
|
||||
@@ -151,6 +153,65 @@ export async function runInit(args: string[]) {
|
||||
return initPostgres({ databaseUrl, jsonOutput, apiKey, aiOpts, schemaPack, skipEmbedCheck });
|
||||
}
|
||||
|
||||
const INIT_BOOLEAN_FLAGS = new Set([
|
||||
'--pglite',
|
||||
'--supabase',
|
||||
'--mcp-only',
|
||||
'--force',
|
||||
'--non-interactive',
|
||||
'--migrate-only',
|
||||
'--json',
|
||||
'--no-embedding',
|
||||
'--skip-embed-check',
|
||||
]);
|
||||
|
||||
const INIT_VALUE_FLAGS = new Set([
|
||||
'--url',
|
||||
'--key',
|
||||
'--path',
|
||||
'--schema-pack',
|
||||
'--embedding-model',
|
||||
'--model',
|
||||
'--embedding-dimensions',
|
||||
'--expansion-model',
|
||||
'--chat-model',
|
||||
'--mcp-url',
|
||||
'--issuer-url',
|
||||
'--oauth-client-id',
|
||||
'--oauth-client-secret',
|
||||
]);
|
||||
|
||||
function validateInitFlags(args: string[]) {
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i];
|
||||
if (!arg.startsWith('-')) continue;
|
||||
|
||||
if (INIT_BOOLEAN_FLAGS.has(arg)) continue;
|
||||
|
||||
if (INIT_VALUE_FLAGS.has(arg)) {
|
||||
if (i + 1 >= args.length || args[i + 1].startsWith('-')) {
|
||||
failInitFlag(`gbrain init: ${arg} requires a value`, args.includes('--json'));
|
||||
}
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg.startsWith('--')) {
|
||||
failInitFlag(`gbrain init: unknown flag ${arg}`, args.includes('--json'));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function failInitFlag(message: string, jsonOutput: boolean): never {
|
||||
if (jsonOutput) {
|
||||
console.log(JSON.stringify({ status: 'error', reason: 'invalid_flag', message }));
|
||||
} else {
|
||||
console.error(message);
|
||||
console.error('Run `gbrain init --help` for supported flags.');
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
interface ResolveAIOptionsArgs {
|
||||
verbose: string | null; // --embedding-model
|
||||
shorthand: string | null; // --model
|
||||
@@ -161,7 +222,7 @@ interface ResolveAIOptionsArgs {
|
||||
nonInteractive: boolean; // --non-interactive (forces D3 fail-loud, no picker)
|
||||
}
|
||||
|
||||
interface ResolvedAIOptions {
|
||||
export interface ResolvedAIOptions {
|
||||
embedding_model?: string;
|
||||
embedding_dimensions?: number;
|
||||
expansion_model?: string;
|
||||
@@ -170,6 +231,41 @@ interface ResolvedAIOptions {
|
||||
noEmbedding?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Seed init's AI options from persisted config, falling back to the raw env
|
||||
* vars when loadConfig() returned null (#1058). On a cold install (no
|
||||
* config.json AND no DATABASE_URL) loadConfig short-circuits BEFORE its env
|
||||
* merge, so GBRAIN_EMBEDDING_MODEL / GBRAIN_EMBEDDING_DIMENSIONS /
|
||||
* GBRAIN_EXPANSION_MODEL / GBRAIN_CHAT_MODEL were silently ignored by init
|
||||
* and Tier-3 detection auto-picked by API key instead. Exported for unit
|
||||
* tests (env injectable).
|
||||
*/
|
||||
export function seedAIOptionsFromConfig(
|
||||
cfg: GBrainConfig | null,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): ResolvedAIOptions {
|
||||
const envDims = env.GBRAIN_EMBEDDING_DIMENSIONS
|
||||
? parseInt(env.GBRAIN_EMBEDDING_DIMENSIONS, 10)
|
||||
: NaN;
|
||||
const seed = cfg ?? {
|
||||
embedding_disabled: undefined,
|
||||
embedding_model: env.GBRAIN_EMBEDDING_MODEL,
|
||||
embedding_dimensions: Number.isFinite(envDims) ? envDims : undefined,
|
||||
expansion_model: env.GBRAIN_EXPANSION_MODEL,
|
||||
chat_model: env.GBRAIN_CHAT_MODEL,
|
||||
};
|
||||
const out: ResolvedAIOptions = {};
|
||||
if (seed.embedding_disabled) {
|
||||
out.noEmbedding = true;
|
||||
} else if (seed.embedding_model) {
|
||||
out.embedding_model = seed.embedding_model;
|
||||
if (seed.embedding_dimensions) out.embedding_dimensions = seed.embedding_dimensions;
|
||||
}
|
||||
if (seed.expansion_model) out.expansion_model = seed.expansion_model;
|
||||
if (seed.chat_model) out.chat_model = seed.chat_model;
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve AI provider options for `gbrain init`.
|
||||
*
|
||||
@@ -203,18 +299,13 @@ async function resolveAIOptions(opts: ResolveAIOptionsArgs): Promise<ResolvedAIO
|
||||
// user already opted into deferred mode.
|
||||
try {
|
||||
const { loadConfig } = await import('../core/config.ts');
|
||||
const cfg = loadConfig();
|
||||
if (cfg?.embedding_disabled) {
|
||||
out.noEmbedding = true;
|
||||
} else if (cfg?.embedding_model) {
|
||||
out.embedding_model = cfg.embedding_model;
|
||||
if (cfg.embedding_dimensions) out.embedding_dimensions = cfg.embedding_dimensions;
|
||||
}
|
||||
if (cfg?.expansion_model) out.expansion_model = cfg.expansion_model;
|
||||
if (cfg?.chat_model) out.chat_model = cfg.chat_model;
|
||||
// #1058: loadConfig() returns null on a cold install (no config.json AND
|
||||
// no DATABASE_URL) — before it ever reaches its env merge. The seed helper
|
||||
// falls back to the same GBRAIN_* env vars directly in that case.
|
||||
Object.assign(out, seedAIOptionsFromConfig(loadConfig()));
|
||||
} catch {
|
||||
// loadConfig throws when no brain configured — first-time install, fall
|
||||
// through to env detection.
|
||||
// loadConfig threw — treat as first-time install, fall through to env
|
||||
// detection.
|
||||
}
|
||||
|
||||
// --- Tier 1+2: explicit flags ---------------------------------------------
|
||||
@@ -246,7 +337,9 @@ async function resolveAIOptions(opts: ResolveAIOptionsArgs): Promise<ResolvedAIO
|
||||
process.exit(1);
|
||||
}
|
||||
out.embedding_model = `${shorthand}:${firstModel}`;
|
||||
out.embedding_dimensions = recipe.touchpoints.embedding!.default_dims;
|
||||
// #2051: width follows the model actually chosen, not the recipe default.
|
||||
const { embeddingDimsForModel } = await import('../core/ai/model-resolver.ts');
|
||||
out.embedding_dimensions = embeddingDimsForModel(recipe, firstModel);
|
||||
}
|
||||
|
||||
if (dimsArg !== null && !Number.isNaN(dimsArg) && dimsArg > 0) {
|
||||
@@ -270,8 +363,13 @@ async function resolveAIOptions(opts: ResolveAIOptionsArgs): Promise<ResolvedAIO
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
if (recipe?.touchpoints.embedding?.default_dims) {
|
||||
out.embedding_dimensions = recipe.touchpoints.embedding.default_dims;
|
||||
// #2051: resolve the width from the SPECIFIC model, not the recipe-wide
|
||||
// default. `--embedding-model ollama:bge-m3` must yield 1024, not Ollama's
|
||||
// nomic-shaped 768.
|
||||
if (recipe) {
|
||||
const { embeddingDimsForModel } = await import('../core/ai/model-resolver.ts');
|
||||
const dims = embeddingDimsForModel(recipe, out.embedding_model);
|
||||
if (dims > 0) out.embedding_dimensions = dims;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -434,9 +532,11 @@ async function resolveEmbeddingByEnv(out: ResolvedAIOptions, nonInteractive: boo
|
||||
// legacy OpenAI 1536), not the recipe's 2560.
|
||||
const { DEFAULT_EMBEDDING_MODEL, DEFAULT_EMBEDDING_DIMENSIONS } =
|
||||
await import('../core/ai/defaults.ts');
|
||||
const { embeddingDimsForModel } = await import('../core/ai/model-resolver.ts');
|
||||
// #2051: non-canonical models resolve per-model, not recipe-wide.
|
||||
const dims = fullModel === DEFAULT_EMBEDDING_MODEL
|
||||
? DEFAULT_EMBEDDING_DIMENSIONS
|
||||
: tp.default_dims;
|
||||
: embeddingDimsForModel(r, model);
|
||||
out.embedding_model = fullModel;
|
||||
out.embedding_dimensions = dims;
|
||||
console.error(
|
||||
@@ -1017,12 +1117,12 @@ async function initPostgres(opts: {
|
||||
|
||||
// v0.37.10.0 T6 (D11) + v0.37.11.0 Lane B.2: ALWAYS configure gateway BEFORE
|
||||
// initSchema. Same preflight contract as PGLite. Refuse to call initSchema
|
||||
// until the gateway-resolved dim is validated. Schema substitution in
|
||||
// src/schema.sql is currently a static `vector(1536)` for Postgres (unlike
|
||||
// PGLite's templated dim), so a Voyage/ZE-configured Postgres brain will
|
||||
// still need a future schema rewrite path — preflight makes the
|
||||
// not-yet-supported case fail loud rather than silently produce a stuck
|
||||
// 1536d column.
|
||||
// until the gateway-resolved dim is validated. PostgresEngine.initSchema()
|
||||
// passes the resolved model and dimensions through getPostgresSchema(),
|
||||
// which templates the static `vector(1536)` source before executing it.
|
||||
// Preflight therefore prevents an invalid dimension from reaching schema
|
||||
// generation, while the post-init assertion below guards against templating
|
||||
// drift.
|
||||
let resolvedDim: number | undefined;
|
||||
let resolvedModel: string | undefined;
|
||||
if (opts.aiOpts?.noEmbedding) {
|
||||
@@ -1078,6 +1178,9 @@ async function initPostgres(opts: {
|
||||
console.warn(' Direct connections are IPv6 only and fail in many environments.');
|
||||
console.warn(' Use the Transaction pooler connection string instead (port 6543):');
|
||||
console.warn(' Supabase Dashboard > Connect (top bar) > Connection String > Transaction pooler');
|
||||
console.warn(' (With a pooler URL, gbrain derives a direct connection for DDL and falls back');
|
||||
console.warn(' to the pooler automatically if that host is unreachable. Power users:');
|
||||
console.warn(' GBRAIN_DIRECT_DATABASE_URL overrides the derived URL; GBRAIN_DISABLE_DIRECT_POOL=1 disables it.)');
|
||||
console.warn('');
|
||||
}
|
||||
|
||||
@@ -1091,6 +1194,9 @@ async function initPostgres(opts: {
|
||||
if (databaseUrl.includes('supabase.co') && (msg.includes('ECONNREFUSED') || msg.includes('ETIMEDOUT'))) {
|
||||
console.error('Connection failed. Supabase direct connections (db.*.supabase.co:5432) are IPv6 only.');
|
||||
console.error('Use the Transaction pooler connection string instead (port 6543).');
|
||||
console.error('(gbrain derives its own direct connection from pooler URLs for DDL; if that host is');
|
||||
console.error('unreachable it falls back to the pooler. GBRAIN_DIRECT_DATABASE_URL overrides the');
|
||||
console.error('derived URL; GBRAIN_DISABLE_DIRECT_POOL=1 disables the direct pool entirely.)');
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
@@ -1455,7 +1561,7 @@ export function reportModStatus(): void {
|
||||
console.log(' cd ~/.claude/skills/gstack && ./setup');
|
||||
}
|
||||
console.log('Resolver: skills/RESOLVER.md');
|
||||
console.log('Soul audit: run `gbrain soul-audit` to customize agent identity');
|
||||
console.log('Soul audit: ask your agent to "run a soul audit" to customize its identity (see skills/soul-audit)');
|
||||
// Retrieval Reflex (#1981): the deterministic pointer layer is ON by default
|
||||
// (no action needed). The policy skill is installed into the HOST repo on
|
||||
// request — we PRINT the command rather than silently mutating the host repo.
|
||||
|
||||
+17
-1
@@ -1885,6 +1885,7 @@ export async function registerBuiltinHandlers(
|
||||
signal: job.signal,
|
||||
deadlineAtMs: job.deadlineAtMs, // #2781: phases budget sub-work from remaining time
|
||||
phases,
|
||||
forceGlobalOrphans: true,
|
||||
yieldBetweenPhases: async () => { await new Promise<void>((r) => setImmediate(r)); },
|
||||
});
|
||||
|
||||
@@ -2061,11 +2062,26 @@ export async function registerBuiltinHandlers(
|
||||
? job.data.repoPath
|
||||
: ((await engine.getConfig('sync.repo_path')) ?? undefined);
|
||||
try {
|
||||
return await runExtractAtomsDrainForSource(engine, {
|
||||
const result = await runExtractAtomsDrainForSource(engine, {
|
||||
sourceId,
|
||||
windowSeconds,
|
||||
brainDir: repoPath,
|
||||
});
|
||||
// issue #3218: every item the drain attempted failed (0 succeeded, >=1
|
||||
// provider error) — completing this job normally would mark the
|
||||
// durable job done while the backlog sits untouched, and no retry
|
||||
// policy would ever fire on it again. Throw so the worker's ordinary
|
||||
// failJob path (attempt+backoff, or dead-letter once exhausted) takes
|
||||
// over instead — matching the existing behavior for every other
|
||||
// handler failure. Partial success (>=1 item extracted) keeps
|
||||
// completing normally, unchanged.
|
||||
if (result.status === 'provider_failure') {
|
||||
throw new Error(
|
||||
`extract-atoms-drain: all provider calls failed this batch ` +
|
||||
`(batches=${result.batches}, remaining=${result.remaining ?? '?'}) — retrying`,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
} catch (e) {
|
||||
if (e instanceof LockUnavailableError) {
|
||||
return { phase: 'extract_atoms', status: 'skipped', deferred: true, reason: 'cycle_already_running' };
|
||||
|
||||
+43
-8
@@ -383,15 +383,30 @@ async function resolveLintContentSanity(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Directories never containing knowledge pages, skipped by default.
|
||||
* Deliberately tiny: only vendored dependency trees qualify. Anything
|
||||
* more opinionated (README.md, CHANGELOG.md, test/) is repo policy —
|
||||
* callers opt in via `--exclude` / `LintOpts.exclude`. Dot- and
|
||||
* underscore-prefixed entries are already skipped by the walk.
|
||||
*/
|
||||
const DEFAULT_LINT_EXCLUDE_DIRS = new Set(['node_modules']);
|
||||
|
||||
/** Collect markdown files from a directory */
|
||||
function collectPages(dir: string): string[] {
|
||||
function collectPages(dir: string, extraExcludes: string[] = []): string[] {
|
||||
const extra = new Set(extraExcludes);
|
||||
const pages: string[] = [];
|
||||
function walk(d: string) {
|
||||
for (const entry of readdirSync(d)) {
|
||||
if (entry.startsWith('.') || entry.startsWith('_')) continue;
|
||||
const full = join(d, entry);
|
||||
if (lstatSync(full).isDirectory()) walk(full);
|
||||
else if (entry.endsWith('.md')) pages.push(full);
|
||||
if (lstatSync(full).isDirectory()) {
|
||||
if (DEFAULT_LINT_EXCLUDE_DIRS.has(entry) || extra.has(entry)) continue;
|
||||
walk(full);
|
||||
} else if (entry.endsWith('.md')) {
|
||||
if (extra.has(entry)) continue;
|
||||
pages.push(full);
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(dir);
|
||||
@@ -419,6 +434,13 @@ export interface LintOpts {
|
||||
* yields + checks this every 200 pages.
|
||||
*/
|
||||
signal?: AbortSignal;
|
||||
/**
|
||||
* #2649: extra dir/file basenames to skip while collecting pages, in
|
||||
* addition to node_modules and dot/underscore entries. For mixed-content
|
||||
* repos (knowledge pages alongside software trees). Ignored for
|
||||
* single-file targets.
|
||||
*/
|
||||
exclude?: string[];
|
||||
}
|
||||
|
||||
export interface LintResult {
|
||||
@@ -445,7 +467,7 @@ export async function runLintCore(opts: LintOpts): Promise<LintResult> {
|
||||
}
|
||||
|
||||
const isSingleFile = statSync(opts.target).isFile();
|
||||
const pages = isSingleFile ? [opts.target] : collectPages(opts.target);
|
||||
const pages = isSingleFile ? [opts.target] : collectPages(opts.target, opts.exclude ?? []);
|
||||
|
||||
// Resolve content-sanity config once for this lint run (D1: lift DB
|
||||
// config when reachable). Caller can pre-pass via opts.contentSanity
|
||||
@@ -496,14 +518,27 @@ export async function runLintCore(opts: LintOpts): Promise<LintResult> {
|
||||
}
|
||||
|
||||
export async function runLint(args: string[]) {
|
||||
const target = args.find(a => !a.startsWith('--'));
|
||||
// #2649: --exclude=a,b or --exclude a,b — extra basenames to skip.
|
||||
const extraExcludes: string[] = [];
|
||||
const skipIdx = new Set<number>();
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const a = args[i];
|
||||
if (a.startsWith('--exclude=')) {
|
||||
extraExcludes.push(...a.slice('--exclude='.length).split(',').map(s => s.trim()).filter(Boolean));
|
||||
} else if (a === '--exclude' && i + 1 < args.length) {
|
||||
extraExcludes.push(...args[i + 1].split(',').map(s => s.trim()).filter(Boolean));
|
||||
skipIdx.add(i + 1);
|
||||
}
|
||||
}
|
||||
const target = args.find((a, i) => !a.startsWith('--') && !skipIdx.has(i));
|
||||
const doFix = args.includes('--fix');
|
||||
const dryRun = args.includes('--dry-run');
|
||||
|
||||
if (!target) {
|
||||
console.error('Usage: gbrain lint <dir|file.md> [--fix] [--dry-run]');
|
||||
console.error('Usage: gbrain lint <dir|file.md> [--fix] [--dry-run] [--exclude a,b]');
|
||||
console.error(' --fix Auto-fix fixable issues (LLM preambles, code fences)');
|
||||
console.error(' --dry-run Preview fixes without writing');
|
||||
console.error(' --exclude Comma-separated dir/file basenames to skip (in addition to node_modules)');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -515,7 +550,7 @@ export async function runLint(args: string[]) {
|
||||
// Single file or directory — print human detail as we go, then rely on
|
||||
// Core for the aggregate numbers at the end.
|
||||
const isSingleFile = statSync(target).isFile();
|
||||
const pages = isSingleFile ? [target] : collectPages(target);
|
||||
const pages = isSingleFile ? [target] : collectPages(target, extraExcludes);
|
||||
|
||||
// Progress on stderr. Stdout keeps the per-issue human output it always had.
|
||||
const { createProgress } = await import('../core/progress.ts');
|
||||
@@ -562,7 +597,7 @@ export async function runLint(args: string[]) {
|
||||
// produces canonical numbers for the summary line).
|
||||
// Pass contentSanity through so runLintCore skips its own resolve
|
||||
// (we already resolved once for the human-detail loop above).
|
||||
const result = await runLintCore({ target, fix: doFix, dryRun, contentSanity });
|
||||
const result = await runLintCore({ target, fix: doFix, dryRun, contentSanity, exclude: extraExcludes });
|
||||
console.log(`\n${result.pages_scanned} pages scanned. ${result.total_issues} issue(s) in ${result.pages_with_issues} page(s).`);
|
||||
if (doFix) {
|
||||
console.log(`${dryRun ? '(dry run) ' : ''}${result.total_fixed} auto-fixed.`);
|
||||
|
||||
@@ -0,0 +1,402 @@
|
||||
/**
|
||||
* `gbrain migrate embeddings --to <provider:model>` (#3390) — the
|
||||
* provider-agnostic forward migration off any embedding provider, built for
|
||||
* the ZeroEntropy 2026-09-04 sunset but not keyed to it.
|
||||
*
|
||||
* Also reachable as `gbrain retrieval-upgrade` — the command README.md and
|
||||
* doctor.ts have promised since v0.36 but which never had a dispatch branch.
|
||||
*
|
||||
* Flow (everything heavy is reused, see src/core/embedding-migration.ts):
|
||||
* 1. plan — chunk/char counts via the widened stale predicates,
|
||||
* cost estimate from embedding-pricing.ts
|
||||
* 2. preflight— print estimate; require --yes or interactive confirm
|
||||
* (non-TTY without --yes refuses with exit 2, mirroring the
|
||||
* reindex-code cost gate in docs/operations/spend-controls.md)
|
||||
* 3. probe — one live embed against the TARGET provider BEFORE any
|
||||
* mutation (validates key + model + dims in one shot)
|
||||
* 4. apply — schema transition (dim change), config (DB + file plane),
|
||||
* #3391 NULL-signature-inclusive invalidation, cache purge
|
||||
* 5. re-embed — runEmbedCore --stale --catch-up with single-flight locks,
|
||||
* pacing (--pace), progress reporting. Resumable: a killed
|
||||
* run re-runs the SAME command; the NULL-embedding cursor is
|
||||
* the checkpoint and steps 3-4 no-op on the second pass.
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { serr, slog } from '../core/console-prefix.ts';
|
||||
import {
|
||||
planEmbeddingMigration,
|
||||
applyEmbeddingMigration,
|
||||
completeEmbeddingMigration,
|
||||
reconcilePageSignatures,
|
||||
MIGRATION_STATE_KEY,
|
||||
type EmbeddingMigrationPlan,
|
||||
} from '../core/embedding-migration.ts';
|
||||
import { formatEnvOverrideWarning } from '../core/retrieval-upgrade-planner.ts';
|
||||
import { parsePaceArgs, runEmbedCore } from './embed.ts';
|
||||
|
||||
export interface MigrateEmbeddingsFlags {
|
||||
to?: string;
|
||||
dim?: number;
|
||||
yes: boolean;
|
||||
dryRun: boolean;
|
||||
json: boolean;
|
||||
noEmbed: boolean;
|
||||
ignoreEnvOverride: boolean;
|
||||
batchSize?: number;
|
||||
pace?: ReturnType<typeof parsePaceArgs>;
|
||||
}
|
||||
|
||||
export function parseMigrateEmbeddingsFlags(args: string[]): MigrateEmbeddingsFlags {
|
||||
const toIdx = args.indexOf('--to');
|
||||
const dimIdx = args.indexOf('--dim');
|
||||
const dimRaw = dimIdx >= 0 ? parseInt(args[dimIdx + 1] ?? '', 10) : NaN;
|
||||
const bsIdx = args.indexOf('--batch-size');
|
||||
const bsRaw = bsIdx >= 0 ? parseInt(args[bsIdx + 1] ?? '', 10) : NaN;
|
||||
const batchSize = Number.isFinite(bsRaw) && bsRaw > 0 ? Math.min(10_000, bsRaw) : undefined;
|
||||
return {
|
||||
to: toIdx >= 0 ? args[toIdx + 1] : undefined,
|
||||
dim: Number.isFinite(dimRaw) && dimRaw > 0 ? dimRaw : undefined,
|
||||
yes: args.includes('--yes') || args.includes('--non-interactive'),
|
||||
dryRun: args.includes('--dry-run'),
|
||||
json: args.includes('--json'),
|
||||
noEmbed: args.includes('--no-embed'),
|
||||
ignoreEnvOverride: args.includes('--ignore-env-override'),
|
||||
...(batchSize !== undefined && { batchSize }),
|
||||
pace: parsePaceArgs(args),
|
||||
};
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
process.stdout.write(`Usage: gbrain migrate embeddings --to <provider:model> [flags]
|
||||
|
||||
Re-embed the whole brain onto a different embedding provider/model. Handles
|
||||
dimension changes (schema transition), pages without a recorded embedding
|
||||
signature (#3391), the query cache, and resume-after-kill. The forward path
|
||||
off a sunsetting provider.
|
||||
|
||||
Flags:
|
||||
--to <provider:model> Target embedding model (e.g. openai:text-embedding-3-small).
|
||||
--dim <N> Target dimensions. Defaults to the provider recipe's
|
||||
declared width; required when the recipe declares none.
|
||||
--dry-run Plan + cost estimate only; change nothing.
|
||||
--yes Skip the confirm prompt (required non-interactively).
|
||||
--json Machine-readable envelope on stdout.
|
||||
--no-embed Apply schema + config + invalidation, but skip the
|
||||
re-embed pass (run \`gbrain embed --stale --include-null-signature\`
|
||||
or \`... --background\` yourself).
|
||||
--batch-size <N> Stale-chunk batch size for the re-embed (default 2000).
|
||||
--pace[=mode] DB-contention pacing for the re-embed (off|gentle|balanced|aggressive).
|
||||
--ignore-env-override Proceed even when GBRAIN_EMBEDDING_* env vars would
|
||||
override the target at runtime (you know why).
|
||||
--help Show this help.
|
||||
|
||||
A killed run is resumable: re-run the same command. Already-migrated chunks
|
||||
are never re-embedded twice.
|
||||
`);
|
||||
}
|
||||
|
||||
function renderPlan(plan: EmbeddingMigrationPlan): string {
|
||||
const lines: string[] = [];
|
||||
lines.push('Embedding migration plan');
|
||||
lines.push(` From: ${plan.from_model} (${plan.from_dims}d${plan.column_dims !== null && plan.column_dims !== plan.from_dims ? `; column is actually ${plan.column_dims}d` : ''})`);
|
||||
lines.push(` To: ${plan.to_model} (${plan.to_dims}d)`);
|
||||
if (plan.dim_change) {
|
||||
lines.push(` DESTRUCTIVE: the embedding column is rebuilt at ${plan.to_dims}d, which DELETES`);
|
||||
lines.push(' every stored embedding vector in this brain. They are not recoverable —');
|
||||
lines.push(' going back to the old provider means paying for a second full re-embed.');
|
||||
lines.push(' Until the re-embed finishes, semantic search is degraded to lexical-only.');
|
||||
lines.push(` The query cache and fact embeddings are rebuilt at ${plan.to_dims}d too`);
|
||||
lines.push(' (cache refills on next query; facts re-embed on their next write).');
|
||||
}
|
||||
lines.push(` Chunks to re-embed: ${plan.chunks_to_embed}${plan.null_signature_chunks > 0 ? ` (includes ${plan.null_signature_chunks} on pages with no recorded embedding signature)` : ''}`);
|
||||
lines.push(
|
||||
plan.price_known
|
||||
? ` Estimated cost: $${plan.est_cost_usd.toFixed(2)} (${plan.total_chars} chars at the ${plan.to_model} rate)`
|
||||
: ` Estimated cost: unknown — no pricing entry for ${plan.to_model}. Check the provider's pricing before proceeding.`,
|
||||
);
|
||||
if (plan.resuming) {
|
||||
lines.push(' Resuming: a prior migration to this target was interrupted; continuing it.');
|
||||
}
|
||||
if (plan.reranker_warning) {
|
||||
lines.push(` WARNING: ${plan.reranker_warning}`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/** Single-keypress y/N confirm on stdin. Injectable for tests. */
|
||||
async function defaultConfirm(question: string): Promise<boolean> {
|
||||
process.stderr.write(`${question} [y/N] `);
|
||||
const stdin = process.stdin;
|
||||
stdin.setRawMode?.(true);
|
||||
stdin.resume();
|
||||
const key: string = await new Promise((resolve) => {
|
||||
stdin.once('data', (d) => resolve(d.toString()));
|
||||
});
|
||||
stdin.setRawMode?.(false);
|
||||
stdin.pause();
|
||||
process.stderr.write('\n');
|
||||
return key.trim().toLowerCase().startsWith('y');
|
||||
}
|
||||
|
||||
/**
|
||||
* One tiny embed against the TARGET provider, BEFORE any mutation: validates
|
||||
* the API key, the model id, and dimension support in a single call, so a bad
|
||||
* target fails with the brain untouched instead of after the column is
|
||||
* dropped. Shared by the CLI and the `migrate_embeddings` op (the op used to
|
||||
* skip it, which let `yes:true` drop the column against a bad key).
|
||||
*/
|
||||
export async function probeTargetProvider(
|
||||
toModel: string,
|
||||
toDims: number,
|
||||
): Promise<{ ok: true } | { ok: false; message: string }> {
|
||||
try {
|
||||
const { embed } = await import('../core/ai/gateway.ts');
|
||||
const vecs = await embed(['gbrain embedding migration probe'], {
|
||||
embeddingModel: toModel,
|
||||
dimensions: toDims,
|
||||
});
|
||||
const got = vecs[0]?.length ?? 0;
|
||||
if (got !== toDims) {
|
||||
return {
|
||||
ok: false,
|
||||
message: `Target provider returned ${got}-dim vectors, expected ${toDims}. Pass a valid --dim for ${toModel}.`,
|
||||
};
|
||||
}
|
||||
return { ok: true };
|
||||
} catch (e) {
|
||||
return {
|
||||
ok: false,
|
||||
message: `Preflight embed against ${toModel} failed — nothing was changed:\n ${e instanceof Error ? e.message : String(e)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist the target model+dims to the FILE plane and reconfigure the
|
||||
* in-process gateway. The gateway reads file/env config, not the DB plane —
|
||||
* without this the re-embed would silently run against the OLD provider.
|
||||
* Shared by the CLI command and the `migrate_embeddings` op handler.
|
||||
*/
|
||||
export async function persistEmbeddingFileConfig(
|
||||
toModel: string,
|
||||
toDims: number,
|
||||
): Promise<void> {
|
||||
const { loadConfig, saveConfig } = await import('../core/config.ts');
|
||||
const { configureGateway } = await import('../core/ai/gateway.ts');
|
||||
const { buildGatewayConfig } = await import('../core/ai/build-gateway-config.ts');
|
||||
const cfg = loadConfig();
|
||||
if (!cfg) {
|
||||
// REFUSE rather than warn-and-proceed. Without a file plane to write, the
|
||||
// switch would not survive this process: the next `gbrain` invocation
|
||||
// reads file/env config, sees the OLD provider, and re-embeds the brain
|
||||
// back into the old space (paying twice) — or fails outright against a
|
||||
// column that is now the new width. Thrown from inside
|
||||
// applyEmbeddingMigration's try, so it surfaces as status: 'failed'
|
||||
// BEFORE the config/cache steps and the caller exits non-zero.
|
||||
throw new Error(
|
||||
'No ~/.gbrain/config.json found — refusing to migrate.\n' +
|
||||
' The embed pipeline reads file/env config, so without a file plane this switch\n' +
|
||||
' would not survive the process and the next run would re-embed into the old space.\n' +
|
||||
' Fix: run `gbrain init` (or set GBRAIN_EMBEDDING_MODEL + GBRAIN_EMBEDDING_DIMENSIONS\n' +
|
||||
' in the environment of every gbrain process) and re-run.',
|
||||
);
|
||||
}
|
||||
cfg.embedding_model = toModel;
|
||||
cfg.embedding_dimensions = toDims;
|
||||
saveConfig(cfg);
|
||||
configureGateway(buildGatewayConfig(cfg));
|
||||
}
|
||||
|
||||
export interface RunMigrateEmbeddingsOpts {
|
||||
/** Test seams. */
|
||||
confirm?: (question: string) => Promise<boolean>;
|
||||
isTTY?: boolean;
|
||||
exit?: (code: number) => never;
|
||||
}
|
||||
|
||||
export async function runMigrateEmbeddings(
|
||||
engine: BrainEngine,
|
||||
args: string[],
|
||||
opts: RunMigrateEmbeddingsOpts = {},
|
||||
): Promise<void> {
|
||||
// Explicit `never` annotation so TS control-flow analysis treats every
|
||||
// exit() call as terminal (required for narrowing after the guard blocks).
|
||||
const exit: (code: number) => never = opts.exit ?? ((code: number) => process.exit(code));
|
||||
if (args.includes('--help') || args.includes('-h')) {
|
||||
printHelp();
|
||||
exit(0);
|
||||
}
|
||||
const flags = parseMigrateEmbeddingsFlags(args);
|
||||
if (!flags.to) {
|
||||
serr('Missing --to <provider:model>. Example: gbrain migrate embeddings --to openai:text-embedding-3-small');
|
||||
serr('Run with --help for all flags.');
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// From-state as the gateway resolved it (file/env config + defaults) —
|
||||
// the truth for what embeds run under TODAY.
|
||||
let fromModel: string | undefined;
|
||||
let fromDims: number | undefined;
|
||||
try {
|
||||
const { getEmbeddingModel, getEmbeddingDimensions } = await import('../core/ai/gateway.ts');
|
||||
fromModel = getEmbeddingModel();
|
||||
fromDims = getEmbeddingDimensions();
|
||||
} catch {
|
||||
// Gateway unconfigured — plan falls back to shipped defaults.
|
||||
}
|
||||
|
||||
let plan: EmbeddingMigrationPlan;
|
||||
try {
|
||||
plan = await planEmbeddingMigration(engine, {
|
||||
to: flags.to!,
|
||||
...(flags.dim !== undefined && { dim: flags.dim }),
|
||||
...(fromModel !== undefined && { fromModel }),
|
||||
...(fromDims !== undefined && { fromDims }),
|
||||
});
|
||||
} catch (e) {
|
||||
serr(e instanceof Error ? e.message : String(e));
|
||||
exit(1);
|
||||
return; // unreachable; keeps TS happy for injected exit seams
|
||||
}
|
||||
|
||||
if (flags.json) {
|
||||
// Human plan goes to stderr so stdout stays JSON-clean.
|
||||
serr(renderPlan(plan));
|
||||
} else {
|
||||
console.log(renderPlan(plan));
|
||||
}
|
||||
|
||||
if (plan.chunks_to_embed === 0 && !plan.dim_change && plan.from_model === plan.to_model) {
|
||||
if (flags.json) console.log(JSON.stringify({ status: 'skipped_no_work', plan }, null, 2));
|
||||
else console.log('Nothing to migrate — brain is already on the target model.');
|
||||
exit(0);
|
||||
}
|
||||
|
||||
if (flags.dryRun) {
|
||||
if (flags.json) console.log(JSON.stringify({ status: 'planned', plan }, null, 2));
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// ── Consent gate. Unlike the pure cost gates in
|
||||
// docs/operations/spend-controls.md, `spend.posture=tokenmax` does NOT
|
||||
// bypass this one: posture waives the SPEND ceiling, and this gate also
|
||||
// guards a destructive schema rebuild (existing vectors are dropped, and
|
||||
// retrieval is degraded until the re-embed finishes). We honor the posture
|
||||
// by marking the dollar figure informational, and still ask.
|
||||
if (!flags.yes) {
|
||||
const { resolveSpendPosture } = await import('../core/spend-posture.ts');
|
||||
const posture = await resolveSpendPosture(engine);
|
||||
if (posture === 'tokenmax') {
|
||||
serr(' [migrate] spend.posture=tokenmax: the cost estimate above is informational.');
|
||||
serr(' [migrate] Confirmation is still required — this rebuilds the embedding column (destructive, not just costly).');
|
||||
}
|
||||
const isTTY = opts.isTTY ?? Boolean(process.stdin.isTTY);
|
||||
if (!isTTY) {
|
||||
serr('Refusing to migrate without confirmation in a non-TTY environment. Re-run with --yes.');
|
||||
exit(2);
|
||||
}
|
||||
const confirm = opts.confirm ?? defaultConfirm;
|
||||
const priceNote = plan.price_known ? `~$${plan.est_cost_usd.toFixed(2)}` : 'an UNKNOWN amount';
|
||||
const ok = await confirm(`Re-embed ${plan.chunks_to_embed} chunks (${priceNote})?`);
|
||||
if (!ok) {
|
||||
serr('Aborted. Nothing was changed.');
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Live probe BEFORE any mutation: one tiny embed against the TARGET
|
||||
// provider validates API key, model id, and dimension support in one call.
|
||||
const probe = await probeTargetProvider(plan.to_model, plan.to_dims);
|
||||
if (!probe.ok) {
|
||||
serr(probe.message);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// ── Apply: schema + config + invalidation + cache purge.
|
||||
const applied = await applyEmbeddingMigration(engine, plan, {
|
||||
ignoreEnvOverride: flags.ignoreEnvOverride,
|
||||
persistConfig: (toModel, toDims) => persistEmbeddingFileConfig(toModel, toDims),
|
||||
});
|
||||
|
||||
if (applied.status === 'refused') {
|
||||
if (flags.json) console.log(JSON.stringify(applied, null, 2));
|
||||
else serr(formatEnvOverrideWarning(applied.warning));
|
||||
exit(1);
|
||||
}
|
||||
if (applied.status === 'failed') {
|
||||
if (flags.json) console.log(JSON.stringify(applied, null, 2));
|
||||
else serr(`Migration apply failed: ${applied.reason}`);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
serr(` [migrate] schema ${applied.schema_transitioned ? `rebuilt at ${plan.to_dims}d` : 'unchanged'}; ` +
|
||||
`${applied.invalidated} chunk(s) invalidated; query cache purged (${applied.cache_cleared} row(s)).`);
|
||||
|
||||
if (flags.noEmbed) {
|
||||
const msg = 'Config + schema migrated. Re-embed deferred — run: gbrain embed --stale --catch-up --include-null-signature';
|
||||
if (flags.json) console.log(JSON.stringify({ ...applied, status: 'applied_no_embed', plan }, null, 2));
|
||||
else console.log(msg);
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// ── Re-embed. All the machinery (locks, pacing, backoff, progress,
|
||||
// signature stamping) is the standard embed pipeline.
|
||||
const { createProgress } = await import('../core/progress.ts');
|
||||
const { getCliOptions, cliOptsToProgressOptions } = await import('../core/cli-options.ts');
|
||||
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
|
||||
let progressStarted = false;
|
||||
const embedResult = await runEmbedCore(engine, {
|
||||
stale: true,
|
||||
catchUp: true,
|
||||
singleFlight: true,
|
||||
includeNullSignature: true,
|
||||
quiet: flags.json,
|
||||
...(flags.batchSize !== undefined && { batchSize: flags.batchSize }),
|
||||
...(flags.pace && { pace: flags.pace }),
|
||||
onProgress: (done, total) => {
|
||||
if (!progressStarted) {
|
||||
progress.start('migrate.reembed', total);
|
||||
progressStarted = true;
|
||||
}
|
||||
progress.tick(1);
|
||||
},
|
||||
});
|
||||
if (progressStarted) progress.finish();
|
||||
|
||||
// Reconcile signatures BEFORE the completion probe: pages straddling a
|
||||
// stale-batch boundary are embedded correctly but left unstamped by the
|
||||
// embed loop's all-or-nothing stamp rule. Without this the probe would call
|
||||
// a fully-migrated brain "incomplete" and the re-run would pay again.
|
||||
const reconciled = await reconcilePageSignatures(engine, plan);
|
||||
if (reconciled > 0) {
|
||||
serr(` [migrate] reconciled the embedding signature on ${reconciled} fully-embedded page(s) (batch-boundary pages).`);
|
||||
}
|
||||
|
||||
const remaining = await engine.countStaleChunks({
|
||||
signature: `${plan.to_model}:${plan.to_dims}`,
|
||||
includeNullSignature: true,
|
||||
});
|
||||
|
||||
if (remaining === 0) {
|
||||
await completeEmbeddingMigration(engine, plan);
|
||||
if (flags.json) {
|
||||
console.log(JSON.stringify({ status: 'completed', plan, embedded: embedResult.embedded, remaining: 0 }, null, 2));
|
||||
} else {
|
||||
slog(`Migration complete: ${embedResult.embedded} chunk(s) embedded on ${plan.to_model} (${plan.to_dims}d).`);
|
||||
if (plan.reranker_warning) serr(` [migrate] reminder: ${plan.reranker_warning}`);
|
||||
}
|
||||
exit(0);
|
||||
} else {
|
||||
if (flags.json) {
|
||||
console.log(JSON.stringify({ status: 'incomplete', plan, embedded: embedResult.embedded, remaining }, null, 2));
|
||||
} else {
|
||||
serr(`Migration incomplete: ${remaining} chunk(s) still stale (embed failures or an interrupted run).`);
|
||||
serr('Re-run the same command to resume — completed chunks are never re-embedded.');
|
||||
}
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/** Re-export for the op handler + tests. */
|
||||
export { MIGRATION_STATE_KEY };
|
||||
+216
-98
@@ -10,12 +10,13 @@
|
||||
import { createEngine } from '../core/engine-factory.ts';
|
||||
import { loadConfig, saveConfig, toEngineConfig, gbrainPath, effectiveEnvDatabaseUrl, type GBrainConfig } from '../core/config.ts';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import type { EngineConfig } from '../core/types.ts';
|
||||
import type { EngineConfig, Page } from '../core/types.ts';
|
||||
import { writeFileSync, readFileSync, existsSync, unlinkSync } from 'fs';
|
||||
import { createHash } from 'crypto';
|
||||
import { resolve } from 'path';
|
||||
import { createProgress } from '../core/progress.ts';
|
||||
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
|
||||
import { setCliExitVerdict } from '../core/cli-force-exit.ts';
|
||||
|
||||
interface MigrateOpts {
|
||||
targetEngine: 'postgres' | 'pglite';
|
||||
@@ -143,6 +144,99 @@ export async function copyMigrationSources(source: BrainEngine, target: BrainEng
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* postgres.js's UNDEFINED_VALUE guard rejects any bound parameter that is JS
|
||||
* `undefined` — unlike PGLite, it will not silently treat it as SQL NULL.
|
||||
* A page read back from a PGLite source can carry `undefined` for a column
|
||||
* that is legitimately empty/NULL (a read-side driver-shape difference, not
|
||||
* a data problem), and passing that value straight into a Postgres
|
||||
* `putPage` throws mid-insert (#3194). Normalizing at this migrate-only
|
||||
* boundary — rather than inside `putPage` itself, which many non-migrate
|
||||
* callers also use — turns that driver-shape difference into an explicit
|
||||
* SQL NULL, so only a genuine NOT-NULL constraint violation (an actual data
|
||||
* problem) still surfaces as a page-copy failure.
|
||||
*/
|
||||
function nullifyUndefinedColumns<T extends Record<string, unknown>>(row: T): T {
|
||||
const normalized = { ...row };
|
||||
for (const key of Object.keys(normalized) as (keyof T)[]) {
|
||||
if (normalized[key] === undefined) normalized[key] = null as T[typeof key];
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy one page's full row (page body, chunks, tags, timeline, raw data)
|
||||
* from source to target. Throws on any failure — the caller (the per-page
|
||||
* loop in runMigrateEngine) decides how to account for that: track it as a
|
||||
* failed page and keep going, rather than letting one bad row silently
|
||||
* disappear from the progress count (#3194). Exported so unit tests can
|
||||
* inject fake engines and exercise the failure path without a live
|
||||
* DATABASE_URL.
|
||||
*/
|
||||
export async function copyPageToTarget(
|
||||
source: BrainEngine,
|
||||
target: BrainEngine,
|
||||
page: Page,
|
||||
): Promise<void> {
|
||||
const sourceOpts = { sourceId: page.source_id };
|
||||
|
||||
// Copy page (preserve source_id). v0.32.8 F8: thread source_id end-to-end
|
||||
// so multi-source pages migrate intact.
|
||||
await target.putPage(page.slug, nullifyUndefinedColumns({
|
||||
type: page.type,
|
||||
title: page.title,
|
||||
compiled_truth: page.compiled_truth,
|
||||
timeline: page.timeline,
|
||||
frontmatter: page.frontmatter,
|
||||
content_hash: page.content_hash,
|
||||
}), sourceOpts);
|
||||
|
||||
// Copy chunks with embeddings.
|
||||
const chunks = await source.getChunksWithEmbeddings(page.slug, sourceOpts);
|
||||
if (chunks.length > 0) {
|
||||
await target.upsertChunks(page.slug, chunks.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
embedding: c.embedding || undefined,
|
||||
model: c.model,
|
||||
token_count: c.token_count || undefined,
|
||||
})), sourceOpts);
|
||||
}
|
||||
|
||||
// Copy tags
|
||||
const tags = await source.getTags(page.slug, sourceOpts);
|
||||
for (const tag of tags) {
|
||||
await target.addTag(page.slug, tag, sourceOpts);
|
||||
}
|
||||
|
||||
// Copy timeline
|
||||
const timeline = await source.getTimeline(page.slug, sourceOpts);
|
||||
for (const entry of timeline) {
|
||||
await target.addTimelineEntry(page.slug, {
|
||||
date: entry.date,
|
||||
source: entry.source,
|
||||
summary: entry.summary,
|
||||
detail: entry.detail,
|
||||
}, sourceOpts);
|
||||
}
|
||||
|
||||
// Copy raw data
|
||||
const rawData = await source.getRawData(page.slug, undefined, sourceOpts);
|
||||
for (const rd of rawData) {
|
||||
await target.putRawData(page.slug, rd.source, rd.data, sourceOpts);
|
||||
}
|
||||
}
|
||||
|
||||
/** A page that failed to copy during migrate — tracked so the run's final
|
||||
* summary reports it honestly instead of letting the "N copied" counter
|
||||
* imply every page landed (#3194). */
|
||||
export interface MigratePageFailure {
|
||||
source_id: string;
|
||||
slug: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]): Promise<void> {
|
||||
const opts = parseArgs(args);
|
||||
const config = loadConfig();
|
||||
@@ -177,32 +271,47 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
|
||||
await targetEngine.connect(targetConfig);
|
||||
await targetEngine.initSchema();
|
||||
|
||||
// Check if target has data
|
||||
const targetStats = await targetEngine.getStats();
|
||||
if (targetStats.page_count > 0 && !opts.force) {
|
||||
console.error(`Target brain is not empty (${targetStats.page_count} pages).`);
|
||||
console.error('Run with --force to overwrite, or migrate to an empty brain.');
|
||||
await targetEngine.disconnect();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (targetStats.page_count > 0 && opts.force) {
|
||||
console.log('--force: wiping target brain...');
|
||||
// v0.18.0+ multi-source: deletePage(slug) is now source-scoped (defaults
|
||||
// to 'default'), so per-page iteration would skip non-default-source
|
||||
// rows. migrate-engine --force is a destructive wipe across the entire
|
||||
// brain — all sources, all pages — so we issue a raw DELETE that matches
|
||||
// the original semantic. Cascades through content_chunks / page_links /
|
||||
// tags / timeline_entries / page_versions via existing FKs.
|
||||
await targetEngine.executeRaw('DELETE FROM pages');
|
||||
}
|
||||
|
||||
// Load or create manifest for resume
|
||||
// Load or create manifest for resume. Checked BEFORE the non-empty-target
|
||||
// guard below: a manifest matching this exact target means the target's
|
||||
// existing rows came from OUR OWN in-progress migration (#3194's per-page
|
||||
// failures now leave the target non-empty by design instead of crashing),
|
||||
// so a resume must not be treated as "attempting to migrate into a
|
||||
// foreign non-empty brain".
|
||||
let manifest = loadManifest();
|
||||
if (manifest && !manifestMatchesTarget(manifest, targetId)) {
|
||||
console.log('Previous migration was to a different target. Starting fresh.');
|
||||
manifest = null;
|
||||
}
|
||||
const resumingMatchingManifest = manifest !== null;
|
||||
|
||||
// Check if target has data
|
||||
const targetStats = await targetEngine.getStats();
|
||||
if (opts.force) {
|
||||
if (targetStats.page_count > 0) {
|
||||
console.log('--force: wiping target brain...');
|
||||
// v0.18.0+ multi-source: deletePage(slug) is now source-scoped (defaults
|
||||
// to 'default'), so per-page iteration would skip non-default-source
|
||||
// rows. migrate-engine --force is a destructive wipe across the entire
|
||||
// brain — all sources, all pages — so we issue a raw DELETE that matches
|
||||
// the original semantic. Cascades through content_chunks / page_links /
|
||||
// tags / timeline_entries / page_versions via existing FKs.
|
||||
await targetEngine.executeRaw('DELETE FROM pages');
|
||||
}
|
||||
// --force always starts this exact migration fresh against this target:
|
||||
// a manifest tracking a previous attempt must not be trusted to skip
|
||||
// pages, regardless of whether the target LOOKED non-empty just now
|
||||
// (e.g. the target DB file was recreated out-of-band but
|
||||
// ~/.gbrain/migrate-manifest.json survived) — round 2 of #3194.
|
||||
manifest = null;
|
||||
} else if (targetStats.page_count > 0 && !resumingMatchingManifest) {
|
||||
console.error(`Target brain is not empty (${targetStats.page_count} pages).`);
|
||||
console.error('Run with --force to overwrite, or migrate to an empty brain.');
|
||||
await targetEngine.disconnect();
|
||||
process.exit(1);
|
||||
} else if (targetStats.page_count > 0 && resumingMatchingManifest) {
|
||||
console.log(`Resuming previous migration: ${manifest!.completed_slugs.length} page(s) already copied.`);
|
||||
}
|
||||
|
||||
// v0.32.8 F8: manifest keys are now `${source_id}::${slug}` so multi-source
|
||||
// migrations don't collide on same-slug-different-source pages. Pre-v0.32.8
|
||||
// entries were bare slugs; we keep treating those as default-source for
|
||||
@@ -219,6 +328,13 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
|
||||
started_at: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
// Persist immediately, before any page copy runs. Otherwise a run where
|
||||
// EVERY page fails after its putPage lands (but before completed_slugs
|
||||
// ever gets a successful entry) leaves the target non-empty with no
|
||||
// manifest file on disk at all — the next invocation can't tell this
|
||||
// was a resumable in-progress migration and hits the non-empty guard
|
||||
// above requiring --force (round 2 of #3194).
|
||||
saveManifest(manifest);
|
||||
|
||||
// Pages.source_id is a foreign key. Copy the complete source catalog first,
|
||||
// including archived rows and sync/routing metadata, so every page write has
|
||||
@@ -235,82 +351,68 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
|
||||
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
|
||||
progress.start('migrate.copy_pages', pagesToMigrate.length);
|
||||
|
||||
// v0.32.8 F8: thread source_id end-to-end so multi-source pages migrate
|
||||
// intact. Pre-fix: putPage / getTags / getTimeline / getRawData / getLinks
|
||||
// all silently defaulted to source_id='default', so non-default-source
|
||||
// tags / timeline / raw / links were either dropped or attached to the
|
||||
// wrong row.
|
||||
let migrated = 0;
|
||||
const failures: MigratePageFailure[] = [];
|
||||
for (const page of pagesToMigrate) {
|
||||
// v0.32.8 F8: thread source_id end-to-end so multi-source pages migrate
|
||||
// intact. Pre-fix: putPage / getTags / getTimeline / getRawData / getLinks
|
||||
// all silently defaulted to source_id='default', so non-default-source
|
||||
// tags / timeline / raw / links were either dropped or attached to the
|
||||
// wrong row.
|
||||
const sourceOpts = { sourceId: page.source_id };
|
||||
|
||||
// Copy page (preserve source_id)
|
||||
await targetEngine.putPage(page.slug, {
|
||||
type: page.type,
|
||||
title: page.title,
|
||||
compiled_truth: page.compiled_truth,
|
||||
timeline: page.timeline,
|
||||
frontmatter: page.frontmatter,
|
||||
content_hash: page.content_hash,
|
||||
}, sourceOpts);
|
||||
|
||||
// Copy chunks with embeddings.
|
||||
const chunks = await sourceEngine.getChunksWithEmbeddings(page.slug, sourceOpts);
|
||||
if (chunks.length > 0) {
|
||||
await targetEngine.upsertChunks(page.slug, chunks.map(c => ({
|
||||
chunk_index: c.chunk_index,
|
||||
chunk_text: c.chunk_text,
|
||||
chunk_source: c.chunk_source,
|
||||
embedding: c.embedding || undefined,
|
||||
model: c.model,
|
||||
token_count: c.token_count || undefined,
|
||||
})), sourceOpts);
|
||||
try {
|
||||
await copyPageToTarget(sourceEngine, targetEngine, page);
|
||||
// Track progress with composite key so multi-source resume is correct.
|
||||
manifest!.completed_slugs.push(makeManifestKey(page.source_id, page.slug));
|
||||
saveManifest(manifest!);
|
||||
migrated++;
|
||||
} catch (e) {
|
||||
// #3194: a per-page write failure must never be swallowed into the
|
||||
// success count. Leave it OUT of completed_slugs (a resume retries
|
||||
// it — putPage/upsertChunks/etc. are all upserts, so re-running the
|
||||
// whole page copy is safe) and surface it in the final summary below
|
||||
// instead of letting "N pages copied" imply everything landed.
|
||||
failures.push({
|
||||
source_id: page.source_id,
|
||||
slug: page.slug,
|
||||
reason: e instanceof Error ? e.message : String(e),
|
||||
});
|
||||
}
|
||||
|
||||
// Copy tags
|
||||
const tags = await sourceEngine.getTags(page.slug, sourceOpts);
|
||||
for (const tag of tags) {
|
||||
await targetEngine.addTag(page.slug, tag, sourceOpts);
|
||||
}
|
||||
|
||||
// Copy timeline
|
||||
const timeline = await sourceEngine.getTimeline(page.slug, sourceOpts);
|
||||
for (const entry of timeline) {
|
||||
await targetEngine.addTimelineEntry(page.slug, {
|
||||
date: entry.date,
|
||||
source: entry.source,
|
||||
summary: entry.summary,
|
||||
detail: entry.detail,
|
||||
}, sourceOpts);
|
||||
}
|
||||
|
||||
// Copy raw data
|
||||
const rawData = await sourceEngine.getRawData(page.slug, undefined, sourceOpts);
|
||||
for (const rd of rawData) {
|
||||
await targetEngine.putRawData(page.slug, rd.source, rd.data, sourceOpts);
|
||||
}
|
||||
|
||||
// Copy versions
|
||||
const versions = await sourceEngine.getVersions(page.slug, sourceOpts);
|
||||
// Versions are snapshots, we recreate them on the target
|
||||
// (createVersion takes a snapshot of current state, which we just set)
|
||||
|
||||
// Track progress with composite key so multi-source resume is correct.
|
||||
manifest!.completed_slugs.push(makeManifestKey(page.source_id, page.slug));
|
||||
saveManifest(manifest!);
|
||||
migrated++;
|
||||
progress.tick(1, page.slug);
|
||||
}
|
||||
progress.finish();
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(`\n${failures.length} of ${pagesToMigrate.length} page(s) FAILED to copy and were NOT migrated:`);
|
||||
for (const f of failures) {
|
||||
const key = f.source_id === 'default' ? f.slug : `${f.source_id}::${f.slug}`;
|
||||
console.error(` - ${key}: ${f.reason}`);
|
||||
}
|
||||
console.error('Re-run `gbrain migrate` to retry the failed pages (already-copied pages resume via the manifest).');
|
||||
// Non-fatal so the run still copies links + config for everything that
|
||||
// DID land, but the process must exit non-zero — a partial migration
|
||||
// must never look identical to a clean one.
|
||||
setCliExitVerdict(1);
|
||||
}
|
||||
|
||||
// Copy links (after all pages exist in target).
|
||||
// v0.32.8 F8: thread source_id so cross-source links migrate correctly.
|
||||
// #3194: a page that failed to copy above does NOT exist on the target,
|
||||
// so any link touching it would violate the target's FK and abort this
|
||||
// whole phase (the exact "addLink failed: page ... not found" crash from
|
||||
// the original report). Skip links on either end of a known-failed page —
|
||||
// a retry that successfully copies the page also re-copies its links.
|
||||
const failedKeys = new Set(failures.map(f => makeManifestKey(f.source_id, f.slug)));
|
||||
console.log('Copying links...');
|
||||
progress.start('migrate.copy_links', allPages.length);
|
||||
for (const page of allPages) {
|
||||
if (failedKeys.has(makeManifestKey(page.source_id, page.slug))) {
|
||||
progress.tick(1);
|
||||
continue;
|
||||
}
|
||||
const sourceOpts = { sourceId: page.source_id };
|
||||
const links = await sourceEngine.getLinks(page.slug, sourceOpts);
|
||||
for (const link of links) {
|
||||
if (failedKeys.has(makeManifestKey(page.source_id, link.to_slug))) continue;
|
||||
await targetEngine.addLink(
|
||||
link.from_slug, link.to_slug,
|
||||
link.context, link.link_type,
|
||||
@@ -342,22 +444,38 @@ export async function runMigrateEngine(sourceEngine: BrainEngine, args: string[]
|
||||
// Update local config. v0.37 fix wave: preserve existing file-plane
|
||||
// embedding/expansion/chat config across the engine migration; only
|
||||
// the engine + connection target should change.
|
||||
const existingFile = (await import('../core/config.ts')).loadConfigFileOnly() ?? ({} as GBrainConfig);
|
||||
const newConfig: GBrainConfig = {
|
||||
...existingFile,
|
||||
engine: opts.targetEngine,
|
||||
...(opts.targetEngine === 'postgres'
|
||||
? { database_url: targetConfig.database_url, database_path: undefined }
|
||||
: { database_path: targetConfig.database_path, database_url: undefined }),
|
||||
};
|
||||
saveConfig(newConfig);
|
||||
//
|
||||
// #3194: only flip the ACTIVE config when the migration is fully clean.
|
||||
// A partial migration leaves the target's data incomplete; auto-switching
|
||||
// every subsequent `gbrain` invocation onto that incomplete target would
|
||||
// (a) make the failure invisible behind otherwise-normal usage and (b)
|
||||
// break the natural retry — `gbrain migrate --to X` again would hit the
|
||||
// "Already using X engine" guard even though the migration never actually
|
||||
// finished. Leaving the file-plane config untouched keeps the source the
|
||||
// active engine, so a retry (which resumes via the still-intact manifest)
|
||||
// is a same-shaped command, not a special case.
|
||||
if (failures.length === 0) {
|
||||
const existingFile = (await import('../core/config.ts')).loadConfigFileOnly() ?? ({} as GBrainConfig);
|
||||
const newConfig: GBrainConfig = {
|
||||
...existingFile,
|
||||
engine: opts.targetEngine,
|
||||
...(opts.targetEngine === 'postgres'
|
||||
? { database_url: targetConfig.database_url, database_path: undefined }
|
||||
: { database_path: targetConfig.database_path, database_url: undefined }),
|
||||
};
|
||||
saveConfig(newConfig);
|
||||
// Clean up the resume manifest — only safe once nothing is left pending.
|
||||
clearManifest();
|
||||
}
|
||||
|
||||
// Clean up
|
||||
clearManifest();
|
||||
|
||||
console.log(`\nMigration complete. ${migrated} pages transferred.`);
|
||||
console.log(`Config updated to engine: ${opts.targetEngine}`);
|
||||
if (config.engine === 'pglite' && config.database_path) {
|
||||
if (failures.length > 0) {
|
||||
console.log(`\nMigration completed with errors. ${migrated} of ${pagesToMigrate.length} pages copied, ${failures.length} failed (${completedSet.size} already done from a prior run). See failure list above.`);
|
||||
console.log(`Config NOT switched — still using engine: ${config.engine}. Re-run \`gbrain migrate --to ${opts.targetEngine}\` to retry; already-copied pages resume via the manifest.`);
|
||||
} else {
|
||||
console.log(`\nMigration complete. ${migrated} pages transferred.`);
|
||||
console.log(`Config updated to engine: ${opts.targetEngine}`);
|
||||
}
|
||||
if (failures.length === 0 && config.engine === 'pglite' && config.database_path) {
|
||||
console.log(`Original PGLite brain preserved at ${config.database_path} (backup).`);
|
||||
}
|
||||
|
||||
|
||||
@@ -186,17 +186,6 @@ async function phaseBFenceFacts(
|
||||
const localPathById = new Map<string, string | null>();
|
||||
for (const s of sources) localPathById.set(s.id, s.local_path);
|
||||
|
||||
// Dirty-tree refusal: check every source's local_path before writing.
|
||||
for (const [id, localPath] of localPathById) {
|
||||
if (localPath && isLocalPathDirty(localPath)) {
|
||||
return {
|
||||
name: 'fence_facts',
|
||||
status: 'failed',
|
||||
detail: `source "${id}" has uncommitted changes in ${localPath}. Commit or stash, then re-run.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Walk legacy rows in (source_id, entity_slug) groups for per-page
|
||||
// atomic writes.
|
||||
const legacy = await engine.executeRaw<LegacyFactRow>(
|
||||
@@ -235,6 +224,21 @@ async function phaseBFenceFacts(
|
||||
groups.set(key, list);
|
||||
}
|
||||
|
||||
// Dirty-tree refusal: check ONLY the sources we are about to write
|
||||
// into. A dirty tree in an unrelated source (or zero fenceable rows
|
||||
// at all) must not block a no-op or a targeted backfill (#927).
|
||||
const targetSourceIds = new Set([...groups.keys()].map(k => k.split('\0')[0]));
|
||||
for (const id of targetSourceIds) {
|
||||
const localPath = localPathById.get(id);
|
||||
if (localPath && isLocalPathDirty(localPath)) {
|
||||
return {
|
||||
name: 'fence_facts',
|
||||
status: 'failed',
|
||||
detail: `source "${id}" has uncommitted changes in ${localPath}. Commit or stash, then re-run.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
for (const [key, group] of groups) {
|
||||
const [sourceId, entitySlug] = key.split('\0');
|
||||
const localPath = localPathById.get(sourceId)!;
|
||||
|
||||
@@ -142,12 +142,16 @@ export async function runOnboard(engine: BrainEngine, args: string[]): Promise<v
|
||||
|
||||
// --auto path: runs through the T2 library orchestrator. Hooks emit CLI
|
||||
// progress to stderr; the final result lands as JSON on stdout (or human
|
||||
// summary).
|
||||
// summary). extraRemediations (gathered above from runAllOnboardChecks)
|
||||
// is threaded into the runner so the onboard-check remediations
|
||||
// (extract-ner, extract-timeline-from-meetings, etc.) reach the planner
|
||||
// — the same wiring the --check path uses above.
|
||||
const result = await runRemediation(
|
||||
engine,
|
||||
{
|
||||
targetScore,
|
||||
maxUsd,
|
||||
extraRemediations,
|
||||
// --auto --yes opts into the prompt_required tier too; library
|
||||
// doesn't distinguish auto_apply vs prompt_required, it just runs
|
||||
// every remediation in the plan. The plan-building side (T12 render)
|
||||
|
||||
@@ -134,7 +134,7 @@ EXAMPLES
|
||||
gbrain providers list
|
||||
gbrain providers test --model openai:text-embedding-3-large
|
||||
gbrain providers test --touchpoint chat --model anthropic:claude-haiku-4-5
|
||||
gbrain providers test --touchpoint chat --model deepseek:deepseek-chat
|
||||
gbrain providers test --touchpoint chat --model deepseek:deepseek-v4-flash
|
||||
gbrain providers env ollama
|
||||
gbrain providers explain --json
|
||||
`);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { VERSION } from '../version.ts';
|
||||
import { isMinorOrMajorBump, isValidVersionString } from '../core/semver.ts';
|
||||
import { isNewerVersion, isValidVersionString } from '../core/semver.ts';
|
||||
import { fetchChangelog, fetchLatestRelease } from './check-update.ts';
|
||||
import { detectInstallMethod, runUpgrade } from './upgrade.ts';
|
||||
import { writeUpdateCache } from '../core/self-upgrade.ts';
|
||||
@@ -37,7 +37,7 @@ export async function runSelfUpgrade(args: string[]): Promise<void> {
|
||||
|
||||
const release = await fetchLatestRelease();
|
||||
const latest = release ? release.tag.replace(/^v/, '') : null;
|
||||
const behind = !!latest && isValidVersionString(latest) && isMinorOrMajorBump(VERSION, latest);
|
||||
const behind = !!latest && isValidVersionString(latest) && isNewerVersion(VERSION, latest);
|
||||
|
||||
// Warm the cache so the next invocation's startup hook can emit without a fetch.
|
||||
try {
|
||||
|
||||
+115
-10
@@ -45,6 +45,7 @@ import {
|
||||
type IngestionContentType,
|
||||
type IngestionEvent,
|
||||
} from '../core/ingestion/types.ts';
|
||||
import { resolveOwnerHolder } from '../core/owner-holder.ts';
|
||||
|
||||
/**
|
||||
* /health endpoint timeout. 3s rather than 5s: Fly.io's default
|
||||
@@ -112,6 +113,24 @@ export function shouldSuppressBootstrapPrint(opts: {
|
||||
return !opts.isTty;
|
||||
}
|
||||
|
||||
export type OAuthTokenRateLimitConfig = {
|
||||
windowMs: number;
|
||||
max: number;
|
||||
};
|
||||
|
||||
function parsePositiveIntEnv(value: string | undefined, fallback: number): number {
|
||||
if (value === undefined) return fallback;
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
export function resolveOAuthTokenRateLimit(env: NodeJS.ProcessEnv = process.env): OAuthTokenRateLimitConfig {
|
||||
return {
|
||||
windowMs: parsePositiveIntEnv(env.GBRAIN_OAUTH_TOKEN_RATE_LIMIT_WINDOW_MS, 15 * 60 * 1000),
|
||||
max: parsePositiveIntEnv(env.GBRAIN_OAUTH_TOKEN_RATE_LIMIT_MAX, 50),
|
||||
};
|
||||
}
|
||||
|
||||
export type ProbeHealthResult =
|
||||
| { ok: true; status: 200; body: { status: 'ok'; version: string; engine: string; [k: string]: unknown } }
|
||||
| { ok: false; status: 503; body: { error: 'service_unavailable'; error_description: string } };
|
||||
@@ -430,6 +449,34 @@ export function skillPublishStatus(publishSkills: boolean): { bannerValue: strin
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* #1196: startup embedding-width guard for stateless host deployments.
|
||||
*
|
||||
* `embedding_model` / `embedding_dimensions` are file/env-plane only, so a
|
||||
* container booted WITHOUT a config.json (stateless host) resolves the
|
||||
* compiled-in default embedding width. Against an existing brain whose
|
||||
* `content_chunks.embedding` is a different `vector(N)`, every write then
|
||||
* fails with an opaque dim mismatch. Run doctor's existing
|
||||
* embedding_width_consistency check at serve startup and return a loud
|
||||
* banner (with the paste-ready recipe) when it isn't ok. Fail-open: a check
|
||||
* error never blocks serving read traffic.
|
||||
*/
|
||||
export async function embeddingWidthStartupWarning(engine: BrainEngine): Promise<string | null> {
|
||||
try {
|
||||
const { checkEmbeddingWidthConsistency } = await import('./doctor.ts');
|
||||
const check = await checkEmbeddingWidthConsistency(engine);
|
||||
if (check.status === 'ok') return null;
|
||||
return (
|
||||
`[serve-http] WARNING: embedding width check failed — writes that embed will fail until fixed.\n` +
|
||||
`${check.message}\n` +
|
||||
`Stateless hosts: embedding_model/embedding_dimensions resolve from env/config.json only — ` +
|
||||
`set GBRAIN_EMBEDDING_MODEL / GBRAIN_EMBEDDING_DIMENSIONS (or mount config.json) to match the brain's schema.`
|
||||
);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function runServeHttp(engine: BrainEngine, options: ServeHttpOptions) {
|
||||
const { port, tokenTtl, enableDcr, enableDcrInsecure, publicUrl, logFullParams } = options;
|
||||
// v0.34.1 (#864, D11): default bind flipped from 0.0.0.0 to 127.0.0.1.
|
||||
@@ -454,6 +501,14 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
);
|
||||
}
|
||||
|
||||
// #1196: fail-loud at startup when the resolved embedding width diverges
|
||||
// from the brain's actual vector(N) column (stateless containers falling
|
||||
// through to the compiled-in default). Non-fatal: reads still work.
|
||||
{
|
||||
const widthWarn = await embeddingWidthStartupWarning(engine);
|
||||
if (widthWarn) console.error(widthWarn);
|
||||
}
|
||||
|
||||
// Skill-publishing status for the banner + nudge. Mirrors readMcpPublishSkills
|
||||
// (skill-catalog.ts): the DB plane (`gbrain config set`) wins over the file
|
||||
// plane. When OFF, a connected coding agent can't see the host's skill
|
||||
@@ -632,12 +687,13 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// Custom client_credentials handler (before mcpAuthRouter)
|
||||
// SDK's token handler only supports authorization_code and refresh_token
|
||||
// ---------------------------------------------------------------------------
|
||||
const oauthTokenRateLimit = resolveOAuthTokenRateLimit();
|
||||
const ccRateLimiter = rateLimit({
|
||||
windowMs: 15 * 60 * 1000,
|
||||
max: 50,
|
||||
windowMs: oauthTokenRateLimit.windowMs,
|
||||
max: oauthTokenRateLimit.max,
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
message: { error: 'too_many_requests', error_description: 'Rate limit exceeded. Try again in 15 minutes.' },
|
||||
message: { error: 'too_many_requests', error_description: 'Rate limit exceeded. Try again later.' },
|
||||
});
|
||||
|
||||
// Magic-link rate limiter: 10 requests/min/IP. The bootstrap token is
|
||||
@@ -1100,7 +1156,8 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// Unified view: OAuth clients + legacy API keys
|
||||
const oauthClients = await sql`
|
||||
SELECT c.client_id as id, c.client_name as name, 'oauth' as auth_type,
|
||||
c.grant_types, c.scope, c.created_at, c.token_ttl,
|
||||
c.grant_types, c.scope, c.source_id, c.federated_read,
|
||||
c.created_at, c.token_ttl,
|
||||
CASE WHEN c.deleted_at IS NOT NULL THEN 'revoked' ELSE 'active' END as status,
|
||||
(SELECT max(created_at) FROM mcp_request_log WHERE token_name = c.client_id) as last_used_at,
|
||||
(SELECT count(*)::int FROM mcp_request_log WHERE token_name = c.client_id) as total_requests,
|
||||
@@ -1116,12 +1173,25 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
(SELECT count(*)::int FROM mcp_request_log WHERE token_name = a.name AND created_at > now() - interval '24 hours') as requests_today
|
||||
FROM access_tokens a ORDER BY a.created_at DESC
|
||||
`;
|
||||
res.json([...oauthClients, ...legacyKeys]);
|
||||
res.json([
|
||||
...oauthClients,
|
||||
...legacyKeys.map((key) => ({ ...key, source_id: null, federated_read: [] })),
|
||||
]);
|
||||
} catch (e) {
|
||||
res.status(503).json({ error: 'service_unavailable' });
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/admin/api/sources', requireAdmin, async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const { listSources } = await import('../core/sources-ops.ts');
|
||||
const sources = await listSources(engine);
|
||||
res.json(sources.map(({ id, name, federated }) => ({ id, name, federated })));
|
||||
} catch {
|
||||
res.status(503).json({ error: 'service_unavailable' });
|
||||
}
|
||||
});
|
||||
|
||||
// v0.38 Slice 4 — per-OAuth-client agent spend viewer. Pre-computes today's
|
||||
// spend (committed + pending reservations) per client so the Agents tab
|
||||
// can render a "$X / $Y today" cell. Read-side endpoint only — no mutation.
|
||||
@@ -1205,7 +1275,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
app.get('/admin/api/calibration/pattern/:id', requireAdmin, async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { getLatestProfile } = await import('./calibration.ts');
|
||||
const holder = (req.query.holder as string) || 'garry';
|
||||
const holder = resolveOwnerHolder({ override: (req.query.holder as string) || undefined, configValue: await engine.getConfig('emotional_weight.user_holder') });
|
||||
const profile = await getLatestProfile(engine, { holder });
|
||||
if (!profile) {
|
||||
res.status(404).json({ error: 'no_profile' });
|
||||
@@ -1255,7 +1325,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
app.get('/admin/api/calibration/profile', requireAdmin, async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { getLatestProfile } = await import('./calibration.ts');
|
||||
const holder = (req.query.holder as string) || 'garry';
|
||||
const holder = resolveOwnerHolder({ override: (req.query.holder as string) || undefined, configValue: await engine.getConfig('emotional_weight.user_holder') });
|
||||
const profile = await getLatestProfile(engine, { holder });
|
||||
res.json(profile);
|
||||
} catch (err) {
|
||||
@@ -1272,7 +1342,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
renderAbandonedThreadsCard,
|
||||
renderPatternStatementsCard,
|
||||
} = await import('../core/calibration/svg-renderer.ts');
|
||||
const holder = (req.query.holder as string) || 'garry';
|
||||
const holder = resolveOwnerHolder({ override: (req.query.holder as string) || undefined, configValue: await engine.getConfig('emotional_weight.user_holder') });
|
||||
const type = req.params.type;
|
||||
const profile = await getLatestProfile(engine, { holder });
|
||||
|
||||
@@ -1511,6 +1581,38 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
}
|
||||
});
|
||||
|
||||
// v0.42.x (#1914): rescope an OAuth client's write source / federated read
|
||||
// scope. Admin-gated on purpose — DCR clients must never self-widen their
|
||||
// scope (fail-closed trust); only the operator rescopes, here or via
|
||||
// `gbrain auth rescope-client`. Source ids are validated by the canonical
|
||||
// validator inside rescopeClient.
|
||||
app.post('/admin/api/rescope-client', requireAdmin, express.json(), async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { clientId, sourceId, federatedRead } = req.body ?? {};
|
||||
if (!clientId || typeof clientId !== 'string') {
|
||||
res.status(400).json({ error: 'clientId required' });
|
||||
return;
|
||||
}
|
||||
if (federatedRead !== undefined &&
|
||||
!(Array.isArray(federatedRead) && federatedRead.every((s: unknown) => typeof s === 'string'))) {
|
||||
res.status(400).json({ error: 'federatedRead must be an array of source id strings' });
|
||||
return;
|
||||
}
|
||||
if (sourceId !== undefined && typeof sourceId !== 'string') {
|
||||
res.status(400).json({ error: 'sourceId must be a string' });
|
||||
return;
|
||||
}
|
||||
const result = await oauthProvider.rescopeClient(clientId, { sourceId, federatedRead });
|
||||
res.json(result);
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : 'Rescope failed';
|
||||
const status = /No OAuth client found/.test(message) ? 404
|
||||
: /Invalid source_id|requires --source|cannot be empty|does not exist/.test(message) ? 400
|
||||
: 500;
|
||||
res.status(status).json({ error: message });
|
||||
}
|
||||
});
|
||||
|
||||
// Revoke OAuth client
|
||||
app.post('/admin/api/revoke-client', requireAdmin, express.json(), async (req: Request, res: Response) => {
|
||||
try {
|
||||
@@ -2161,8 +2263,10 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
// Other event types (ping, pull_request, etc.) return 202 'ignored'
|
||||
// so GitHub doesn't retry.
|
||||
// D15.5: HMAC compare uses the shared safeHexEqual helper.
|
||||
// D18: submits 'sync' job with auto_embed_backfill=true and priority -10
|
||||
// (above autopilot's 0).
|
||||
// D18: submits 'sync' job with extraction + auto_embed_backfill enabled and
|
||||
// priority -10 (above autopilot's 0). This opts normal incremental pushes
|
||||
// into sync's inline extraction while pagesAffected still identifies the
|
||||
// changed pages. The sync core can still defer large (>100) changes.
|
||||
// ---------------------------------------------------------------------------
|
||||
const githubWebhookLimiter = rateLimit({
|
||||
windowMs: 60_000,
|
||||
@@ -2282,6 +2386,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
|
||||
'sync',
|
||||
{
|
||||
sourceId: source.id,
|
||||
noExtract: false,
|
||||
auto_embed_backfill: true,
|
||||
embed_reason: 'webhook',
|
||||
},
|
||||
|
||||
+68
-1
@@ -9,6 +9,17 @@ import { startMcpServer } from '../mcp/server.ts';
|
||||
// the dir, sees a dead PID, and removes it).
|
||||
const CLEANUP_DEADLINE_MS = 5_000;
|
||||
|
||||
// Boot-readiness deadline (#3273). A serve process that wedges mid-boot
|
||||
// (e.g. an MCP boot step that never completes because a configured
|
||||
// upstream is unreachable) holds the PGLite write lock indefinitely: the
|
||||
// post-#2348 lock discipline never steals from a live holder, so every
|
||||
// CLI consumer times out until someone hunts down and kills the PID. If
|
||||
// startMcpServer hasn't finished connecting the transport within this
|
||||
// window, we release the engine (dropping the lock) and exit non-zero so
|
||||
// a supervisor can restart with backoff. Env-tunable via
|
||||
// GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS; 0 disables.
|
||||
const DEFAULT_BOOT_TIMEOUT_SECONDS = 60;
|
||||
|
||||
// How often the parent-process watchdog polls the live kernel parent PID
|
||||
// (via `readLiveParentPid`, NOT the cached `process.ppid` — see that
|
||||
// helper's comment). We don't receive a signal when our parent dies (the
|
||||
@@ -67,6 +78,10 @@ export interface ServeOptions {
|
||||
// transport.onclose still cover legitimate shutdown.
|
||||
// Defaults to `process.env.MCP_STDIO === '1'` when omitted.
|
||||
mcpStdio?: boolean;
|
||||
// Test seam for the boot-readiness deadline (#3273). Milliseconds.
|
||||
// Defaults to GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS (seconds; 60 when
|
||||
// unset, 0 disables) when omitted.
|
||||
bootTimeoutMs?: number;
|
||||
}
|
||||
|
||||
export async function runServe(
|
||||
@@ -142,7 +157,43 @@ export async function runServe(
|
||||
installStdioLifecycle(engine, args, opts);
|
||||
|
||||
const start = opts.startMcpServer ?? startMcpServer;
|
||||
await start(engine);
|
||||
|
||||
// Boot-readiness deadline (#3273): never sit on the PGLite write lock
|
||||
// forever with a boot that never completes. On expiry: log, release the
|
||||
// engine (drops the lock), exit non-zero so supervisors restart with
|
||||
// backoff. The disconnect itself is raced against CLEANUP_DEADLINE_MS,
|
||||
// same as the graceful-shutdown path, so a wedged WASM close can't trap
|
||||
// us either.
|
||||
const bootTimeoutMs = opts.bootTimeoutMs ?? resolveBootTimeoutMs();
|
||||
let bootDeadline: ReturnType<typeof setTimeout> | null = null;
|
||||
if (bootTimeoutMs > 0) {
|
||||
const log = opts.log ?? ((msg: string) => console.error(msg));
|
||||
const exit = opts.exit ?? ((code?: number) => { process.exit(code); });
|
||||
bootDeadline = setTimeout(() => {
|
||||
log(
|
||||
`GBrain MCP server: boot did not complete within ${bootTimeoutMs}ms — releasing DB lock and exiting so other consumers unblock (check configured provider endpoints; tune via GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS, 0 disables)`,
|
||||
);
|
||||
const cleanup = setTimeout(() => { exit(1); }, CLEANUP_DEADLINE_MS);
|
||||
cleanup.unref?.();
|
||||
Promise.resolve()
|
||||
.then(() => engine.disconnect())
|
||||
.catch((err: unknown) => {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
log(`GBrain MCP server: boot-deadline cleanup error: ${msg}`);
|
||||
})
|
||||
.finally(() => {
|
||||
clearTimeout(cleanup);
|
||||
exit(1);
|
||||
});
|
||||
}, bootTimeoutMs);
|
||||
bootDeadline.unref?.();
|
||||
}
|
||||
|
||||
try {
|
||||
await start(engine);
|
||||
} finally {
|
||||
if (bootDeadline) clearTimeout(bootDeadline);
|
||||
}
|
||||
// startMcpServer's `await server.connect(transport)` resolves once the
|
||||
// SDK has wired up its stdin 'data' listener; that listener keeps the
|
||||
// event loop alive. We deliberately do NOT add `await new Promise(() =>
|
||||
@@ -150,6 +201,22 @@ export async function runServe(
|
||||
// hooks from being able to call process.exit() cleanly.
|
||||
}
|
||||
|
||||
// Env resolution for the boot deadline. Lenient (warn + default) rather
|
||||
// than throw: this is an incident-time escape hatch, and a typo'd env var
|
||||
// must not turn a boot-safety net into a boot failure of its own.
|
||||
function resolveBootTimeoutMs(): number {
|
||||
const raw = process.env.GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS;
|
||||
if (raw === undefined || raw.trim() === '') return DEFAULT_BOOT_TIMEOUT_SECONDS * 1000;
|
||||
const n = Number(raw);
|
||||
if (!Number.isFinite(n) || n < 0) {
|
||||
console.error(
|
||||
`[gbrain serve] ignoring invalid GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS=${JSON.stringify(raw)} — using default ${DEFAULT_BOOT_TIMEOUT_SECONDS}s`,
|
||||
);
|
||||
return DEFAULT_BOOT_TIMEOUT_SECONDS * 1000;
|
||||
}
|
||||
return n * 1000;
|
||||
}
|
||||
|
||||
interface StdioLifecycleDeps {
|
||||
stdin: NodeJS.ReadableStream & { isTTY?: boolean };
|
||||
signals: Pick<NodeJS.Process, 'on'>;
|
||||
|
||||
@@ -53,6 +53,7 @@ import {
|
||||
import {
|
||||
loadAllSources,
|
||||
parseSourceConfig,
|
||||
normalizeSourceConfig,
|
||||
isSourceFederated,
|
||||
type SourceRow as LoadedSourceRow,
|
||||
} from '../core/sources-load.ts';
|
||||
@@ -711,7 +712,7 @@ async function runFederate(engine: BrainEngine, args: string[], value: boolean):
|
||||
config.federated = value;
|
||||
await engine.executeRaw(
|
||||
`UPDATE sources SET config = $1::text::jsonb WHERE id = $2`,
|
||||
[JSON.stringify(config), id],
|
||||
[JSON.stringify(normalizeSourceConfig(config)), id],
|
||||
);
|
||||
console.log(`Source "${id}" is now ${value ? 'federated (appears in cross-source default search)' : 'isolated (only searched when explicitly named)'}.`);
|
||||
|
||||
@@ -898,7 +899,7 @@ async function runWebhookSet(engine: BrainEngine, args: string[]): Promise<void>
|
||||
cfg.github_repo = githubRepo;
|
||||
await engine.executeRaw(
|
||||
`UPDATE sources SET config = $1::text::jsonb WHERE id = $2`,
|
||||
[JSON.stringify(cfg), id],
|
||||
[JSON.stringify(normalizeSourceConfig(cfg)), id],
|
||||
);
|
||||
|
||||
console.log(`Webhook configured for source "${id}":`);
|
||||
@@ -954,7 +955,7 @@ async function runWebhookRotate(engine: BrainEngine, args: string[]): Promise<vo
|
||||
cfg.webhook_secret = secret;
|
||||
await engine.executeRaw(
|
||||
`UPDATE sources SET config = $1::text::jsonb WHERE id = $2`,
|
||||
[JSON.stringify(cfg), id],
|
||||
[JSON.stringify(normalizeSourceConfig(cfg)), id],
|
||||
);
|
||||
console.log(`New webhook secret for source "${id}":`);
|
||||
console.log(` ${secret}`);
|
||||
@@ -978,7 +979,7 @@ async function runWebhookClear(engine: BrainEngine, args: string[]): Promise<voi
|
||||
delete cfg.github_repo;
|
||||
await engine.executeRaw(
|
||||
`UPDATE sources SET config = $1::text::jsonb WHERE id = $2`,
|
||||
[JSON.stringify(cfg), id],
|
||||
[JSON.stringify(normalizeSourceConfig(cfg)), id],
|
||||
);
|
||||
console.log(`Webhook configuration cleared for source "${id}".`);
|
||||
}
|
||||
@@ -1003,7 +1004,7 @@ async function runTrackedBranch(engine: BrainEngine, args: string[]): Promise<vo
|
||||
cfg.tracked_branch = setArg;
|
||||
await engine.executeRaw(
|
||||
`UPDATE sources SET config = $1::text::jsonb WHERE id = $2`,
|
||||
[JSON.stringify(cfg), id],
|
||||
[JSON.stringify(normalizeSourceConfig(cfg)), id],
|
||||
);
|
||||
console.log(`Tracked branch for source "${id}" set to "${setArg}".`);
|
||||
return;
|
||||
@@ -1019,7 +1020,7 @@ async function runTrackedBranch(engine: BrainEngine, args: string[]): Promise<vo
|
||||
cfg.tracked_branch = branch;
|
||||
await engine.executeRaw(
|
||||
`UPDATE sources SET config = $1::text::jsonb WHERE id = $2`,
|
||||
[JSON.stringify(cfg), id],
|
||||
[JSON.stringify(normalizeSourceConfig(cfg)), id],
|
||||
);
|
||||
console.log(`Detected branch "${branch}" for source "${id}"; persisted to config.tracked_branch.`);
|
||||
} catch (e) {
|
||||
|
||||
+365
-36
@@ -1,6 +1,6 @@
|
||||
import { existsSync, readFileSync, writeFileSync, statSync, realpathSync } from 'fs';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { join, relative } from 'path';
|
||||
import { isAbsolute, join, relative, sep } from 'path';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { DELETE_BATCH_SIZE } from '../core/engine-constants.ts';
|
||||
import { importFile } from '../core/import-file.ts';
|
||||
@@ -213,7 +213,7 @@ export interface SyncResult {
|
||||
* cron operators can disambiguate timeout vs pull-timeout in monitoring.
|
||||
*/
|
||||
filesImported?: number;
|
||||
reason?: 'timeout' | 'pull_timeout' | 'stall_timeout' | 'checkpoint_unavailable';
|
||||
reason?: 'timeout' | 'pull_timeout' | 'pull_failed' | 'stall_timeout' | 'checkpoint_unavailable';
|
||||
/**
|
||||
* v0.42.x (#1794): cumulative file paths durably banked to the checkpoint
|
||||
* across THIS run + prior resumed runs. Surfaced on every partial/blocked
|
||||
@@ -239,11 +239,12 @@ export interface SyncResult {
|
||||
export function estimateSourceTreeTokens(
|
||||
localPath: string,
|
||||
strategy: 'markdown' | 'code' | 'auto',
|
||||
opts: { includeGitignored?: boolean } = {},
|
||||
): { tokens: number; files: number } {
|
||||
let tokens = 0;
|
||||
let files = 0;
|
||||
try {
|
||||
const fileList = collectSyncableFiles(localPath, { strategy });
|
||||
const fileList = collectSyncableFiles(localPath, { strategy, includeGitignored: opts.includeGitignored });
|
||||
for (const fullPath of fileList) {
|
||||
try {
|
||||
const stat = statSync(fullPath);
|
||||
@@ -376,6 +377,7 @@ export function estimateInlineNewTokens(
|
||||
chunker_version: string | null;
|
||||
}>,
|
||||
currentChunkerVersion: string,
|
||||
opts: { forceFullTree?: boolean } = {},
|
||||
): InlineEstimate {
|
||||
let tokens = 0;
|
||||
let changedSources = 0;
|
||||
@@ -398,6 +400,14 @@ export function estimateInlineNewTokens(
|
||||
const strategy = cfg.strategy ?? 'markdown';
|
||||
const localPath = src.local_path;
|
||||
|
||||
if (opts.forceFullTree) {
|
||||
tokens += estimateSourceTreeTokens(localPath, strategy, { includeGitignored: true }).tokens;
|
||||
changedSources++;
|
||||
hadCeiling = true;
|
||||
ceilingReasons.push('include_gitignored');
|
||||
continue;
|
||||
}
|
||||
|
||||
// Rung 2: chunker drift forces a full re-chunk → full re-embed. CEILING.
|
||||
if (src.chunker_version !== currentChunkerVersion) {
|
||||
ceiling(localPath, strategy, 'chunker_drift');
|
||||
@@ -542,6 +552,7 @@ interface CostGateContext {
|
||||
jsonOut: boolean;
|
||||
yesFlag: boolean;
|
||||
full: boolean;
|
||||
includeGitignored?: boolean;
|
||||
/** Message prefix ('sync --all' | 'sync'). */
|
||||
label: string;
|
||||
}
|
||||
@@ -626,7 +637,9 @@ async function runInlineCostGate(
|
||||
}
|
||||
|
||||
// ── Inline path ───────────────────────────────────────────────
|
||||
const inline = estimateInlineNewTokens(sources, String(CHUNKER_VERSION));
|
||||
const inline = estimateInlineNewTokens(sources, String(CHUNKER_VERSION), {
|
||||
forceFullTree: ctx.includeGitignored === true,
|
||||
});
|
||||
// D7A: `--full` runs `performFullSync` → `runEmbedCore({stale:true})`, which
|
||||
// sweeps the pre-existing stale backlog INLINE on top of the delta. Price it.
|
||||
const costUsd = estimateEmbeddingCostUsd(inline.tokens) + (full ? staleCostUsd : 0);
|
||||
@@ -764,6 +777,11 @@ export interface SyncOpts {
|
||||
* matching the #1433 metafile posture).
|
||||
*/
|
||||
exclude?: string[];
|
||||
/**
|
||||
* Include files matched by .gitignore. Git cannot report untracked ignored
|
||||
* changes in diffs, so sync uses the full filesystem walker when this is set.
|
||||
*/
|
||||
includeGitignored?: boolean;
|
||||
/**
|
||||
* Number of parallel workers for the import phase. When > 1, each worker
|
||||
* gets its own small Postgres connection pool and files are dispatched via
|
||||
@@ -909,6 +927,25 @@ export function buildAutoEmbedArgs(slugs: string[], sourceId?: string): string[]
|
||||
return sourceId ? ['--source', sourceId, '--slugs', ...slugs] : ['--slugs', ...slugs];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve sync's effective no-embed mode from CLI args + config.
|
||||
*
|
||||
* The deferred-setup sentinel (`embedding_disabled: true`, written by
|
||||
* `gbrain init --no-embedding`) is an implicit `--no-embed`: without this,
|
||||
* the embed credential preflight demands provider credentials the user
|
||||
* deliberately deferred at init, and every `gbrain sync` on a keyless
|
||||
* brain exits 1. See embed-preflight.ts's skip protocol — the sentinel is
|
||||
* meant to be honored before the credential check ever runs.
|
||||
*
|
||||
* Exported for `test/sync-no-embed-sentinel.test.ts`.
|
||||
*/
|
||||
export function resolveNoEmbed(
|
||||
args: string[],
|
||||
cfg: { embedding_disabled?: boolean } | null,
|
||||
): boolean {
|
||||
return args.includes('--no-embed') || cfg?.embedding_disabled === true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shell out to git with a generous maxBuffer.
|
||||
*
|
||||
@@ -918,12 +955,28 @@ export function buildAutoEmbedArgs(slugs: string[], sourceId?: string): string[]
|
||||
*
|
||||
* 100 MiB is generous but still bounded — a 100K-file diff with long
|
||||
* paths tops out around 10–20 MiB in practice.
|
||||
*
|
||||
* `silenceStderr`: Node's `execFileSync` writes the child's stderr straight
|
||||
* through to the parent's real stderr by default (in addition to attaching
|
||||
* it to the thrown error's `.stderr`) *unless* an explicit `stdio` array is
|
||||
* given. Callers that treat a failure as an expected, self-handled outcome
|
||||
* (rather than a crash to surface) pass `silenceStderr: true` so git's raw
|
||||
* `fatal: ...` line never reaches the process's own stderr — only the
|
||||
* caller's own (usually friendlier) handling of the caught error does.
|
||||
* Default `false` preserves today's passthrough for every other call site.
|
||||
*/
|
||||
function git(repoPath: string, args: string[], configs: string[] = [], timeoutMs = 30000): string {
|
||||
function git(
|
||||
repoPath: string,
|
||||
args: string[],
|
||||
configs: string[] = [],
|
||||
timeoutMs = 30000,
|
||||
{ silenceStderr = false }: { silenceStderr?: boolean } = {},
|
||||
): string {
|
||||
return execFileSync('git', buildGitInvocation(repoPath, args, configs), {
|
||||
encoding: 'utf-8',
|
||||
timeout: timeoutMs,
|
||||
maxBuffer: 100 * 1024 * 1024,
|
||||
...(silenceStderr ? { stdio: ['ignore', 'pipe', 'pipe'] as const } : {}),
|
||||
}).trim();
|
||||
}
|
||||
|
||||
@@ -932,10 +985,19 @@ function git(repoPath: string, args: string[], configs: string[] = [], timeoutMs
|
||||
* `git -C <path> rev-parse --show-toplevel`. Handles worktrees and submodules
|
||||
* natively (git itself resolves them). Throws a user-friendly error when no
|
||||
* git repo is found.
|
||||
*
|
||||
* The probe's failure is expected and routine (a non-git-yet brain dir, a
|
||||
* scratch dir, a caller checking "is this a repo?") — `sync.ts` self-heals
|
||||
* it (git-init) or surfaces the message below, never the raw git stderr.
|
||||
* `silenceStderr: true` keeps git's own `fatal: not a git repository ...`
|
||||
* off the process's real stderr so operator log-scanning for `fatal:` as a
|
||||
* crash signature doesn't false-alarm on every routine probe miss (#2964
|
||||
* auto-recovery made the *outcome* self-healing; this keeps the *log* quiet
|
||||
* about the expected miss that triggered it).
|
||||
*/
|
||||
export function discoverGitRoot(inputPath: string): string {
|
||||
try {
|
||||
return git(inputPath, ['rev-parse', '--show-toplevel']);
|
||||
return git(inputPath, ['rev-parse', '--show-toplevel'], [], 30000, { silenceStderr: true });
|
||||
} catch {
|
||||
throw new Error(
|
||||
`Not inside a git repository: ${inputPath}. GBrain sync requires a git-initialized repo (or a subdirectory of one).`,
|
||||
@@ -1108,6 +1170,20 @@ function createSyncBaselineCommit(repoPath: string): void {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* True when `childReal` is `rootReal` itself or lives inside it. Both arguments
|
||||
* must already be realpath-resolved. Containment is decided by `relative()`
|
||||
* rather than a string prefix, so it holds on Windows too: `realpathSync`
|
||||
* returns backslash paths there, and a literal `rootReal + '/'` prefix can
|
||||
* never match one. A sibling (`root-evil`) is rejected because `relative`
|
||||
* yields `../root-evil`, and a cross-drive path because it yields an absolute.
|
||||
*/
|
||||
export function isWithinRoot(childReal: string, rootReal: string): boolean {
|
||||
if (childReal === rootReal) return true;
|
||||
const rel = relative(rootReal, childReal);
|
||||
return rel !== '' && rel !== '..' && !rel.startsWith('..' + sep) && !isAbsolute(rel);
|
||||
}
|
||||
|
||||
/**
|
||||
* #774 NAV-1 TOCTOU: true only if filePath realpath-resolves inside gitRoot.
|
||||
* Guards symlink escape at the per-file level (a committed symlink whose
|
||||
@@ -1115,9 +1191,7 @@ function createSyncBaselineCommit(repoPath: string): void {
|
||||
*/
|
||||
function isPathSafe(filePath: string, gitRoot: string): boolean {
|
||||
try {
|
||||
const real = realpathSync(filePath);
|
||||
const rootReal = realpathSync(gitRoot);
|
||||
return real === rootReal || real.startsWith(rootReal + '/');
|
||||
return isWithinRoot(realpathSync(filePath), realpathSync(gitRoot));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
@@ -1379,6 +1453,7 @@ See also:
|
||||
{
|
||||
sourceId: sourceIdArg,
|
||||
repoPath: source.local_path,
|
||||
noExtract: false,
|
||||
auto_embed_backfill: true,
|
||||
embed_reason: 'sync_trigger',
|
||||
},
|
||||
@@ -1717,7 +1792,7 @@ function buildPartialResult(opts: {
|
||||
modified: number;
|
||||
deleted: number;
|
||||
renamed: number;
|
||||
reason: 'timeout' | 'pull_timeout' | 'stall_timeout' | 'checkpoint_unavailable';
|
||||
reason: 'timeout' | 'pull_timeout' | 'pull_failed' | 'stall_timeout' | 'checkpoint_unavailable';
|
||||
bankedFiles?: number;
|
||||
}): SyncResult {
|
||||
return {
|
||||
@@ -1887,7 +1962,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// NAV-1/NAV-2 scope-entry guard: the realpath-resolved scope must live
|
||||
// inside the realpath-resolved git root. Catches `--src-subpath ../escape`
|
||||
// AND a symlinked subdir pointing outside the repo, before any git op runs.
|
||||
if (syncScopeRoot !== gitContextRoot && !syncScopeRoot.startsWith(gitContextRoot + '/')) {
|
||||
if (!isWithinRoot(syncScopeRoot, gitContextRoot)) {
|
||||
throw new Error(
|
||||
`Sync scope ${syncScopeRoot} resolves outside git repo ${gitContextRoot}. ` +
|
||||
`Refusing to sync: possible path traversal via --src-subpath.`,
|
||||
@@ -1948,6 +2023,15 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
});
|
||||
}
|
||||
|
||||
// #3068: remember a warn-and-continue pull failure. The fall-through-to-
|
||||
// working-tree design stays (local commits still import when the remote is
|
||||
// unreachable), but a ZERO-import sync after a failed pull must not report
|
||||
// `up_to_date` / bump the freshness heartbeat — that is what made a
|
||||
// permanently-failing pull (e.g. a local-path origin rejected by
|
||||
// protocol.file.allow=never, #1315) invisible forever: every nightly run
|
||||
// exited 0 with "Already up to date" and doctor's sync_freshness never
|
||||
// fired because last_sync_at kept advancing.
|
||||
let pullFailed = false;
|
||||
if (!opts.noPull && !detachedHead && originRemotePresent) {
|
||||
const _t0 = Date.now();
|
||||
serr(`[gbrain phase] sync.git_pull start`);
|
||||
@@ -1990,6 +2074,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
reason: 'pull_timeout',
|
||||
});
|
||||
}
|
||||
pullFailed = true;
|
||||
if (msg.includes('non-fast-forward') || msg.includes('diverged')) {
|
||||
serr(`Warning: git pull failed (remote diverged). Syncing from local state.`);
|
||||
} else {
|
||||
@@ -2103,6 +2188,14 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
return performFullSync(engine, fullSyncRoots, headCommit, opts);
|
||||
}
|
||||
|
||||
if (opts.includeGitignored) {
|
||||
slog(
|
||||
`[sync] --include-gitignored: running full filesystem reconcile because ` +
|
||||
`git diff cannot report untracked ignored files.`,
|
||||
);
|
||||
return performFullSync(engine, fullSyncRoots, headCommit, opts);
|
||||
}
|
||||
|
||||
// v0.42.x (#1794): resumable incremental sync — resolve the PINNED target.
|
||||
// last_commit advances only at FULL import completion, so a killed run keeps
|
||||
// lastCommit fixed and the checkpoint key stable across every resume even as
|
||||
@@ -2164,6 +2257,29 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
detachedWorkingTreeManifest.renamed.length > 0);
|
||||
|
||||
if (lastCommit === headCommit && !versionMismatch && !versionNeverSet && !hasDetachedWorkingTreeChanges) {
|
||||
// #3068: the pull failed and nothing local advanced — this run imported
|
||||
// NOTHING and the remote may hold commits we could not fetch. Reporting
|
||||
// `up_to_date` here (and bumping the heartbeat below) is exactly the
|
||||
// silent-wedge from the issue: every scheduled sync exits 0 forever while
|
||||
// the source is stale. Return `partial` instead (not a clean status, and
|
||||
// last_sync_at stays frozen so doctor/sources-status staleness fires).
|
||||
// The anchor is untouched; the next sync retries the pull from the same
|
||||
// bookmark.
|
||||
if (pullFailed) {
|
||||
serr(
|
||||
`[sync] git pull failed and no local changes imported — reporting partial ` +
|
||||
`(not up_to_date); sync anchor unchanged at ${lastCommit.slice(0, 8)}.`,
|
||||
);
|
||||
return buildPartialResult({
|
||||
fromCommit: lastCommit,
|
||||
toCommit: lastCommit,
|
||||
filesImported: 0,
|
||||
pagesAffected: [],
|
||||
chunksCreated: 0,
|
||||
added: 0, modified: 0, deleted: 0, renamed: 0,
|
||||
reason: 'pull_failed',
|
||||
});
|
||||
}
|
||||
// v0.42.52.0 (PR #22xx): bump last_sync_at as a heartbeat on every successful
|
||||
// 0-changes sync. D4 invariant ("never advance last_commit on partial") is
|
||||
// preserved: last_sync_at is a monitoring signal (doctor sync_freshness
|
||||
@@ -2348,6 +2464,27 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
}
|
||||
|
||||
if (totalChanges === 0) {
|
||||
// #3068: same guard as the git-HEAD-equality gate above — a failed pull
|
||||
// plus zero imports must not produce a clean `up_to_date` (and must not
|
||||
// advance the anchor past commits this run never looked at remotely).
|
||||
// Reached when local-only commits landed with no syncable content while
|
||||
// the pull kept failing. Nothing is written; the next sync re-diffs the
|
||||
// same trivial range and retries the pull.
|
||||
if (pullFailed) {
|
||||
serr(
|
||||
`[sync] git pull failed and no syncable changes imported — reporting partial ` +
|
||||
`(not up_to_date); sync anchor unchanged at ${lastCommit.slice(0, 8)}.`,
|
||||
);
|
||||
return buildPartialResult({
|
||||
fromCommit: lastCommit,
|
||||
toCommit: lastCommit,
|
||||
filesImported: 0,
|
||||
pagesAffected: [],
|
||||
chunksCreated: 0,
|
||||
added: 0, modified: 0, deleted: 0, renamed: 0,
|
||||
reason: 'pull_failed',
|
||||
});
|
||||
}
|
||||
// Update sync state even with no syncable changes (git advanced). v0.42.x
|
||||
// (#1794): advance to the PINNED target, and clear any checkpoint (a resume
|
||||
// whose remaining range turned out to have no syncable changes still
|
||||
@@ -2456,6 +2593,13 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
};
|
||||
|
||||
const pagesAffected: string[] = [];
|
||||
// #1284: slugs deleted this run (delete loop, or renamed-away old slugs are
|
||||
// NOT pushed — only confirmed deletes land here). pagesAffected stays the
|
||||
// full manifest for extract/report paths, but the auto-embed at the end
|
||||
// must NOT be handed deleted slugs: embedPage throws 'Page not found' for
|
||||
// each one and serr-logs noise. A slug re-imported later in the same run
|
||||
// (delete + re-add) is removed from this set at its push site.
|
||||
const deletedSlugs = new Set<string>();
|
||||
// issue #1939: file paths that imported cleanly this run. The failure-ledger
|
||||
// gate clears these so a previously-failing file's `attempts` streak resets
|
||||
// on success (consecutive-failure semantics for the auto-skip valve).
|
||||
@@ -2616,6 +2760,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// slugs (paths in filtered.deleted but with no DB row) so
|
||||
// downstream extract/embed don't waste lookups.
|
||||
pagesAffected.push(...deleted);
|
||||
for (const s of deleted) deletedSlugs.add(s);
|
||||
// v0.42.x (#1794): the whole batch is handled (deleted or already
|
||||
// gone); checkpoint every path so a resume skips it.
|
||||
for (const p of batch) await markCompleted(p);
|
||||
@@ -2628,6 +2773,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
try {
|
||||
await engine.deletePage(slugs[j], deleteScopedOpts);
|
||||
pagesAffected.push(slugs[j]);
|
||||
deletedSlugs.add(slugs[j]);
|
||||
await markCompleted(batch[j]);
|
||||
} catch (perSlugErr) {
|
||||
failedFiles.push({
|
||||
@@ -2655,6 +2801,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
try {
|
||||
await engine.deletePage(slug, deleteOpts);
|
||||
pagesAffected.push(slug);
|
||||
deletedSlugs.add(slug);
|
||||
await markCompleted(path);
|
||||
} catch (err) {
|
||||
failedFiles.push({
|
||||
@@ -2755,6 +2902,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
}
|
||||
}
|
||||
pagesAffected.push(newSlug);
|
||||
deletedSlugs.delete(newSlug); // #1284: rename landed on a previously-deleted slug → embeddable again
|
||||
await markCompleted(to);
|
||||
progress.tick(1, newSlug);
|
||||
}
|
||||
@@ -2955,6 +3103,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
if (result.status === 'imported') {
|
||||
chunksCreated += result.chunks;
|
||||
pagesAffected.push(result.slug);
|
||||
deletedSlugs.delete(result.slug); // #1284: deleted-then-re-added in the same run → embeddable again
|
||||
// issue #1939: record the file path (not slug) so the gate clears any
|
||||
// prior failure-ledger row — success resets the auto-skip attempt streak.
|
||||
succeededPaths.push(path);
|
||||
@@ -3129,6 +3278,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// pin..HEAD diff. Advance to pin.
|
||||
// - pin NOT an ancestor of HEAD (history REWRITE / reset / force-push) →
|
||||
// the tree we imported against is gone. Block; do not advance.
|
||||
let headVerificationSucceeded = false;
|
||||
try {
|
||||
const currentHead = git(gitContextRoot, ['rev-parse', 'HEAD']);
|
||||
if (currentHead !== pin) {
|
||||
@@ -3144,8 +3294,12 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
path: '<head>',
|
||||
error: `git history rewritten during sync: pinned target ${pin.slice(0, 8)} is no longer an ancestor of HEAD ${currentHead.slice(0, 8)}`,
|
||||
});
|
||||
} else {
|
||||
headVerificationSucceeded = true;
|
||||
}
|
||||
// else: forward progress (enrich committed on top) — safe, advance to pin.
|
||||
} else {
|
||||
headVerificationSucceeded = true;
|
||||
}
|
||||
} catch (e) {
|
||||
// rev-parse failure is itself a drift signal (worktree disappeared).
|
||||
@@ -3191,6 +3345,10 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
...succeededPaths,
|
||||
...filtered.deleted,
|
||||
...filtered.renamed.map(r => r.from),
|
||||
// A prior transient rev-parse timeout records a hard-blocking sentinel that
|
||||
// operators cannot acknowledge manually. Once pin ancestry is verified on
|
||||
// a later run, clear that stale sentinel through the ordinary success path.
|
||||
...(headVerificationSucceeded ? ['<head>'] : []),
|
||||
];
|
||||
|
||||
const gate = await applySyncFailureGate({
|
||||
@@ -3265,6 +3423,9 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
|
||||
// Log ingest
|
||||
await engine.logIngest({
|
||||
// #3242 (attribution sub-bug): credit the sync to the source it wrote
|
||||
// to, not the shared 'default' bucket.
|
||||
...(opts.sourceId ? { source_id: opts.sourceId } : {}),
|
||||
source_type: 'git_sync',
|
||||
source_ref: `${repoPath} @ ${headCommit.slice(0, 8)}`,
|
||||
pages_updated: pagesAffected,
|
||||
@@ -3368,14 +3529,19 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// sync. Non-mismatch errors stay best-effort (rate limits, transient
|
||||
// network) — those shouldn't break sync.
|
||||
let embedded = 0;
|
||||
if (!noEmbed && pagesAffected.length > 0 && pagesAffected.length <= 100) {
|
||||
// #1284: never hand deleted slugs to the embedder — embedPage throws
|
||||
// 'Page not found' per deleted slug and logs one error line each. Filter
|
||||
// against this run's confirmed-deleted set (slugs re-imported later in the
|
||||
// run were removed from it at their push sites).
|
||||
const embedSlugs = pagesAffected.filter((s) => !deletedSlugs.has(s));
|
||||
if (!noEmbed && embedSlugs.length > 0 && pagesAffected.length <= 100) {
|
||||
try {
|
||||
const { runEmbedCore } = await import('./embed.ts');
|
||||
const embedOpts = opts.sourceId
|
||||
? { slugs: pagesAffected, sourceId: opts.sourceId }
|
||||
: { slugs: pagesAffected };
|
||||
? { slugs: embedSlugs, sourceId: opts.sourceId }
|
||||
: { slugs: embedSlugs };
|
||||
await runEmbedCore(engine, embedOpts);
|
||||
embedded = pagesAffected.length;
|
||||
embedded = embedSlugs.length;
|
||||
} catch (e: unknown) {
|
||||
const { EmbeddingDimMismatchError } = await import('./embed.ts');
|
||||
if (e instanceof EmbeddingDimMismatchError) {
|
||||
@@ -3392,7 +3558,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
return {
|
||||
status: 'synced',
|
||||
fromCommit: lastCommit,
|
||||
toCommit: headCommit,
|
||||
toCommit: pin,
|
||||
added: filtered.added.length,
|
||||
modified: filtered.modified.length,
|
||||
deleted: filtered.deleted.length,
|
||||
@@ -3429,7 +3595,10 @@ async function performFullSync(
|
||||
// code --dry-run` always reported zero files even when ~1500 code
|
||||
// files were waiting.
|
||||
if (opts.dryRun) {
|
||||
let allFiles = collectSyncableFiles(syncScopeRoot, { strategy: opts.strategy ?? 'markdown' });
|
||||
let allFiles = collectSyncableFiles(syncScopeRoot, {
|
||||
strategy: opts.strategy ?? 'markdown',
|
||||
includeGitignored: opts.includeGitignored,
|
||||
});
|
||||
if (opts.exclude && opts.exclude.length > 0) {
|
||||
allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(syncScopeRoot, abs), opts.exclude));
|
||||
}
|
||||
@@ -3463,6 +3632,7 @@ async function performFullSync(
|
||||
const { runImport } = await import('./import.ts');
|
||||
const importArgs = [syncScopeRoot];
|
||||
if (opts.noEmbed) importArgs.push('--no-embed');
|
||||
if (opts.includeGitignored) importArgs.push('--include-gitignored');
|
||||
if (fullConcurrency > 1) importArgs.push('--workers', String(fullConcurrency));
|
||||
// v0.31.2: thread strategy through so code-strategy first sync
|
||||
// actually enumerates code files (closes bug 1).
|
||||
@@ -3476,6 +3646,7 @@ async function performFullSync(
|
||||
strategy: opts.strategy,
|
||||
sourceId: opts.sourceId,
|
||||
exclude: opts.exclude,
|
||||
includeGitignored: opts.includeGitignored,
|
||||
slugRoot,
|
||||
// issue #1939: performFullSync owns the failure ledger + bookmark via the
|
||||
// shared gate below; don't let runImport double-record or write its own.
|
||||
@@ -3588,7 +3759,10 @@ async function performFullSync(
|
||||
// #774: scoped syncs store git-root-relative source_paths (slugRoot), so
|
||||
// relativize the walk to the same base — otherwise every page mismatches
|
||||
// and the mass-delete valve trips on a perfectly healthy scoped source.
|
||||
const currentFiles = collectSyncableFiles(syncScopeRoot, { strategy: opts.strategy ?? 'markdown' })
|
||||
const currentFiles = collectSyncableFiles(syncScopeRoot, {
|
||||
strategy: opts.strategy ?? 'markdown',
|
||||
includeGitignored: opts.includeGitignored,
|
||||
})
|
||||
.map(abs => relative(slugRoot ?? syncScopeRoot, abs));
|
||||
const rows = await engine.executeRaw<{ slug: string; source_path: string | null }>(
|
||||
`SELECT slug, source_path FROM pages WHERE source_id = $1 AND source_path IS NOT NULL AND deleted_at IS NULL`,
|
||||
@@ -3969,6 +4143,9 @@ Options:
|
||||
subdirectory directly as --repo also works.
|
||||
--exclude <glob> Exclude files matching the glob from sync (repeatable;
|
||||
matched against the scope-relative path).
|
||||
--include-gitignored Include otherwise-syncable files matched by .gitignore.
|
||||
Forces a full filesystem walk so periodic syncs see
|
||||
ignored untracked content.
|
||||
--dry-run Show what would be synced without writing.
|
||||
--skip-failed Acknowledge previously-recorded sync failures so
|
||||
the bookmark can advance past unparseable files.
|
||||
@@ -3992,12 +4169,22 @@ Options:
|
||||
connections per wave ≈ parallel × workers × 2
|
||||
(per-file pool) + parent pool. Pass --parallel 1
|
||||
to force serial.
|
||||
--missing-path M (with --all) What to do when a source's local_path
|
||||
does not exist on this machine: 'fail' (default —
|
||||
loud, current behavior) or 'skip' (classify as
|
||||
skipped_missing_path: ⊘ in the aggregate, excluded
|
||||
from error_count and the rc=1 gate). Use skip on
|
||||
brains whose sources were registered from more
|
||||
than one machine.
|
||||
--json Emit a structured JSON envelope on stdout
|
||||
({schema_version: 1, sources, parallel,
|
||||
ok_count, error_count}). Human banners route to
|
||||
stderr so '--json | jq' parses cleanly.
|
||||
Exit codes: 0 = all sources ok, 1 = any error,
|
||||
2 = cost-prompt-not-confirmed.
|
||||
ok_count, error_count, skipped_count}). Sources
|
||||
skipped by --missing-path skip appear with
|
||||
status 'skipped_missing_path' and their
|
||||
local_path. Human banners route to stderr so
|
||||
'--json | jq' parses cleanly.
|
||||
Exit codes: 0 = all sources ok or skipped,
|
||||
1 = any error, 2 = cost-prompt-not-confirmed.
|
||||
--yes Accept any interactive prompts (CI / non-TTY).
|
||||
|
||||
See also:
|
||||
@@ -4014,12 +4201,26 @@ See also:
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const full = args.includes('--full');
|
||||
const noPull = args.includes('--no-pull');
|
||||
const noEmbed = args.includes('--no-embed');
|
||||
const noEmbed = resolveNoEmbed(args, loadConfig());
|
||||
const noExtract = args.includes('--no-extract'); // v0.42.7 #1696
|
||||
const skipFailed = args.includes('--skip-failed');
|
||||
const retryFailed = args.includes('--retry-failed');
|
||||
const noSchemaPack = args.includes('--no-schema-pack'); // v0.41.37.0 #1569
|
||||
const includeGitignored = args.includes('--include-gitignored');
|
||||
const syncAll = args.includes('--all');
|
||||
let missingPathMode: MissingPathMode = 'fail';
|
||||
try {
|
||||
missingPathMode = parseMissingPathMode(args);
|
||||
} catch (e) {
|
||||
console.error(e instanceof Error ? e.message : String(e));
|
||||
process.exit(2);
|
||||
}
|
||||
if (missingPathMode !== 'fail' && !syncAll) {
|
||||
// Single-source sync on a missing path should stay loud — an explicit
|
||||
// `--source X` naming an absent checkout is an operator error, not a
|
||||
// multi-machine artifact. Warn instead of silently ignoring the flag.
|
||||
console.error('[gbrain] WARN: --missing-path only applies to `sync --all`; ignored here.');
|
||||
}
|
||||
const jsonOut = args.includes('--json');
|
||||
const yesFlag = args.includes('--yes');
|
||||
// v0.41.6.0 D3: lock-recovery flags. --break-lock (safe) verifies the
|
||||
@@ -4275,7 +4476,7 @@ See also:
|
||||
if (!noEmbed) {
|
||||
const mode = willEmbedSynchronously({ v2Enabled, serialFlag, noEmbed });
|
||||
const gate = await runInlineCostGate(engine, {
|
||||
sources, mode, dryRun, jsonOut, yesFlag, full, label: 'sync --all',
|
||||
sources, mode, dryRun, jsonOut, yesFlag, full, includeGitignored, label: 'sync --all',
|
||||
});
|
||||
if (gate.action === 'stop') return;
|
||||
autoDeferEmbeds = gate.autoDeferEmbeds;
|
||||
@@ -4306,14 +4507,40 @@ See also:
|
||||
writeHuman(`Skipping ${disabledCount} disabled source(s).`);
|
||||
}
|
||||
|
||||
if (activeSources.length === 0) {
|
||||
// --missing-path skip: classify sources whose checkout is not on this
|
||||
// machine instead of failing them (see parseMissingPathMode's rationale).
|
||||
// Under the default 'fail' this is a no-op and behavior is unchanged.
|
||||
let skippedMissingPath: typeof activeSources = [];
|
||||
let runnableSources = activeSources;
|
||||
if (missingPathMode === 'skip') {
|
||||
const parts = partitionMissingPathSources(activeSources, existsSync);
|
||||
runnableSources = parts.runnable;
|
||||
skippedMissingPath = parts.missing;
|
||||
for (const src of skippedMissingPath) {
|
||||
writeHuman(` ⊘ ${src.name}: skipped — local_path not present on this host (${src.local_path})`);
|
||||
}
|
||||
if (skippedMissingPath.length > 0) {
|
||||
writeHuman(`Skipped ${skippedMissingPath.length} source(s) whose local_path is not present on this host (--missing-path skip).`);
|
||||
}
|
||||
}
|
||||
|
||||
if (runnableSources.length === 0) {
|
||||
if (jsonOut) {
|
||||
console.log(JSON.stringify({
|
||||
schema_version: 1,
|
||||
sources: [],
|
||||
sources: skippedMissingPath
|
||||
.slice()
|
||||
.sort((a, b) => a.id.localeCompare(b.id))
|
||||
.map((s) => ({
|
||||
source_id: s.id,
|
||||
name: s.name,
|
||||
status: 'skipped_missing_path',
|
||||
local_path: s.local_path,
|
||||
})),
|
||||
parallel: 0,
|
||||
ok_count: 0,
|
||||
error_count: 0,
|
||||
skipped_count: skippedMissingPath.length,
|
||||
}));
|
||||
}
|
||||
return;
|
||||
@@ -4323,11 +4550,20 @@ See also:
|
||||
type PerSourceResult = {
|
||||
sourceId: string;
|
||||
sourceName: string;
|
||||
status: 'ok' | 'error';
|
||||
status: 'ok' | 'error' | 'skipped_missing_path';
|
||||
result?: SyncResult;
|
||||
error?: string;
|
||||
localPath?: string;
|
||||
};
|
||||
const perSourceResults: PerSourceResult[] = [];
|
||||
for (const src of skippedMissingPath) {
|
||||
perSourceResults.push({
|
||||
sourceId: src.id,
|
||||
sourceName: src.name,
|
||||
status: 'skipped_missing_path',
|
||||
localPath: src.local_path ?? undefined,
|
||||
});
|
||||
}
|
||||
|
||||
// #1633 (Part B): one shared SIGINT controller for the whole --all fan-out.
|
||||
// process-cleanup.ts doesn't own SIGINT, so without this Ctrl-C hard-cuts the
|
||||
@@ -4373,6 +4609,7 @@ See also:
|
||||
noEmbed: effectiveNoEmbed,
|
||||
noExtract,
|
||||
skipFailed, retryFailed, noSchemaPack,
|
||||
includeGitignored,
|
||||
sourceId: src.id,
|
||||
strategy: cfg.strategy,
|
||||
concurrency,
|
||||
@@ -4436,7 +4673,7 @@ See also:
|
||||
};
|
||||
|
||||
const parallelEligible =
|
||||
v2Enabled && !serialFlag && engine.kind !== 'pglite' && activeSources.length > 1;
|
||||
v2Enabled && !serialFlag && engine.kind !== 'pglite' && runnableSources.length > 1;
|
||||
|
||||
// v0.42.42.0 (#2139, D13C): the v0.40.6.0 (D15) refusal of --skip-failed /
|
||||
// --retry-failed under parallel sync is LIFTED. It existed because the
|
||||
@@ -4450,7 +4687,7 @@ See also:
|
||||
// know how the run was actually dispatched. 1 in the serial fallback,
|
||||
// capped at min(sourceCount, --max-sources, 8) in the parallel path.
|
||||
const effectiveParallel = parallelEligible
|
||||
? Math.min(activeSources.length, maxSources ?? 8)
|
||||
? Math.min(runnableSources.length, maxSources ?? 8)
|
||||
: 1;
|
||||
|
||||
process.on('SIGINT', onAllSigint);
|
||||
@@ -4474,8 +4711,8 @@ See also:
|
||||
);
|
||||
}
|
||||
|
||||
writeHuman(`\nParallel sync: ${activeSources.length} sources, ${cap} concurrent workers.\n`);
|
||||
const results = await pMapAllSettled(activeSources, cap, async (src) => {
|
||||
writeHuman(`\nParallel sync: ${runnableSources.length} sources, ${cap} concurrent workers.\n`);
|
||||
const results = await pMapAllSettled(runnableSources, cap, async (src) => {
|
||||
const r = await runOne(src);
|
||||
return { name: src.name, result: r };
|
||||
});
|
||||
@@ -4483,7 +4720,7 @@ See also:
|
||||
writeHuman('\n--- sync --all aggregate ---');
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
const r = results[i];
|
||||
const src = activeSources[i];
|
||||
const src = runnableSources[i];
|
||||
if (r.status === 'fulfilled') {
|
||||
writeHuman(` ✓ ${src.name}: ${r.value.result.status} (added=${r.value.result.added}, modified=${r.value.result.modified}, deleted=${r.value.result.deleted})`);
|
||||
perSourceResults.push({
|
||||
@@ -4504,7 +4741,7 @@ See also:
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const src of activeSources) {
|
||||
for (const src of runnableSources) {
|
||||
writeHuman(`\n--- Syncing source: ${src.name} ---`);
|
||||
try {
|
||||
const result = await runOne(src);
|
||||
@@ -4544,8 +4781,12 @@ See also:
|
||||
source_id: r.sourceId,
|
||||
name: r.sourceName,
|
||||
status: r.status,
|
||||
...(r.localPath ? { local_path: r.localPath } : {}),
|
||||
...(r.result ? {
|
||||
sync_status: r.result.status,
|
||||
// #3068: surface the partial reason (e.g. pull_failed) so JSON
|
||||
// consumers can distinguish a self-healing timeout from a wedge.
|
||||
...(r.result.reason ? { reason: r.result.reason } : {}),
|
||||
added: r.result.added,
|
||||
modified: r.result.modified,
|
||||
deleted: r.result.deleted,
|
||||
@@ -4560,6 +4801,7 @@ See also:
|
||||
parallel: effectiveParallel,
|
||||
ok_count: okCount,
|
||||
error_count: errCount,
|
||||
skipped_count: perSourceResults.filter((r) => r.status === 'skipped_missing_path').length,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -4567,7 +4809,14 @@ See also:
|
||||
// Best-effort, stderr-only; skipped on dry-run.
|
||||
if (!dryRun) await maybeExtractionNudge(engine);
|
||||
|
||||
if (errCount > 0) process.exit(1);
|
||||
// #3068: any source wedged on a failed pull (partial/pull_failed) makes
|
||||
// the whole --all run non-zero — it will not self-heal on retry, so a
|
||||
// green exit would hide it from cron/monitoring. Timeout-class partials
|
||||
// keep the pre-existing exit-0 behavior (they converge on retry).
|
||||
const pullFailedCount = perSourceResults.filter(
|
||||
(r) => r.status === 'ok' && r.result?.status === 'partial' && r.result.reason === 'pull_failed',
|
||||
).length;
|
||||
if (errCount > 0 || pullFailedCount > 0) process.exit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -4587,7 +4836,7 @@ See also:
|
||||
const singleSourceInterrupt = new AbortController();
|
||||
const onSingleSourceSigint = () => { try { singleSourceInterrupt.abort(new Error('SIGINT')); } catch { /* */ } };
|
||||
const opts: SyncOpts = {
|
||||
repoPath, dryRun, full, noPull, noEmbed, noExtract, skipFailed, retryFailed, noSchemaPack, sourceId,
|
||||
repoPath, dryRun, full, noPull, noEmbed, noExtract, skipFailed, retryFailed, noSchemaPack, includeGitignored, sourceId,
|
||||
strategy: strategyArg, concurrency,
|
||||
srcSubpath,
|
||||
exclude: excludePatterns.length > 0 ? excludePatterns : undefined,
|
||||
@@ -4616,7 +4865,7 @@ See also:
|
||||
chunker_version: gateRows[0].chunker_version,
|
||||
}];
|
||||
const gate = await runInlineCostGate(engine, {
|
||||
sources: gateSources, mode: 'inline', dryRun: false, jsonOut, yesFlag, full, label: 'sync',
|
||||
sources: gateSources, mode: 'inline', dryRun: false, jsonOut, yesFlag, full, includeGitignored, label: 'sync',
|
||||
});
|
||||
if (gate.action === 'stop') return;
|
||||
if (gate.autoDeferEmbeds) {
|
||||
@@ -4655,6 +4904,16 @@ See also:
|
||||
process.off('SIGINT', onSingleSourceSigint);
|
||||
}
|
||||
printSyncResult(result);
|
||||
// #3068: a pull_failed partial is NOT a success — unlike timeout-class
|
||||
// partials (which converge on retry), a failing pull will not self-heal.
|
||||
// Exit non-zero so cron/monitoring sees the wedge instead of a green run.
|
||||
// Routed through the owned verdict channel (NOT bare `process.exitCode`,
|
||||
// which PGLite's Emscripten runtime clobbers mid-run — see
|
||||
// src/core/cli-force-exit.ts).
|
||||
if (result.status === 'partial' && result.reason === 'pull_failed') {
|
||||
const { setCliExitVerdict } = await import('../core/cli-force-exit.ts');
|
||||
setCliExitVerdict(1);
|
||||
}
|
||||
// v0.42.7 (#1696, D5): extraction-lag nudge after a completed single-source
|
||||
// sync. Fire on every non-error completion (synced | first_sync | up_to_date)
|
||||
// — NOT just 'synced'; a fresh/--full import (`first_sync`) is the biggest
|
||||
@@ -4739,6 +4998,63 @@ See also:
|
||||
}
|
||||
}
|
||||
|
||||
/** Mode for `sync --all --missing-path`: what to do when a source's
|
||||
* local_path does not exist on this machine. */
|
||||
export type MissingPathMode = 'fail' | 'skip';
|
||||
|
||||
/**
|
||||
* Parse `--missing-path <fail|skip>` (default: fail).
|
||||
*
|
||||
* Why the flag exists: `sources.local_path` is machine-specific state in a
|
||||
* brain-wide table. Any brain whose sources were registered from more than
|
||||
* one machine — or a sanctioned setup mid-migration (topologies.md Topology 2,
|
||||
* or the system-of-record git flow before every repo is cloned here) — has
|
||||
* sources whose checkout simply is not present on the machine running
|
||||
* `sync --all`. Each used to surface as a hard failure ("Not a git
|
||||
* repository: <path>") and force rc=1 on every run; on one observed fleet
|
||||
* that was 12 phantom failures per hour, which trains operators to ignore
|
||||
* the exit code.
|
||||
*
|
||||
* The DEFAULT stays `fail`: on a single-machine brain a missing local_path
|
||||
* usually means an unmounted volume or a deleted checkout, and silently
|
||||
* skipping it would hide real data loss. Skip is an explicit opt-in.
|
||||
*
|
||||
* Throws on a bad/absent value with a paste-ready hint (caller converts to
|
||||
* stderr + exit 2, same as other flag-misuse exits).
|
||||
*/
|
||||
export function parseMissingPathMode(args: string[]): MissingPathMode {
|
||||
const idx = args.indexOf('--missing-path');
|
||||
if (idx === -1) return 'fail';
|
||||
const val = args[idx + 1];
|
||||
if (val === 'fail' || val === 'skip') return val;
|
||||
throw new Error(
|
||||
`--missing-path expects 'fail' or 'skip', got: ${val ?? '(nothing)'}. ` +
|
||||
`Use \`--missing-path skip\` to classify sources whose local_path is not ` +
|
||||
`present on this machine as skipped instead of failed, or \`--missing-path ` +
|
||||
`fail\` (the default) to keep them loud.`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Partition `--all` sources by whether their local_path exists on THIS
|
||||
* machine. Classification is driven only by the injected predicate so tests
|
||||
* never touch the filesystem. A null local_path passes through as runnable —
|
||||
* pure-DB sources are already excluded from `--all` by the
|
||||
* `local_path IS NOT NULL` SELECT; this is defensive, not load-bearing.
|
||||
*/
|
||||
export function partitionMissingPathSources<T extends { local_path: string | null }>(
|
||||
sources: T[],
|
||||
pathExists: (p: string) => boolean,
|
||||
): { runnable: T[]; missing: T[] } {
|
||||
const runnable: T[] = [];
|
||||
const missing: T[] = [];
|
||||
for (const s of sources) {
|
||||
if (s.local_path != null && !pathExists(s.local_path)) missing.push(s);
|
||||
else runnable.push(s);
|
||||
}
|
||||
return { runnable, missing };
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.40.3.0 — resolve effective per-source concurrency for `sync --all`.
|
||||
*
|
||||
@@ -4816,6 +5132,7 @@ export async function syncOneSource(
|
||||
noSchemaPack?: boolean;
|
||||
/** v0.42.7 #1696: propagate --no-extract into every per-source sync. */
|
||||
noExtract?: boolean;
|
||||
includeGitignored?: boolean;
|
||||
},
|
||||
): Promise<{ result: SyncResult; log: string }> {
|
||||
const cfg = (src.config || {}) as { strategy?: 'markdown' | 'code' | 'auto' };
|
||||
@@ -4830,6 +5147,7 @@ export async function syncOneSource(
|
||||
skipFailed: shared.skipFailed,
|
||||
retryFailed: shared.retryFailed,
|
||||
noSchemaPack: shared.noSchemaPack,
|
||||
includeGitignored: shared.includeGitignored,
|
||||
sourceId: src.id,
|
||||
strategy: cfg.strategy,
|
||||
concurrency: shared.concurrency,
|
||||
@@ -5360,6 +5678,17 @@ function printSyncResult(result: SyncResult, sink: NodeJS.WriteStream = process.
|
||||
write(` Fix the files then re-run 'gbrain sync', or 'gbrain sync --skip-failed' to move on.`);
|
||||
break;
|
||||
case 'partial':
|
||||
// #3068: a failed (non-timeout) pull with zero imports gets its own
|
||||
// message — "imported 0 of 0" reads like success, but the local
|
||||
// checkout may be behind a remote we could not fetch.
|
||||
if (result.reason === 'pull_failed') {
|
||||
write(
|
||||
`Sync INCOMPLETE at ${result.fromCommit?.slice(0, 8) ?? '<initial>'}: ` +
|
||||
`git pull failed — the local checkout may be behind its remote.`,
|
||||
);
|
||||
write(` Fix the pull (see the warning above), then re-run 'gbrain sync' (last_commit unchanged; safe to retry).`);
|
||||
break;
|
||||
}
|
||||
// v0.41.13.0 (T7 / D-V3-5): --timeout fired before the bookmark write
|
||||
// so last_commit is UNCHANGED. The next sync re-walks the same diff
|
||||
// and content_hash short-circuits already-imported files at ~10ms each.
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
} from '../core/takes-fence.ts';
|
||||
import { withPageLock } from '../core/page-lock.ts';
|
||||
import { resolveSourceId } from '../core/source-resolver.ts';
|
||||
import { resolveOwnerHolder } from '../core/owner-holder.ts';
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
@@ -291,7 +292,7 @@ async function cmdSupersede(engine: BrainEngine, args: string[], sourceId?: stri
|
||||
const pageId = await getPageId(engine, slug, sourceId);
|
||||
|
||||
// Read existing row to inherit kind/holder unless overridden
|
||||
const existing = await engine.listTakes({ page_id: pageId, active: false, limit: 500 });
|
||||
const existing = await engine.listTakes({ page_id: pageId, active: true, limit: 500 });
|
||||
const target = existing.find(t => t.row_num === rowNum);
|
||||
if (!target) {
|
||||
console.error(`Row #${rowNum} not found on ${slug}.`);
|
||||
@@ -364,7 +365,7 @@ async function cmdResolve(engine: BrainEngine, args: string[], sourceId?: string
|
||||
// --evidence is the v0.30.0 alias for --source on the resolve subcommand
|
||||
// (semantic clarity: "what evidence resolved this bet?").
|
||||
const source = flagValue(args, '--evidence') ?? flagValue(args, '--source');
|
||||
const resolvedBy = flagValue(args, '--by') ?? 'garry';
|
||||
const resolvedBy = flagValue(args, '--by') ?? resolveOwnerHolder({ configValue: await engine.getConfig('emotional_weight.user_holder') });
|
||||
const dirArg = flagValue(args, '--dir');
|
||||
|
||||
const pageId = await getPageId(engine, slug, sourceId);
|
||||
|
||||
+30
-1
@@ -9,6 +9,7 @@ import type { BrainEngine } from '../core/engine.ts';
|
||||
import { runThink, persistSynthesis, stripGapsSection } from '../core/think/index.ts';
|
||||
import { loadConfig, isThinClient } from '../core/config.ts';
|
||||
import { callRemoteTool, unpackToolResult } from '../core/mcp-client.ts';
|
||||
import { canonicalLookup } from '../core/model-pricing.ts';
|
||||
|
||||
function flagValue(args: string[], name: string): string | undefined {
|
||||
const i = args.indexOf(name);
|
||||
@@ -20,6 +21,27 @@ function flagPresent(args: string[], name: string): boolean {
|
||||
return args.includes(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* think's own cost was previously unsurfaced anywhere: not in this CLI's own
|
||||
* `--json` output, not in `budget_ledger`, and invisible to a wrapping
|
||||
* caller's own token accounting (the LLM call `think` makes is its own,
|
||||
* separate API call). Returns undefined when `usage` is absent (no-client/
|
||||
* stub paths, or a remote-MCP call that didn't forward it) or when the
|
||||
* resolved model has no entry in the canonical pricing table.
|
||||
*/
|
||||
export function computeThinkCostUsd(
|
||||
usage: { input_tokens: number; output_tokens: number } | undefined,
|
||||
modelUsed: string,
|
||||
): number | undefined {
|
||||
if (!usage) return undefined;
|
||||
const pricing = canonicalLookup(modelUsed);
|
||||
if (!pricing) return undefined;
|
||||
return Number(
|
||||
((usage.input_tokens / 1_000_000) * pricing.input
|
||||
+ (usage.output_tokens / 1_000_000) * pricing.output).toFixed(4),
|
||||
);
|
||||
}
|
||||
|
||||
export async function runThinkCli(engine: BrainEngine, args: string[]): Promise<void> {
|
||||
if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
|
||||
console.log(`Usage: gbrain think "<question>" [options]
|
||||
@@ -146,9 +168,15 @@ prints what would have been the input (exit 0).
|
||||
}
|
||||
}
|
||||
|
||||
const costUsd = computeThinkCostUsd(
|
||||
(result as { usage?: { input_tokens: number; output_tokens: number } }).usage,
|
||||
result.modelUsed,
|
||||
);
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify({
|
||||
...result,
|
||||
cost_usd: costUsd ?? null,
|
||||
saved_slug: savedSlug ?? null,
|
||||
evidence_inserted: evidenceInserted,
|
||||
}, null, 2));
|
||||
@@ -165,7 +193,8 @@ prints what would have been the input (exit 0).
|
||||
console.log('');
|
||||
}
|
||||
console.log('---');
|
||||
console.log(`Model: ${result.modelUsed} | Pages: ${result.pagesGathered} | Takes: ${result.takesGathered} | Graph: ${result.graphHits} | Citations: ${result.citations.length}`);
|
||||
const costSuffix = costUsd !== undefined ? ` | Cost: $${costUsd.toFixed(4)}` : '';
|
||||
console.log(`Model: ${result.modelUsed} | Pages: ${result.pagesGathered} | Takes: ${result.takesGathered} | Graph: ${result.graphHits} | Citations: ${result.citations.length}${costSuffix}`);
|
||||
if (savedSlug) {
|
||||
console.log(`Saved: ${savedSlug} (${evidenceInserted} evidence rows)`);
|
||||
}
|
||||
|
||||
@@ -462,6 +462,53 @@ export async function runPostUpgrade(args: string[] = []): Promise<void> {
|
||||
// Banner is cosmetic; never block the upgrade.
|
||||
}
|
||||
|
||||
// #3390: ZeroEntropy sunset notice. ZE announced (2026-07-24) that
|
||||
// its hosted endpoints — including /models/embed and /models/rerank —
|
||||
// shut down on 2026-09-04. Any brain resolving to a zeroentropyai:*
|
||||
// embedding model (including default-config brains that never set
|
||||
// one) loses SEMANTIC RETRIEVAL ENTIRELY on that date: the query
|
||||
// embedding uses the same endpoint, so existing vectors become
|
||||
// unqueryable. One-shot per install, gated by
|
||||
// `ze_sunset_notice_shown` (same pattern as the search-mode banner).
|
||||
try {
|
||||
const shown = await engine.getConfig('ze_sunset_notice_shown');
|
||||
const { DEFAULT_EMBEDDING_MODEL } = await import('../core/ai/defaults.ts');
|
||||
const effectiveModel = cfgSchema.embedding_model ?? DEFAULT_EMBEDDING_MODEL;
|
||||
const rerankerModel = await engine.getConfig('search.reranker.model');
|
||||
const onZeEmbedding = effectiveModel.startsWith('zeroentropyai:');
|
||||
const onZeReranker = !!rerankerModel?.startsWith('zeroentropyai:');
|
||||
if (shown !== 'true' && (onZeEmbedding || onZeReranker)) {
|
||||
console.log('');
|
||||
console.log('═══════════════════════════════════════════════════════════════');
|
||||
console.log('[gbrain] ACTION REQUIRED: ZeroEntropy hosted API sunsets 2026-09-04.');
|
||||
if (onZeEmbedding) {
|
||||
console.log(`[gbrain] This brain embeds with ${effectiveModel}. After the sunset,`);
|
||||
console.log('[gbrain] semantic retrieval STOPS WORKING (queries can no longer be');
|
||||
console.log('[gbrain] embedded against your existing vectors).');
|
||||
}
|
||||
if (onZeReranker) {
|
||||
console.log(`[gbrain] The reranker (${rerankerModel}) also sunsets; search falls`);
|
||||
console.log('[gbrain] back to unreranked ordering.');
|
||||
}
|
||||
console.log('═══════════════════════════════════════════════════════════════');
|
||||
console.log('');
|
||||
console.log('Migrate before the sunset (resumable; preview cost first):');
|
||||
console.log(' gbrain migrate embeddings --to <provider:model> --dry-run');
|
||||
console.log(' gbrain migrate embeddings --to <provider:model>');
|
||||
console.log('');
|
||||
console.log('Self-hosting zembed-1 (weights are Apache-2.0) via llama-server /');
|
||||
console.log('ollama also works and preserves your existing vectors — point');
|
||||
console.log('embedding at the local endpoint instead of migrating.');
|
||||
if (onZeReranker) {
|
||||
console.log('Reranker: gbrain config set search.reranker.enabled false (or pick another).');
|
||||
}
|
||||
console.log('');
|
||||
await engine.setConfig('ze_sunset_notice_shown', 'true');
|
||||
}
|
||||
} catch {
|
||||
// Banner is cosmetic; never block the upgrade.
|
||||
}
|
||||
|
||||
// PR1: skill-catalog publish consent. New installs default ON at
|
||||
// `gbrain init`; EXISTING installs stay OFF (default-OFF runtime = no
|
||||
// silent capability grant on upgrade) until the owner opts in HERE.
|
||||
|
||||
@@ -18,12 +18,22 @@
|
||||
import { loadConfig } from '../config.ts';
|
||||
|
||||
export function hasAnthropicKey(): boolean {
|
||||
if (process.env.ANTHROPIC_API_KEY) return true;
|
||||
return resolveAnthropicKey() !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the actual key value: env first, then the gbrain config file.
|
||||
* Callers constructing an Anthropic client directly (e.g. the legacy
|
||||
* subagent path) must pass this as `apiKey` — a bare `new Anthropic()`
|
||||
* only sees env, so launchd/MCP workers with config-stored keys fail.
|
||||
*/
|
||||
export function resolveAnthropicKey(): string | undefined {
|
||||
if (process.env.ANTHROPIC_API_KEY) return process.env.ANTHROPIC_API_KEY;
|
||||
try {
|
||||
const cfg = loadConfig();
|
||||
if (cfg?.anthropic_api_key) return true;
|
||||
if (cfg?.anthropic_api_key) return cfg.anthropic_api_key;
|
||||
} catch {
|
||||
// loadConfig may throw on first-run installs; treat as no key available.
|
||||
}
|
||||
return false;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
* import it from `../../src/cli.ts`.
|
||||
*
|
||||
* The single ownership site for: (a) folding file-plane API keys
|
||||
* (openai/anthropic/zeroentropy) into the gateway env, and (b) threading
|
||||
* (openai/anthropic/zeroentropy/openrouter/voyage) into the gateway env, and (b) threading
|
||||
* local-server `*_BASE_URL` env vars into base_urls. Both matter for the
|
||||
* init-time embedding-key probe — without (a) it would false-warn on
|
||||
* config.json-keyed users, and without (b) a live probe could hit the wrong
|
||||
@@ -38,6 +38,19 @@ export function buildGatewayConfig(c: GBrainConfig): AIGatewayConfig {
|
||||
// config.json) must reach the openrouter recipe's OPENROUTER_API_KEY.
|
||||
// process.env still wins via the later spread.
|
||||
if (c.openrouter_api_key) envFromConfig.OPENROUTER_API_KEY = c.openrouter_api_key;
|
||||
// #2662: same seam for Voyage. Before this, config.json's voyage_api_key
|
||||
// was accepted at the file plane but never threaded into the gateway env,
|
||||
// so launchd/daemon/MCP contexts (no process-env export) silently failed
|
||||
// multimodal/image embeds despite config.json looking complete. process.env
|
||||
// still wins via the later spread.
|
||||
if (c.voyage_api_key) envFromConfig.VOYAGE_API_KEY = c.voyage_api_key;
|
||||
// Azure OpenAI (keyless/Entra): fold the non-secret endpoint/deployment + the
|
||||
// Entra opt-in into the gateway env so the azure-openai recipe works in any
|
||||
// shell (incl. non-interactive agent shells). The bearer token is minted at
|
||||
// request time via `az`; no secret is stored in config.json.
|
||||
if (c.azure_openai_endpoint) envFromConfig.AZURE_OPENAI_ENDPOINT = c.azure_openai_endpoint;
|
||||
if (c.azure_openai_deployment) envFromConfig.AZURE_OPENAI_DEPLOYMENT = c.azure_openai_deployment;
|
||||
if (c.azure_openai_use_entra) envFromConfig.AZURE_OPENAI_USE_ENTRA = c.azure_openai_use_entra;
|
||||
|
||||
// v0.32 codex finding #4+#5 fix: thread local-server _BASE_URL env vars
|
||||
// into base_urls so the gateway hits the user's configured port. Without
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
*/
|
||||
|
||||
import { resolveRecipe } from './model-resolver.ts';
|
||||
import { listRecipes } from './recipes/index.ts';
|
||||
import { AIConfigError } from './errors.ts';
|
||||
|
||||
export interface ProviderCapabilities {
|
||||
@@ -77,7 +78,10 @@ export function getProviderCapabilities(modelString: string): ProviderCapabiliti
|
||||
if (!chat) {
|
||||
throw new AIConfigError(
|
||||
`Provider "${recipe.id}" does not offer a chat touchpoint.`,
|
||||
`Known providers with chat: openai, anthropic, google, openrouter, litellm-proxy, deepseek, groq, together, azure-openai, dashscope, minimax, zhipu, ollama, llama-server. Pick one for models.tier.subagent.`,
|
||||
// Computed from the registry so the hint can't drift into listing
|
||||
// chat-less providers (the pre-fix list falsely included embedding-only
|
||||
// recipes, sending users in circles — #1157).
|
||||
`Known providers with chat: ${listRecipes().filter(r => r.touchpoints.chat).map(r => r.id).join(', ')}. Pick one for models.tier.subagent.`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -88,9 +92,13 @@ export function getProviderCapabilities(modelString: string): ProviderCapabiliti
|
||||
// boundary; this function returns capabilities for whatever the user asked
|
||||
// for, on the assumption it'll be validated elsewhere.
|
||||
|
||||
const promptCache = chat.supports_prompt_cache;
|
||||
|
||||
return {
|
||||
supportsToolCalling: chat.supports_tools === true,
|
||||
supportsPromptCaching: chat.supports_prompt_cache === true,
|
||||
supportsPromptCaching: typeof promptCache === 'function'
|
||||
? promptCache(parsed.modelId)
|
||||
: promptCache === true,
|
||||
// No recipe exposes parallel-tools-specifically yet; gate on supports_tools.
|
||||
// Subsequent waves can split this into its own recipe field if a provider
|
||||
// ever supports tools without parallel dispatch.
|
||||
@@ -101,11 +109,6 @@ export function getProviderCapabilities(modelString: string): ProviderCapabiliti
|
||||
supportsThinking: false,
|
||||
maxContext: chat.max_context_tokens ?? 128_000,
|
||||
};
|
||||
|
||||
// The `parsed` binding is intentionally unused — `resolveRecipe` is called
|
||||
// here for its validation side-effects (throws on unknown provider). Keeping
|
||||
// the destructure makes future per-model capability overrides cheap.
|
||||
void parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+45
-3
@@ -90,6 +90,30 @@ export function isValidOpenAITextEmbedding3Dim(modelId: string, dims: number): b
|
||||
return Number.isInteger(dims) && dims >= 1 && dims <= max;
|
||||
}
|
||||
|
||||
// Perplexity hosted embeddings (#1046): Matryoshka-style flexible dims,
|
||||
// any integer from 128 up to the model's native size. `dimensions` is the
|
||||
// native wire field (no translation needed); output encoding divergence
|
||||
// (base64 int8) is handled by perplexityCompatFetch in gateway.ts.
|
||||
const PERPLEXITY_EMBEDDING_MAX_DIMS: Record<string, number> = {
|
||||
'pplx-embed-v1-0.6b': 1024,
|
||||
'pplx-embed-v1-4b': 2560,
|
||||
};
|
||||
export const PERPLEXITY_MIN_DIMS = 128;
|
||||
|
||||
export function isPerplexityEmbeddingModel(modelId: string): boolean {
|
||||
return modelId in PERPLEXITY_EMBEDDING_MAX_DIMS;
|
||||
}
|
||||
|
||||
export function maxPerplexityEmbeddingDim(modelId: string): number | undefined {
|
||||
return PERPLEXITY_EMBEDDING_MAX_DIMS[modelId];
|
||||
}
|
||||
|
||||
export function isValidPerplexityDim(modelId: string, dims: number): boolean {
|
||||
const max = PERPLEXITY_EMBEDDING_MAX_DIMS[modelId];
|
||||
if (max === undefined) return false;
|
||||
return Number.isInteger(dims) && dims >= PERPLEXITY_MIN_DIMS && dims <= max;
|
||||
}
|
||||
|
||||
// NVIDIA NIM hosted embedding models use asymmetric input_type values. Most
|
||||
// emit fixed natural dimensions, but llama-nemotron-embed-1b-v2 accepts
|
||||
// Matryoshka-style dimension overrides (e.g. matching an existing 1280d
|
||||
@@ -226,6 +250,23 @@ export function dimsProviderOptions(
|
||||
},
|
||||
};
|
||||
}
|
||||
// Perplexity pplx-embed-v1-* — flexible dims via the native
|
||||
// `dimensions` field. Fail-loud when the configured dim is outside
|
||||
// the model's range (same rationale as the Voyage/ZE guards: the
|
||||
// upstream HTTP 400 misroutes as a transient network error).
|
||||
// Symmetric retrieval — inputType is never emitted.
|
||||
if (isPerplexityEmbeddingModel(modelId)) {
|
||||
if (!isValidPerplexityDim(modelId, dims)) {
|
||||
const max = maxPerplexityEmbeddingDim(modelId)!;
|
||||
throw new AIConfigError(
|
||||
`Perplexity model "${modelId}" supports embedding_dimensions in ` +
|
||||
`${PERPLEXITY_MIN_DIMS}..${max}, got ${dims}.`,
|
||||
`Set \`embedding_dimensions\` to a value between ${PERPLEXITY_MIN_DIMS} and ${max} ` +
|
||||
`in your gbrain config.`,
|
||||
);
|
||||
}
|
||||
return { openaiCompatible: { dimensions: dims } };
|
||||
}
|
||||
// NVIDIA NIM hosted embeddings are OpenAI-compatible but require
|
||||
// asymmetric input_type. Use passage for indexing/document-side vectors
|
||||
// and query for search-side vectors. Only llama-nemotron-embed-1b-v2
|
||||
@@ -244,9 +285,10 @@ export function dimsProviderOptions(
|
||||
// configured for a smaller width (e.g. 1536) hard-fail at first embed.
|
||||
// Azure/OpenAI-compat embeddings are symmetric — inputType ignored.
|
||||
// v0.36.0.0 (D13): same range validation as native-openai path.
|
||||
if (modelId.startsWith('text-embedding-3')) {
|
||||
if (isOpenAITextEmbedding3Model(modelId) && !isValidOpenAITextEmbedding3Dim(modelId, dims)) {
|
||||
const max = maxOpenAITextEmbedding3Dim(modelId)!;
|
||||
const bareModelId = modelId.includes('/') ? modelId.split('/').pop()! : modelId;
|
||||
if (bareModelId.startsWith('text-embedding-3')) {
|
||||
if (isOpenAITextEmbedding3Model(bareModelId) && !isValidOpenAITextEmbedding3Dim(bareModelId, dims)) {
|
||||
const max = maxOpenAITextEmbedding3Dim(bareModelId)!;
|
||||
throw new AIConfigError(
|
||||
`OpenAI model "${modelId}" supports embedding_dimensions in 1..${max}, got ${dims}.`,
|
||||
`Set \`embedding_dimensions\` to a value between 1 and ${max} ` +
|
||||
|
||||
+258
-18
@@ -47,7 +47,12 @@ import type {
|
||||
TouchpointKind,
|
||||
} from './types.ts';
|
||||
import { resolveRecipe, assertTouchpoint, parseModelId } from './model-resolver.ts';
|
||||
import {
|
||||
OPENROUTER_CACHE_HEADER,
|
||||
openrouterRequiresExplicitPromptCache,
|
||||
} from './recipes/openrouter.ts';
|
||||
import { resolveModel, TIER_DEFAULTS } from '../model-config.ts';
|
||||
import { parseLlmJson } from '../llm-json.ts';
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
import { dimsProviderOptions } from './dims.ts';
|
||||
import { hasAnthropicKey } from './anthropic-key.ts';
|
||||
@@ -263,6 +268,18 @@ export class ZeroEntropyResponseTooLargeError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/** Perplexity twin of the Voyage/ZE OOM caps (#1046). Int8 components are
|
||||
* 1 byte each, so a real response (512 texts × 2560 dims) is ~1.3 MB —
|
||||
* anything near this cap is unambiguously not legitimate. */
|
||||
const MAX_PERPLEXITY_RESPONSE_BYTES = 256 * 1024 * 1024;
|
||||
|
||||
export class PerplexityResponseTooLargeError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'PerplexityResponseTooLargeError';
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Unified auth resolution (D12=A) ----
|
||||
//
|
||||
// Pre-v0.32, openai-compatible auth was duplicated across instantiateEmbedding,
|
||||
@@ -435,6 +452,20 @@ export function resolveNativeBaseUrl(
|
||||
return /\/v1$/.test(trimmed) ? trimmed : `${trimmed}/v1`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether an openai-compatible recipe's backend honors OpenAI structured
|
||||
* outputs. Threaded into `createOpenAICompatible`'s `supportsStructuredOutputs`
|
||||
* at the chat + expansion build sites, and consulted by `expand()` to pick the
|
||||
* strict `generateObject` path over the schemaless text path. Single source of
|
||||
* truth read from the chat touchpoint: the backend serves both chat and
|
||||
* expansion, so the capability is declared once.
|
||||
*
|
||||
* @internal exported for tests.
|
||||
*/
|
||||
export function recipeSupportsStructuredOutputs(recipe: Recipe): boolean {
|
||||
return recipe.touchpoints.chat?.supports_structured_outputs === true;
|
||||
}
|
||||
|
||||
/** Configure the gateway. Called by cli.ts#connectEngine. Clears cached models. */
|
||||
export function configureGateway(config: AIGatewayConfig): void {
|
||||
_config = {
|
||||
@@ -1294,6 +1325,103 @@ const openAICompatAsymmetricFetch = (async (input: RequestInfo | URL, init?: Req
|
||||
return fetch(typeof input === 'string' ? input : input.toString(), baseInit);
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
/**
|
||||
* Perplexity compatibility shim (#1046). Perplexity's `/v1/embeddings`
|
||||
* endpoint is OpenAI-shaped but diverges on two points that break the AI
|
||||
* SDK's openai-compatible adapter:
|
||||
* - `encoding_format` only accepts 'base64_int8' (default) or
|
||||
* 'base64_binary'; the SDK sends 'float', which Perplexity rejects.
|
||||
* Force 'base64_int8' on the wire.
|
||||
* - The response `embedding` is a base64 string encoding SIGNED INT8
|
||||
* components (natively quantized output). The SDK schema expects
|
||||
* `number[]` — decode Int8Array → number[] here. Cosine similarity is
|
||||
* scale-invariant, so the raw int8 components rank correctly.
|
||||
* `dimensions` is Perplexity's native field name — no translation needed
|
||||
* (dims.ts emits it directly). Layer 1/Layer 2 OOM caps mirror the Voyage
|
||||
* pattern.
|
||||
*
|
||||
* Exported for tests (behavioral coverage of the int8 decode); not part of
|
||||
* the public gateway API.
|
||||
*/
|
||||
export const perplexityCompatFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
// OUTBOUND: force the encoding Perplexity actually accepts.
|
||||
if (init?.body && typeof init.body === 'string') {
|
||||
try {
|
||||
const parsed = JSON.parse(init.body);
|
||||
if (parsed && typeof parsed === 'object' && parsed.encoding_format !== 'base64_int8') {
|
||||
parsed.encoding_format = 'base64_int8';
|
||||
// Drop Content-Length so fetch recomputes from the new body.
|
||||
const headers = new Headers(init.headers ?? {});
|
||||
headers.delete('content-length');
|
||||
init = { ...init, body: JSON.stringify(parsed), headers };
|
||||
}
|
||||
} catch {
|
||||
// Body wasn't JSON — pass through untouched.
|
||||
}
|
||||
}
|
||||
|
||||
const resp = await fetch(input as any, init);
|
||||
if (!resp.ok) return resp;
|
||||
const ct = resp.headers.get('content-type') ?? '';
|
||||
if (!ct.toLowerCase().includes('application/json')) return resp;
|
||||
|
||||
// Layer 1: Content-Length pre-check BEFORE the body is parsed.
|
||||
const contentLengthHeader = resp.headers.get('content-length');
|
||||
if (contentLengthHeader) {
|
||||
const len = parseInt(contentLengthHeader, 10);
|
||||
if (Number.isFinite(len) && len > MAX_PERPLEXITY_RESPONSE_BYTES) {
|
||||
throw new PerplexityResponseTooLargeError(
|
||||
`Perplexity response Content-Length=${len} exceeds ${MAX_PERPLEXITY_RESPONSE_BYTES} bytes — ` +
|
||||
`likely compromised endpoint or misconfiguration`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// INBOUND: decode base64 int8 embeddings to number[] so the SDK's Zod
|
||||
// schema validates.
|
||||
try {
|
||||
const json: any = await resp.clone().json();
|
||||
if (!json || typeof json !== 'object') return resp;
|
||||
let modified = false;
|
||||
if (Array.isArray(json.data)) {
|
||||
for (const item of json.data) {
|
||||
if (item && typeof item.embedding === 'string') {
|
||||
// Layer 2: per-embedding cap for chunked responses that skipped
|
||||
// Layer 1. base64 → bytes is the canonical 0.75 ratio.
|
||||
const estDecoded = Math.ceil(item.embedding.length * 0.75);
|
||||
if (estDecoded > MAX_PERPLEXITY_RESPONSE_BYTES) {
|
||||
throw new PerplexityResponseTooLargeError(
|
||||
`Perplexity embedding base64 exceeds ${MAX_PERPLEXITY_RESPONSE_BYTES} bytes ` +
|
||||
`(estimated ${estDecoded} bytes from ${item.embedding.length} base64 chars)`,
|
||||
);
|
||||
}
|
||||
// base64_int8: one signed int8 per component.
|
||||
const bytes = Buffer.from(item.embedding, 'base64');
|
||||
item.embedding = Array.from(new Int8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength));
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (json.usage && typeof json.usage === 'object' && json.usage.prompt_tokens === undefined) {
|
||||
json.usage.prompt_tokens = typeof json.usage.total_tokens === 'number'
|
||||
? json.usage.total_tokens
|
||||
: 0;
|
||||
modified = true;
|
||||
}
|
||||
if (!modified) return resp;
|
||||
return new Response(JSON.stringify(json), {
|
||||
status: resp.status,
|
||||
statusText: resp.statusText,
|
||||
headers: resp.headers,
|
||||
});
|
||||
} catch (err) {
|
||||
// OOM-cap throws MUST propagate; anything else falls back to the
|
||||
// original response (same contract as voyageCompatFetch).
|
||||
if (err instanceof PerplexityResponseTooLargeError) throw err;
|
||||
return resp;
|
||||
}
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
async function resolveEmbeddingProvider(modelStr: string): Promise<{ model: any; recipe: Recipe; modelId: string }> {
|
||||
const { parsed, recipe } = resolveRecipe(modelStr);
|
||||
assertTouchpoint(recipe, 'embedding', parsed.modelId, getExtendedModelsForProvider(parsed.providerId));
|
||||
@@ -1338,6 +1466,10 @@ function instantiateEmbedding(recipe: Recipe, modelId: string, cfg: AIGatewayCon
|
||||
throw new AIConfigError(
|
||||
`Anthropic has no embedding model. Use openai or google for embeddings.`,
|
||||
);
|
||||
case 'claude-cli':
|
||||
throw new AIConfigError(
|
||||
`claude-cli has no embedding model. Use openai or google for embeddings.`,
|
||||
);
|
||||
case 'openai-compatible': {
|
||||
// D12=A: unified auth via Recipe.resolveAuth (or default).
|
||||
const auth = applyResolveAuth(recipe, cfg, 'embedding');
|
||||
@@ -1362,6 +1494,8 @@ function instantiateEmbedding(recipe: Recipe, modelId: string, cfg: AIGatewayCon
|
||||
? zeroEntropyCompatFetch
|
||||
: recipe.id === 'nvidia'
|
||||
? nvidiaCompatFetch
|
||||
: recipe.id === 'perplexity'
|
||||
? perplexityCompatFetch
|
||||
: openAICompatAsymmetricFetch);
|
||||
const client = createOpenAICompatible({
|
||||
name: recipe.id,
|
||||
@@ -2280,6 +2414,15 @@ function instantiateExpansion(recipe: Recipe, modelId: string, cfg: AIGatewayCon
|
||||
const baseURL = resolveNativeBaseUrl('anthropic', cfg);
|
||||
return createAnthropic({ apiKey, ...(baseURL ? { baseURL } : {}) }).languageModel(modelId);
|
||||
}
|
||||
case 'claude-cli': {
|
||||
// The CLI handles its own auth (OAuth session); spawn the subprocess
|
||||
// directly via the same LanguageModelV2 implementation chat uses. There
|
||||
// is no separate expansion path because claude-cli does not declare a
|
||||
// separate expansion touchpoint — but routing here keeps the switch
|
||||
// exhaustive and lets a future expansion touchpoint use the same code.
|
||||
const { ClaudeCliLanguageModel } = require('./providers/claude-cli-language-model.ts');
|
||||
return new ClaudeCliLanguageModel(modelId);
|
||||
}
|
||||
case 'openai-compatible': {
|
||||
// D12=A: unified auth via Recipe.resolveAuth (or default).
|
||||
const auth = applyResolveAuth(recipe, cfg, 'expansion');
|
||||
@@ -2290,6 +2433,7 @@ function instantiateExpansion(recipe: Recipe, modelId: string, cfg: AIGatewayCon
|
||||
baseURL: compat.baseURL,
|
||||
...(compat.fetch ? { fetch: compat.fetch } : {}),
|
||||
...auth,
|
||||
supportsStructuredOutputs: recipeSupportsStructuredOutputs(recipe),
|
||||
}).languageModel(modelId);
|
||||
}
|
||||
}
|
||||
@@ -2299,6 +2443,20 @@ const ExpansionSchema = z.object({
|
||||
queries: z.array(z.string()).min(1).max(5),
|
||||
});
|
||||
|
||||
/**
|
||||
* Recover expansion queries from a schemaless model response. Used by the
|
||||
* openai-compatible expansion paths: a tolerant JSON decode plus schema
|
||||
* validation pulls the `queries` array out of the model's text (the prompt
|
||||
* pins it to a bare JSON object). Returns null when the text carries no valid
|
||||
* `{ queries: string[] }` object.
|
||||
*
|
||||
* @internal exported for tests.
|
||||
*/
|
||||
export function parseExpansionResponse(text: string): string[] | null {
|
||||
const parsed = ExpansionSchema.safeParse(parseLlmJson<unknown>(text));
|
||||
return parsed.success ? parsed.data.queries : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Expand a search query into up to 4 related queries.
|
||||
* Returns the original query PLUS expansions. On failure, returns just the original.
|
||||
@@ -2315,24 +2473,63 @@ export async function expand(query: string): Promise<string[]> {
|
||||
metadata: { query_chars: query.length },
|
||||
});
|
||||
|
||||
const expansionPrompt = [
|
||||
'Rewrite the search query below into 3-4 different, related queries that would help find relevant documents. Respond with a JSON object in exactly this shape: {"queries": ["rewrite1", "rewrite2", "rewrite3"]}. The JSON key MUST be exactly "queries" (not "rewrites" or any other variation).',
|
||||
'Return ONLY the JSON object. Do NOT include the original query in the result.',
|
||||
'Each rewrite should emphasize different aspects, synonyms, or framings.',
|
||||
'',
|
||||
`Query: ${query}`,
|
||||
].join('\n');
|
||||
|
||||
try {
|
||||
const { model, recipe, modelId } = await resolveExpansionProvider(getExpansionModel());
|
||||
const result = await generateObject({
|
||||
model,
|
||||
schema: ExpansionSchema,
|
||||
// v0.42.20.0 (codex P0) — expansion had NO abortSignal; same stalled-socket
|
||||
// class as chat. Default the chat timeout.
|
||||
abortSignal: withDefaultTimeout(undefined, AI_CHAT_TIMEOUT_MS),
|
||||
prompt: [
|
||||
'Rewrite the search query below into 3-4 different, related queries that would help find relevant documents.',
|
||||
'Return ONLY the JSON object. Do NOT include the original query in the result.',
|
||||
'Each rewrite should emphasize different aspects, synonyms, or framings.',
|
||||
'',
|
||||
`Query: ${query}`,
|
||||
].join('\n'),
|
||||
});
|
||||
|
||||
const expansions = result.object?.queries ?? [];
|
||||
let expansions: string[];
|
||||
|
||||
// Schemaless text path for openai-compatible backends whose structured-output
|
||||
// support is unknown: the AI SDK can't send a json_schema response_format
|
||||
// there, so generateObject would warn and silently degrade. generateText + a
|
||||
// tolerant parse recovers the queries instead. Fresh abortSignal per call.
|
||||
const viaText = async (): Promise<string[]> => {
|
||||
const { text } = await generateText({
|
||||
model,
|
||||
abortSignal: withDefaultTimeout(undefined, AI_CHAT_TIMEOUT_MS),
|
||||
prompt: expansionPrompt,
|
||||
});
|
||||
return parseExpansionResponse(text) ?? [];
|
||||
};
|
||||
|
||||
if (recipe.implementation !== 'openai-compatible') {
|
||||
// Native providers (Anthropic, OpenAI, Google) support generateObject's
|
||||
// structured output natively — unchanged path.
|
||||
const result = await generateObject({
|
||||
model,
|
||||
schema: ExpansionSchema,
|
||||
abortSignal: withDefaultTimeout(undefined, AI_CHAT_TIMEOUT_MS),
|
||||
prompt: expansionPrompt,
|
||||
});
|
||||
expansions = result.object?.queries ?? [];
|
||||
} else if (recipeSupportsStructuredOutputs(recipe)) {
|
||||
// openai-compatible backend that honors strict json_schema: request the
|
||||
// schema (strict validation), and fall back to the text path if it is
|
||||
// rejected at call time so a mis-declared capability never drops expansion.
|
||||
try {
|
||||
const result = await generateObject({
|
||||
model,
|
||||
schema: ExpansionSchema,
|
||||
abortSignal: withDefaultTimeout(undefined, AI_CHAT_TIMEOUT_MS),
|
||||
prompt: expansionPrompt,
|
||||
});
|
||||
expansions = result.object?.queries ?? [];
|
||||
} catch {
|
||||
expansions = await viaText();
|
||||
}
|
||||
} else {
|
||||
// openai-compatible backend, structured-output support unknown: skip the
|
||||
// json_schema attempt entirely (no SDK warning, no silent degradation).
|
||||
expansions = await viaText();
|
||||
}
|
||||
|
||||
// Deduplicate + include the original query
|
||||
const seen = new Set<string>();
|
||||
const all = [query, ...expansions].filter(q => {
|
||||
@@ -2735,6 +2932,17 @@ export function probeChatModel(modelStr: string): ChatModelProbe {
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-model prompt-cache capability: `supports_prompt_cache` may be a static
|
||||
* boolean (native providers) or a per-model-id predicate (OpenRouter's
|
||||
* family-scoped caching).
|
||||
*/
|
||||
function chatSupportsPromptCache(recipe: Recipe, modelId: string): boolean {
|
||||
const support = recipe.touchpoints.chat?.supports_prompt_cache;
|
||||
if (typeof support === 'function') return support(modelId);
|
||||
return support === true;
|
||||
}
|
||||
|
||||
async function resolveChatProvider(modelStr: string): Promise<{ model: any; recipe: Recipe; modelId: string }> {
|
||||
const { parsed, recipe } = resolveRecipe(modelStr);
|
||||
assertTouchpoint(recipe, 'chat', parsed.modelId, getExtendedModelsForProvider(parsed.providerId));
|
||||
@@ -2768,6 +2976,15 @@ function instantiateChat(recipe: Recipe, modelId: string, cfg: AIGatewayConfig):
|
||||
const baseURL = resolveNativeBaseUrl('anthropic', cfg);
|
||||
return createAnthropic({ apiKey, ...(baseURL ? { baseURL } : {}) }).languageModel(modelId);
|
||||
}
|
||||
case 'claude-cli': {
|
||||
// The CLI handles its own auth (OAuth session managed by `claude`
|
||||
// login). Subprocess-based LanguageModelV2 dispatches via the recipe
|
||||
// path so per-call routing works: `claude-cli:claude-sonnet-4-6` lands
|
||||
// here, while sibling `litellm:gpt-5.4` continues through the
|
||||
// openai-compatible path below. No env-var switch, no global flag.
|
||||
const { ClaudeCliLanguageModel } = require('./providers/claude-cli-language-model.ts');
|
||||
return new ClaudeCliLanguageModel(modelId);
|
||||
}
|
||||
case 'openai-compatible': {
|
||||
// D12=A: unified auth via Recipe.resolveAuth (or default).
|
||||
const auth = applyResolveAuth(recipe, cfg, 'chat');
|
||||
@@ -2778,6 +2995,7 @@ function instantiateChat(recipe: Recipe, modelId: string, cfg: AIGatewayConfig):
|
||||
baseURL: compat.baseURL,
|
||||
...(compat.fetch ? { fetch: compat.fetch } : {}),
|
||||
...auth,
|
||||
supportsStructuredOutputs: recipeSupportsStructuredOutputs(recipe),
|
||||
}).languageModel(modelId);
|
||||
}
|
||||
default:
|
||||
@@ -3056,9 +3274,19 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
|
||||
const { model, recipe, modelId } = await resolveChatProvider(modelStr);
|
||||
const cfg = requireConfig();
|
||||
|
||||
const supportsCache = recipe.touchpoints.chat?.supports_prompt_cache === true;
|
||||
const supportsCache = chatSupportsPromptCache(recipe, modelId);
|
||||
const useCache = !!opts.cacheSystem && supportsCache;
|
||||
|
||||
// OpenRouter Claude routes need an explicit `cache_control` on the system
|
||||
// content block, but the openai-compatible adapter drops anthropic-namespace
|
||||
// providerOptions before building the wire body. Signal intent via a private
|
||||
// header; the recipe's compat fetch shim rewrites the body and strips the
|
||||
// header before the request leaves the process. OpenAI routes through
|
||||
// OpenRouter cache automatically — no marker needed.
|
||||
const requestHeaders = useCache && recipe.id === 'openrouter' && openrouterRequiresExplicitPromptCache(modelId)
|
||||
? { [OPENROUTER_CACHE_HEADER]: '1' }
|
||||
: undefined;
|
||||
|
||||
const tools = toAISDKTools(opts.tools);
|
||||
|
||||
const providerOptions: Record<string, any> = {};
|
||||
@@ -3170,6 +3398,7 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
|
||||
// shorter wins). Covers native-anthropic (the default provider + facts Haiku).
|
||||
abortSignal: withDefaultTimeout(opts.abortSignal, AI_CHAT_TIMEOUT_MS),
|
||||
providerOptions: Object.keys(providerOptions).length > 0 ? providerOptions : undefined,
|
||||
...(requestHeaders ? { headers: requestHeaders } : {}),
|
||||
});
|
||||
|
||||
// Normalize blocks. Vercel SDK gives us `result.content` (an array of typed
|
||||
@@ -3218,7 +3447,10 @@ export async function chat(opts: ChatOpts): Promise<ChatResult> {
|
||||
usage: {
|
||||
input_tokens: inTok,
|
||||
output_tokens: outTok,
|
||||
cache_read_tokens: Number(anthropicCache.cacheReadInputTokens ?? anthropicCache.cache_read_input_tokens ?? 0),
|
||||
// `usage.cachedInputTokens` is the AI SDK's provider-neutral cache-read
|
||||
// count — it's how OpenAI-compatible routes (OpenRouter's
|
||||
// prompt_tokens_details.cached_tokens) surface cache hits.
|
||||
cache_read_tokens: Number(anthropicCache.cacheReadInputTokens ?? anthropicCache.cache_read_input_tokens ?? usage.cachedInputTokens ?? 0),
|
||||
cache_creation_tokens: Number(anthropicCache.cacheCreationInputTokens ?? anthropicCache.cache_creation_input_tokens ?? 0),
|
||||
},
|
||||
model: `${recipe.id}:${modelId}`,
|
||||
@@ -3682,7 +3914,15 @@ export async function rerank(input: RerankInput): Promise<RerankResult[]> {
|
||||
// whose request/response shape differs from ZE/llama.cpp (e.g. Voyage with
|
||||
// `top_k` / `data[]`) needs separate adapter hooks in a follow-up plan.
|
||||
const url = `${compat.baseURL.replace(/\/$/, '')}${tp.path ?? '/models/rerank'}`;
|
||||
const auth = applyResolveAuth(recipe, cfg, 'reranker');
|
||||
let auth: { apiKey?: string; headers?: Record<string, string> };
|
||||
try {
|
||||
auth = applyResolveAuth(recipe, cfg, 'reranker');
|
||||
} catch (err) {
|
||||
if (err instanceof AIConfigError) {
|
||||
throw new RerankError(err.message, 'auth');
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
// applyResolveAuth returns { apiKey } for Bearer-style auth (SDK's native
|
||||
// path) or { headers } for custom-header providers (Azure). v0.37.6.0:
|
||||
// recipes can ALSO declare default_headers (attribution etc.) which flow
|
||||
|
||||
@@ -144,3 +144,35 @@ export function assertTouchpoint(
|
||||
export function knownProviderIds(): string[] {
|
||||
return [...RECIPES.keys()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Native embedding width for `modelId` under `recipe`.
|
||||
*
|
||||
* Resolution: the recipe's `model_dims` entry for this model, else the
|
||||
* recipe-wide `default_dims`. Returns 0 when neither is known (the
|
||||
* user-provided-model recipes declare `default_dims: 0` to force an explicit
|
||||
* `--embedding-dimensions`), so callers keep their existing falsy checks.
|
||||
*
|
||||
* Accepts a bare model id (`bge-m3`) or a qualified one (`ollama:bge-m3`);
|
||||
* the provider prefix is stripped before lookup so call sites can pass
|
||||
* whichever they hold.
|
||||
*
|
||||
* Fixes #2051: a recipe-wide default silently picked 768 for every Ollama
|
||||
* model, so `init --embedding-model ollama:bge-m3` built a 768-wide column
|
||||
* for a model that emits 1024 and only failed at first insert.
|
||||
*/
|
||||
export function embeddingDimsForModel(
|
||||
recipe: Recipe,
|
||||
modelId: string | undefined,
|
||||
): number {
|
||||
const tp = recipe.touchpoints.embedding;
|
||||
if (!tp) return 0;
|
||||
if (!modelId) return tp.default_dims ?? 0;
|
||||
// Strip a leading `provider:` so both forms resolve. Slash-form ids
|
||||
// (openrouter nested) are left intact — they're the model id.
|
||||
const colon = modelId.indexOf(':');
|
||||
const bare = colon === -1 ? modelId : modelId.slice(colon + 1);
|
||||
const declared = tp.model_dims?.[bare];
|
||||
if (typeof declared === 'number' && declared > 0) return declared;
|
||||
return tp.default_dims ?? 0;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,444 @@
|
||||
/**
|
||||
* ai-sdk LanguageModelV2 implementation that dispatches via the `claude --print`
|
||||
* CLI subprocess. Used by the `claude-cli` recipe to route gateway.toolLoop /
|
||||
* gateway.chat calls through Claude Code's OAuth session instead of the
|
||||
* Anthropic SDK + ANTHROPIC_API_KEY.
|
||||
*
|
||||
* Per-call routing is the contract: the gateway resolves the model string
|
||||
* to this recipe based on the `claude-cli:` prefix, instantiates one of
|
||||
* these objects per modelId, and dispatches doGenerate. Sibling subagent
|
||||
* jobs with `litellm:gpt-5.4` continue routing through litellm-proxy in
|
||||
* the same worker; no env-var switch, no global state.
|
||||
*
|
||||
* Tool use is supported via system-prompt-instructed JSON emission:
|
||||
* The recipe injects a fenced instruction block into the system prompt
|
||||
* that teaches the model the `<use_tools>[{id,name,input}, ...]</use_tools>`
|
||||
* emission format. The adapter parses those blocks back into ai-sdk
|
||||
* `tool-call` content parts. Parallel tool calls (multiple entries in
|
||||
* the JSON array) round-trip cleanly — this is the case that breaks
|
||||
* on the codex-proxy / litellm GPT-5.x bridge today.
|
||||
*
|
||||
* Context isolation:
|
||||
* The subprocess is spawned from a dedicated tmpdir so claude-cli's
|
||||
* CLAUDE.md auto-discovery has no local files to find. `--system-prompt`
|
||||
* replaces the default system prompt; `--disable-slash-commands` skips
|
||||
* skill resolution. User-level ~/.claude/CLAUDE.md still loads because
|
||||
* the only way to skip it is `--bare`, which forces ANTHROPIC_API_KEY
|
||||
* auth and defeats the whole point of this provider. The ~42k cached
|
||||
* tokens from user-level instructions are accepted as a cost-trivial
|
||||
* trade-off on the subscription path.
|
||||
*
|
||||
* doStream is not yet implemented; the model declares no streaming. Callers
|
||||
* (gateway.toolLoop primarily) use doGenerate.
|
||||
*/
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdirSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type {
|
||||
LanguageModelV2,
|
||||
LanguageModelV2CallOptions,
|
||||
LanguageModelV2Content,
|
||||
LanguageModelV2FunctionTool,
|
||||
LanguageModelV2Prompt,
|
||||
LanguageModelV2Message,
|
||||
LanguageModelV2ProviderDefinedTool,
|
||||
} from '@ai-sdk/provider';
|
||||
|
||||
function claudeBin(): string {
|
||||
return process.env.GBRAIN_CLAUDE_CLI_BIN ?? 'claude';
|
||||
}
|
||||
const CLAUDE_CWD = join(tmpdir(), `gbrain-claude-cli-cwd-${process.pid}`);
|
||||
let cwdEnsured = false;
|
||||
function ensureCleanCwd(): string {
|
||||
if (!cwdEnsured) {
|
||||
mkdirSync(CLAUDE_CWD, { recursive: true });
|
||||
cwdEnsured = true;
|
||||
}
|
||||
return CLAUDE_CWD;
|
||||
}
|
||||
|
||||
/** Parsed shape of `claude --print --output-format json`. */
|
||||
interface ClaudeJsonResult {
|
||||
type: 'result';
|
||||
subtype: 'success' | string;
|
||||
is_error: boolean;
|
||||
result: string;
|
||||
stop_reason: string | null;
|
||||
session_id: string;
|
||||
num_turns: number;
|
||||
usage?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
cache_read_input_tokens?: number;
|
||||
cache_creation_input_tokens?: number;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the system-prompt addendum that teaches the model the
|
||||
* `<use_tools>...</use_tools>` emission format. Returns the empty string
|
||||
* when no tools are registered for this turn so the model gets a normal
|
||||
* text-completion prompt without protocol noise.
|
||||
*/
|
||||
function buildToolUseInstructions(
|
||||
tools: ReadonlyArray<LanguageModelV2FunctionTool | LanguageModelV2ProviderDefinedTool> | undefined,
|
||||
): string {
|
||||
if (!tools || tools.length === 0) return '';
|
||||
|
||||
const functionTools = tools.filter((t): t is LanguageModelV2FunctionTool => t.type === 'function');
|
||||
if (functionTools.length === 0) return '';
|
||||
|
||||
const toolSpecs = functionTools.map(t => ({
|
||||
name: t.name,
|
||||
description: t.description ?? '',
|
||||
input_schema: t.inputSchema ?? { type: 'object', properties: {} },
|
||||
}));
|
||||
|
||||
return [
|
||||
'',
|
||||
'## Tool Use Protocol',
|
||||
'',
|
||||
'You have access to these tools:',
|
||||
'',
|
||||
'```json',
|
||||
JSON.stringify(toolSpecs, null, 2),
|
||||
'```',
|
||||
'',
|
||||
'To call one or more tools in this turn, emit EXACTLY ONE block of this form, ' +
|
||||
'with no other text outside the block on its own lines:',
|
||||
'',
|
||||
'<use_tools>',
|
||||
'[',
|
||||
' {"id": "<unique tool call id, like toolu_01ABC>", "name": "<tool name>", "input": <input object matching the tool\'s input_schema>}',
|
||||
']',
|
||||
'</use_tools>',
|
||||
'',
|
||||
'Multiple tool calls go in the array. Tool results are returned to you on the ' +
|
||||
'next turn as [tool_result <text>] entries. You may then call more tools or emit a final response.',
|
||||
'',
|
||||
'When you are ready to give a final answer instead of calling tools, respond with prose text only — ' +
|
||||
'do not include a <use_tools> block in that case.',
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Render the ai-sdk message array into a single text prompt for `claude --print`
|
||||
* stdin. System messages are extracted up-front and concatenated into the
|
||||
* `--system-prompt` flag value. Tool calls and tool results are rendered as
|
||||
* placeholders so the model sees the conversation in a coherent shape even
|
||||
* though the adapter does not natively round-trip tool calls through claude-cli.
|
||||
*/
|
||||
function renderPrompt(prompt: LanguageModelV2Prompt): { systemText: string; userPrompt: string } {
|
||||
const systemParts: string[] = [];
|
||||
const convo: string[] = [];
|
||||
|
||||
for (const msg of prompt as ReadonlyArray<LanguageModelV2Message>) {
|
||||
if (msg.role === 'system') {
|
||||
systemParts.push(msg.content);
|
||||
continue;
|
||||
}
|
||||
if (msg.role === 'user') {
|
||||
const text = msg.content
|
||||
.map(p => {
|
||||
if (p.type === 'text') return p.text;
|
||||
// File parts get a stub — multimodal is not supported via subprocess yet.
|
||||
if (p.type === 'file') return `[file ${p.mediaType ?? 'unknown'}]`;
|
||||
return '';
|
||||
})
|
||||
.filter(s => s.length > 0)
|
||||
.join('\n');
|
||||
if (text) convo.push(`User: ${text}`);
|
||||
continue;
|
||||
}
|
||||
if (msg.role === 'assistant') {
|
||||
const rendered = msg.content
|
||||
.map(p => {
|
||||
if (p.type === 'text') return p.text;
|
||||
if (p.type === 'reasoning') return ''; // dropped on replay
|
||||
if (p.type === 'tool-call') {
|
||||
return `[tool_use ${p.toolName}(${p.input})]`;
|
||||
}
|
||||
if (p.type === 'tool-result') {
|
||||
const out = typeof p.output === 'string' ? p.output : JSON.stringify(p.output);
|
||||
return `[tool_result ${out}]`;
|
||||
}
|
||||
return '';
|
||||
})
|
||||
.filter(s => s.length > 0)
|
||||
.join('\n');
|
||||
if (rendered) convo.push(`Assistant: ${rendered}`);
|
||||
continue;
|
||||
}
|
||||
if (msg.role === 'tool') {
|
||||
const rendered = msg.content
|
||||
.map(p => {
|
||||
const out = typeof p.output === 'string' ? p.output : JSON.stringify(p.output);
|
||||
return `[tool_result ${out}]`;
|
||||
})
|
||||
.join('\n');
|
||||
if (rendered) convo.push(`User: ${rendered}`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return { systemText: systemParts.join('\n'), userPrompt: convo.join('\n\n') };
|
||||
}
|
||||
|
||||
/**
|
||||
* Spawn `claude --print` with the contamination-suppression flags and return
|
||||
* the parsed `--output-format json` envelope. Aborts propagate to SIGTERM on
|
||||
* the child.
|
||||
*/
|
||||
function runClaude(
|
||||
systemPrompt: string,
|
||||
userPrompt: string,
|
||||
model: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ClaudeJsonResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const args = [
|
||||
'--print',
|
||||
'--output-format', 'json',
|
||||
'--model', model,
|
||||
'--disable-slash-commands',
|
||||
// Agent isolation: this subprocess must behave like a raw LLM, not a
|
||||
// full Claude Code agent. `--tools ""` disables every built-in tool
|
||||
// (Bash/Read/WebSearch/...); `--strict-mcp-config` ignores all user-level
|
||||
// MCP servers (without it, each call would boot the user's MCP servers —
|
||||
// including gbrain's own MCP → recursion + PGLite single-writer lock
|
||||
// contention). Verified against claude CLI 2.1.145 --help.
|
||||
'--tools', '',
|
||||
'--strict-mcp-config',
|
||||
];
|
||||
if (systemPrompt) {
|
||||
args.push('--system-prompt', systemPrompt);
|
||||
}
|
||||
// Env scrub: guarantee the CLI authenticates via its own OAuth session
|
||||
// (subscription), never via an inherited API key. Without this, an
|
||||
// ANTHROPIC_API_KEY in gbrain's env (the exact setup this recipe is meant
|
||||
// to replace) silently flips billing to per-token API usage.
|
||||
const env = { ...process.env };
|
||||
delete env.ANTHROPIC_API_KEY;
|
||||
delete env.ANTHROPIC_AUTH_TOKEN;
|
||||
delete env.ANTHROPIC_BASE_URL;
|
||||
const child = spawn(claudeBin(), args, {
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
cwd: ensureCleanCwd(),
|
||||
env,
|
||||
});
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
child.stdout.on('data', chunk => { stdout += String(chunk); });
|
||||
child.stderr.on('data', chunk => { stderr += String(chunk); });
|
||||
|
||||
const onAbort = () => {
|
||||
child.kill('SIGTERM');
|
||||
reject(new Error('claude-cli adapter aborted'));
|
||||
};
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
}
|
||||
|
||||
child.on('error', err => {
|
||||
if (signal) signal.removeEventListener('abort', onAbort);
|
||||
reject(new Error(`claude-cli spawn failed: ${err instanceof Error ? err.message : String(err)}`));
|
||||
});
|
||||
|
||||
child.on('close', code => {
|
||||
if (signal) signal.removeEventListener('abort', onAbort);
|
||||
if (code !== 0) {
|
||||
reject(new Error(`claude-cli exited ${code}: ${stderr.trim() || stdout.trim()}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
let parsed = JSON.parse(stdout) as unknown;
|
||||
// Compat: when the user has `"verbose": true` in ~/.claude/settings.json,
|
||||
// `--print --output-format json` emits an ARRAY of events
|
||||
// ([{type:"system",subtype:"init",...}, ..., {type:"result",...}])
|
||||
// instead of the bare result object. There is no CLI flag to force it
|
||||
// off (no --no-verbose; --settings '{}' merges, does not replace), so
|
||||
// tolerate both shapes and pick the result event. Verified on CLI 2.1.145.
|
||||
if (Array.isArray(parsed)) {
|
||||
const resultEvent = parsed.find(
|
||||
(ev): ev is ClaudeJsonResult =>
|
||||
!!ev && typeof ev === 'object' && (ev as { type?: unknown }).type === 'result',
|
||||
);
|
||||
if (!resultEvent) {
|
||||
reject(new Error(`claude-cli JSON event array had no "result" event\n--- raw ---\n${stdout.slice(0, 500)}`));
|
||||
return;
|
||||
}
|
||||
parsed = resultEvent;
|
||||
}
|
||||
const envelope = parsed as ClaudeJsonResult;
|
||||
if (envelope.is_error) {
|
||||
reject(new Error(`claude-cli reported error: ${envelope.result || envelope.subtype}`));
|
||||
return;
|
||||
}
|
||||
resolve(envelope);
|
||||
} catch (e) {
|
||||
reject(new Error(`claude-cli output not JSON: ${e instanceof Error ? e.message : String(e)}\n--- raw ---\n${stdout.slice(0, 500)}`));
|
||||
}
|
||||
});
|
||||
|
||||
// stdin error handler: if the binary does not exist (ENOENT) or the child
|
||||
// dies before draining stdin, write/end can emit an unhandled 'error'
|
||||
// (EPIPE) that would crash the worker. The spawn-level 'error' / non-zero
|
||||
// 'close' handlers above already surface the real failure, so the stdin
|
||||
// error itself is safe to swallow.
|
||||
child.stdin.on('error', () => { /* surfaced via child 'error'/'close' */ });
|
||||
try {
|
||||
child.stdin.write(userPrompt);
|
||||
child.stdin.end();
|
||||
} catch (e) {
|
||||
if (signal) signal.removeEventListener('abort', onAbort);
|
||||
reject(new Error(`claude-cli stdin write failed (is the claude binary installed?): ${e instanceof Error ? e.message : String(e)}`));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
interface ParsedToolCall {
|
||||
id: string;
|
||||
name: string;
|
||||
/** Stringified JSON, matching the ai-sdk LanguageModelV2ToolCall.input contract. */
|
||||
input: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Locate and parse the `<use_tools>...</use_tools>` block in the assistant's
|
||||
* raw text response. Returns the parsed tool calls plus whatever prose
|
||||
* surrounded the block. Returns an empty `toolCalls` array when no block is
|
||||
* present, malformed, or unterminated — the caller then treats the full
|
||||
* raw text as a final text response.
|
||||
*/
|
||||
function extractToolCalls(raw: string): {
|
||||
toolCalls: ParsedToolCall[];
|
||||
beforeText: string;
|
||||
afterText: string;
|
||||
} {
|
||||
const openTag = '<use_tools>';
|
||||
const closeTag = '</use_tools>';
|
||||
const openIdx = raw.indexOf(openTag);
|
||||
if (openIdx === -1) {
|
||||
return { toolCalls: [], beforeText: raw.trim(), afterText: '' };
|
||||
}
|
||||
const closeIdx = raw.indexOf(closeTag, openIdx + openTag.length);
|
||||
if (closeIdx === -1) {
|
||||
// Unterminated block — recover gracefully.
|
||||
return { toolCalls: [], beforeText: raw.trim(), afterText: '' };
|
||||
}
|
||||
|
||||
const beforeText = raw.slice(0, openIdx).trim();
|
||||
const afterText = raw.slice(closeIdx + closeTag.length).trim();
|
||||
let inner = raw.slice(openIdx + openTag.length, closeIdx).trim();
|
||||
|
||||
if (inner.startsWith('```')) {
|
||||
inner = inner.replace(/^```(?:json|JSON)?\s*\n?/, '').replace(/\n?```$/, '').trim();
|
||||
}
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(inner);
|
||||
} catch {
|
||||
return { toolCalls: [], beforeText: raw.trim(), afterText: '' };
|
||||
}
|
||||
if (!Array.isArray(parsed)) {
|
||||
return { toolCalls: [], beforeText: raw.trim(), afterText: '' };
|
||||
}
|
||||
|
||||
const toolCalls: ParsedToolCall[] = [];
|
||||
for (const entry of parsed) {
|
||||
if (!entry || typeof entry !== 'object') continue;
|
||||
const e = entry as Record<string, unknown>;
|
||||
const name = typeof e.name === 'string' ? e.name : null;
|
||||
if (!name) continue;
|
||||
const id = typeof e.id === 'string' && e.id.length > 0
|
||||
? e.id
|
||||
: `toolu_claude_cli_${Math.random().toString(36).slice(2, 12)}`;
|
||||
const inputJson = JSON.stringify(e.input ?? {});
|
||||
toolCalls.push({ id, name, input: inputJson });
|
||||
}
|
||||
|
||||
return { toolCalls, beforeText, afterText };
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip provider prefixes (`anthropic:`, `litellm:`, `claude-cli:`) that the
|
||||
* underlying CLI does not understand. The gateway hands us a bare model id
|
||||
* via `recipe.aliases` resolution, but defensive normalization here keeps
|
||||
* direct LanguageModelV2 construction (in tests, for example) ergonomic.
|
||||
*/
|
||||
function normalizeModel(model: string): string {
|
||||
const idx = model.indexOf(':');
|
||||
return idx >= 0 ? model.slice(idx + 1) : model;
|
||||
}
|
||||
|
||||
export class ClaudeCliLanguageModel implements LanguageModelV2 {
|
||||
readonly specificationVersion = 'v2' as const;
|
||||
readonly provider = 'claude-cli';
|
||||
readonly modelId: string;
|
||||
readonly supportedUrls = {};
|
||||
|
||||
constructor(modelId: string) {
|
||||
this.modelId = normalizeModel(modelId);
|
||||
}
|
||||
|
||||
async doGenerate(options: LanguageModelV2CallOptions): Promise<{
|
||||
content: LanguageModelV2Content[];
|
||||
finishReason: 'stop' | 'length' | 'content-filter' | 'tool-calls' | 'error' | 'other' | 'unknown';
|
||||
usage: { inputTokens: number | undefined; outputTokens: number | undefined; totalTokens: number | undefined };
|
||||
warnings: never[];
|
||||
}> {
|
||||
const { systemText, userPrompt } = renderPrompt(options.prompt);
|
||||
const toolInstructions = buildToolUseInstructions(options.tools);
|
||||
const systemPrompt = [systemText, toolInstructions].filter(s => s.length > 0).join('\n');
|
||||
|
||||
const result = await runClaude(systemPrompt, userPrompt, this.modelId, options.abortSignal);
|
||||
const { toolCalls, beforeText, afterText } = extractToolCalls(result.result);
|
||||
|
||||
const content: LanguageModelV2Content[] = [];
|
||||
if (beforeText) content.push({ type: 'text', text: beforeText });
|
||||
for (const call of toolCalls) {
|
||||
content.push({
|
||||
type: 'tool-call',
|
||||
toolCallId: call.id,
|
||||
toolName: call.name,
|
||||
input: call.input,
|
||||
});
|
||||
}
|
||||
if (afterText) content.push({ type: 'text', text: afterText });
|
||||
if (content.length === 0) {
|
||||
// Empty response — still hand the caller a well-formed content array.
|
||||
content.push({ type: 'text', text: result.result ?? '' });
|
||||
}
|
||||
|
||||
const finishReason = toolCalls.length > 0 ? 'tool-calls' as const : 'stop' as const;
|
||||
const inputTokens = result.usage?.input_tokens;
|
||||
const outputTokens = result.usage?.output_tokens;
|
||||
const totalTokens = (inputTokens ?? 0) + (outputTokens ?? 0);
|
||||
|
||||
return {
|
||||
content,
|
||||
finishReason,
|
||||
usage: {
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens: inputTokens !== undefined && outputTokens !== undefined ? totalTokens : undefined,
|
||||
},
|
||||
warnings: [],
|
||||
};
|
||||
}
|
||||
|
||||
async doStream(): Promise<never> {
|
||||
throw new Error(
|
||||
'claude-cli LanguageModel does not support streaming. Use doGenerate or set ' +
|
||||
'the model on a non-streaming chat surface (gateway.toolLoop is non-streaming).',
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ export const anthropic: Recipe = {
|
||||
chat: {
|
||||
models: [
|
||||
'claude-fable-5',
|
||||
'claude-opus-5',
|
||||
'claude-opus-4-8',
|
||||
'claude-opus-4-7',
|
||||
'claude-sonnet-5',
|
||||
|
||||
@@ -1,8 +1,60 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
import { AIConfigError } from '../errors.ts';
|
||||
import { execSync } from 'node:child_process';
|
||||
|
||||
const DEFAULT_API_VERSION = '2024-10-21'; // stable Azure OpenAI version as of 2026-05
|
||||
|
||||
// Entra (keyless) auth support. Subscriptions that enforce disableLocalAuth via
|
||||
// Azure Policy reject api-key auth, so when no AZURE_OPENAI_API_KEY is present
|
||||
// (or AZURE_OPENAI_USE_ENTRA=1) we mint a short-lived AAD bearer token via the
|
||||
// Azure CLI and cache it. resolveAuth is synchronous, so execSync is the seam.
|
||||
// The caller needs `az login` + the "Cognitive Services OpenAI User" role.
|
||||
let _entraToken: { token: string; fetchedAt: number } | null = null;
|
||||
const ENTRA_TOKEN_TTL_MS = 45 * 60 * 1000; // refresh well before the ~60-90min expiry
|
||||
|
||||
function fetchEntraToken(): string {
|
||||
const now = Date.now();
|
||||
if (_entraToken && now - _entraToken.fetchedAt < ENTRA_TOKEN_TTL_MS) {
|
||||
return _entraToken.token;
|
||||
}
|
||||
let token = '';
|
||||
try {
|
||||
token = execSync(
|
||||
'az account get-access-token --resource https://cognitiveservices.azure.com --query accessToken -o tsv',
|
||||
{ encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 30_000 },
|
||||
).trim();
|
||||
} catch {
|
||||
throw new AIConfigError(
|
||||
'Azure OpenAI (Entra/keyless): could not get an access token via `az account get-access-token`.',
|
||||
'Run `az login` and ensure your identity has the "Cognitive Services OpenAI User" role on the resource.',
|
||||
);
|
||||
}
|
||||
if (!token) {
|
||||
throw new AIConfigError(
|
||||
'Azure OpenAI (Entra/keyless): `az account get-access-token` returned an empty token.',
|
||||
'Run `az login` and verify the active subscription owns the Azure OpenAI resource.',
|
||||
);
|
||||
}
|
||||
_entraToken = { token, fetchedAt: now };
|
||||
return token;
|
||||
}
|
||||
|
||||
/** @internal test seam: pre-populate (or clear) the Entra token cache so unit
|
||||
* tests never shell out to `az`. */
|
||||
export function __setEntraTokenForTests(token: string | null): void {
|
||||
_entraToken = token === null ? null : { token, fetchedAt: Date.now() };
|
||||
}
|
||||
|
||||
/** Entra/keyless mode: EXPLICIT opt-in only (AZURE_OPENAI_USE_ENTRA=1 /
|
||||
* config azure_openai_use_entra). A missing api-key must NOT silently shell
|
||||
* out to `az` — that surprises CI boxes and every non-Azure-CLI environment,
|
||||
* and it broke the cross-recipe auth iron-rule test. Keyless subscriptions
|
||||
* (disableLocalAuth) set the flag; missing key without the flag keeps the
|
||||
* original loud AIConfigError. */
|
||||
function isEntraMode(env: Record<string, string | undefined>): boolean {
|
||||
return env.AZURE_OPENAI_USE_ENTRA === '1';
|
||||
}
|
||||
|
||||
/**
|
||||
* Azure OpenAI. The first recipe in v0.32 to exercise both seams:
|
||||
* - resolveAuth returns `{headerName: 'api-key', token: <key>}` instead of
|
||||
@@ -30,11 +82,13 @@ export const azureOpenAI: Recipe = {
|
||||
// base_url_default omitted: Azure URLs are env-templated only.
|
||||
auth_env: {
|
||||
required: [
|
||||
'AZURE_OPENAI_API_KEY',
|
||||
'AZURE_OPENAI_ENDPOINT',
|
||||
'AZURE_OPENAI_DEPLOYMENT',
|
||||
],
|
||||
optional: ['AZURE_OPENAI_API_VERSION'],
|
||||
// AZURE_OPENAI_API_KEY optional: when absent (or AZURE_OPENAI_USE_ENTRA=1)
|
||||
// the recipe uses a refreshing Entra/AAD bearer token via the Azure CLI,
|
||||
// required on subscriptions that enforce disableLocalAuth (keyless).
|
||||
optional: ['AZURE_OPENAI_API_KEY', 'AZURE_OPENAI_USE_ENTRA', 'AZURE_OPENAI_API_VERSION'],
|
||||
setup_url:
|
||||
'https://learn.microsoft.com/en-us/azure/ai-services/openai/quickstart',
|
||||
},
|
||||
@@ -54,17 +108,18 @@ export const azureOpenAI: Recipe = {
|
||||
},
|
||||
},
|
||||
resolveAuth(env) {
|
||||
const key = env.AZURE_OPENAI_API_KEY;
|
||||
if (!key) {
|
||||
throw new AIConfigError(
|
||||
`Azure OpenAI requires AZURE_OPENAI_API_KEY.`,
|
||||
'Get a key from your Azure portal: https://learn.microsoft.com/en-us/azure/ai-services/openai/quickstart',
|
||||
);
|
||||
// Entra/keyless mode: no api-key (disableLocalAuth) or opt-in via
|
||||
// AZURE_OPENAI_USE_ENTRA=1. Mint a refreshing AAD bearer token. Returning
|
||||
// an `Authorization: Bearer …` pair makes the gateway use the SDK's native
|
||||
// bearer path (it strips the prefix and re-adds it), so no double-auth.
|
||||
if (isEntraMode(env)) {
|
||||
return { headerName: 'Authorization', token: `Bearer ${fetchEntraToken()}` };
|
||||
}
|
||||
// Azure uses `api-key:` (no Bearer); the unified seam routes this
|
||||
// Key mode: Azure uses `api-key:` (no Bearer); the unified seam routes this
|
||||
// through `headers` instead of the SDK's apiKey field to avoid any
|
||||
// double-auth Authorization header sneaking in.
|
||||
return { headerName: 'api-key', token: key };
|
||||
// double-auth Authorization header sneaking in. The key is present here:
|
||||
// !isEntraMode(env) implies AZURE_OPENAI_API_KEY is set.
|
||||
return { headerName: 'api-key', token: env.AZURE_OPENAI_API_KEY! };
|
||||
},
|
||||
resolveOpenAICompatConfig(env) {
|
||||
const endpoint = env.AZURE_OPENAI_ENDPOINT?.replace(/\/+$/, '');
|
||||
@@ -82,6 +137,7 @@ export const azureOpenAI: Recipe = {
|
||||
);
|
||||
}
|
||||
const apiVersion = env.AZURE_OPENAI_API_VERSION ?? DEFAULT_API_VERSION;
|
||||
const entra = isEntraMode(env);
|
||||
const baseURL = `${endpoint}/openai/deployments/${deployment}`;
|
||||
// Custom fetch wrapper splices ?api-version=... onto every request.
|
||||
// Azure rejects requests without it.
|
||||
@@ -102,10 +158,23 @@ export const azureOpenAI: Recipe = {
|
||||
typeof input === 'string' || input instanceof URL
|
||||
? finalUrl
|
||||
: new Request(finalUrl, input as Request);
|
||||
if (entra) {
|
||||
// Entra mode: refresh the AAD bearer on every request. The gateway
|
||||
// caches model instances (auth is baked in at instantiation), so a
|
||||
// long-running process would otherwise send an expired token after
|
||||
// ~1h. fetchEntraToken()'s 45-min TTL cache keeps `az` invocations
|
||||
// rare; the override here keeps the header fresh.
|
||||
const headers = new Headers(
|
||||
init?.headers ??
|
||||
(typeof finalInput !== 'string' ? (finalInput as Request).headers : undefined),
|
||||
);
|
||||
headers.set('Authorization', `Bearer ${fetchEntraToken()}`);
|
||||
init = { ...init, headers };
|
||||
}
|
||||
return fetch(finalInput, init);
|
||||
}) as unknown as typeof fetch;
|
||||
return { baseURL, fetch: wrappedFetch };
|
||||
},
|
||||
setup_hint:
|
||||
'Azure portal → Azure OpenAI resource. Set AZURE_OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_DEPLOYMENT. Optionally AZURE_OPENAI_API_VERSION (default 2024-10-21).',
|
||||
'Azure portal → Azure OpenAI resource. Set AZURE_OPENAI_ENDPOINT + AZURE_OPENAI_DEPLOYMENT, and either AZURE_OPENAI_API_KEY or keyless Entra auth (`az login` + "Cognitive Services OpenAI User" role; force with AZURE_OPENAI_USE_ENTRA=1). Optionally AZURE_OPENAI_API_VERSION (default 2024-10-21).',
|
||||
};
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Claude via the local `claude` CLI binary, using its built-in OAuth session
|
||||
* (Claude Code / Claude Max subscription). No ANTHROPIC_API_KEY needed — the
|
||||
* CLI manages its own auth state and the gateway dispatches via subprocess.
|
||||
*
|
||||
* Solves the #334 case where Max subscribers want Minions subagent dispatch
|
||||
* to run against their existing subscription instead of paying per-token API
|
||||
* charges. The recipe sits alongside the existing `anthropic` recipe so users
|
||||
* pick per call: `anthropic:claude-sonnet-4-6` (API key + per-token billing)
|
||||
* vs `claude-cli:claude-sonnet-4-6` (OAuth subscription, no API key).
|
||||
*
|
||||
* Chat-only. Claude has no first-party embedding model; users wanting an
|
||||
* Anthropic chat path with embeddings still combine this with openai/google/
|
||||
* voyage for embedding the way the existing `anthropic` recipe documents.
|
||||
*
|
||||
* Auth: `auth_env.required: []` because the CLI handles auth itself. The
|
||||
* `claude` binary on PATH (or `GBRAIN_CLAUDE_CLI_BIN`) IS the auth surface;
|
||||
* there is nothing for the gateway to forward.
|
||||
*
|
||||
* Setup expectation: `claude` CLI installed and logged in (Claude Code
|
||||
* onboarding does this), or `GBRAIN_CLAUDE_CLI_BIN` pointing at the binary.
|
||||
*/
|
||||
export const claudeCli: Recipe = {
|
||||
id: 'claude-cli',
|
||||
name: 'Claude (via CLI)',
|
||||
tier: 'native',
|
||||
implementation: 'claude-cli',
|
||||
// The CLI owns auth; no env vars are required from the gateway side.
|
||||
auth_env: {
|
||||
required: [],
|
||||
},
|
||||
touchpoints: {
|
||||
// No embedding or expansion touchpoints — chat-only.
|
||||
chat: {
|
||||
models: [
|
||||
'claude-opus-4-7',
|
||||
'claude-sonnet-4-6',
|
||||
'claude-haiku-4-5-20251001',
|
||||
],
|
||||
supports_tools: true,
|
||||
supports_subagent_loop: true,
|
||||
// The CLI handles caching internally and does not surface it via the
|
||||
// standard cache_control control plane. From the gateway's POV the
|
||||
// model does not support prompt caching.
|
||||
supports_prompt_cache: false,
|
||||
max_context_tokens: 200000,
|
||||
// Cost figures match the underlying Claude API tier, but the actual
|
||||
// bill is borne by the subscription. We report them for the budget
|
||||
// ledger's per-call accounting; operators on flat-rate subscriptions
|
||||
// can treat the numbers as nominal.
|
||||
cost_per_1m_input_usd: 3.0,
|
||||
cost_per_1m_output_usd: 15.0,
|
||||
price_last_verified: '2026-06-17',
|
||||
},
|
||||
},
|
||||
// Friendly aliases mirror the `anthropic` recipe so config strings stay
|
||||
// portable: switching `anthropic:claude-sonnet-4-6` to `claude-cli:claude-sonnet-4-6`
|
||||
// is a one-token edit. Reverse aliases rewrite legacy IDs back to canonical.
|
||||
aliases: {
|
||||
'claude-haiku-4-5': 'claude-haiku-4-5-20251001',
|
||||
'claude-sonnet-4-6-20250929': 'claude-sonnet-4-6',
|
||||
'sonnet': 'claude-sonnet-4-6',
|
||||
'haiku': 'claude-haiku-4-5-20251001',
|
||||
'opus': 'claude-opus-4-7',
|
||||
},
|
||||
setup_hint:
|
||||
'Install Claude Code (`claude` CLI) and run `claude` once to log in. ' +
|
||||
'Set GBRAIN_CLAUDE_CLI_BIN if the binary is not on PATH.',
|
||||
};
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Alibaba DashScope (灵积) reranker. DashScope's OpenAI-compatible surface
|
||||
* splits by capability: embeddings live under `/compatible-mode/v1` (see the
|
||||
* sibling `dashscope` recipe) while rerank lives under `/compatible-api/v1`
|
||||
* with a PLURAL leaf — `POST {base}/reranks`. Wire shape matches ZeroEntropy:
|
||||
* request `{model, query, documents, top_n?}`, response
|
||||
* `{results: [{index, relevance_score}]}` — so it rides gateway.rerank()'s
|
||||
* native path with only the recipe-pluggable `path` override (v0.40.6.1).
|
||||
*
|
||||
* This is a SEPARATE recipe rather than a reranker touchpoint on `dashscope`
|
||||
* because the two capabilities need different base URLs (`compatible-mode`
|
||||
* vs `compatible-api`) and `provider_base_urls` is keyed by recipe id — one
|
||||
* recipe can't point embeddings and rerank at different prefixes. Same
|
||||
* topology precedent as llama-server vs llama-server-reranker.
|
||||
*
|
||||
* Live-verified against the China endpoint (2026-07): `/reranks` with
|
||||
* `qwen3-rerank` → 200 `results[].relevance_score`; `/rerank` (singular)
|
||||
* → 404; `gte-rerank-v2` → 404 "Unsupported model for OpenAI compatibility
|
||||
* mode" (native-API only, so it is deliberately NOT listed here).
|
||||
*
|
||||
* Note: the international endpoint requires a region-aware DASHSCOPE_API_KEY.
|
||||
* China-region users point at https://dashscope.aliyuncs.com/compatible-api/v1
|
||||
* via `provider_base_urls['dashscope-rerank']`, mirroring the embedding
|
||||
* recipe's convention.
|
||||
*/
|
||||
export const dashscopeRerank: Recipe = {
|
||||
id: 'dashscope-rerank',
|
||||
name: 'Alibaba DashScope (灵积, reranker)',
|
||||
tier: 'openai-compat',
|
||||
implementation: 'openai-compatible',
|
||||
base_url_default: 'https://dashscope-intl.aliyuncs.com/compatible-api/v1',
|
||||
auth_env: {
|
||||
required: ['DASHSCOPE_API_KEY'],
|
||||
setup_url: 'https://help.aliyun.com/zh/model-studio/getting-started/',
|
||||
},
|
||||
touchpoints: {
|
||||
reranker: {
|
||||
// Only the model verified live on the OpenAI-compat /reranks surface.
|
||||
// gte-rerank-v2 exists on DashScope's native API but the compat path
|
||||
// rejects it ("Unsupported model for OpenAI compatibility mode").
|
||||
models: ['qwen3-rerank'],
|
||||
default_model: 'qwen3-rerank',
|
||||
// Mirror ZE's defensive per-request ceiling; gateway.rerank()
|
||||
// pre-flights body size and fails open.
|
||||
max_payload_bytes: 5_000_000,
|
||||
// PLURAL leaf under compatible-api — the whole reason this recipe
|
||||
// exists. `${base_url}${path}` → `…/compatible-api/v1/reranks`.
|
||||
path: '/reranks',
|
||||
// Hosted API: no local warmup, but cross-region latency can exceed
|
||||
// the 5s gateway default (same rationale as llama-server-reranker).
|
||||
default_timeout_ms: 30_000,
|
||||
},
|
||||
},
|
||||
setup_hint:
|
||||
'Get an API key at https://help.aliyun.com/zh/model-studio/getting-started/, then ' +
|
||||
'`export DASHSCOPE_API_KEY=...` and `gbrain config set search.reranker.model ' +
|
||||
'dashscope-rerank:qwen3-rerank`. China-region accounts: `gbrain config set ' +
|
||||
'provider_base_urls.dashscope-rerank https://dashscope.aliyuncs.com/compatible-api/v1`.',
|
||||
};
|
||||
@@ -1,9 +1,10 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* `deepseek-reasoner` returns its answer in a separate `reasoning_content`
|
||||
* field and leaves `content` empty/whitespace when the whole response was
|
||||
* reasoning. The AI SDK's openai-compatible adapter reads only `content`, so
|
||||
* DeepSeek's thinking mode (default on `deepseek-v4-flash`/`deepseek-v4-pro`;
|
||||
* formerly the `deepseek-reasoner` model, retired 2026-07-24) returns its
|
||||
* answer in a separate `reasoning_content` field and leaves `content`
|
||||
* empty/whitespace when the whole response was reasoning. The AI SDK's openai-compatible adapter reads only `content`, so
|
||||
* the model appears to answer with nothing. This transport shim promotes
|
||||
* `reasoning_content` into `content` when `content` is empty, before the
|
||||
* adapter parses the body. Fail-open: any error returns the original response.
|
||||
@@ -76,15 +77,29 @@ export const deepseek: Recipe = {
|
||||
setup_url: 'https://platform.deepseek.com/api_keys',
|
||||
},
|
||||
touchpoints: {
|
||||
// Query expansion reuses the same OpenAI-compatible chat endpoint (the
|
||||
// gateway's expansion path is a plain languageModel call). Without this
|
||||
// declaration an explicit `expansion_model: deepseek:...` silently
|
||||
// yields no expansion (#1135).
|
||||
// `deepseek-chat` / `deepseek-reasoner` were retired by DeepSeek on
|
||||
// 2026-07-24 (#1255); both map to `deepseek-v4-flash` (non-thinking /
|
||||
// thinking mode). Do not re-add the old names — the API 404s them.
|
||||
// openai-compat tier means user-configured legacy names still pass
|
||||
// validation locally; the provider rejects them at call time.
|
||||
expansion: {
|
||||
models: ['deepseek-v4-flash'],
|
||||
cost_per_1m_tokens_usd: 0.14,
|
||||
price_last_verified: '2026-07-27',
|
||||
},
|
||||
chat: {
|
||||
models: ['deepseek-chat', 'deepseek-reasoner'],
|
||||
models: ['deepseek-v4-flash', 'deepseek-v4-pro'],
|
||||
supports_tools: true,
|
||||
supports_subagent_loop: true,
|
||||
supports_prompt_cache: false,
|
||||
max_context_tokens: 128000,
|
||||
cost_per_1m_input_usd: 0.14, // deepseek-chat off-peak baseline
|
||||
max_context_tokens: 1_000_000,
|
||||
cost_per_1m_input_usd: 0.14, // deepseek-v4-flash cache-miss baseline
|
||||
cost_per_1m_output_usd: 0.28,
|
||||
price_last_verified: '2026-04-20',
|
||||
price_last_verified: '2026-07-27',
|
||||
},
|
||||
},
|
||||
setup_hint: 'Get an API key at https://platform.deepseek.com/api_keys, then `export DEEPSEEK_API_KEY=...`',
|
||||
|
||||
@@ -16,6 +16,14 @@ export const groq: Recipe = {
|
||||
setup_url: 'https://console.groq.com/keys',
|
||||
},
|
||||
touchpoints: {
|
||||
// Same OpenAI-compatible endpoint as chat; declared so an explicit
|
||||
// `expansion_model: groq:...` resolves instead of silently dropping
|
||||
// expansion (#1135). 8b-instant is the natural expansion pick (cheap,
|
||||
// fast, no tool-calling needed for multi-query rewrites).
|
||||
expansion: {
|
||||
models: ['llama-3.1-8b-instant', 'llama-3.3-70b-versatile'],
|
||||
price_last_verified: '2026-04-20',
|
||||
},
|
||||
chat: {
|
||||
models: [
|
||||
'llama-3.3-70b-versatile',
|
||||
|
||||
@@ -9,6 +9,7 @@ import type { Recipe } from '../types.ts';
|
||||
import { openai } from './openai.ts';
|
||||
import { google } from './google.ts';
|
||||
import { anthropic } from './anthropic.ts';
|
||||
import { claudeCli } from './claude-cli.ts';
|
||||
import { ollama } from './ollama.ts';
|
||||
import { openrouter } from './openrouter.ts';
|
||||
import { voyage } from './voyage.ts';
|
||||
@@ -19,6 +20,7 @@ import { together } from './together.ts';
|
||||
import { llamaServer } from './llama-server.ts';
|
||||
import { minimax } from './minimax.ts';
|
||||
import { dashscope } from './dashscope.ts';
|
||||
import { dashscopeRerank } from './dashscope-rerank.ts';
|
||||
import { zhipu } from './zhipu.ts';
|
||||
import { azureOpenAI } from './azure-openai.ts';
|
||||
import { zeroentropyai } from './zeroentropyai.ts';
|
||||
@@ -26,11 +28,13 @@ import { llamaServerReranker } from './llama-server-reranker.ts';
|
||||
import { moonshot } from './moonshot.ts';
|
||||
import { mistral } from './mistral.ts';
|
||||
import { nvidia } from './nvidia.ts';
|
||||
import { perplexity } from './perplexity.ts';
|
||||
|
||||
const ALL: Recipe[] = [
|
||||
openai,
|
||||
google,
|
||||
anthropic,
|
||||
claudeCli,
|
||||
ollama,
|
||||
openrouter,
|
||||
voyage,
|
||||
@@ -42,12 +46,14 @@ const ALL: Recipe[] = [
|
||||
llamaServerReranker,
|
||||
minimax,
|
||||
dashscope,
|
||||
dashscopeRerank,
|
||||
zhipu,
|
||||
azureOpenAI,
|
||||
zeroentropyai,
|
||||
moonshot,
|
||||
mistral,
|
||||
nvidia,
|
||||
perplexity,
|
||||
];
|
||||
|
||||
/** Map from `provider:id` key to recipe. */
|
||||
|
||||
@@ -1,8 +1,100 @@
|
||||
import type { Recipe } from '../types.ts';
|
||||
|
||||
/**
|
||||
* MiniMax (海螺AI). OpenAI-compatible /embeddings endpoint at
|
||||
* api.minimax.chat. The flagship embedding model is `embo-01` (1536 dims).
|
||||
* MiniMax transport shim (#1977). MiniMax's `/v1/embeddings` endpoint is NOT
|
||||
* OpenAI-compatible at the wire level despite the recipe's
|
||||
* `implementation: 'openai-compatible'`:
|
||||
* - Request: requires `texts` (the AI SDK sends `input`) plus an optional
|
||||
* `type: 'db' | 'query'` asymmetric-retrieval field, and rejects OpenAI's
|
||||
* `encoding_format`.
|
||||
* - Response: returns `{vectors: number[][], total_tokens}` where the AI
|
||||
* SDK's Zod schema expects `{data: [{embedding, index}], usage}`.
|
||||
*
|
||||
* Chat (`/chat/completions`) IS OpenAI-compatible, and this same fetch is
|
||||
* applied to every openai-compatible touchpoint by `applyOpenAICompatConfig`,
|
||||
* so everything outside the embeddings path passes through untouched — and
|
||||
* the response rewrite parses via `resp.clone()` only (never consume the
|
||||
* body of a response we return as-is; the DeepSeek shim rule). Fail-open:
|
||||
* any rewrite error returns the original request/response.
|
||||
*
|
||||
* @internal exported for tests.
|
||||
*/
|
||||
// Cast through `unknown` because Bun's `typeof fetch` carries a `preconnect`
|
||||
// member the arrow function does not implement (matches deepseek.ts).
|
||||
export const minimaxCompatFetch = (async (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
): Promise<Response> => {
|
||||
const url =
|
||||
typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
|
||||
const isEmbeddings = url.includes('/embeddings');
|
||||
|
||||
// OUTBOUND (embeddings only): `input` → `texts`, default `type: 'db'`
|
||||
// (the recipe's documented symmetric default — the AI SDK adapter strips
|
||||
// the `type` threaded via providerOptions before it reaches the wire,
|
||||
// same class as #1400), and drop `encoding_format` (not a MiniMax param).
|
||||
if (isEmbeddings && init?.body && typeof init.body === 'string') {
|
||||
try {
|
||||
const parsed = JSON.parse(init.body);
|
||||
if (
|
||||
parsed && typeof parsed === 'object' &&
|
||||
parsed.input !== undefined && parsed.texts === undefined
|
||||
) {
|
||||
parsed.texts = Array.isArray(parsed.input) ? parsed.input : [parsed.input];
|
||||
delete parsed.input;
|
||||
delete parsed.encoding_format;
|
||||
if (parsed.type === undefined) parsed.type = 'db';
|
||||
// Drop Content-Length so fetch recomputes from the new body.
|
||||
const headers = new Headers(init.headers ?? {});
|
||||
headers.delete('content-length');
|
||||
init = { ...init, body: JSON.stringify(parsed), headers };
|
||||
}
|
||||
} catch {
|
||||
// Body wasn't JSON — pass through untouched.
|
||||
}
|
||||
}
|
||||
|
||||
const res = await fetch(input as any, init as any);
|
||||
|
||||
// INBOUND (embeddings only): `{vectors: [[...]]}` → `{data: [{embedding}]}`.
|
||||
// Anything else (chat completions, MiniMax base_resp errors, non-JSON)
|
||||
// returns the ORIGINAL response with its body unread.
|
||||
if (!isEmbeddings || !res.ok) return res;
|
||||
const ctype = res.headers.get('content-type') ?? '';
|
||||
if (!ctype.toLowerCase().includes('application/json')) return res;
|
||||
try {
|
||||
const json = await res.clone().json();
|
||||
if (!json || typeof json !== 'object' || !Array.isArray(json.vectors)) return res;
|
||||
const totalTokens = typeof json.total_tokens === 'number' ? json.total_tokens : 0;
|
||||
const rewritten = {
|
||||
object: 'list',
|
||||
data: (json.vectors as number[][]).map((embedding, index) => ({
|
||||
object: 'embedding',
|
||||
embedding,
|
||||
index,
|
||||
})),
|
||||
model: typeof json.model === 'string' ? json.model : 'embo-01',
|
||||
usage: { prompt_tokens: totalTokens, total_tokens: totalTokens },
|
||||
};
|
||||
// Fresh header set: the body changed, so upstream content-length /
|
||||
// content-encoding would now be wrong.
|
||||
const headers = new Headers(res.headers);
|
||||
headers.delete('content-length');
|
||||
headers.delete('content-encoding');
|
||||
return new Response(JSON.stringify(rewritten), {
|
||||
status: res.status,
|
||||
statusText: res.statusText,
|
||||
headers,
|
||||
});
|
||||
} catch {
|
||||
return res;
|
||||
}
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
/**
|
||||
* MiniMax (海螺AI). `/embeddings` endpoint at api.minimaxi.com (wire shape
|
||||
* normalized by `minimaxCompatFetch` above); OpenAI-compatible
|
||||
* `/chat/completions`. The flagship embedding model is `embo-01` (1536 dims).
|
||||
*
|
||||
* MiniMax's API takes an extra `type: 'db' | 'query'` field for asymmetric
|
||||
* retrieval. gbrain currently has no notion of "this is a document vs a
|
||||
@@ -38,7 +130,25 @@ export const minimax: Recipe = {
|
||||
// halving in the gateway catches token-limit errors at runtime.
|
||||
max_batch_tokens: 4096,
|
||||
},
|
||||
chat: {
|
||||
// Model list from MiniMax's /v1/models (#1977). Chat is genuinely
|
||||
// OpenAI-compatible — no wire rewrite needed (minimaxCompatFetch
|
||||
// passes non-embedding requests through untouched).
|
||||
models: [
|
||||
'MiniMax-M3',
|
||||
'MiniMax-M2.7',
|
||||
'MiniMax-M2.7-highspeed',
|
||||
'MiniMax-M2.5',
|
||||
'MiniMax-M2.5-highspeed',
|
||||
'MiniMax-M2.1',
|
||||
'MiniMax-M2.1-highspeed',
|
||||
'MiniMax-M2',
|
||||
],
|
||||
supports_tools: false,
|
||||
supports_subagent_loop: false,
|
||||
},
|
||||
},
|
||||
setup_hint:
|
||||
'Get an API key at https://www.minimaxi.com, then `export MINIMAX_API_KEY=...`',
|
||||
compat: { fetch: minimaxCompatFetch },
|
||||
};
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user