mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-16 09:52:22 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Garry TanandClaude Opus 4.7
|
4d5c4772e2 | ||
|
Garry TanandClaude Opus 4.7
|
3f1f1e2601 | ||
|
Garry TanandClaude Opus 4.7
|
0f93cb23c4 | ||
|
root
|
3c012bce23 |
@@ -21,22 +21,11 @@ jobs:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test:
|
||||
# ubuntu-latest is free 2-core/7GB. Larger runners (16-cores, etc.) require
|
||||
# a provisioned runner pool in repo settings. Falling back to default keeps
|
||||
# the matrix shard speedup (~5-6x via parallelism) at zero cost.
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4]
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: latest
|
||||
- run: bun install
|
||||
- name: Pre-test gates (shard 1 only — they're not test files)
|
||||
if: matrix.shard == 1
|
||||
run: scripts/check-jsonb-pattern.sh && scripts/check-progress-to-stdout.sh && scripts/check-wasm-embedded.sh && bun run typecheck
|
||||
- name: Run test shard ${{ matrix.shard }}/4
|
||||
run: scripts/test-shard.sh ${{ matrix.shard }} 4
|
||||
- run: bun run test
|
||||
|
||||
@@ -17,4 +17,3 @@ eval/data/world-v1/world.html
|
||||
|
||||
# BrainBench amara-life-v1 Opus cache (regenerate via eval:generate-amara-life)
|
||||
eval/data/amara-life-v1/_cache/
|
||||
.claude/
|
||||
|
||||
-323
@@ -2,329 +2,6 @@
|
||||
|
||||
All notable changes to GBrain will be documented in this file.
|
||||
|
||||
## [0.22.9] - 2026-04-29
|
||||
|
||||
**Sync failures now tell you why, not just how many.**
|
||||
**`gbrain sync --skip-failed` and `gbrain doctor` group failures by error code, so 2,685 silent SLUG_MISMATCH files don't hide behind a single count.**
|
||||
|
||||
Before this release, when sync hit per-file parse errors the only signal was a number:
|
||||
|
||||
```
|
||||
Sync blocked: 2688 file(s) failed to parse. Fix the YAML frontmatter...
|
||||
```
|
||||
|
||||
That count is useless when you're staring at 2,688 files and don't know what's wrong. On a real 81K-page brain, 2,685 of those turned out to be `SLUG_MISMATCH` from a posterous import — a single root cause hiding behind a giant number. It took manual `cat ~/.gbrain/sync-failures.jsonl | jq` to figure that out.
|
||||
|
||||
After:
|
||||
|
||||
```
|
||||
Sync blocked: 2688 file(s) failed to parse:
|
||||
SLUG_MISMATCH: 2685
|
||||
YAML_DUPLICATE_KEY: 3
|
||||
|
||||
Fix the YAML frontmatter in the files above and re-run, or use 'gbrain sync --skip-failed' to acknowledge and move on.
|
||||
|
||||
# gbrain sync --skip-failed
|
||||
Acknowledged 2688 failure(s) and advancing past them:
|
||||
SLUG_MISMATCH: 2685
|
||||
YAML_DUPLICATE_KEY: 3
|
||||
```
|
||||
|
||||
`gbrain doctor` shows the same breakdown for unacknowledged AND historical entries:
|
||||
|
||||
```
|
||||
[WARN] sync_failures: 2688 unacknowledged sync failure(s) [SLUG_MISMATCH=2685, YAML_DUPLICATE_KEY=3].
|
||||
[OK] sync_failures: 500544 historical sync failure(s), all acknowledged [SLUG_MISMATCH=2685, ...].
|
||||
```
|
||||
|
||||
The classifier knows the canonical messages from `collectValidationErrors()` in `src/core/markdown.ts` (8 frontmatter codes), Postgres unique-constraint violations (`DB_DUPLICATE_KEY`), statement-timeout errors (`STATEMENT_TIMEOUT`), invalid UTF-8, and YAML duplicates. DB-layer errors check before YAML-layer ones — so a Postgres `duplicate key value violates unique constraint` no longer mislabels as a YAML duplicate. Unrecognized errors fall through to `UNKNOWN`.
|
||||
|
||||
### What this means for you
|
||||
|
||||
If `gbrain sync` blocks with parse failures, the breakdown tells you what to fix first. SLUG_MISMATCH is one fix-pattern (frontmatter says one slug, path says another); YAML_PARSE is a different one (malformed YAML); STATEMENT_TIMEOUT means a DB timeout, not a parse problem. You stop staring at counts and start fixing root causes.
|
||||
|
||||
### For contributors
|
||||
|
||||
`acknowledgeSyncFailures()` in `src/core/sync.ts` now returns `{count, summary}` instead of `number`. If you import this directly from `gbrain/sync`, replace `n` with `result.count` and use `result.summary` (an `Array<{code, count}>`) for the new code-grouped breakdown. The function is reachable via the package exports map; this is a deliberate, non-shimmed breaking change. There is a new `formatCodeBreakdown()` helper in the same module that accepts either raw failures or pre-summarized input — use it instead of building breakdown strings inline.
|
||||
|
||||
### Itemized changes
|
||||
|
||||
#### Added
|
||||
|
||||
- `classifyErrorCode(errorMsg)` in `src/core/sync.ts` — best-effort error-code extraction from sync failure messages. Codes: `SLUG_MISMATCH`, `YAML_PARSE`, `YAML_DUPLICATE_KEY`, `MISSING_OPEN`, `MISSING_CLOSE`, `EMPTY_FRONTMATTER`, `NULL_BYTES`, `NESTED_QUOTES`, `DB_DUPLICATE_KEY`, `STATEMENT_TIMEOUT`, `INVALID_UTF8`, `UNKNOWN`.
|
||||
- `summarizeFailuresByCode(failures)` — groups failures by code and returns a sorted `Array<{code, count}>`.
|
||||
- `formatCodeBreakdown(input)` — renders a multi-line `code: count` string from either raw failures or a pre-computed summary. Single helper, two input shapes.
|
||||
- `code?: string` field on the `SyncFailure` JSONL row in `~/.gbrain/sync-failures.jsonl`. Populated at write-time so the classifier runs once per failure, not on every load.
|
||||
- `AcknowledgeResult` interface as the new return shape of `acknowledgeSyncFailures()`.
|
||||
- 15 new test cases in `test/sync-failures.test.ts`: DB-vs-YAML duplicate-key disambiguation, canonical-message coverage for all 7 frontmatter codes, `acknowledgeSyncFailures()` legacy-entry backfill branch, `formatCodeBreakdown()` dual-input shape.
|
||||
|
||||
#### Changed
|
||||
|
||||
- `gbrain sync` blocked-message: now lists code breakdown above the fix instructions (both incremental and full-sync paths).
|
||||
- `gbrain sync --skip-failed` ack message: now lists what was skipped, grouped by code.
|
||||
- `gbrain doctor` `sync_failures` check: warn-and-ok messages both include `[code=count, ...]` breakdown.
|
||||
- `recordSyncFailures()` now stores `code` alongside `error` so downstream readers don't re-classify.
|
||||
- `acknowledgeSyncFailures()` backfills `code` on legacy rows that predate the field — upgrade-safe for users with existing `~/.gbrain/sync-failures.jsonl`.
|
||||
- DB-layer error patterns (`DB_DUPLICATE_KEY`, `STATEMENT_TIMEOUT`) check BEFORE YAML patterns in the classifier, so Postgres errors don't get YAML-labeled.
|
||||
- Frontmatter regex patterns rewritten to match canonical messages from `collectValidationErrors()` (`File is empty...`, `No closing --- delimiter found`, `Frontmatter block is empty`) instead of aspirational code-token strings (`missing.*open`) that never appeared in practice.
|
||||
|
||||
Closes #500. Eng-review plan: `~/.claude/plans/then-codex-synchronous-toucan.md` (codex outside-voice agreed on all 7 findings).
|
||||
|
||||
## [0.22.8] - 2026-04-28
|
||||
|
||||
## **Doctor stops timing out on Supabase. Integrity scan finishes in ~6s, multi-source brains get correct counts.**
|
||||
|
||||
If you've been hitting the 60-second `gbrain doctor` timeout on Supabase or any pooled-connection deployment, this fixes it. The integrity check used to call `getPage()` 500 times sequentially through PgBouncer transaction-mode pooling. Each call required a full connection acquire/release cycle, which doctor couldn't finish before CI killed it. The new path batch-loads all 500 pages in a single SQL query, finishing in ~6s.
|
||||
|
||||
While shipping the perf fix, codex review caught a correctness regression for multi-source brains: the batch SQL was scanning raw `(source_id, slug)` rows while the sequential path scanned unique slugs. Multi-source brains were getting inflated counts. `SELECT DISTINCT ON (slug)` mirrors the sequential path's `Set<string>` semantics; parity tests against real Postgres pin both paths to the same output.
|
||||
|
||||
Plus a Linux CI fix: `gbrain skillpack` lockfile checks were intermittently failing on ext4's sub-millisecond `mtimeMs` timestamps when `Date.now()` returned an integer ms behind the file's recorded mtime. Lock age now clamps to zero.
|
||||
|
||||
### The numbers that matter
|
||||
|
||||
Measured against the real failure mode on a Supabase PgBouncer deployment that hit the 60s CI timeout pre-fix.
|
||||
|
||||
| Behavior | Before v0.22.8 | After v0.22.8 |
|
||||
|---|---|---|
|
||||
| `gbrain doctor` wall-clock (Postgres + PgBouncer) | 60s+ timeout (killed) | ~6s |
|
||||
| `integrity_sample` query round-trips | ~500 (sequential `getPage`) | 1 (`SELECT DISTINCT ON`) |
|
||||
| Multi-source brain scan accuracy | Overcounted by `source_id` | Exact per unique slug |
|
||||
|
||||
### What this means for Supabase deployments
|
||||
|
||||
If you've been avoiding `gbrain doctor` because it timed out, run it again. If you maintain a multi-source brain (imported pages from another gbrain deployment under a non-default `source_id`), the scan now treats each slug once instead of once-per-source — your output is exact, not inflated. Single-source users see no behavior change; PGLite users were never affected (the batch path is Postgres-only).
|
||||
|
||||
## To take advantage of v0.22.8
|
||||
|
||||
`gbrain upgrade` should do this automatically. If it didn't, or if `gbrain doctor` warns about anything afterwards:
|
||||
|
||||
1. **Run the upgrade:**
|
||||
```bash
|
||||
gbrain upgrade
|
||||
```
|
||||
2. **Verify doctor finishes cleanly (especially relevant if you hit timeouts before):**
|
||||
```bash
|
||||
gbrain doctor
|
||||
```
|
||||
On Postgres + PgBouncer deployments, you should see `integrity_sample` finish in ~6s instead of timing out at 60s.
|
||||
3. **If `doctor` still times out or output looks wrong,** please file an issue:
|
||||
https://github.com/garrytan/gbrain/issues with:
|
||||
- output of `gbrain doctor` (full)
|
||||
- which engine (Postgres vs PGLite)
|
||||
- whether you use multi-source brains
|
||||
|
||||
### Itemized changes
|
||||
|
||||
#### Performance
|
||||
- `gbrain doctor` integrity sample now batch-loads via a single SQL query on Postgres deployments (60s+ timeout → ~6s wall-clock, 500 round-trips → 1).
|
||||
- Batch path explicitly gated to Postgres via `engine.kind` so PGLite never attempts it (clean fallback signal).
|
||||
|
||||
#### Correctness
|
||||
- `scanIntegrity` batch path uses `SELECT DISTINCT ON (slug)` to scope by unique slug, matching `engine.getAllSlugs()`'s `Set<string>` semantics. Multi-source brains (UNIQUE(source_id, slug) since v0.18.0) now get correct counts instead of one-scan-per-source-row.
|
||||
- `IntegrityScanResult.pagesScanned` now reflects unique slugs scanned, not raw row count. Single-source brains: unchanged. Multi-source brains: counts now match expected distinct-page semantics.
|
||||
- Batch-path fallback narrowed: real Postgres errors (deadlock, connection drop, SQL bug) surface via `GBRAIN_DEBUG=1` instead of being silently swallowed.
|
||||
|
||||
#### Tests
|
||||
- New `test/e2e/integrity-batch.test.ts` — four parity cases (dedup, hits, validate, topPages) asserting batch ≡ sequential against real Postgres. Pinning the multi-source dedup case requires a raw-SQL fixture for the alt-source row since `engine.putPage` doesn't take a `source_id`.
|
||||
|
||||
#### Infrastructure
|
||||
- `src/core/skillpack/installer.ts` — clamp negative lock-age to 0, fixing intermittent Linux ext4 CI flakes from sub-millisecond `mtimeMs` precision (Date.now is integer ms; mtime can be ~0.3ms ahead). New regression test in `test/skillpack-install.test.ts` deterministically reproduces via `utimesSync`.
|
||||
- `CLAUDE.md` test inventory updated for the new test files.
|
||||
|
||||
## [0.22.7] - 2026-04-28
|
||||
|
||||
## **Built-in HTTP transport with bearer auth for remote MCP.**
|
||||
## **Postgres-backed tokens, default-deny CORS, two-bucket rate limit, body cap, per-request audit.**
|
||||
|
||||
v0.22.7 ships `gbrain serve --http`: a built-in HTTP transport for remote MCP, authenticating via the existing `access_tokens` table that `gbrain auth create/list/revoke` already manages. Bearer-only, no OAuth surface, no registration endpoint, no self-service tokens. SECURITY.md is the canonical reference for the hardening posture and recommended deployment.
|
||||
|
||||
The hardening lives inside the transport, not in the doc:
|
||||
|
||||
| Layer | Default | Configurable via |
|
||||
|---|---|---|
|
||||
| CORS | default-deny (no `Access-Control-Allow-Origin`) | `GBRAIN_HTTP_CORS_ORIGIN=a.com,b.com` |
|
||||
| Pre-auth IP rate limit | 30 req / 60s | `GBRAIN_HTTP_RATE_LIMIT_IP` |
|
||||
| Post-auth token rate limit | 60 req / 60s | `GBRAIN_HTTP_RATE_LIMIT_TOKEN` |
|
||||
| Body cap | 1 MiB, stream-counted | `GBRAIN_HTTP_MAX_BODY_BYTES` |
|
||||
| `last_used_at` debounce | once per token per 60s | (SQL-level WHERE clause, race-tolerant) |
|
||||
| Per-request audit | `mcp_request_log` row per `/mcp` | (existing schema, since v4) |
|
||||
| Reverse-proxy trust | off | `GBRAIN_HTTP_TRUST_PROXY=1` to honor X-Forwarded-For |
|
||||
|
||||
The IP rate-limit fires **before** the auth lookup so the limit caps load on the auth path itself, not just response codes. The token-id rate limit fires after auth so a runaway authenticated client gets throttled at the right principal. Both buckets live in a bounded LRU map (default 10K keys, TTL prune at 2× window) so unique-key growth can't drift into memory pressure.
|
||||
|
||||
### What changed for users
|
||||
|
||||
You can now expose GBrain remotely with the built-in transport:
|
||||
|
||||
```bash
|
||||
gbrain auth create my-laptop # tokens managed via the existing CLI
|
||||
gbrain serve --http --port 8787 # Postgres-only; PGLite users see a clear fail-fast
|
||||
ngrok http 8787 --url your-brain.ngrok.app # any tunnel works
|
||||
```
|
||||
|
||||
Then point Claude Desktop, claude.ai/code, or any MCP client at `http://your-tunnel/mcp` with `Authorization: Bearer <token>`. CORS, rate limits, and body caps are on by default. `gbrain auth` is now wired into the main CLI, so it works from the compiled binary the same as `gbrain doctor` or `gbrain serve`.
|
||||
|
||||
### For contributors
|
||||
|
||||
- `src/mcp/dispatch.ts` (new) — shared `dispatchToolCall(engine, name, params, opts)` consumed by both stdio (`server.ts`) and HTTP (`http-transport.ts`). One source of truth for `validateParams`, `OperationContext` construction, and handler invocation, so the two transports can't drift apart.
|
||||
- `src/mcp/rate-limit.ts` (new) — bounded-LRU token-bucket. Tracks `lastTouchedMs` separately from `lastRefillMs` so an exhausted key can't be reset by hammering past the TTL.
|
||||
- `src/mcp/http-transport.ts` — built on the new dispatch + rate-limit modules. `application/json` response shape (gbrain MCP tools are synchronous; the Streamable HTTP transport spec allows JSON for non-streaming responses).
|
||||
- `src/cli.ts` + `src/commands/auth.ts` — `auth` is now a wired CLI subcommand. Direct-script usage (`bun run src/commands/auth.ts ...`) still works for environments without a compiled binary.
|
||||
- 23 unit cases in `test/http-transport.test.ts`, 8 E2E cases in `test/e2e/http-transport.test.ts`. Unit covers the full dispatch round-trip with a real operation; E2E covers `last_used_at` debounce against real Postgres semantics.
|
||||
|
||||
### Known limits
|
||||
|
||||
- `gbrain serve --http` is **Postgres-only**. PGLite has no `access_tokens` or `mcp_request_log` table by design (`src/core/pglite-schema.ts:5-6`). Local agents continue to use stdio (`gbrain serve`).
|
||||
- Behind a tunnel (ngrok, Tailscale Funnel, Cloudflare Tunnel), all requests share one egress IP. The pre-auth IP bucket becomes effectively shared by all clients on that tunnel; the token-id bucket is the load-bearing limiter for tunnel deployments. Documented in SECURITY.md.
|
||||
|
||||
### Itemized changes
|
||||
|
||||
- New: `gbrain serve --http [--port N]` ships the built-in HTTP transport
|
||||
- New: `gbrain auth create/list/revoke/test` wired into the main CLI (was a standalone script)
|
||||
- New: SECURITY.md documents the disclosure path, the recommended remote-MCP setup, and the full hardening reference
|
||||
- New: `src/mcp/dispatch.ts` — shared dispatch path for stdio + HTTP
|
||||
- New: `src/mcp/rate-limit.ts` — bounded-LRU token-bucket limiter
|
||||
- Hardening: CORS default-deny, two-bucket rate limit (per-IP pre-auth + per-token post-auth), 1 MiB body cap with stream-counted enforcement, `mcp_request_log` per-request audit, `last_used_at` SQL-level debounce
|
||||
- Tests: 23 unit + 8 E2E covering auth, dispatch, CORS, body cap, rate limit, and audit
|
||||
- Docs: SECURITY.md, DEPLOY.md, and per-client setup guides updated to recommend `--http` and document the env vars
|
||||
|
||||
## To take advantage of v0.22.7
|
||||
|
||||
`gbrain upgrade` should do this automatically. If it didn't, or if you want to expose your brain over HTTP:
|
||||
|
||||
1. **Confirm migrations are at v4 or higher** (the `access_tokens` + `mcp_request_log` tables were added in migration v4):
|
||||
```bash
|
||||
gbrain doctor # schema_version check should pass
|
||||
gbrain apply-migrations --yes # if not, run this
|
||||
```
|
||||
2. **Create a token for each remote client:**
|
||||
```bash
|
||||
gbrain auth create my-laptop # prints the token once — copy it
|
||||
```
|
||||
3. **Start the HTTP server:**
|
||||
```bash
|
||||
gbrain serve --http --port 8787
|
||||
```
|
||||
4. **(Optional) configure CORS allowlist if a browser client will hit it:**
|
||||
```bash
|
||||
GBRAIN_HTTP_CORS_ORIGIN=https://claude.ai gbrain serve --http --port 8787
|
||||
```
|
||||
5. **(Optional) audit who's hitting your brain:**
|
||||
```bash
|
||||
psql $DATABASE_URL -c "SELECT created_at, token_name, operation, status, latency_ms
|
||||
FROM mcp_request_log ORDER BY created_at DESC LIMIT 50"
|
||||
```
|
||||
6. **If `gbrain serve --http` exits with "Postgres engine required":** PGLite is local-only by design. Either keep using stdio (`gbrain serve`) for local agents, or migrate to Postgres (`gbrain migrate --to supabase`).
|
||||
|
||||
If anything breaks: `gbrain doctor`, `~/.gbrain/upgrade-errors.jsonl` (if present), and please file an issue at https://github.com/garrytan/gbrain/issues with both.
|
||||
|
||||
|
||||
|
||||
## [0.22.6.1] - 2026-04-26
|
||||
|
||||
**Old brains can upgrade again.**
|
||||
**Two-year, ten-issue wedge cycle ends. Pre-v0.13/v0.18/v0.19 brains all upgrade clean.**
|
||||
|
||||
If you've been pinned to an older gbrain because `gbrain upgrade` wedges your brain
|
||||
with `column "source_id" does not exist` or `column "link_source" does not exist`,
|
||||
v0.22.6.1 unblocks you. The fix lives in `initSchema()` itself, where it should
|
||||
have lived all along.
|
||||
|
||||
The bug class is structural: gbrain ships an "embedded latest schema" SQL blob
|
||||
that runs before numbered migrations on every connect. The blob references
|
||||
columns that newer migrations introduce. On any brain older than the migration
|
||||
that adds those columns, the blob crashes before the migration can run. This
|
||||
incident family hit users 10+ times across 6 schema versions over 2 years
|
||||
(issues #239, #243, #266, #357, #366, #374, #375, #378, #395, #396).
|
||||
|
||||
The fix is a narrow pre-schema bootstrap. `initSchema()` now probes for the
|
||||
specific forward-referenced state the schema blob needs (`pages.source_id`,
|
||||
`links.link_source`, `links.origin_page_id`, `content_chunks.symbol_name`,
|
||||
`content_chunks.language`, plus the `sources` FK target table) and adds only
|
||||
that state if missing. Then SCHEMA_SQL replays cleanly. Then the normal
|
||||
migration chain runs as usual. Fresh installs and modern brains both no-op.
|
||||
|
||||
A test guard prevents this incident family from recurring. Every future
|
||||
migration that adds a column-with-index to PGLITE_SCHEMA_SQL must extend the
|
||||
bootstrap; the CI guard fails loudly if not. The pattern that broke gbrain ten
|
||||
times in two years is now structurally prevented.
|
||||
|
||||
Also includes the v24 PGLite RLS fix from #395 (community PR by @jdcastro2):
|
||||
`rls_backfill_missing_tables` now no-ops on PGLite via `sqlFor.pglite: ''`,
|
||||
since PGLite has no RLS engine and is single-tenant by definition.
|
||||
|
||||
### The numbers that matter
|
||||
|
||||
| Metric | v0.22.0 | v0.22.6.1 | Δ |
|
||||
|---|---|---|---|
|
||||
| Pre-v0.13 brain upgrades cleanly | wedges on `link_source` | passes | ✓ |
|
||||
| Pre-v0.18 brain upgrades cleanly | wedges on `source_id` | passes | ✓ |
|
||||
| Pre-v0.21 brain upgrades cleanly | wedges on `symbol_name` | passes | ✓ |
|
||||
| v24 RLS migration on PGLite | wedges (table doesn't exist) | no-op | ✓ |
|
||||
| Issues closed | — | #366, #375, #378, #395, #396 | 5 |
|
||||
| Issue families resolved | — | wedge-cycle | the whole class |
|
||||
|
||||
### What this means for you
|
||||
|
||||
If you've been on v0.13.x, v0.14.x, v0.17.x, v0.18.x, v0.19.x, v0.20.x, or v0.22.0 and
|
||||
your `gbrain upgrade` failed, run it again. It should walk to v0.22.6.1 cleanly.
|
||||
If you wedged on the v24 RLS migration on a PGLite brain, the same thing.
|
||||
|
||||
If you're on a fresh install or already on v0.22.0, this patch is invisible.
|
||||
The bootstrap probe runs once per connect, sees nothing to do, and returns.
|
||||
|
||||
### Itemized changes
|
||||
|
||||
#### Fixed
|
||||
- `gbrain upgrade` no longer wedges on pre-v0.18 brains that lack `pages.source_id`. The schema blob's `CREATE INDEX idx_pages_source_id` previously crashed before migration v21 could add the column. Closes #366, #375, #378, #396.
|
||||
- `gbrain upgrade` no longer wedges on pre-v0.13 brains that lack `links.link_source` or `links.origin_page_id`. The schema blob's `CREATE INDEX idx_links_source/origin` previously crashed before migration v11 could add the columns. Closes #266, #357.
|
||||
- `gbrain upgrade` no longer wedges on pre-v0.19 brains that lack `content_chunks.symbol_name` or `content_chunks.language`. The schema blob's partial indexes previously crashed before migration v26 could add the columns.
|
||||
- Migration v24 (`rls_backfill_missing_tables`) no-ops on PGLite via `sqlFor.pglite: ''`. PGLite has no RLS engine and is single-tenant. The migration previously tried to ALTER subagent tables that don't exist in pglite-schema.ts. Closes #395. Contributed by @jdcastro2.
|
||||
|
||||
#### Changed
|
||||
- `PGLiteEngine.initSchema()` and `PostgresEngine.initSchema()` now call a new private `applyForwardReferenceBootstrap()` before running the embedded schema blob. The bootstrap probes for missing forward-referenced state and adds only what's needed. No-op on fresh installs and modern brains.
|
||||
|
||||
#### For contributors
|
||||
- New CI guard `test/schema-bootstrap-coverage.test.ts` enforces that `applyForwardReferenceBootstrap` covers every forward reference in PGLITE_SCHEMA_SQL. When you add a new column-with-index in the schema blob, extend `REQUIRED_BOOTSTRAP_COVERAGE` and the bootstrap function. The test fails loudly if you skip step one.
|
||||
- New `test/bootstrap.test.ts` covers the bootstrap contract: no-op on fresh install, idempotent, no-op on modern brain, full path pre-v0.18, fresh-install regression, pre-v0.13 links shape.
|
||||
- New `test/e2e/postgres-bootstrap.test.ts` exercises `PostgresEngine.initSchema()` directly (not the standalone `db.initSchema` from `src/core/db.ts`, which only runs SCHEMA_SQL and would have produced false-positive coverage). Codex caught this E2E shape gap during plan review.
|
||||
- Wave PRs incorporated with attribution: @vinsew (#398), @jdcastro2 (#399), @schnubb-web (#402). The narrow-bootstrap shape supersedes #402's broader "run all migrations early" approach, which would have crashed on v24 trying to alter tables that the schema blob hadn't created yet (codex finding during plan review).
|
||||
|
||||
## To take advantage of v0.22.6.1
|
||||
|
||||
`gbrain upgrade` should do this automatically. If you're currently wedged on a
|
||||
prior version's upgrade attempt:
|
||||
|
||||
1. **Run the upgrade:**
|
||||
```bash
|
||||
gbrain upgrade
|
||||
```
|
||||
2. **Verify the outcome:**
|
||||
```bash
|
||||
gbrain doctor
|
||||
```
|
||||
Expected: `schema_version: Version 29 (latest: 29)` clean, no
|
||||
`column "..." does not exist` errors, no wedged migration ledger.
|
||||
|
||||
3. **If wedged after upgrade,** run the migration runner directly:
|
||||
```bash
|
||||
gbrain apply-migrations --yes
|
||||
```
|
||||
|
||||
4. **If any step still fails,** please file an issue:
|
||||
https://github.com/garrytan/gbrain/issues with:
|
||||
- output of `gbrain doctor`
|
||||
- your prior gbrain version (`gbrain --version`)
|
||||
- which step broke
|
||||
|
||||
## [0.22.6] - 2026-04-28
|
||||
|
||||
### Schema verification after migrations
|
||||
|
||||
- Post-migration schema verification catches columns that were defined in migrations but silently failed to create (common with PgBouncer transaction-mode poolers).
|
||||
- Self-healing: automatically adds missing columns via ALTER TABLE when detected.
|
||||
- Prevents the "column X does not exist" embed failures that occur when schema version is ahead of actual table state.
|
||||
|
||||
## [0.22.5] - 2026-04-27
|
||||
|
||||
## **Autopilot stops re-importing your whole brain when a commit gets garbage-collected.**
|
||||
|
||||
@@ -25,9 +25,9 @@ strict behavior when unset.
|
||||
- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Also exports upload validators: `validateUploadPath`, `validatePageSlug`, `validateFilename`. `OperationContext.remote` flags untrusted callers.
|
||||
- `src/core/engine.ts` — Pluggable engine interface (BrainEngine). `clampSearchLimit(limit, default, cap)` takes an explicit cap so per-operation caps can be tighter than `MAX_SEARCH_LIMIT`. Exports `LinkBatchInput` / `TimelineBatchInput` for the v0.12.1 bulk-insert API (`addLinksBatch` / `addTimelineEntriesBatch`). As of v0.13.1, `BrainEngine` has a `readonly kind: 'postgres' | 'pglite'` discriminator so migrations (`src/core/migrate.ts`) and other consumers can branch on engine without `instanceof` + dynamic imports.
|
||||
- `src/core/engine-factory.ts` — Engine factory with dynamic imports (`'pglite'` | `'postgres'`)
|
||||
- `src/core/pglite-engine.ts` — PGLite (embedded Postgres 17.5 via WASM) implementation, all 40 BrainEngine methods. `addLinksBatch` / `addTimelineEntriesBatch` use multi-row `unnest()` with manual `$N` placeholders. As of v0.13.1, `connect()` wraps `PGlite.create()` in a try/catch that emits an actionable error naming the macOS 26.3 WASM bug (#223) and pointing at `gbrain doctor`; the lock is released on failure so the next process can retry cleanly. v0.22.0: `searchKeyword` and `searchKeywordChunks` multiply `ts_rank` by the source-factor CASE expression at the chunk-grain level; `searchVector` becomes a two-stage CTE — inner CTE keeps `ORDER BY cc.embedding <=> vec` so HNSW stays usable, outer SELECT re-ranks by `raw_score * source_factor`. Inner LIMIT scales with offset to preserve pagination contract. As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL — probes for the specific forward-referenced state the embedded schema blob needs (`pages.source_id`, `links.link_source`, `links.origin_page_id`, `content_chunks.symbol_name`, `content_chunks.language`, `sources` FK target table) and adds only what's missing. Closes the upgrade-wedge bug class that bit users 10+ times across 6 schema versions over 2 years (#239/#243/#266/#357/#366/#374/#375/#378/#395/#396). No-op on fresh installs and modern brains.
|
||||
- `src/core/pglite-engine.ts` — PGLite (embedded Postgres 17.5 via WASM) implementation, all 40 BrainEngine methods. `addLinksBatch` / `addTimelineEntriesBatch` use multi-row `unnest()` with manual `$N` placeholders. As of v0.13.1, `connect()` wraps `PGlite.create()` in a try/catch that emits an actionable error naming the macOS 26.3 WASM bug (#223) and pointing at `gbrain doctor`; the lock is released on failure so the next process can retry cleanly. v0.22.0: `searchKeyword` and `searchKeywordChunks` multiply `ts_rank` by the source-factor CASE expression at the chunk-grain level; `searchVector` becomes a two-stage CTE — inner CTE keeps `ORDER BY cc.embedding <=> vec` so HNSW stays usable, outer SELECT re-ranks by `raw_score * source_factor`. Inner LIMIT scales with offset to preserve pagination contract.
|
||||
- `src/core/pglite-schema.ts` — PGLite-specific DDL (pgvector, pg_trgm, triggers)
|
||||
- `src/core/postgres-engine.ts` — Postgres + pgvector implementation (Supabase / self-hosted). `addLinksBatch` / `addTimelineEntriesBatch` use `INSERT ... SELECT FROM unnest($1::text[], ...) JOIN pages ON CONFLICT DO NOTHING RETURNING 1` — 4-5 array params regardless of batch size, sidesteps the 65535-parameter cap. As of v0.12.3, `searchKeyword` / `searchVector` scope `statement_timeout` via `sql.begin` + `SET LOCAL` so the GUC dies with the transaction instead of leaking across the pooled postgres.js connection (contributed by @garagon). `getEmbeddingsByChunkIds` uses `tryParseEmbedding` so one corrupt row skips+warns instead of killing the query. v0.22.0: `searchKeyword`, `searchKeywordChunks`, and `searchVector` apply source-aware ranking by inlining the source-factor CASE and `NOT (col LIKE …)` hard-exclude clause from `src/core/search/sql-ranking.ts`. `searchVector` switches to a two-stage CTE (HNSW-safe inner ORDER BY, source-boost re-rank in the outer SELECT) and carries `p.source_id` through inner→outer for v0.18 multi-source callers. v0.22.1 (#406): `_savedConfig` retains the connect config; `reconnect()` tears down + recreates the pool from saved config (called by supervisor watchdog after 3 consecutive health-check failures). `executeRaw` is a single-statement passthrough — no per-call retry (D3 dropped that as unsound for non-idempotent statements; recovery is supervisor-driven). v0.22.1 (#363, contributed by @orendi84): `connect()` applies `resolveSessionTimeouts()` from `db.ts` as connection-time startup parameters (`statement_timeout`, `idle_in_transaction_session_timeout`) so orphan pgbouncer backends can't hold locks for hours. v0.22.1 (#409, contributed by @atrevino47): `countStaleChunks()` + `listStaleChunks()` server-side-filter on `embedding IS NULL` for `embed --stale`, eliminating ~76 MB/call client-side pull on a fully-embedded brain; `upsertChunks()` resets both `embedding` AND `embedded_at` to NULL when chunk_text changes without a new embedding (consistency). As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL on the same forward-reference probe set as the PGLite engine, so old Postgres brains pinned at v0.13/v0.18/v0.19 walk forward cleanly instead of wedging on `column "..." does not exist`.
|
||||
- `src/core/postgres-engine.ts` — Postgres + pgvector implementation (Supabase / self-hosted). `addLinksBatch` / `addTimelineEntriesBatch` use `INSERT ... SELECT FROM unnest($1::text[], ...) JOIN pages ON CONFLICT DO NOTHING RETURNING 1` — 4-5 array params regardless of batch size, sidesteps the 65535-parameter cap. As of v0.12.3, `searchKeyword` / `searchVector` scope `statement_timeout` via `sql.begin` + `SET LOCAL` so the GUC dies with the transaction instead of leaking across the pooled postgres.js connection (contributed by @garagon). `getEmbeddingsByChunkIds` uses `tryParseEmbedding` so one corrupt row skips+warns instead of killing the query. v0.22.0: `searchKeyword`, `searchKeywordChunks`, and `searchVector` apply source-aware ranking by inlining the source-factor CASE and `NOT (col LIKE …)` hard-exclude clause from `src/core/search/sql-ranking.ts`. `searchVector` switches to a two-stage CTE (HNSW-safe inner ORDER BY, source-boost re-rank in the outer SELECT) and carries `p.source_id` through inner→outer for v0.18 multi-source callers. v0.22.1 (#406): `_savedConfig` retains the connect config; `reconnect()` tears down + recreates the pool from saved config (called by supervisor watchdog after 3 consecutive health-check failures). `executeRaw` is a single-statement passthrough — no per-call retry (D3 dropped that as unsound for non-idempotent statements; recovery is supervisor-driven). v0.22.1 (#363, contributed by @orendi84): `connect()` applies `resolveSessionTimeouts()` from `db.ts` as connection-time startup parameters (`statement_timeout`, `idle_in_transaction_session_timeout`) so orphan pgbouncer backends can't hold locks for hours. v0.22.1 (#409, contributed by @atrevino47): `countStaleChunks()` + `listStaleChunks()` server-side-filter on `embedding IS NULL` for `embed --stale`, eliminating ~76 MB/call client-side pull on a fully-embedded brain; `upsertChunks()` resets both `embedding` AND `embedded_at` to NULL when chunk_text changes without a new embedding (consistency).
|
||||
- `src/core/utils.ts` — Shared SQL utilities extracted from postgres-engine.ts. Exports `parseEmbedding(value)` (throws on unknown input, used by migration + ingest paths where data integrity matters) and as of v0.12.3 `tryParseEmbedding(value)` (returns `null` + warns once per process, used by search/rescore paths where availability matters more than strictness).
|
||||
- `src/core/db.ts` — Connection management, schema initialization. v0.22.1 (#363, contributed by @orendi84): `resolveSessionTimeouts()` returns `statement_timeout` + `idle_in_transaction_session_timeout` (defaults: 5min each, env-overridable via `GBRAIN_STATEMENT_TIMEOUT` / `GBRAIN_IDLE_TX_TIMEOUT` / `GBRAIN_CLIENT_CHECK_INTERVAL`). Both `connect()` (module singleton) and `PostgresEngine.connect()` (worker pool) consume the result via postgres.js's `connection` option, sending GUCs as startup parameters that survive PgBouncer transaction mode (unlike the prior `setSessionDefaults` post-pool SET, kept as a back-compat no-op shim).
|
||||
- `src/commands/migrate-engine.ts` — Bidirectional engine migration (`gbrain migrate --to supabase/pglite`)
|
||||
@@ -91,18 +91,14 @@ strict behavior when unset.
|
||||
- `src/commands/jobs.ts` — `gbrain jobs` CLI subcommands + `gbrain jobs work` daemon. v0.13.1 surfaces the full `MinionJobInput` retry/backoff/timeout/idempotency surface as first-class CLI flags on `jobs submit`: `--max-stalled`, `--backoff-type fixed|exponential`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key`. `jobs smoke --sigkill-rescue` is the opt-in regression guard for #219. v0.16 wires `registerBuiltinHandlers` to always register `subagent` + `subagent_aggregator` (no env flag — `ANTHROPIC_API_KEY` is the natural cost gate, trust is via `PROTECTED_JOB_NAMES`) and loads `GBRAIN_PLUGIN_PATH` plugins at worker startup with a loud startup-line per plugin. `shell` handler still gated by `GBRAIN_ALLOW_SHELL_JOBS=1` (RCE surface, separate concern).
|
||||
- `src/commands/features.ts` — `gbrain features --json --auto-fix`: usage scan + feature adoption salesman
|
||||
- `src/commands/autopilot.ts` — `gbrain autopilot --install`: self-maintaining brain daemon (sync+extract+embed)
|
||||
- `src/mcp/server.ts` — MCP stdio server (generated from operations). v0.22.7: tool-call handler delegates to `dispatchToolCall` from `src/mcp/dispatch.ts` so stdio + HTTP transports share one validation, context-build, and error-format path.
|
||||
- `src/mcp/dispatch.ts` (v0.22.7) — Shared tool-call dispatch consumed by both stdio (`server.ts`) and HTTP (`http-transport.ts`). Exports `dispatchToolCall(engine, name, params, opts)`, `buildOperationContext(engine, params, opts)`, and `validateParams(op, params)`. Single source of truth for `(ctx, params)` handler arg order and the 5-field `OperationContext` shape (engine + config + logger + dryRun + remote). Defaults to `remote: true` (untrusted); local CLI callers pass `remote: false`. Closed F1 (reversed handler args) + F2 (incomplete OperationContext) + F3 (no param validation) drift bugs in the original v0.22.5 HTTP transport.
|
||||
- `src/mcp/rate-limit.ts` (v0.22.7) — Bounded-LRU token-bucket limiter for `gbrain serve --http`. `buildDefaultLimiters()` returns the two-bucket pipeline used by http-transport: pre-auth IP (default 30/60s, fires BEFORE the DB lookup so brute-force load against `access_tokens` is actually capped) + post-auth token-id (default 60/60s). Tracks `lastTouchedMs` separately from `lastRefillMs` so an exhausted key can't be reset by hammering past the TTL. LRU cap (default 10K keys) bounds memory under attacker-controlled key growth; TTL prune at 2× window evicts abandoned buckets.
|
||||
- `src/mcp/http-transport.ts` (v0.22.7, rewrite) — `gbrain serve --http` HTTP transport. Postgres-only — fails fast at startup on PGLite (the `access_tokens` table only exists on Postgres). Bearer auth against SHA-256 hashes in `access_tokens`. CORS default-deny via `GBRAIN_HTTP_CORS_ORIGIN` allowlist. Body cap stream-counted (1 MiB default via `GBRAIN_HTTP_MAX_BODY_BYTES`) so chunked transfers without Content-Length still hit the cap. `last_used_at` SQL-level debounce (one UPDATE per token per 60s). Per-request audit row in `mcp_request_log` with token_name + operation + status + latency. Optional `GBRAIN_HTTP_TRUST_PROXY=1` honors `X-Forwarded-For` — only safe when bound to a private interface AND the proxy strips client-supplied XFF (otherwise enables IP spoofing past the pre-auth rate limit). `/health` does `SELECT 1` against Postgres and returns 503 + `status:unhealthy` when the DB is unreachable so orchestration doesn't see green pods while clients get misleading 401s. Replaces the standalone OAuth wrapper that was vulnerable to unauthenticated client registration.
|
||||
- `src/commands/auth.ts` — Token management for the HTTP transport. `gbrain auth create/list/revoke/test`. As of v0.22.7 wired into the main CLI (`src/cli.ts`); also runs standalone via `bun run src/commands/auth.ts ...` for environments without a compiled binary. Tokens stored as SHA-256 hashes in `access_tokens` (Postgres-only).
|
||||
- `src/mcp/server.ts` — MCP stdio server (generated from operations)
|
||||
- `src/commands/auth.ts` — Standalone token management (create/list/revoke/test)
|
||||
- `src/commands/upgrade.ts` — Self-update CLI. `runPostUpgrade()` enumerates migrations from the TS registry (src/commands/migrations/index.ts) and tail-calls `runApplyMigrations(['--yes', '--non-interactive'])` so the mechanical side of every outstanding migration runs unconditionally.
|
||||
- `src/commands/migrations/` — TS migration registry (compiled into the binary; no filesystem walk of `skills/migrations/*.md` needed at runtime). `index.ts` lists migrations in semver order. `v0_11_0.ts` = Minions adoption orchestrator (8 phases). `v0_12_0.ts` = Knowledge Graph auto-wire orchestrator (5 phases: schema → config check → backfill links → backfill timeline → verify). `phaseASchema` has a 600s timeout (bumped from 60s in v0.12.1 for duplicate-heavy brains). `v0_12_2.ts` = JSONB double-encode repair orchestrator (4 phases: schema → repair-jsonb → verify → record). `v0_14_0.ts` = shell-jobs + autopilot cooperative (2 phases: schema ALTER minion_jobs.max_stalled SET DEFAULT 3 — superseded by v0.14.3's schema-level DEFAULT 5 + UPDATE backfill; pending-host-work ping for skills/migrations/v0.14.0.md). All orchestrators are idempotent and resumable from `partial` status. As of v0.14.2 (Bug 3), the RUNNER owns all ledger writes — orchestrators return `OrchestratorResult` and `apply-migrations.ts` persists a canonical `{version, status, phases}` shape after return. Orchestrators no longer call `appendCompletedMigration` directly. `statusForVersion` prefers `complete` over `partial` (never regresses). 3 consecutive partials → wedged → `--force-retry <version>` writes a `'retry'` reset marker. v0.14.3 (fix wave) ships schema-only migrations v14 (`pages_updated_at_index`) + v15 (`minion_jobs_max_stalled_default_5` with UPDATE backfill) via the `MIGRATIONS` array in `src/core/migrate.ts` — no orchestrator phases needed.
|
||||
- `src/commands/repair-jsonb.ts` — `gbrain repair-jsonb [--dry-run] [--json]`: rewrites `jsonb_typeof='string'` rows in place across 5 affected columns (pages.frontmatter, raw_data.data, ingest_log.pages_updated, files.metadata, page_versions.frontmatter). Fixes v0.12.0 double-encode bug on Postgres; PGLite no-ops. Idempotent.
|
||||
- `src/commands/orphans.ts` — `gbrain orphans [--json] [--count] [--include-pseudo]`: surfaces pages with zero inbound wikilinks, grouped by domain. Auto-generated/raw/pseudo pages filtered by default. Also exposed as `find_orphans` MCP operation. Shipped in v0.12.3 (contributed by @knee5).
|
||||
- `src/commands/integrity.ts` — `gbrain integrity check|auto|review|extract`: bare-tweet detection, dead-link detection, three-bucket repair (auto-repair / review-queue / skip). `scanIntegrity()` is the shared library function called from `gbrain doctor` (sampled at limit=500) and `cmdCheck` (full scan). v0.22.8: batch-load fast path on Postgres uses `SELECT DISTINCT ON (slug)` in a single SQL query to fix the PgBouncer round-trip timeout (60s → ~6s) while preserving `engine.getAllSlugs()`'s `Set<string>` semantics on multi-source brains. Gated by `engine.kind === 'postgres'` at the call site so PGLite never enters batch; fallback `catch` logs at `GBRAIN_DEBUG=1` so real Postgres errors are diagnosable.
|
||||
- `src/commands/doctor.ts` — `gbrain doctor [--json] [--fast] [--fix] [--dry-run] [--index-audit]`: health checks. v0.12.3 added `jsonb_integrity` + `markdown_body_completeness` reliability checks. v0.14.1: `--fix` delegates inlined cross-cutting rules to `> **Convention:** see [path](path).` callouts (pipes DRY violations into `src/core/dry-fix.ts`); `--fix --dry-run` previews without writing. v0.14.2: `schema_version` check fails loudly when `version=0` (migrations never ran — the #218 `bun install -g` signature) and routes users to `gbrain apply-migrations --yes`; new opt-in `--index-audit` flag (Postgres-only) reports zero-scan indexes from `pg_stat_user_indexes` (informational only, no auto-drop). v0.15.2: every DB check is wrapped in a progress phase; `markdown_body_completeness` runs under a 1s heartbeat timer so 10+ min scans are observable on 50K-page brains. v0.19.1 added `queue_health` (Postgres-only) with two subchecks: stalled-forever active jobs (started_at > 1h) and waiting-depth-per-name > threshold (default 10, override via `GBRAIN_QUEUE_WAITING_THRESHOLD`). Worker-heartbeat subcheck intentionally deferred to follow-up B7 because it needs a `minion_workers` table to produce ground-truth signal. Fix hints point at `gbrain repair-jsonb`, `gbrain sync --force`, `gbrain apply-migrations`, and `gbrain jobs get/cancel <id>`.
|
||||
- `src/core/migrate.ts` — schema-migration runner. Owns the `MIGRATIONS` array (source of truth for schema DDL). v0.14.2 extended the `Migration` interface with `sqlFor?: { postgres?, pglite? }` (engine-specific SQL overrides `sql`) and `transaction?: boolean` (set to false for `CREATE INDEX CONCURRENTLY`, which Postgres refuses inside a transaction; ignored on PGLite since it has no concurrent writers). Migration v14 (fix wave) uses a handler branching on `engine.kind` to run CONCURRENTLY on Postgres (with a pre-drop of any invalid remnant via `pg_index.indisvalid`) and plain `CREATE INDEX` on PGLite. v15 bumps `minion_jobs.max_stalled` default 1→5 and backfills existing non-terminal rows. v0.22.6.1: migration v24 (`rls_backfill_missing_tables`) uses `sqlFor: { pglite: '' }` to no-op on PGLite — PGLite has no RLS engine and is single-tenant by definition, and the v24 ALTERs target subagent tables that don't exist in pglite-schema.ts. Closes #395 (contributed by @jdcastro2).
|
||||
- `src/core/migrate.ts` — schema-migration runner. Owns the `MIGRATIONS` array (source of truth for schema DDL). v0.14.2 extended the `Migration` interface with `sqlFor?: { postgres?, pglite? }` (engine-specific SQL overrides `sql`) and `transaction?: boolean` (set to false for `CREATE INDEX CONCURRENTLY`, which Postgres refuses inside a transaction; ignored on PGLite since it has no concurrent writers). Migration v14 (fix wave) uses a handler branching on `engine.kind` to run CONCURRENTLY on Postgres (with a pre-drop of any invalid remnant via `pg_index.indisvalid`) and plain `CREATE INDEX` on PGLite. v15 bumps `minion_jobs.max_stalled` default 1→5 and backfills existing non-terminal rows.
|
||||
- `src/core/progress.ts` — Shared bulk-action progress reporter. Writes to stderr. Modes: `auto` (TTY: `\r`-rewriting; non-TTY: plain lines), `human`, `json` (JSONL), `quiet`. Rate-gated by `minIntervalMs` and `minItems`. `startHeartbeat(reporter, note)` helper for single long queries. `child()` composes phase paths. Singleton SIGINT/SIGTERM coordinator emits `abort` events for every live phase. EPIPE defense on both sync throws and stream `'error'` events. Zero dependencies. Introduced in v0.15.2.
|
||||
- `src/core/cli-options.ts` — Global CLI flag parser. `parseGlobalFlags(argv)` returns `{cliOpts, rest}` with `--quiet` / `--progress-json` / `--progress-interval=<ms>` stripped. `getCliOptions()` / `setCliOptions()` expose a module-level singleton so commands reach the resolved flags without parameter threading. `cliOptsToProgressOptions()` maps to reporter options. `childGlobalFlags()` returns the flag suffix to append to `execSync('gbrain ...')` calls in migration orchestrators. `OperationContext.cliOpts` extends shared-op dispatch for MCP callers.
|
||||
- `src/core/cycle.ts` — v0.17 brain maintenance cycle primitive. `runCycle(engine: BrainEngine | null, opts: CycleOpts): Promise<CycleReport>` composes 6 phases in semantically-driven order (lint → backlinks → sync → extract → embed → orphans). Three callers: `gbrain dream` CLI, `gbrain autopilot` daemon's inline path, and the Minions `autopilot-cycle` handler (`src/commands/jobs.ts`). One source of truth for what the brain does overnight. Coordination via `gbrain_cycle_locks` DB table (TTL-based; works through PgBouncer transaction pooling, unlike session-scoped `pg_try_advisory_lock`) + `~/.gbrain/cycle.lock` file lock with PID-liveness for PGLite / engine=null mode. `CycleReport.schema_version: "1"` is the stable agent-consumable shape. `PhaseResult.error: { class, code, message, hint?, docs_url? }` is Stripe-API-tier structured failure info. `yieldBetweenPhases` hook awaited between every phase — Minions handler uses this to renew its job lock and prevent v0.14 stall-death regression. Engine nullable: filesystem phases (lint, backlinks) run without DB; DB phases skip with `status: "skipped", reason: "no_database"`. Lock-skip: read-only phase selections (`--phase orphans`) bypass the cycle lock. v0.22.1 (#403): `CycleOpts.signal?: AbortSignal` propagates the worker's abort signal; `checkAborted()` fires between every phase and throws if the signal is aborted (cooperative — can't interrupt a phase mid-execution). v0.22.1 (#417): `runPhaseSync` returns `pagesAffected` via `SyncPhaseResult`; `runCycle` captures it and threads to `runPhaseExtract` as the 4th arg, enabling incremental extract on the cycle path. v0.22.1 (Codex F2): `runPhaseSync` takes `willRunExtractPhase: boolean` and sets `noExtract: phases.includes('extract')` so `gbrain dream --phase sync` doesn't silently lose extraction. v0.22.5 (#475): new `resolveSourceForDir(engine, brainDir)` helper queries `SELECT id FROM sources WHERE local_path = $1 LIMIT 1`; `runPhaseSync` threads result as `sourceId` to `performSync()` so sync reads the per-source `sources.last_commit` anchor instead of the drift-prone global `config.sync.last_commit` key. Bare try/catch lets pre-v0.18 brains fall through to the global key. Closes the prod hang where every autopilot cycle ran a 30-min full reimport because the global anchor commit had been GC'd from git history.
|
||||
@@ -227,9 +223,7 @@ parity), `test/cli.test.ts` (CLI structure), `test/config.test.ts` (config redac
|
||||
`test/files.test.ts` (MIME/hash), `test/import-file.test.ts` (import pipeline),
|
||||
`test/upgrade.test.ts` (schema migrations),
|
||||
`test/file-migration.test.ts` (file migration), `test/file-resolver.test.ts` (file resolution),
|
||||
`test/import-resume.test.ts` (import checkpoints), `test/migrate.test.ts` (migration; v8/v9 helper-btree-index SQL structural assertions + 1000-row wall-clock fixtures that guard the O(n²)→O(n log n) fix + v0.13.1 assertions on v12/v13 SQL shape, `sqlFor` + `transaction:false` runner semantics, the `max_stalled DEFAULT 1` regression guard, and v0.22.6.1 v24 `sqlFor.pglite: ''` no-op assertion),
|
||||
`test/bootstrap.test.ts` (v0.22.6.1 — bootstrap contract: no-op on fresh install, idempotent across two `initSchema()` calls, no-op on modern brain that already has every probed column, full bootstrap path on simulated pre-v0.18 brain, fresh-install regression guard, pre-v0.13 `links` shape coverage),
|
||||
`test/schema-bootstrap-coverage.test.ts` (v0.22.6.1 CI guard — `REQUIRED_BOOTSTRAP_COVERAGE` lists every forward reference in PGLITE_SCHEMA_SQL; the test fails loudly if `applyForwardReferenceBootstrap` skips one. When you add a column-with-index to the embedded schema blob, you extend both arrays or this guard fails. The pattern that broke gbrain ten times in two years is now structurally prevented.),
|
||||
`test/import-resume.test.ts` (import checkpoints), `test/migrate.test.ts` (migration; v8/v9 helper-btree-index SQL structural assertions + 1000-row wall-clock fixtures that guard the O(n²)→O(n log n) fix + v0.13.1 assertions on v12/v13 SQL shape, `sqlFor` + `transaction:false` runner semantics, and the `max_stalled DEFAULT 1` regression guard),
|
||||
`test/setup-branching.test.ts` (setup flow), `test/slug-validation.test.ts` (slug validation),
|
||||
`test/storage.test.ts` (storage backends), `test/supabase-admin.test.ts` (Supabase admin),
|
||||
`test/yaml-lite.test.ts` (YAML parsing), `test/check-update.test.ts` (version check + update CLI),
|
||||
@@ -281,15 +275,13 @@ parity), `test/cli.test.ts` (CLI structure), `test/config.test.ts` (config redac
|
||||
`test/skill-manifest.test.ts` (v0.19 skill manifest parser: drift detection, managed-block markers),
|
||||
`test/skillify-scaffold.test.ts` (v0.19 `gbrain skillify scaffold` stubs: SKILL.md, script, tests, routing-eval fixtures),
|
||||
`test/skillpack-install.test.ts` (v0.19 `gbrain skillpack install` managed-block install / update / no-clobber semantics),
|
||||
`test/skillpack-sync-guard.test.ts` (v0.19 sync-guard: bundled skills stay byte-identical to `skills/` source),
|
||||
`test/http-transport.test.ts` (v0.22.7 HTTP transport: 23 unit cases covering bearer auth + missing/no-Bearer/unknown/revoked + `/health` bypass, F1+F2 round-trip via dispatch.ts, F3 invalid_params, application/json response shape (not SSE), CORS default-deny + allowlist, body cap on Content-Length AND chunked, two-bucket rate limit (refill, exhaust+Retry-After, LRU eviction, TTL prune, pre-auth IP fires before DB), and `mcp_request_log` audit on success + auth_failed).
|
||||
`test/skillpack-sync-guard.test.ts` (v0.19 sync-guard: bundled skills stay byte-identical to `skills/` source).
|
||||
|
||||
E2E tests (`test/e2e/`): Run against real Postgres+pgvector. Require `DATABASE_URL`.
|
||||
- `bun run test:e2e` runs Tier 1 (mechanical, all operations, no API keys). Includes 9 dedicated cases for the postgres-engine `addLinksBatch` / `addTimelineEntriesBatch` bind path — postgres-js's `unnest()` binding is structurally different from PGLite's and gets its own coverage.
|
||||
- `test/e2e/search-quality.test.ts` runs search quality E2E against PGLite (no API keys, in-memory)
|
||||
- `test/e2e/graph-quality.test.ts` runs the v0.10.3 knowledge graph pipeline (auto-link via put_page, reconciliation, traversePaths) against PGLite in-memory
|
||||
- `test/e2e/postgres-jsonb.test.ts` — v0.12.2 regression test. Round-trips all 5 JSONB write sites (pages.frontmatter, raw_data.data, ingest_log.pages_updated, files.metadata, page_versions.frontmatter) against real Postgres and asserts `jsonb_typeof='object'` plus `->>'key'` returns the expected scalar. The test that should have caught the original double-encode bug.
|
||||
- `test/e2e/integrity-batch.test.ts` (v0.22.8) — parity tests for `scanIntegrity`'s batch-load fast path vs sequential. Four cases (dedup, hits, validate, topPages) seed a fixture and assert both paths return identical results. Dedup case uses raw SQL via `getConn().unsafe()` to seed a `(test-source-2, people/alice)` row alongside the default-source row, since `engine.putPage` doesn't take a `source_id`. Pins the codex-caught multi-source overcounting regression.
|
||||
- `test/e2e/jsonb-roundtrip.test.ts` — v0.12.3 companion regression against the 4 doctor-scanned JSONB sites. Assertion-level overlap with `postgres-jsonb.test.ts` is intentional defense-in-depth: if doctor's scan surface ever drifts from the actual write surface, one of these tests catches it.
|
||||
- `test/e2e/upgrade.test.ts` runs check-update E2E against real GitHub API (network required)
|
||||
- `test/e2e/minions-shell-pglite.test.ts` (v0.20.4) exercises the PGLite `--follow` inline shell-job path (in-memory, no `DATABASE_URL` required) — the path the consolidated minion-orchestrator skill documents for dev use
|
||||
@@ -297,8 +289,6 @@ E2E tests (`test/e2e/`): Run against real Postgres+pgvector. Require `DATABASE_U
|
||||
- `test/e2e/search-swamp.test.ts` (v0.22.0) — reproduces the headline source-swamp case. Seeds a curated `originals/talks/article-outline-fat-code` page against two `wintermute/chat/` pages stuffed with the same multi-word phrase. Asserts the article wins keyword AND vector ranking, that `detail=high` lets the chat swamp re-surface (temporal-query workflow preserved), and that `source_id` passes through the two-stage CTE intact. PGLite in-memory.
|
||||
- `test/e2e/search-exclude.test.ts` (v0.22.0) — verifies `test/` + `archive/` pages are hidden by default, that `include_slug_prefixes` opts back in, and that caller-supplied `exclude_slug_prefixes` adds to defaults. Both keyword and vector search paths covered.
|
||||
- `test/e2e/engine-parity.test.ts` (v0.22.0) — Postgres ↔ PGLite top-result and result-set parity for `searchKeyword` + `searchVector`. Codex flagged that Postgres ranks pages then picks best chunk while PGLite returns chunks directly — without parity coverage the source-boost fix could pass on PGLite and fail on Postgres. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/postgres-bootstrap.test.ts` (v0.22.6.1) — exercises `PostgresEngine.initSchema()` directly against a fresh real Postgres database. Asserts the bootstrap path is no-op on fresh installs and that SCHEMA_SQL replays cleanly through the engine path (not via the standalone `db.initSchema` from `src/core/db.ts`, which would have produced false-positive coverage). Codex caught the E2E-shape gap during plan review.
|
||||
- `test/e2e/http-transport.test.ts` (v0.22.7) — 8 cases against real Postgres covering `gbrain serve --http` end-to-end: bearer auth round-trip, `last_used_at` SQL-level debounce semantics, `mcp_request_log` row insertion on success and auth_failed paths, `/health` DB-down → 503 (DB-probing health check), and the F1+F2+F3 dispatch round-trip with a real operation. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- Tier 2 (`skills.test.ts`) requires OpenClaw + API keys, runs nightly in CI
|
||||
- If `.env.testing` doesn't exist in this directory, check sibling worktrees for one:
|
||||
`find ../ -maxdepth 2 -name .env.testing -print -quit` and copy it here if found.
|
||||
|
||||
@@ -80,13 +80,12 @@ Add to `~/.claude/server.json` (Claude Code), Settings > MCP Servers (Cursor), o
|
||||
### Remote MCP (Claude Desktop, Cowork, Perplexity)
|
||||
|
||||
```bash
|
||||
gbrain auth create "claude-desktop" # tokens via the existing CLI
|
||||
gbrain serve --http --port 8787 # built-in HTTP transport (Postgres-only)
|
||||
ngrok http 8787 --url your-brain.ngrok.app # any tunnel works
|
||||
ngrok http 8787 --url your-brain.ngrok.app
|
||||
bun run src/commands/auth.ts create "claude-desktop"
|
||||
claude mcp add gbrain -t http https://your-brain.ngrok.app/mcp -H "Authorization: Bearer TOKEN"
|
||||
```
|
||||
|
||||
Per-client guides: [`docs/mcp/`](docs/mcp/DEPLOY.md). Hardening defaults, env vars, and threat model: [SECURITY.md](SECURITY.md). ChatGPT requires OAuth 2.1 (not yet implemented).
|
||||
Per-client guides: [`docs/mcp/`](docs/mcp/DEPLOY.md). ChatGPT requires OAuth 2.1 (not yet implemented).
|
||||
|
||||
### Using gbrain with GStack
|
||||
|
||||
@@ -658,8 +657,6 @@ ADMIN
|
||||
gbrain doctor --locks List idle-in-tx backends (57014 diagnostic, Postgres only)
|
||||
gbrain stats Brain statistics
|
||||
gbrain serve MCP server (stdio)
|
||||
gbrain serve --http --port 8787 MCP server (HTTP, Postgres-only, bearer auth)
|
||||
gbrain auth create|list|revoke|test Token management for the HTTP transport
|
||||
gbrain integrations Integration recipe dashboard
|
||||
gbrain sources list|add|remove|... Multi-source brain management (v0.18)
|
||||
gbrain dream [--dry-run] [--phase N] One maintenance cycle then exit (cron-friendly)
|
||||
|
||||
-168
@@ -1,168 +0,0 @@
|
||||
# Security
|
||||
|
||||
## Reporting Vulnerabilities
|
||||
|
||||
If you discover a security issue in GBrain, please report it privately by opening
|
||||
a [private security advisory](https://github.com/garrytan/gbrain/security/advisories/new)
|
||||
on GitHub.
|
||||
|
||||
Do not open a public issue for security vulnerabilities.
|
||||
|
||||
## Remote MCP Security
|
||||
|
||||
### ⚠️ Do NOT use open OAuth client registration for remote MCP
|
||||
|
||||
If you deploy GBrain's MCP server behind an HTTP wrapper with OAuth 2.1
|
||||
support, **never allow unauthenticated client registration**. An attacker
|
||||
who discovers your server URL can:
|
||||
|
||||
1. Register a new OAuth client via `POST /register`
|
||||
2. Use `client_credentials` grant to obtain a bearer token
|
||||
3. Access all brain data via the MCP tools
|
||||
|
||||
### Recommended: `gbrain serve --http`
|
||||
|
||||
As of v0.22.7, GBrain ships a built-in HTTP transport that uses the
|
||||
existing `access_tokens` table for authentication:
|
||||
|
||||
```bash
|
||||
# Create a token
|
||||
gbrain auth create "my-client"
|
||||
|
||||
# Start the HTTP server
|
||||
gbrain serve --http --port 8787
|
||||
|
||||
# Connect via ngrok, Tailscale, or any tunnel
|
||||
ngrok http 8787 --url your-brain.ngrok.app
|
||||
```
|
||||
|
||||
This is the recommended way to expose GBrain remotely. No OAuth, no
|
||||
registration endpoint, no self-service tokens. Tokens are managed
|
||||
exclusively via `gbrain auth create/list/revoke`.
|
||||
|
||||
### If you must use a custom HTTP wrapper
|
||||
|
||||
1. **Require a secret for client registration** — check a header or body
|
||||
parameter before creating new OAuth clients
|
||||
2. **Disable `client_credentials` grant** — only allow `authorization_code`
|
||||
with browser-based approval
|
||||
3. **Restrict scopes** — never issue tokens with unlimited scope
|
||||
4. **Log all token issuance** — alert on unexpected registrations
|
||||
5. **Rate-limit registration and token endpoints**
|
||||
|
||||
### Token Management
|
||||
|
||||
```bash
|
||||
gbrain auth create "claude-desktop" # Create a new token
|
||||
gbrain auth list # List all tokens
|
||||
gbrain auth revoke "claude-desktop" # Revoke a token
|
||||
gbrain auth test <url> --token <tok> # Smoke-test a remote server
|
||||
```
|
||||
|
||||
Tokens are stored as SHA-256 hashes in the `access_tokens` table. The
|
||||
plaintext token is shown once at creation and never stored.
|
||||
|
||||
## `gbrain serve --http` hardening (v0.22.7+)
|
||||
|
||||
The built-in HTTP transport ships with several layers of hardening on by
|
||||
default. All env vars below are optional; the defaults are intentionally
|
||||
conservative.
|
||||
|
||||
### Postgres-only
|
||||
|
||||
`gbrain serve --http` requires a Postgres engine. PGLite is local-only by
|
||||
design and the `access_tokens` / `mcp_request_log` tables don't exist in
|
||||
the PGLite schema. Local agents continue to use stdio (`gbrain serve`).
|
||||
Running `--http` against a PGLite-backed install fails fast with a clear
|
||||
error message at startup.
|
||||
|
||||
### CORS
|
||||
|
||||
Default-deny: no `Access-Control-Allow-Origin` header is sent unless an
|
||||
allowlist is configured. To allow browser-based MCP clients:
|
||||
|
||||
```bash
|
||||
GBRAIN_HTTP_CORS_ORIGIN=https://claude.ai gbrain serve --http --port 8787
|
||||
# Multiple origins: comma-separated
|
||||
GBRAIN_HTTP_CORS_ORIGIN=https://claude.ai,https://your.app gbrain serve --http
|
||||
```
|
||||
|
||||
When the request `Origin` matches the allowlist, the server echoes it
|
||||
back in `Access-Control-Allow-Origin` (with `Vary: Origin`). Otherwise no
|
||||
CORS header is sent and the browser blocks the request.
|
||||
|
||||
### Rate limiting
|
||||
|
||||
Two buckets, both stored in a bounded LRU map (default 10K keys, evicts
|
||||
least-recently-used on overflow, prunes entries older than 2× the
|
||||
window):
|
||||
|
||||
| Bucket | When it fires | Default | Env var |
|
||||
|---|---|---|---|
|
||||
| Pre-auth IP | Before the DB lookup, on every `/mcp` request | 30 req / 60s | `GBRAIN_HTTP_RATE_LIMIT_IP` |
|
||||
| Post-auth token | After a valid token is resolved | 60 req / 60s | `GBRAIN_HTTP_RATE_LIMIT_TOKEN` |
|
||||
| LRU cap | Maximum distinct keys across both buckets | 10000 | `GBRAIN_HTTP_RATE_LIMIT_LRU` |
|
||||
|
||||
On exhaustion the server returns `429 Too Many Requests` with a
|
||||
`Retry-After` header.
|
||||
|
||||
**Caveat for tunneled deployments (ngrok, Tailscale Funnel, Cloudflare
|
||||
Tunnel):** all requests share one egress IP, so the pre-auth IP bucket
|
||||
becomes effectively shared by all clients on that tunnel. The
|
||||
post-auth token-id bucket is the load-bearing limiter for tunnel-fronted
|
||||
deployments.
|
||||
|
||||
### Reverse-proxy trust
|
||||
|
||||
Disabled by default. To honor `X-Forwarded-For` (or `X-Real-IP`) when
|
||||
gbrain runs behind a trusted reverse proxy:
|
||||
|
||||
```bash
|
||||
GBRAIN_HTTP_TRUST_PROXY=1 gbrain serve --http --port 8787
|
||||
```
|
||||
|
||||
**Critical safety contract:** only set `GBRAIN_HTTP_TRUST_PROXY=1` when
|
||||
**both** of these are true:
|
||||
|
||||
1. gbrain is reachable only via a trusted reverse proxy (not directly
|
||||
exposed to the internet on the configured port). The simplest
|
||||
guarantee is to bind gbrain to `127.0.0.1` or a private interface
|
||||
and have the proxy forward to it.
|
||||
2. The proxy strips any client-supplied `X-Forwarded-For` and `X-Real-IP`
|
||||
headers, then sets them itself. (nginx with `proxy_set_header
|
||||
X-Forwarded-For $remote_addr` does this; Cloudflare and most cloud
|
||||
load balancers handle it automatically.)
|
||||
|
||||
If gbrain is reachable directly AND `GBRAIN_HTTP_TRUST_PROXY=1` is set,
|
||||
clients can spoof their IP by sending arbitrary `X-Forwarded-For`
|
||||
headers, defeating the pre-auth IP rate limit. Without the flag, gbrain
|
||||
ignores all forwarded-for headers and uses the socket peer address,
|
||||
which is the safe default for direct-exposure deployments.
|
||||
|
||||
### Body size cap
|
||||
|
||||
Default 1 MiB, stream-counted (chunked transfers without
|
||||
`Content-Length` are still capped). Override:
|
||||
|
||||
```bash
|
||||
GBRAIN_HTTP_MAX_BODY_BYTES=2097152 gbrain serve --http # 2 MiB
|
||||
```
|
||||
|
||||
Over-cap requests get `413 Payload Too Large` immediately, before any
|
||||
body is materialized in memory.
|
||||
|
||||
### Audit log
|
||||
|
||||
Every `/mcp` request writes one row to `mcp_request_log`:
|
||||
|
||||
```bash
|
||||
psql "$DATABASE_URL" -c \
|
||||
"SELECT created_at, token_name, operation, status, latency_ms
|
||||
FROM mcp_request_log
|
||||
ORDER BY created_at DESC LIMIT 100"
|
||||
```
|
||||
|
||||
`status` is one of: `success`, `error`, `auth_failed`, `rate_limited`,
|
||||
`body_too_large`, `parse_error`, `unknown_method`. Failed-auth rows have
|
||||
`token_name = NULL`. Inserts are fire-and-forget so audit failures
|
||||
never block requests.
|
||||
@@ -1,114 +1,5 @@
|
||||
# TODOS
|
||||
|
||||
## sync error-code classification (PR #501 follow-ups)
|
||||
|
||||
### Plumb structured `ParseValidationCode` through `ImportResult`
|
||||
**Priority:** P2
|
||||
|
||||
**What:** Replace the regex-on-error-message path in `src/core/sync.ts:classifyErrorCode`
|
||||
with a structured `code` field threaded through `ImportResult` from the parse layer.
|
||||
|
||||
Three changes:
|
||||
1. `src/core/import-file.ts:362` — call `parseMarkdown(content, relativePath, { validate: true, expectedSlug })`
|
||||
so `parsed.errors[0].code` is populated.
|
||||
2. `src/core/import-file.ts` — add `code?: string` to `ImportResult`. Promote the
|
||||
structured code (or `'SLUG_MISMATCH'` when the existing expectedSlug check trips)
|
||||
into the result envelope alongside `error`.
|
||||
3. `src/commands/sync.ts:488` — extend `failedFiles` shape with `code?: string`.
|
||||
`recordSyncFailures` already accepts the field; the only thing missing is the
|
||||
capture site populating it.
|
||||
4. `src/core/sync.ts:classifyErrorCode` — keep as a fallback for un-coded errors
|
||||
(DB exceptions, generic catches). Primary path reads the structured code.
|
||||
|
||||
**Why:** The repo already has `ParseValidationCode` + `ParseValidationError` in
|
||||
`src/core/markdown.ts:5-18`, and three other consumers (`src/commands/lint.ts:72`,
|
||||
`src/commands/frontmatter.ts:148`, `src/core/brain-writer.ts:314`) read structured
|
||||
errors directly. Sync is the outlier — it calls `parseMarkdown` without validation
|
||||
and reverse-engineers codes via regex. PR #501 shipped that regex out of pragmatism;
|
||||
this TODO removes ~50% of `classifyErrorCode` and eliminates a class of false-positives.
|
||||
|
||||
**Pros:**
|
||||
- One source of truth for parse codes (the enum in `markdown.ts`).
|
||||
- Eliminates regex fragility — adding a new validation code in `markdown.ts`
|
||||
automatically flows to sync without a new regex.
|
||||
- Closes the case where canonical messages (`File is empty...`, `No closing ---...`)
|
||||
don't match aspirational regex patterns.
|
||||
|
||||
**Cons:** Touches `ImportResult` interface, which ripples through `src/commands/import.ts:105`,
|
||||
`src/commands/sync.ts:498-510`, `src/core/cycle.ts`, brain-writer reconciler.
|
||||
|
||||
**Context:** PR #501 documented this as P3 in the eng review at
|
||||
`~/.claude/plans/then-codex-synchronous-toucan.md`. Codex's outside-voice review
|
||||
agreed independently. The fix is small — ~50 lines including tests + downstream
|
||||
call sites — and it's the correct architectural endpoint.
|
||||
|
||||
**Effort:** M (human: ~2 hr / CC: ~20 min).
|
||||
|
||||
**Depends on / blocked by:** Nothing.
|
||||
|
||||
### CHANGELOG migration note for `acknowledgeSyncFailures()` shape change
|
||||
**Priority:** P0 — required at /ship time
|
||||
|
||||
**What:** When PR #501 ships, the release CHANGELOG entry MUST include this
|
||||
`### For contributors` block:
|
||||
|
||||
```markdown
|
||||
### For contributors
|
||||
|
||||
`acknowledgeSyncFailures()` now returns `{count, summary}` instead of `number`.
|
||||
If you import this directly from `gbrain/sync`, replace `n` with `result.count`
|
||||
and use `result.summary` for the new code-grouped breakdown.
|
||||
```
|
||||
|
||||
**Why:** The function is exported from `src/core/sync.ts:433` and reachable via
|
||||
the package exports map. External TS consumers (gbrain-evals, host agent forks)
|
||||
that imported it got `number` and now get an object — silent type break.
|
||||
|
||||
**Effort:** XS (human: ~1 min). Just don't forget.
|
||||
|
||||
**Depends on / blocked by:** PR #501 ship.
|
||||
|
||||
### Concurrent-safe ack of `~/.gbrain/sync-failures.jsonl`
|
||||
**Priority:** P3
|
||||
|
||||
**What:** Two concurrent `gbrain sync` runs hitting `acknowledgeSyncFailures()`
|
||||
can clobber each other. The function does a whole-file `writeFileSync` rewrite
|
||||
(`src/core/sync.ts:433-455`); `recordSyncFailures()` does independent
|
||||
`appendFileSync` (`src/core/sync.ts:395-416`). Concurrent ack + append can lose rows.
|
||||
|
||||
**Why:** Pre-existing — predates PR #501. Real risk only on autopilot setups where
|
||||
multiple sync invocations might overlap (rare today, more likely as multi-source
|
||||
sync matures).
|
||||
|
||||
**Fix sketch:** Atomic rename pattern (write to `sync-failures.jsonl.tmp`, then
|
||||
`renameSync`) plus a file lock for the read-modify-write cycle. Or move the
|
||||
acknowledged-set to the DB.
|
||||
|
||||
**Effort:** S (human: ~1 hr / CC: ~10 min).
|
||||
|
||||
**Depends on / blocked by:** Nothing.
|
||||
|
||||
## test-infra
|
||||
|
||||
### Parallel-load timeout flake on v0.21 PGLite-heavy tests
|
||||
**Priority:** P0
|
||||
|
||||
**What:** 22 tests added in v0.21.0 (Code Cathedral II) consistently fail in the full `bun test` run with timeout-pattern elapsed times of 7-10s, but pass in isolation. Every failing test calls `engine.initSchema()` in `beforeAll` without a timeout extension. Under parallel load (168 test files now run concurrently after v0.21 added ~24 new files), `initSchema` exceeds bun's default 5s `beforeAll` timeout.
|
||||
|
||||
Affected files include (non-exhaustive): `test/sync-strategy.test.ts`, `test/cathedral-ii-brainbench.test.ts`, `test/code-edges.test.ts`, `test/reindex-code.test.ts`, `test/reconcile-links.test.ts`, `test/two-pass.test.ts`, `test/parent-symbol-path.test.ts`, `test/pglite-v0_19.test.ts`.
|
||||
|
||||
**Why:** Currently triaged as "skip pre-existing, ship anyway" but that's not a real fix. Blocks /ship for anyone whose CHANGELOG-time test run sees them.
|
||||
|
||||
**Pros:** Fixing it lets /ship run cleanly without manual triage every release.
|
||||
|
||||
**Cons:** ~22 file edits adding `beforeAll(async () => {...}, 30000)` is mechanical but dull.
|
||||
|
||||
**Context:** Same pattern fixed in v0.20.5 wave for `test/e2e/minions-shell-pglite.test.ts`. Single-file repro: each fails in `bun test`, passes in `bun test <file>`. Reproduces with my changes stashed, so it's on master.
|
||||
|
||||
**Effort:** S (human: ~30 min / CC: ~5 min). Mechanical: grep for `beforeAll(async () => {` in affected files, add `, 30000)` argument.
|
||||
|
||||
**Depends on / blocked by:** Nothing.
|
||||
|
||||
## resolver / check-resolvable (v0.22.4 follow-ups)
|
||||
|
||||
### D10 — Extend `check-resolvable` to parse RESOLVER.md disambiguation rules
|
||||
@@ -673,90 +564,3 @@ iteration's residuals.
|
||||
**Effort estimate:** S (human: ~2 hr / CC: ~10 min).
|
||||
**Priority:** P3.
|
||||
**Depends on:** The above caller-opt-in retry (#1) is the natural co-lander since both touch the same error-classification surface.
|
||||
|
||||
## remote MCP / HTTP transport (v0.22.7 follow-ups)
|
||||
|
||||
### Audit-log write amplification on rejected `/mcp` traffic
|
||||
**What:** `src/mcp/http-transport.ts` writes a row to `mcp_request_log` for every
|
||||
incoming `/mcp` request, including rate-limited (429), oversized (413), and
|
||||
auth-failed (401) traffic. Under sustained attack the IP rate limit caps audit
|
||||
writes per IP at 30/min, but at scale (10K distinct IPs) that's still 300K
|
||||
inserts/min. Two follow-ups: (1) instrument the audit-write rate so we can see
|
||||
the actual production volume; (2) consider a separate "rejected" table or
|
||||
sampling for failed-auth rows so the success-path audit table doesn't get
|
||||
swamped.
|
||||
|
||||
**Why:** Codex flagged this during the v0.22.7 ship adversarial review. We kept
|
||||
the full audit on purpose — forensic data of an attack is valuable — but want
|
||||
to revisit once we have real volume numbers.
|
||||
|
||||
**Pros:** Bounds DB write volume under attack. Keeps the success-path audit
|
||||
table small enough for fast queries.
|
||||
|
||||
**Cons:** Adds a second table or a sampling rule. Not free complexity. Probably
|
||||
not worth it until production hits a real attack pattern.
|
||||
|
||||
**Context:** `src/mcp/http-transport.ts:222,235,245` (the three audit-on-reject
|
||||
call sites) + `src/schema.sql:342` (the unbounded table).
|
||||
|
||||
**Effort estimate:** M (human: ~half day / CC: ~30 min once we have volume data).
|
||||
**Priority:** P3 — wait for evidence.
|
||||
**Depends on:** Production telemetry on `mcp_request_log` insert rate.
|
||||
|
||||
### `validateParams` doesn't check enum values or array item types
|
||||
**What:** `src/mcp/dispatch.ts:27` (extracted from `src/mcp/server.ts` in
|
||||
v0.22.7) only checks top-level JS types. Operations declare `enum` constraints
|
||||
(e.g. `direction: 'in' | 'out' | 'both'`) and array `items: { type: ... }`
|
||||
schemas in `src/core/operations.ts`, but `validateParams` ignores both. Bad
|
||||
inputs still reach handlers — concretely, an invalid `direction` falls through
|
||||
the engine's else branch at `src/core/postgres-engine.ts:954`, widening
|
||||
traversal unexpectedly; malformed `pages_updated` arrays could be written as
|
||||
garbage JSONB.
|
||||
|
||||
**Why:** Codex flagged this during the v0.22.7 ship adversarial review. The
|
||||
validator was lifted verbatim from the pre-existing stdio path during the
|
||||
dispatch.ts extraction — same gap exists on the stdio MCP server today, so
|
||||
this isn't a v0.22.7 regression. Still worth tightening, since "shared
|
||||
validation" is now the architectural guarantee both transports rely on.
|
||||
|
||||
**Pros:** Better defense-in-depth at the MCP boundary. Catches malformed agent
|
||||
inputs before the engine layer has to.
|
||||
|
||||
**Cons:** Need to walk every operation's param schema and decide which enum
|
||||
violations are user-facing errors vs internal bugs. May need a typed Zod-style
|
||||
schema layer to do this cleanly.
|
||||
|
||||
**Context:** `src/mcp/dispatch.ts:27` + `src/core/operations.ts` (param defs).
|
||||
Same gap pre-existed on stdio MCP path.
|
||||
|
||||
**Effort estimate:** M (human: ~half day / CC: ~30 min if we use the existing
|
||||
ParamDef shape; XL if a Zod migration is the chosen direction).
|
||||
**Priority:** P2.
|
||||
**Depends on:** Whether we want to keep the lightweight ParamDef shape or
|
||||
migrate to typed schemas.
|
||||
|
||||
### Streaming MCP tool support (re-add SSE based on Accept header)
|
||||
**What:** v0.22.7 dropped SSE entirely from `gbrain serve --http` because no
|
||||
current MCP tool streams. When the first streaming tool ships (long-running
|
||||
agent delegation as an MCP tool, `resources/subscribe`, `sampling/createMessage`),
|
||||
re-add SSE in `/mcp` based on the `Accept` header per the Streamable HTTP
|
||||
transport spec. ~30 lines + spec compliance test.
|
||||
|
||||
**Why:** Removing SSE simplified the v0.22.7 transport (one response path,
|
||||
fewer test cases). Adding it back when actually needed is cheap and keeps the
|
||||
code lean in the meantime.
|
||||
|
||||
**Effort estimate:** S (human: ~2 hr / CC: ~15 min).
|
||||
**Priority:** P3 — wait for the first streaming tool.
|
||||
**Depends on:** A streaming MCP tool actually existing.
|
||||
|
||||
### `access_tokens.scopes` enforcement
|
||||
**What:** The `access_tokens` schema has had a `scopes TEXT[]` column since
|
||||
migration v4 (`src/core/migrate.ts:84`), but nothing enforces it. v0.22.7's
|
||||
`gbrain auth create` doesn't accept a `--scopes` flag, and `dispatchToolCall`
|
||||
doesn't gate on scopes. Adding per-tool scope enforcement would let
|
||||
"claude-desktop-readonly" and "ingest-only" tokens exist.
|
||||
|
||||
**Effort estimate:** M (human: ~1 day / CC: ~30 min for the schema-aware gate).
|
||||
**Priority:** P3.
|
||||
**Depends on:** Nothing.
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
# Remote MCP Deployment Options
|
||||
|
||||
GBrain's MCP server runs via `gbrain serve` (stdio transport). To make it
|
||||
accessible from other devices and AI clients, run `gbrain serve --http`
|
||||
(built-in HTTP transport with bearer auth, Postgres-only ... see
|
||||
[DEPLOY.md](DEPLOY.md)) behind a public tunnel. Here are your tunnel options.
|
||||
accessible from other devices and AI clients, you need an HTTP wrapper and
|
||||
a public tunnel. Here are your options.
|
||||
|
||||
## ngrok (recommended)
|
||||
|
||||
@@ -14,9 +13,8 @@ accessible from other devices and AI clients, run `gbrain serve --http`
|
||||
# 1. Install ngrok
|
||||
brew install ngrok
|
||||
|
||||
# 2. Start the built-in HTTP transport
|
||||
gbrain serve --http --port 8787
|
||||
# See docs/mcp/DEPLOY.md for token setup
|
||||
# 2. Start your MCP server (behind an HTTP wrapper)
|
||||
# See docs/mcp/DEPLOY.md for the server setup
|
||||
|
||||
# 3. Expose via ngrok
|
||||
ngrok http 8787 --url your-brain.ngrok.app
|
||||
@@ -61,7 +59,6 @@ Both run Bun natively. No bundling, no Deno, no cold start, no timeout limits.
|
||||
| All 30 operations | Yes | Yes | Yes |
|
||||
| Setup time | 5 min | 10 min | 15 min |
|
||||
|
||||
**Note:** `gbrain serve --http` is the built-in HTTP transport (v0.22.7+). Bearer auth
|
||||
against the `access_tokens` table, default-deny CORS, two-bucket rate limit, body cap,
|
||||
per-request audit log. Postgres-only by design (PGLite is local-only). See
|
||||
[DEPLOY.md](DEPLOY.md) and [SECURITY.md](../../SECURITY.md) for env vars and tunables.
|
||||
**Note:** `gbrain serve --http` (built-in HTTP transport) is planned but not yet
|
||||
implemented. Currently, remote MCP requires a custom HTTP wrapper around `gbrain serve`.
|
||||
See [DEPLOY.md](DEPLOY.md) for details.
|
||||
|
||||
@@ -21,7 +21,7 @@ claude mcp add gbrain -t http \
|
||||
```
|
||||
|
||||
Replace `YOUR-DOMAIN` with your ngrok domain and `YOUR_TOKEN` with a token
|
||||
from `gbrain auth create "claude-code"`.
|
||||
from `bun run src/commands/auth.ts create "claude-code"`.
|
||||
|
||||
## Verify
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ For Team/Enterprise plans, an org Owner adds the connector:
|
||||
https://YOUR-DOMAIN.ngrok.app/mcp
|
||||
```
|
||||
3. Add Bearer token authentication in Advanced Settings
|
||||
(create one with `gbrain auth create "cowork"`)
|
||||
(create one with `bun run src/commands/auth.ts create "cowork"`)
|
||||
4. Save
|
||||
|
||||
Note: Cowork connects from Anthropic's cloud, not your device. Your server
|
||||
|
||||
@@ -16,7 +16,7 @@ Remote HTTP servers must be added through the GUI.
|
||||
Replace `YOUR-DOMAIN` with your ngrok domain (see
|
||||
[ngrok-tunnel recipe](../../recipes/ngrok-tunnel.md) for setup).
|
||||
5. Set authentication to **Bearer Token** and paste your token
|
||||
(create one with `gbrain auth create "claude-desktop"`)
|
||||
(create one with `bun run src/commands/auth.ts create "claude-desktop"`)
|
||||
6. Save
|
||||
|
||||
## Verify
|
||||
|
||||
+14
-21
@@ -1,13 +1,8 @@
|
||||
# Deploy GBrain Remote MCP Server
|
||||
|
||||
> **v0.22.7+:** Use `gbrain serve --http` for remote access. It includes built-in
|
||||
> bearer token auth, default-deny CORS, two-bucket rate limiting, body cap, and
|
||||
> per-request audit log. **Postgres-only** (PGLite is local-only by design).
|
||||
> See [SECURITY.md](../../SECURITY.md) for env vars and tunable defaults.
|
||||
|
||||
Access your brain from any device, any AI client. GBrain's MCP server runs locally
|
||||
via `gbrain serve` (stdio). For remote access, expose it via the built-in HTTP
|
||||
transport behind a public tunnel.
|
||||
via `gbrain serve` (stdio). For remote access, wrap it in an HTTP server behind a
|
||||
public tunnel.
|
||||
|
||||
## Two Paths
|
||||
|
||||
@@ -18,23 +13,21 @@ gbrain serve
|
||||
```
|
||||
|
||||
Works with Claude Code, Cursor, Windsurf, and any MCP client that supports stdio.
|
||||
No server, no tunnel, no token needed. Works on both PGLite and Postgres engines.
|
||||
No server, no tunnel, no token needed.
|
||||
|
||||
### Remote (any device, any AI client) — Postgres only
|
||||
### Remote (any device, any AI client)
|
||||
|
||||
```
|
||||
Your AI client (Claude Desktop, Perplexity, etc.)
|
||||
→ ngrok tunnel (https://YOUR-DOMAIN.ngrok.app)
|
||||
→ gbrain serve --http (built-in transport with bearer auth)
|
||||
→ Postgres (pooler connection or self-hosted)
|
||||
→ Your HTTP server (wraps gbrain serve)
|
||||
→ Supabase Postgres (via pooler connection string)
|
||||
```
|
||||
|
||||
This requires:
|
||||
1. A Postgres-backed brain (the `access_tokens` table only exists on Postgres;
|
||||
running `gbrain serve --http` against a PGLite install fails fast at startup)
|
||||
2. A machine running `gbrain serve --http`
|
||||
3. A public tunnel (ngrok, Tailscale, or cloud host)
|
||||
4. A bearer token created via `gbrain auth create <name>`
|
||||
1. A machine running `gbrain serve` behind an HTTP wrapper
|
||||
2. A public tunnel (ngrok, Tailscale, or cloud host)
|
||||
3. Bearer token auth for security
|
||||
|
||||
## Remote Setup
|
||||
|
||||
@@ -53,13 +46,13 @@ ngrok http 8787 --url your-brain.ngrok.app # Hobby tier for fixed domain
|
||||
|
||||
```bash
|
||||
# Create a token for each client
|
||||
gbrain auth create "claude-desktop"
|
||||
bun run src/commands/auth.ts create "claude-desktop"
|
||||
|
||||
# List all tokens
|
||||
gbrain auth list
|
||||
bun run src/commands/auth.ts list
|
||||
|
||||
# Revoke a token
|
||||
gbrain auth revoke "claude-desktop"
|
||||
bun run src/commands/auth.ts revoke "claude-desktop"
|
||||
```
|
||||
|
||||
Tokens are per-client. Create one for each device/app. Revoke individually
|
||||
@@ -75,7 +68,7 @@ if compromised. Tokens are stored SHA-256 hashed in your database.
|
||||
### 4. Verify
|
||||
|
||||
```bash
|
||||
gbrain auth test \
|
||||
bun run src/commands/auth.ts test \
|
||||
https://YOUR-DOMAIN.ngrok.app/mcp \
|
||||
--token YOUR_TOKEN
|
||||
```
|
||||
@@ -103,7 +96,7 @@ Funnel, and cloud hosts (Fly.io, Railway).
|
||||
Include the Authorization header: `Authorization: Bearer YOUR_TOKEN`
|
||||
|
||||
**"invalid_token" error**
|
||||
Run `gbrain auth list` to see active tokens.
|
||||
Run `bun run src/commands/auth.ts list` to see active tokens.
|
||||
|
||||
**"service_unavailable" error**
|
||||
Database connection failed. Check your Supabase dashboard for outages.
|
||||
|
||||
@@ -10,7 +10,7 @@ Perplexity Computer supports remote MCP servers with bearer token authentication
|
||||
- **URL:** `https://YOUR-DOMAIN.ngrok.app/mcp`
|
||||
- **Authentication:** API Key / Bearer Token
|
||||
- **Token:** your GBrain access token
|
||||
(create one with `gbrain auth create "perplexity"`)
|
||||
(create one with `bun run src/commands/auth.ts create "perplexity"`)
|
||||
4. Save
|
||||
|
||||
Replace `YOUR-DOMAIN` with your ngrok domain (see
|
||||
|
||||
+24
-44
@@ -104,9 +104,9 @@ strict behavior when unset.
|
||||
- `src/core/operations.ts` — Contract-first operation definitions (the foundation). Also exports upload validators: `validateUploadPath`, `validatePageSlug`, `validateFilename`. `OperationContext.remote` flags untrusted callers.
|
||||
- `src/core/engine.ts` — Pluggable engine interface (BrainEngine). `clampSearchLimit(limit, default, cap)` takes an explicit cap so per-operation caps can be tighter than `MAX_SEARCH_LIMIT`. Exports `LinkBatchInput` / `TimelineBatchInput` for the v0.12.1 bulk-insert API (`addLinksBatch` / `addTimelineEntriesBatch`). As of v0.13.1, `BrainEngine` has a `readonly kind: 'postgres' | 'pglite'` discriminator so migrations (`src/core/migrate.ts`) and other consumers can branch on engine without `instanceof` + dynamic imports.
|
||||
- `src/core/engine-factory.ts` — Engine factory with dynamic imports (`'pglite'` | `'postgres'`)
|
||||
- `src/core/pglite-engine.ts` — PGLite (embedded Postgres 17.5 via WASM) implementation, all 40 BrainEngine methods. `addLinksBatch` / `addTimelineEntriesBatch` use multi-row `unnest()` with manual `$N` placeholders. As of v0.13.1, `connect()` wraps `PGlite.create()` in a try/catch that emits an actionable error naming the macOS 26.3 WASM bug (#223) and pointing at `gbrain doctor`; the lock is released on failure so the next process can retry cleanly. v0.22.0: `searchKeyword` and `searchKeywordChunks` multiply `ts_rank` by the source-factor CASE expression at the chunk-grain level; `searchVector` becomes a two-stage CTE — inner CTE keeps `ORDER BY cc.embedding <=> vec` so HNSW stays usable, outer SELECT re-ranks by `raw_score * source_factor`. Inner LIMIT scales with offset to preserve pagination contract. As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL — probes for the specific forward-referenced state the embedded schema blob needs (`pages.source_id`, `links.link_source`, `links.origin_page_id`, `content_chunks.symbol_name`, `content_chunks.language`, `sources` FK target table) and adds only what's missing. Closes the upgrade-wedge bug class that bit users 10+ times across 6 schema versions over 2 years (#239/#243/#266/#357/#366/#374/#375/#378/#395/#396). No-op on fresh installs and modern brains.
|
||||
- `src/core/pglite-engine.ts` — PGLite (embedded Postgres 17.5 via WASM) implementation, all 40 BrainEngine methods. `addLinksBatch` / `addTimelineEntriesBatch` use multi-row `unnest()` with manual `$N` placeholders. As of v0.13.1, `connect()` wraps `PGlite.create()` in a try/catch that emits an actionable error naming the macOS 26.3 WASM bug (#223) and pointing at `gbrain doctor`; the lock is released on failure so the next process can retry cleanly. v0.22.0: `searchKeyword` and `searchKeywordChunks` multiply `ts_rank` by the source-factor CASE expression at the chunk-grain level; `searchVector` becomes a two-stage CTE — inner CTE keeps `ORDER BY cc.embedding <=> vec` so HNSW stays usable, outer SELECT re-ranks by `raw_score * source_factor`. Inner LIMIT scales with offset to preserve pagination contract.
|
||||
- `src/core/pglite-schema.ts` — PGLite-specific DDL (pgvector, pg_trgm, triggers)
|
||||
- `src/core/postgres-engine.ts` — Postgres + pgvector implementation (Supabase / self-hosted). `addLinksBatch` / `addTimelineEntriesBatch` use `INSERT ... SELECT FROM unnest($1::text[], ...) JOIN pages ON CONFLICT DO NOTHING RETURNING 1` — 4-5 array params regardless of batch size, sidesteps the 65535-parameter cap. As of v0.12.3, `searchKeyword` / `searchVector` scope `statement_timeout` via `sql.begin` + `SET LOCAL` so the GUC dies with the transaction instead of leaking across the pooled postgres.js connection (contributed by @garagon). `getEmbeddingsByChunkIds` uses `tryParseEmbedding` so one corrupt row skips+warns instead of killing the query. v0.22.0: `searchKeyword`, `searchKeywordChunks`, and `searchVector` apply source-aware ranking by inlining the source-factor CASE and `NOT (col LIKE …)` hard-exclude clause from `src/core/search/sql-ranking.ts`. `searchVector` switches to a two-stage CTE (HNSW-safe inner ORDER BY, source-boost re-rank in the outer SELECT) and carries `p.source_id` through inner→outer for v0.18 multi-source callers. v0.22.1 (#406): `_savedConfig` retains the connect config; `reconnect()` tears down + recreates the pool from saved config (called by supervisor watchdog after 3 consecutive health-check failures). `executeRaw` is a single-statement passthrough — no per-call retry (D3 dropped that as unsound for non-idempotent statements; recovery is supervisor-driven). v0.22.1 (#363, contributed by @orendi84): `connect()` applies `resolveSessionTimeouts()` from `db.ts` as connection-time startup parameters (`statement_timeout`, `idle_in_transaction_session_timeout`) so orphan pgbouncer backends can't hold locks for hours. v0.22.1 (#409, contributed by @atrevino47): `countStaleChunks()` + `listStaleChunks()` server-side-filter on `embedding IS NULL` for `embed --stale`, eliminating ~76 MB/call client-side pull on a fully-embedded brain; `upsertChunks()` resets both `embedding` AND `embedded_at` to NULL when chunk_text changes without a new embedding (consistency). As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL on the same forward-reference probe set as the PGLite engine, so old Postgres brains pinned at v0.13/v0.18/v0.19 walk forward cleanly instead of wedging on `column "..." does not exist`.
|
||||
- `src/core/postgres-engine.ts` — Postgres + pgvector implementation (Supabase / self-hosted). `addLinksBatch` / `addTimelineEntriesBatch` use `INSERT ... SELECT FROM unnest($1::text[], ...) JOIN pages ON CONFLICT DO NOTHING RETURNING 1` — 4-5 array params regardless of batch size, sidesteps the 65535-parameter cap. As of v0.12.3, `searchKeyword` / `searchVector` scope `statement_timeout` via `sql.begin` + `SET LOCAL` so the GUC dies with the transaction instead of leaking across the pooled postgres.js connection (contributed by @garagon). `getEmbeddingsByChunkIds` uses `tryParseEmbedding` so one corrupt row skips+warns instead of killing the query. v0.22.0: `searchKeyword`, `searchKeywordChunks`, and `searchVector` apply source-aware ranking by inlining the source-factor CASE and `NOT (col LIKE …)` hard-exclude clause from `src/core/search/sql-ranking.ts`. `searchVector` switches to a two-stage CTE (HNSW-safe inner ORDER BY, source-boost re-rank in the outer SELECT) and carries `p.source_id` through inner→outer for v0.18 multi-source callers. v0.22.1 (#406): `_savedConfig` retains the connect config; `reconnect()` tears down + recreates the pool from saved config (called by supervisor watchdog after 3 consecutive health-check failures). `executeRaw` is a single-statement passthrough — no per-call retry (D3 dropped that as unsound for non-idempotent statements; recovery is supervisor-driven). v0.22.1 (#363, contributed by @orendi84): `connect()` applies `resolveSessionTimeouts()` from `db.ts` as connection-time startup parameters (`statement_timeout`, `idle_in_transaction_session_timeout`) so orphan pgbouncer backends can't hold locks for hours. v0.22.1 (#409, contributed by @atrevino47): `countStaleChunks()` + `listStaleChunks()` server-side-filter on `embedding IS NULL` for `embed --stale`, eliminating ~76 MB/call client-side pull on a fully-embedded brain; `upsertChunks()` resets both `embedding` AND `embedded_at` to NULL when chunk_text changes without a new embedding (consistency).
|
||||
- `src/core/utils.ts` — Shared SQL utilities extracted from postgres-engine.ts. Exports `parseEmbedding(value)` (throws on unknown input, used by migration + ingest paths where data integrity matters) and as of v0.12.3 `tryParseEmbedding(value)` (returns `null` + warns once per process, used by search/rescore paths where availability matters more than strictness).
|
||||
- `src/core/db.ts` — Connection management, schema initialization. v0.22.1 (#363, contributed by @orendi84): `resolveSessionTimeouts()` returns `statement_timeout` + `idle_in_transaction_session_timeout` (defaults: 5min each, env-overridable via `GBRAIN_STATEMENT_TIMEOUT` / `GBRAIN_IDLE_TX_TIMEOUT` / `GBRAIN_CLIENT_CHECK_INTERVAL`). Both `connect()` (module singleton) and `PostgresEngine.connect()` (worker pool) consume the result via postgres.js's `connection` option, sending GUCs as startup parameters that survive PgBouncer transaction mode (unlike the prior `setSessionDefaults` post-pool SET, kept as a back-compat no-op shim).
|
||||
- `src/commands/migrate-engine.ts` — Bidirectional engine migration (`gbrain migrate --to supabase/pglite`)
|
||||
@@ -170,18 +170,14 @@ strict behavior when unset.
|
||||
- `src/commands/jobs.ts` — `gbrain jobs` CLI subcommands + `gbrain jobs work` daemon. v0.13.1 surfaces the full `MinionJobInput` retry/backoff/timeout/idempotency surface as first-class CLI flags on `jobs submit`: `--max-stalled`, `--backoff-type fixed|exponential`, `--backoff-delay`, `--backoff-jitter`, `--timeout-ms`, `--idempotency-key`. `jobs smoke --sigkill-rescue` is the opt-in regression guard for #219. v0.16 wires `registerBuiltinHandlers` to always register `subagent` + `subagent_aggregator` (no env flag — `ANTHROPIC_API_KEY` is the natural cost gate, trust is via `PROTECTED_JOB_NAMES`) and loads `GBRAIN_PLUGIN_PATH` plugins at worker startup with a loud startup-line per plugin. `shell` handler still gated by `GBRAIN_ALLOW_SHELL_JOBS=1` (RCE surface, separate concern).
|
||||
- `src/commands/features.ts` — `gbrain features --json --auto-fix`: usage scan + feature adoption salesman
|
||||
- `src/commands/autopilot.ts` — `gbrain autopilot --install`: self-maintaining brain daemon (sync+extract+embed)
|
||||
- `src/mcp/server.ts` — MCP stdio server (generated from operations). v0.22.7: tool-call handler delegates to `dispatchToolCall` from `src/mcp/dispatch.ts` so stdio + HTTP transports share one validation, context-build, and error-format path.
|
||||
- `src/mcp/dispatch.ts` (v0.22.7) — Shared tool-call dispatch consumed by both stdio (`server.ts`) and HTTP (`http-transport.ts`). Exports `dispatchToolCall(engine, name, params, opts)`, `buildOperationContext(engine, params, opts)`, and `validateParams(op, params)`. Single source of truth for `(ctx, params)` handler arg order and the 5-field `OperationContext` shape (engine + config + logger + dryRun + remote). Defaults to `remote: true` (untrusted); local CLI callers pass `remote: false`. Closed F1 (reversed handler args) + F2 (incomplete OperationContext) + F3 (no param validation) drift bugs in the original v0.22.5 HTTP transport.
|
||||
- `src/mcp/rate-limit.ts` (v0.22.7) — Bounded-LRU token-bucket limiter for `gbrain serve --http`. `buildDefaultLimiters()` returns the two-bucket pipeline used by http-transport: pre-auth IP (default 30/60s, fires BEFORE the DB lookup so brute-force load against `access_tokens` is actually capped) + post-auth token-id (default 60/60s). Tracks `lastTouchedMs` separately from `lastRefillMs` so an exhausted key can't be reset by hammering past the TTL. LRU cap (default 10K keys) bounds memory under attacker-controlled key growth; TTL prune at 2× window evicts abandoned buckets.
|
||||
- `src/mcp/http-transport.ts` (v0.22.7, rewrite) — `gbrain serve --http` HTTP transport. Postgres-only — fails fast at startup on PGLite (the `access_tokens` table only exists on Postgres). Bearer auth against SHA-256 hashes in `access_tokens`. CORS default-deny via `GBRAIN_HTTP_CORS_ORIGIN` allowlist. Body cap stream-counted (1 MiB default via `GBRAIN_HTTP_MAX_BODY_BYTES`) so chunked transfers without Content-Length still hit the cap. `last_used_at` SQL-level debounce (one UPDATE per token per 60s). Per-request audit row in `mcp_request_log` with token_name + operation + status + latency. Optional `GBRAIN_HTTP_TRUST_PROXY=1` honors `X-Forwarded-For` — only safe when bound to a private interface AND the proxy strips client-supplied XFF (otherwise enables IP spoofing past the pre-auth rate limit). `/health` does `SELECT 1` against Postgres and returns 503 + `status:unhealthy` when the DB is unreachable so orchestration doesn't see green pods while clients get misleading 401s. Replaces the standalone OAuth wrapper that was vulnerable to unauthenticated client registration.
|
||||
- `src/commands/auth.ts` — Token management for the HTTP transport. `gbrain auth create/list/revoke/test`. As of v0.22.7 wired into the main CLI (`src/cli.ts`); also runs standalone via `bun run src/commands/auth.ts ...` for environments without a compiled binary. Tokens stored as SHA-256 hashes in `access_tokens` (Postgres-only).
|
||||
- `src/mcp/server.ts` — MCP stdio server (generated from operations)
|
||||
- `src/commands/auth.ts` — Standalone token management (create/list/revoke/test)
|
||||
- `src/commands/upgrade.ts` — Self-update CLI. `runPostUpgrade()` enumerates migrations from the TS registry (src/commands/migrations/index.ts) and tail-calls `runApplyMigrations(['--yes', '--non-interactive'])` so the mechanical side of every outstanding migration runs unconditionally.
|
||||
- `src/commands/migrations/` — TS migration registry (compiled into the binary; no filesystem walk of `skills/migrations/*.md` needed at runtime). `index.ts` lists migrations in semver order. `v0_11_0.ts` = Minions adoption orchestrator (8 phases). `v0_12_0.ts` = Knowledge Graph auto-wire orchestrator (5 phases: schema → config check → backfill links → backfill timeline → verify). `phaseASchema` has a 600s timeout (bumped from 60s in v0.12.1 for duplicate-heavy brains). `v0_12_2.ts` = JSONB double-encode repair orchestrator (4 phases: schema → repair-jsonb → verify → record). `v0_14_0.ts` = shell-jobs + autopilot cooperative (2 phases: schema ALTER minion_jobs.max_stalled SET DEFAULT 3 — superseded by v0.14.3's schema-level DEFAULT 5 + UPDATE backfill; pending-host-work ping for skills/migrations/v0.14.0.md). All orchestrators are idempotent and resumable from `partial` status. As of v0.14.2 (Bug 3), the RUNNER owns all ledger writes — orchestrators return `OrchestratorResult` and `apply-migrations.ts` persists a canonical `{version, status, phases}` shape after return. Orchestrators no longer call `appendCompletedMigration` directly. `statusForVersion` prefers `complete` over `partial` (never regresses). 3 consecutive partials → wedged → `--force-retry <version>` writes a `'retry'` reset marker. v0.14.3 (fix wave) ships schema-only migrations v14 (`pages_updated_at_index`) + v15 (`minion_jobs_max_stalled_default_5` with UPDATE backfill) via the `MIGRATIONS` array in `src/core/migrate.ts` — no orchestrator phases needed.
|
||||
- `src/commands/repair-jsonb.ts` — `gbrain repair-jsonb [--dry-run] [--json]`: rewrites `jsonb_typeof='string'` rows in place across 5 affected columns (pages.frontmatter, raw_data.data, ingest_log.pages_updated, files.metadata, page_versions.frontmatter). Fixes v0.12.0 double-encode bug on Postgres; PGLite no-ops. Idempotent.
|
||||
- `src/commands/orphans.ts` — `gbrain orphans [--json] [--count] [--include-pseudo]`: surfaces pages with zero inbound wikilinks, grouped by domain. Auto-generated/raw/pseudo pages filtered by default. Also exposed as `find_orphans` MCP operation. Shipped in v0.12.3 (contributed by @knee5).
|
||||
- `src/commands/integrity.ts` — `gbrain integrity check|auto|review|extract`: bare-tweet detection, dead-link detection, three-bucket repair (auto-repair / review-queue / skip). `scanIntegrity()` is the shared library function called from `gbrain doctor` (sampled at limit=500) and `cmdCheck` (full scan). v0.22.8: batch-load fast path on Postgres uses `SELECT DISTINCT ON (slug)` in a single SQL query to fix the PgBouncer round-trip timeout (60s → ~6s) while preserving `engine.getAllSlugs()`'s `Set<string>` semantics on multi-source brains. Gated by `engine.kind === 'postgres'` at the call site so PGLite never enters batch; fallback `catch` logs at `GBRAIN_DEBUG=1` so real Postgres errors are diagnosable.
|
||||
- `src/commands/doctor.ts` — `gbrain doctor [--json] [--fast] [--fix] [--dry-run] [--index-audit]`: health checks. v0.12.3 added `jsonb_integrity` + `markdown_body_completeness` reliability checks. v0.14.1: `--fix` delegates inlined cross-cutting rules to `> **Convention:** see [path](path).` callouts (pipes DRY violations into `src/core/dry-fix.ts`); `--fix --dry-run` previews without writing. v0.14.2: `schema_version` check fails loudly when `version=0` (migrations never ran — the #218 `bun install -g` signature) and routes users to `gbrain apply-migrations --yes`; new opt-in `--index-audit` flag (Postgres-only) reports zero-scan indexes from `pg_stat_user_indexes` (informational only, no auto-drop). v0.15.2: every DB check is wrapped in a progress phase; `markdown_body_completeness` runs under a 1s heartbeat timer so 10+ min scans are observable on 50K-page brains. v0.19.1 added `queue_health` (Postgres-only) with two subchecks: stalled-forever active jobs (started_at > 1h) and waiting-depth-per-name > threshold (default 10, override via `GBRAIN_QUEUE_WAITING_THRESHOLD`). Worker-heartbeat subcheck intentionally deferred to follow-up B7 because it needs a `minion_workers` table to produce ground-truth signal. Fix hints point at `gbrain repair-jsonb`, `gbrain sync --force`, `gbrain apply-migrations`, and `gbrain jobs get/cancel <id>`.
|
||||
- `src/core/migrate.ts` — schema-migration runner. Owns the `MIGRATIONS` array (source of truth for schema DDL). v0.14.2 extended the `Migration` interface with `sqlFor?: { postgres?, pglite? }` (engine-specific SQL overrides `sql`) and `transaction?: boolean` (set to false for `CREATE INDEX CONCURRENTLY`, which Postgres refuses inside a transaction; ignored on PGLite since it has no concurrent writers). Migration v14 (fix wave) uses a handler branching on `engine.kind` to run CONCURRENTLY on Postgres (with a pre-drop of any invalid remnant via `pg_index.indisvalid`) and plain `CREATE INDEX` on PGLite. v15 bumps `minion_jobs.max_stalled` default 1→5 and backfills existing non-terminal rows. v0.22.6.1: migration v24 (`rls_backfill_missing_tables`) uses `sqlFor: { pglite: '' }` to no-op on PGLite — PGLite has no RLS engine and is single-tenant by definition, and the v24 ALTERs target subagent tables that don't exist in pglite-schema.ts. Closes #395 (contributed by @jdcastro2).
|
||||
- `src/core/migrate.ts` — schema-migration runner. Owns the `MIGRATIONS` array (source of truth for schema DDL). v0.14.2 extended the `Migration` interface with `sqlFor?: { postgres?, pglite? }` (engine-specific SQL overrides `sql`) and `transaction?: boolean` (set to false for `CREATE INDEX CONCURRENTLY`, which Postgres refuses inside a transaction; ignored on PGLite since it has no concurrent writers). Migration v14 (fix wave) uses a handler branching on `engine.kind` to run CONCURRENTLY on Postgres (with a pre-drop of any invalid remnant via `pg_index.indisvalid`) and plain `CREATE INDEX` on PGLite. v15 bumps `minion_jobs.max_stalled` default 1→5 and backfills existing non-terminal rows.
|
||||
- `src/core/progress.ts` — Shared bulk-action progress reporter. Writes to stderr. Modes: `auto` (TTY: `\r`-rewriting; non-TTY: plain lines), `human`, `json` (JSONL), `quiet`. Rate-gated by `minIntervalMs` and `minItems`. `startHeartbeat(reporter, note)` helper for single long queries. `child()` composes phase paths. Singleton SIGINT/SIGTERM coordinator emits `abort` events for every live phase. EPIPE defense on both sync throws and stream `'error'` events. Zero dependencies. Introduced in v0.15.2.
|
||||
- `src/core/cli-options.ts` — Global CLI flag parser. `parseGlobalFlags(argv)` returns `{cliOpts, rest}` with `--quiet` / `--progress-json` / `--progress-interval=<ms>` stripped. `getCliOptions()` / `setCliOptions()` expose a module-level singleton so commands reach the resolved flags without parameter threading. `cliOptsToProgressOptions()` maps to reporter options. `childGlobalFlags()` returns the flag suffix to append to `execSync('gbrain ...')` calls in migration orchestrators. `OperationContext.cliOpts` extends shared-op dispatch for MCP callers.
|
||||
- `src/core/cycle.ts` — v0.17 brain maintenance cycle primitive. `runCycle(engine: BrainEngine | null, opts: CycleOpts): Promise<CycleReport>` composes 6 phases in semantically-driven order (lint → backlinks → sync → extract → embed → orphans). Three callers: `gbrain dream` CLI, `gbrain autopilot` daemon's inline path, and the Minions `autopilot-cycle` handler (`src/commands/jobs.ts`). One source of truth for what the brain does overnight. Coordination via `gbrain_cycle_locks` DB table (TTL-based; works through PgBouncer transaction pooling, unlike session-scoped `pg_try_advisory_lock`) + `~/.gbrain/cycle.lock` file lock with PID-liveness for PGLite / engine=null mode. `CycleReport.schema_version: "1"` is the stable agent-consumable shape. `PhaseResult.error: { class, code, message, hint?, docs_url? }` is Stripe-API-tier structured failure info. `yieldBetweenPhases` hook awaited between every phase — Minions handler uses this to renew its job lock and prevent v0.14 stall-death regression. Engine nullable: filesystem phases (lint, backlinks) run without DB; DB phases skip with `status: "skipped", reason: "no_database"`. Lock-skip: read-only phase selections (`--phase orphans`) bypass the cycle lock. v0.22.1 (#403): `CycleOpts.signal?: AbortSignal` propagates the worker's abort signal; `checkAborted()` fires between every phase and throws if the signal is aborted (cooperative — can't interrupt a phase mid-execution). v0.22.1 (#417): `runPhaseSync` returns `pagesAffected` via `SyncPhaseResult`; `runCycle` captures it and threads to `runPhaseExtract` as the 4th arg, enabling incremental extract on the cycle path. v0.22.1 (Codex F2): `runPhaseSync` takes `willRunExtractPhase: boolean` and sets `noExtract: phases.includes('extract')` so `gbrain dream --phase sync` doesn't silently lose extraction. v0.22.5 (#475): new `resolveSourceForDir(engine, brainDir)` helper queries `SELECT id FROM sources WHERE local_path = $1 LIMIT 1`; `runPhaseSync` threads result as `sourceId` to `performSync()` so sync reads the per-source `sources.last_commit` anchor instead of the drift-prone global `config.sync.last_commit` key. Bare try/catch lets pre-v0.18 brains fall through to the global key. Closes the prod hang where every autopilot cycle ran a 30-min full reimport because the global anchor commit had been GC'd from git history.
|
||||
@@ -306,9 +302,7 @@ parity), `test/cli.test.ts` (CLI structure), `test/config.test.ts` (config redac
|
||||
`test/files.test.ts` (MIME/hash), `test/import-file.test.ts` (import pipeline),
|
||||
`test/upgrade.test.ts` (schema migrations),
|
||||
`test/file-migration.test.ts` (file migration), `test/file-resolver.test.ts` (file resolution),
|
||||
`test/import-resume.test.ts` (import checkpoints), `test/migrate.test.ts` (migration; v8/v9 helper-btree-index SQL structural assertions + 1000-row wall-clock fixtures that guard the O(n²)→O(n log n) fix + v0.13.1 assertions on v12/v13 SQL shape, `sqlFor` + `transaction:false` runner semantics, the `max_stalled DEFAULT 1` regression guard, and v0.22.6.1 v24 `sqlFor.pglite: ''` no-op assertion),
|
||||
`test/bootstrap.test.ts` (v0.22.6.1 — bootstrap contract: no-op on fresh install, idempotent across two `initSchema()` calls, no-op on modern brain that already has every probed column, full bootstrap path on simulated pre-v0.18 brain, fresh-install regression guard, pre-v0.13 `links` shape coverage),
|
||||
`test/schema-bootstrap-coverage.test.ts` (v0.22.6.1 CI guard — `REQUIRED_BOOTSTRAP_COVERAGE` lists every forward reference in PGLITE_SCHEMA_SQL; the test fails loudly if `applyForwardReferenceBootstrap` skips one. When you add a column-with-index to the embedded schema blob, you extend both arrays or this guard fails. The pattern that broke gbrain ten times in two years is now structurally prevented.),
|
||||
`test/import-resume.test.ts` (import checkpoints), `test/migrate.test.ts` (migration; v8/v9 helper-btree-index SQL structural assertions + 1000-row wall-clock fixtures that guard the O(n²)→O(n log n) fix + v0.13.1 assertions on v12/v13 SQL shape, `sqlFor` + `transaction:false` runner semantics, and the `max_stalled DEFAULT 1` regression guard),
|
||||
`test/setup-branching.test.ts` (setup flow), `test/slug-validation.test.ts` (slug validation),
|
||||
`test/storage.test.ts` (storage backends), `test/supabase-admin.test.ts` (Supabase admin),
|
||||
`test/yaml-lite.test.ts` (YAML parsing), `test/check-update.test.ts` (version check + update CLI),
|
||||
@@ -360,15 +354,13 @@ parity), `test/cli.test.ts` (CLI structure), `test/config.test.ts` (config redac
|
||||
`test/skill-manifest.test.ts` (v0.19 skill manifest parser: drift detection, managed-block markers),
|
||||
`test/skillify-scaffold.test.ts` (v0.19 `gbrain skillify scaffold` stubs: SKILL.md, script, tests, routing-eval fixtures),
|
||||
`test/skillpack-install.test.ts` (v0.19 `gbrain skillpack install` managed-block install / update / no-clobber semantics),
|
||||
`test/skillpack-sync-guard.test.ts` (v0.19 sync-guard: bundled skills stay byte-identical to `skills/` source),
|
||||
`test/http-transport.test.ts` (v0.22.7 HTTP transport: 23 unit cases covering bearer auth + missing/no-Bearer/unknown/revoked + `/health` bypass, F1+F2 round-trip via dispatch.ts, F3 invalid_params, application/json response shape (not SSE), CORS default-deny + allowlist, body cap on Content-Length AND chunked, two-bucket rate limit (refill, exhaust+Retry-After, LRU eviction, TTL prune, pre-auth IP fires before DB), and `mcp_request_log` audit on success + auth_failed).
|
||||
`test/skillpack-sync-guard.test.ts` (v0.19 sync-guard: bundled skills stay byte-identical to `skills/` source).
|
||||
|
||||
E2E tests (`test/e2e/`): Run against real Postgres+pgvector. Require `DATABASE_URL`.
|
||||
- `bun run test:e2e` runs Tier 1 (mechanical, all operations, no API keys). Includes 9 dedicated cases for the postgres-engine `addLinksBatch` / `addTimelineEntriesBatch` bind path — postgres-js's `unnest()` binding is structurally different from PGLite's and gets its own coverage.
|
||||
- `test/e2e/search-quality.test.ts` runs search quality E2E against PGLite (no API keys, in-memory)
|
||||
- `test/e2e/graph-quality.test.ts` runs the v0.10.3 knowledge graph pipeline (auto-link via put_page, reconciliation, traversePaths) against PGLite in-memory
|
||||
- `test/e2e/postgres-jsonb.test.ts` — v0.12.2 regression test. Round-trips all 5 JSONB write sites (pages.frontmatter, raw_data.data, ingest_log.pages_updated, files.metadata, page_versions.frontmatter) against real Postgres and asserts `jsonb_typeof='object'` plus `->>'key'` returns the expected scalar. The test that should have caught the original double-encode bug.
|
||||
- `test/e2e/integrity-batch.test.ts` (v0.22.8) — parity tests for `scanIntegrity`'s batch-load fast path vs sequential. Four cases (dedup, hits, validate, topPages) seed a fixture and assert both paths return identical results. Dedup case uses raw SQL via `getConn().unsafe()` to seed a `(test-source-2, people/alice)` row alongside the default-source row, since `engine.putPage` doesn't take a `source_id`. Pins the codex-caught multi-source overcounting regression.
|
||||
- `test/e2e/jsonb-roundtrip.test.ts` — v0.12.3 companion regression against the 4 doctor-scanned JSONB sites. Assertion-level overlap with `postgres-jsonb.test.ts` is intentional defense-in-depth: if doctor's scan surface ever drifts from the actual write surface, one of these tests catches it.
|
||||
- `test/e2e/upgrade.test.ts` runs check-update E2E against real GitHub API (network required)
|
||||
- `test/e2e/minions-shell-pglite.test.ts` (v0.20.4) exercises the PGLite `--follow` inline shell-job path (in-memory, no `DATABASE_URL` required) — the path the consolidated minion-orchestrator skill documents for dev use
|
||||
@@ -376,8 +368,6 @@ E2E tests (`test/e2e/`): Run against real Postgres+pgvector. Require `DATABASE_U
|
||||
- `test/e2e/search-swamp.test.ts` (v0.22.0) — reproduces the headline source-swamp case. Seeds a curated `originals/talks/article-outline-fat-code` page against two `wintermute/chat/` pages stuffed with the same multi-word phrase. Asserts the article wins keyword AND vector ranking, that `detail=high` lets the chat swamp re-surface (temporal-query workflow preserved), and that `source_id` passes through the two-stage CTE intact. PGLite in-memory.
|
||||
- `test/e2e/search-exclude.test.ts` (v0.22.0) — verifies `test/` + `archive/` pages are hidden by default, that `include_slug_prefixes` opts back in, and that caller-supplied `exclude_slug_prefixes` adds to defaults. Both keyword and vector search paths covered.
|
||||
- `test/e2e/engine-parity.test.ts` (v0.22.0) — Postgres ↔ PGLite top-result and result-set parity for `searchKeyword` + `searchVector`. Codex flagged that Postgres ranks pages then picks best chunk while PGLite returns chunks directly — without parity coverage the source-boost fix could pass on PGLite and fail on Postgres. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- `test/e2e/postgres-bootstrap.test.ts` (v0.22.6.1) — exercises `PostgresEngine.initSchema()` directly against a fresh real Postgres database. Asserts the bootstrap path is no-op on fresh installs and that SCHEMA_SQL replays cleanly through the engine path (not via the standalone `db.initSchema` from `src/core/db.ts`, which would have produced false-positive coverage). Codex caught the E2E-shape gap during plan review.
|
||||
- `test/e2e/http-transport.test.ts` (v0.22.7) — 8 cases against real Postgres covering `gbrain serve --http` end-to-end: bearer auth round-trip, `last_used_at` SQL-level debounce semantics, `mcp_request_log` row insertion on success and auth_failed paths, `/health` DB-down → 503 (DB-probing health check), and the F1+F2+F3 dispatch round-trip with a real operation. Skips gracefully when `DATABASE_URL` is unset.
|
||||
- Tier 2 (`skills.test.ts`) requires OpenClaw + API keys, runs nightly in CI
|
||||
- If `.env.testing` doesn't exist in this directory, check sibling worktrees for one:
|
||||
`find ../ -maxdepth 2 -name .env.testing -print -quit` and copy it here if found.
|
||||
@@ -1358,13 +1348,12 @@ Add to `~/.claude/server.json` (Claude Code), Settings > MCP Servers (Cursor), o
|
||||
### Remote MCP (Claude Desktop, Cowork, Perplexity)
|
||||
|
||||
```bash
|
||||
gbrain auth create "claude-desktop" # tokens via the existing CLI
|
||||
gbrain serve --http --port 8787 # built-in HTTP transport (Postgres-only)
|
||||
ngrok http 8787 --url your-brain.ngrok.app # any tunnel works
|
||||
ngrok http 8787 --url your-brain.ngrok.app
|
||||
bun run src/commands/auth.ts create "claude-desktop"
|
||||
claude mcp add gbrain -t http https://your-brain.ngrok.app/mcp -H "Authorization: Bearer TOKEN"
|
||||
```
|
||||
|
||||
Per-client guides: [`docs/mcp/`](docs/mcp/DEPLOY.md). Hardening defaults, env vars, and threat model: [SECURITY.md](SECURITY.md). ChatGPT requires OAuth 2.1 (not yet implemented).
|
||||
Per-client guides: [`docs/mcp/`](docs/mcp/DEPLOY.md). ChatGPT requires OAuth 2.1 (not yet implemented).
|
||||
|
||||
### Using gbrain with GStack
|
||||
|
||||
@@ -1936,8 +1925,6 @@ ADMIN
|
||||
gbrain doctor --locks List idle-in-tx backends (57014 diagnostic, Postgres only)
|
||||
gbrain stats Brain statistics
|
||||
gbrain serve MCP server (stdio)
|
||||
gbrain serve --http --port 8787 MCP server (HTTP, Postgres-only, bearer auth)
|
||||
gbrain auth create|list|revoke|test Token management for the HTTP transport
|
||||
gbrain integrations Integration recipe dashboard
|
||||
gbrain sources list|add|remove|... Multi-source brain management (v0.18)
|
||||
gbrain dream [--dry-run] [--phase N] One maintenance cycle then exit (cron-friendly)
|
||||
@@ -4118,14 +4105,9 @@ Source: https://raw.githubusercontent.com/garrytan/gbrain/master/docs/mcp/DEPLOY
|
||||
|
||||
# Deploy GBrain Remote MCP Server
|
||||
|
||||
> **v0.22.7+:** Use `gbrain serve --http` for remote access. It includes built-in
|
||||
> bearer token auth, default-deny CORS, two-bucket rate limiting, body cap, and
|
||||
> per-request audit log. **Postgres-only** (PGLite is local-only by design).
|
||||
> See [SECURITY.md](../../SECURITY.md) for env vars and tunable defaults.
|
||||
|
||||
Access your brain from any device, any AI client. GBrain's MCP server runs locally
|
||||
via `gbrain serve` (stdio). For remote access, expose it via the built-in HTTP
|
||||
transport behind a public tunnel.
|
||||
via `gbrain serve` (stdio). For remote access, wrap it in an HTTP server behind a
|
||||
public tunnel.
|
||||
|
||||
## Two Paths
|
||||
|
||||
@@ -4136,23 +4118,21 @@ gbrain serve
|
||||
```
|
||||
|
||||
Works with Claude Code, Cursor, Windsurf, and any MCP client that supports stdio.
|
||||
No server, no tunnel, no token needed. Works on both PGLite and Postgres engines.
|
||||
No server, no tunnel, no token needed.
|
||||
|
||||
### Remote (any device, any AI client) — Postgres only
|
||||
### Remote (any device, any AI client)
|
||||
|
||||
```
|
||||
Your AI client (Claude Desktop, Perplexity, etc.)
|
||||
→ ngrok tunnel (https://YOUR-DOMAIN.ngrok.app)
|
||||
→ gbrain serve --http (built-in transport with bearer auth)
|
||||
→ Postgres (pooler connection or self-hosted)
|
||||
→ Your HTTP server (wraps gbrain serve)
|
||||
→ Supabase Postgres (via pooler connection string)
|
||||
```
|
||||
|
||||
This requires:
|
||||
1. A Postgres-backed brain (the `access_tokens` table only exists on Postgres;
|
||||
running `gbrain serve --http` against a PGLite install fails fast at startup)
|
||||
2. A machine running `gbrain serve --http`
|
||||
3. A public tunnel (ngrok, Tailscale, or cloud host)
|
||||
4. A bearer token created via `gbrain auth create <name>`
|
||||
1. A machine running `gbrain serve` behind an HTTP wrapper
|
||||
2. A public tunnel (ngrok, Tailscale, or cloud host)
|
||||
3. Bearer token auth for security
|
||||
|
||||
## Remote Setup
|
||||
|
||||
@@ -4171,13 +4151,13 @@ ngrok http 8787 --url your-brain.ngrok.app # Hobby tier for fixed domain
|
||||
|
||||
```bash
|
||||
# Create a token for each client
|
||||
gbrain auth create "claude-desktop"
|
||||
bun run src/commands/auth.ts create "claude-desktop"
|
||||
|
||||
# List all tokens
|
||||
gbrain auth list
|
||||
bun run src/commands/auth.ts list
|
||||
|
||||
# Revoke a token
|
||||
gbrain auth revoke "claude-desktop"
|
||||
bun run src/commands/auth.ts revoke "claude-desktop"
|
||||
```
|
||||
|
||||
Tokens are per-client. Create one for each device/app. Revoke individually
|
||||
@@ -4193,7 +4173,7 @@ if compromised. Tokens are stored SHA-256 hashed in your database.
|
||||
### 4. Verify
|
||||
|
||||
```bash
|
||||
gbrain auth test \
|
||||
bun run src/commands/auth.ts test \
|
||||
https://YOUR-DOMAIN.ngrok.app/mcp \
|
||||
--token YOUR_TOKEN
|
||||
```
|
||||
@@ -4221,7 +4201,7 @@ Funnel, and cloud hosts (Fly.io, Railway).
|
||||
Include the Authorization header: `Authorization: Bearer YOUR_TOKEN`
|
||||
|
||||
**"invalid_token" error**
|
||||
Run `gbrain auth list` to see active tokens.
|
||||
Run `bun run src/commands/auth.ts list` to see active tokens.
|
||||
|
||||
**"service_unavailable" error**
|
||||
Database connection failed. Check your Supabase dashboard for outages.
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.22.9",
|
||||
"version": "0.22.5",
|
||||
"description": "Postgres-native personal knowledge brain with hybrid RAG search",
|
||||
"type": "module",
|
||||
"main": "src/core/index.ts",
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Partition unit test files into N shards by stable hash and run one shard.
|
||||
#
|
||||
# Usage: scripts/test-shard.sh <shard-index> <total-shards>
|
||||
# shard-index: 1-based (1..N)
|
||||
# total-shards: positive integer
|
||||
#
|
||||
# E2E tests under test/e2e/ are excluded — they need DATABASE_URL and run via
|
||||
# bun run test:e2e separately.
|
||||
#
|
||||
# Stable partitioning: a file's shard is `(hash(path) % N) + 1`. Same file
|
||||
# lands in the same shard on every run, regardless of how many other files
|
||||
# exist, so retries are reproducible. Hash is FNV-1a — pure shell, no jq.
|
||||
set -euo pipefail
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "usage: scripts/test-shard.sh <shard-index> <total-shards>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SHARD_INDEX="$1"
|
||||
TOTAL_SHARDS="$2"
|
||||
|
||||
if ! [[ "$SHARD_INDEX" =~ ^[0-9]+$ ]] || ! [[ "$TOTAL_SHARDS" =~ ^[0-9]+$ ]]; then
|
||||
echo "error: shard index and total must be positive integers" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$SHARD_INDEX" -lt 1 ] || [ "$SHARD_INDEX" -gt "$TOTAL_SHARDS" ]; then
|
||||
echo "error: shard index $SHARD_INDEX out of range 1..$TOTAL_SHARDS" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Find all unit test files, deterministic order. Excludes test/e2e/.
|
||||
# Portable: avoid `mapfile` (bash 4+) so this runs on macOS bash 3.2 too.
|
||||
FILES=()
|
||||
while IFS= read -r line; do
|
||||
FILES+=("$line")
|
||||
done < <(find test -name '*.test.ts' -not -path 'test/e2e/*' | sort)
|
||||
|
||||
if [ "${#FILES[@]}" -eq 0 ]; then
|
||||
echo "no test files found under test/" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# FNV-1a 32-bit hash of a string — implemented in pure bash so we don't depend
|
||||
# on python/openssl/etc on the runner. Output is decimal.
|
||||
fnv1a() {
|
||||
local str="$1"
|
||||
local h=2166136261 # FNV offset basis
|
||||
local i ord
|
||||
for (( i=0; i<${#str}; i++ )); do
|
||||
ord=$(printf '%d' "'${str:$i:1}")
|
||||
h=$(( (h ^ ord) & 0xFFFFFFFF ))
|
||||
h=$(( (h * 16777619) & 0xFFFFFFFF ))
|
||||
done
|
||||
echo "$h"
|
||||
}
|
||||
|
||||
SHARD_FILES=()
|
||||
for f in "${FILES[@]}"; do
|
||||
hash=$(fnv1a "$f")
|
||||
bucket=$(( hash % TOTAL_SHARDS + 1 ))
|
||||
if [ "$bucket" -eq "$SHARD_INDEX" ]; then
|
||||
SHARD_FILES+=("$f")
|
||||
fi
|
||||
done
|
||||
|
||||
echo "shard $SHARD_INDEX/$TOTAL_SHARDS: ${#SHARD_FILES[@]}/${#FILES[@]} files"
|
||||
if [ "${#SHARD_FILES[@]}" -eq 0 ]; then
|
||||
echo "warning: shard $SHARD_INDEX has no files (rehash or reduce shard count)" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
exec bun test --timeout=60000 "${SHARD_FILES[@]}"
|
||||
+2
-7
@@ -19,7 +19,7 @@ for (const op of operations) {
|
||||
}
|
||||
|
||||
// CLI-only commands that bypass the operation layer
|
||||
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'sources', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'repos', 'code-def', 'code-refs', 'reindex-code', 'code-callers', 'code-callees', 'frontmatter', 'auth']);
|
||||
const CLI_ONLY = new Set(['init', 'upgrade', 'post-upgrade', 'check-update', 'integrations', 'publish', 'check-backlinks', 'lint', 'report', 'import', 'export', 'files', 'embed', 'serve', 'call', 'config', 'doctor', 'migrate', 'eval', 'sync', 'extract', 'features', 'autopilot', 'graph-query', 'jobs', 'agent', 'apply-migrations', 'skillpack-check', 'skillpack', 'resolvers', 'integrity', 'repair-jsonb', 'orphans', 'sources', 'dream', 'check-resolvable', 'routing-eval', 'skillify', 'smoke-test', 'repos', 'code-def', 'code-refs', 'reindex-code', 'code-callers', 'code-callees', 'frontmatter']);
|
||||
|
||||
async function main() {
|
||||
// Parse global flags (--quiet / --progress-json / --progress-interval)
|
||||
@@ -285,11 +285,6 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
await runIntegrations(args);
|
||||
return;
|
||||
}
|
||||
if (command === 'auth') {
|
||||
const { runAuth } = await import('./commands/auth.ts');
|
||||
await runAuth(args);
|
||||
return;
|
||||
}
|
||||
if (command === 'resolvers') {
|
||||
const { runResolvers } = await import('./commands/resolvers.ts');
|
||||
await runResolvers(args);
|
||||
@@ -452,7 +447,7 @@ async function handleCliOnly(command: string, args: string[]) {
|
||||
}
|
||||
case 'serve': {
|
||||
const { runServe } = await import('./commands/serve.ts');
|
||||
await runServe(engine, args);
|
||||
await runServe(engine);
|
||||
return; // serve doesn't disconnect
|
||||
}
|
||||
case 'call': {
|
||||
|
||||
+31
-52
@@ -1,29 +1,20 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* GBrain token management.
|
||||
* GBrain token management — standalone script, no gbrain CLI dependency.
|
||||
*
|
||||
* Wired into the CLI as of v0.22.5:
|
||||
* gbrain auth create "claude-desktop"
|
||||
* gbrain auth list
|
||||
* gbrain auth revoke "claude-desktop"
|
||||
* gbrain auth test <url> --token <token>
|
||||
*
|
||||
* Also runs standalone (no compiled binary required):
|
||||
* Usage:
|
||||
* DATABASE_URL=... bun run src/commands/auth.ts create "claude-desktop"
|
||||
*
|
||||
* Both paths require DATABASE_URL or GBRAIN_DATABASE_URL (except `test`,
|
||||
* which only hits the remote URL and doesn't need a local DB).
|
||||
* DATABASE_URL=... bun run src/commands/auth.ts list
|
||||
* DATABASE_URL=... bun run src/commands/auth.ts revoke "claude-desktop"
|
||||
* DATABASE_URL=... bun run src/commands/auth.ts test <url> --token <token>
|
||||
*/
|
||||
import postgres from 'postgres';
|
||||
import { createHash, randomBytes } from 'crypto';
|
||||
|
||||
function getDatabaseUrl(requireDb: boolean): string | undefined {
|
||||
const url = process.env.DATABASE_URL || process.env.GBRAIN_DATABASE_URL;
|
||||
if (!url && requireDb) {
|
||||
console.error('Set DATABASE_URL or GBRAIN_DATABASE_URL environment variable.');
|
||||
process.exit(1);
|
||||
}
|
||||
return url;
|
||||
const DATABASE_URL = process.env.DATABASE_URL || process.env.GBRAIN_DATABASE_URL;
|
||||
if (!DATABASE_URL && process.argv[2] !== 'test') {
|
||||
console.error('Set DATABASE_URL or GBRAIN_DATABASE_URL environment variable.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function hashToken(token: string): string {
|
||||
@@ -36,7 +27,7 @@ function generateToken(): string {
|
||||
|
||||
async function create(name: string) {
|
||||
if (!name) { console.error('Usage: auth create <name>'); process.exit(1); }
|
||||
const sql = postgres(getDatabaseUrl(true)!);
|
||||
const sql = postgres(DATABASE_URL!);
|
||||
const token = generateToken();
|
||||
const hash = hashToken(token);
|
||||
|
||||
@@ -62,7 +53,7 @@ async function create(name: string) {
|
||||
}
|
||||
|
||||
async function list() {
|
||||
const sql = postgres(getDatabaseUrl(true)!);
|
||||
const sql = postgres(DATABASE_URL!);
|
||||
try {
|
||||
const rows = await sql`
|
||||
SELECT name, created_at, last_used_at, revoked_at
|
||||
@@ -89,7 +80,7 @@ async function list() {
|
||||
|
||||
async function revoke(name: string) {
|
||||
if (!name) { console.error('Usage: auth revoke <name>'); process.exit(1); }
|
||||
const sql = postgres(getDatabaseUrl(true)!);
|
||||
const sql = postgres(DATABASE_URL!);
|
||||
try {
|
||||
const result = await sql`
|
||||
UPDATE access_tokens SET revoked_at = now()
|
||||
@@ -225,38 +216,26 @@ async function test(url: string, token: string) {
|
||||
console.log(`\n🧠 Your brain is live! (${elapsed}s)`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Entry point for the `gbrain auth` CLI subcommand. Also reused by the
|
||||
* direct-script path (see bottom of file) so `bun run src/commands/auth.ts`
|
||||
* still works.
|
||||
*/
|
||||
export async function runAuth(args: string[]): Promise<void> {
|
||||
const [cmd, ...rest] = args;
|
||||
switch (cmd) {
|
||||
case 'create': await create(rest[0]); return;
|
||||
case 'list': await list(); return;
|
||||
case 'revoke': await revoke(rest[0]); return;
|
||||
case 'test': {
|
||||
const tokenIdx = rest.indexOf('--token');
|
||||
const url = rest.find(a => !a.startsWith('--') && a !== rest[tokenIdx + 1]);
|
||||
const token = tokenIdx >= 0 ? rest[tokenIdx + 1] : '';
|
||||
await test(url || '', token || '');
|
||||
return;
|
||||
}
|
||||
default:
|
||||
console.log(`GBrain Token Management
|
||||
// CLI dispatch
|
||||
const [cmd, ...args] = process.argv.slice(2);
|
||||
switch (cmd) {
|
||||
case 'create': await create(args[0]); break;
|
||||
case 'list': await list(); break;
|
||||
case 'revoke': await revoke(args[0]); break;
|
||||
case 'test': {
|
||||
const tokenIdx = args.indexOf('--token');
|
||||
const url = args.find(a => !a.startsWith('--') && a !== args[tokenIdx + 1]);
|
||||
const token = tokenIdx >= 0 ? args[tokenIdx + 1] : '';
|
||||
await test(url || '', token || '');
|
||||
break;
|
||||
}
|
||||
default:
|
||||
console.log(`GBrain Token Management
|
||||
|
||||
Usage:
|
||||
gbrain auth create <name> Create a new access token
|
||||
gbrain auth list List all tokens
|
||||
gbrain auth revoke <name> Revoke a token
|
||||
gbrain auth test <url> --token <t> Smoke-test a remote MCP server
|
||||
bun run src/commands/auth.ts create <name> Create a new access token
|
||||
bun run src/commands/auth.ts list List all tokens
|
||||
bun run src/commands/auth.ts revoke <name> Revoke a token
|
||||
bun run src/commands/auth.ts test <url> --token <token> Smoke test a remote MCP server
|
||||
`);
|
||||
}
|
||||
}
|
||||
|
||||
// Direct-script entry point — only runs when this file is invoked as the main module
|
||||
// (e.g. `bun run src/commands/auth.ts ...`). When imported by cli.ts, this block is skipped.
|
||||
if (import.meta.main) {
|
||||
await runAuth(process.argv.slice(2));
|
||||
}
|
||||
|
||||
@@ -249,29 +249,25 @@ export async function runDoctor(engine: BrainEngine | null, args: string[], dbSo
|
||||
// Without this doctor check, users see "sync blocked" and have no
|
||||
// surface showing which files to fix.
|
||||
try {
|
||||
const { unacknowledgedSyncFailures, loadSyncFailures, summarizeFailuresByCode } = await import('../core/sync.ts');
|
||||
const { unacknowledgedSyncFailures, loadSyncFailures } = await import('../core/sync.ts');
|
||||
const unacked = unacknowledgedSyncFailures();
|
||||
const all = loadSyncFailures();
|
||||
if (unacked.length > 0) {
|
||||
const codeSummary = summarizeFailuresByCode(unacked);
|
||||
const codeBreakdown = codeSummary.map(s => `${s.code}=${s.count}`).join(', ');
|
||||
const preview = unacked.slice(0, 3).map(f => `${f.path} (${f.error.slice(0, 60)})`).join('; ');
|
||||
checks.push({
|
||||
name: 'sync_failures',
|
||||
status: 'warn',
|
||||
message:
|
||||
`${unacked.length} unacknowledged sync failure(s) [${codeBreakdown}]. ${preview}` +
|
||||
`${unacked.length} unacknowledged sync failure(s). ${preview}` +
|
||||
`${unacked.length > 3 ? `, and ${unacked.length - 3} more` : ''}. ` +
|
||||
`Fix the file(s) and re-run 'gbrain sync', or use 'gbrain sync --skip-failed' to acknowledge.`,
|
||||
});
|
||||
} else if (all.length > 0) {
|
||||
// Acknowledged-only: show code breakdown for visibility.
|
||||
const ackedSummary = summarizeFailuresByCode(all);
|
||||
const ackedBreakdown = ackedSummary.map(s => `${s.code}=${s.count}`).join(', ');
|
||||
// Acknowledged-only: informational, not a warning.
|
||||
checks.push({
|
||||
name: 'sync_failures',
|
||||
status: 'ok',
|
||||
message: `${all.length} historical sync failure(s), all acknowledged [${ackedBreakdown}].`,
|
||||
message: `${all.length} historical sync failure(s), all acknowledged.`,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
|
||||
@@ -31,7 +31,6 @@ import { join, dirname } from 'path';
|
||||
import { loadConfig, toEngineConfig } from '../core/config.ts';
|
||||
import { createEngine } from '../core/engine-factory.ts';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import * as db from '../core/db.ts';
|
||||
import { BrainWriter } from '../core/output/writer.ts';
|
||||
import {
|
||||
getDefaultRegistry,
|
||||
@@ -267,12 +266,6 @@ export interface IntegrityScanOptions {
|
||||
limit?: number;
|
||||
/** Slug prefix filter (e.g. "people") — matches slugs starting with `${typeFilter}/`. */
|
||||
typeFilter?: string;
|
||||
/**
|
||||
* When true (default), batch-load pages via a single SQL query instead of
|
||||
* sequential getPage() calls. Falls back to sequential on error (e.g. PGLite).
|
||||
* Eliminates 500 round-trips through PgBouncer that caused doctor timeouts.
|
||||
*/
|
||||
batchLoad?: boolean;
|
||||
}
|
||||
|
||||
export interface IntegrityScanResult {
|
||||
@@ -294,29 +287,7 @@ export async function scanIntegrity(
|
||||
engine: BrainEngine,
|
||||
opts: IntegrityScanOptions = {},
|
||||
): Promise<IntegrityScanResult> {
|
||||
const { limit = Infinity, typeFilter, batchLoad = true } = opts;
|
||||
|
||||
// Fast path: single SQL query instead of N sequential getPage() calls.
|
||||
// Eliminates ~500 round-trips through PgBouncer that caused doctor to
|
||||
// timeout on transaction-mode pooling. Postgres-only: PGLite has no
|
||||
// postgres.js connection, so the gate keeps the GBRAIN_DEBUG fallback
|
||||
// log clean for real Postgres errors instead of expected PGLite skips.
|
||||
if (batchLoad && limit !== Infinity && engine.kind === 'postgres') {
|
||||
try {
|
||||
return await scanIntegrityBatch(limit, typeFilter);
|
||||
} catch (err) {
|
||||
// GBRAIN_DEBUG=1 surfaces real Postgres errors (deadlock, connection
|
||||
// drop, SQL bug) that would otherwise vanish into the sequential
|
||||
// fallback. Quiet by default since the fallback is harmless.
|
||||
if (process.env.GBRAIN_DEBUG) {
|
||||
console.error(
|
||||
'[integrity] batch path failed, falling back to sequential:',
|
||||
err instanceof Error ? err.message : err,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const { limit = Infinity, typeFilter } = opts;
|
||||
const allSlugs = [...(await engine.getAllSlugs())].sort();
|
||||
|
||||
const bareHits: BareTweetHit[] = [];
|
||||
@@ -345,52 +316,6 @@ export async function scanIntegrity(
|
||||
return { pagesScanned, bareHits, externalHits, topPages };
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch-load integrity scan: fetches all candidate pages in a single SQL
|
||||
* query, then scans in-memory. Reduces PgBouncer round-trips from ~500 to 1.
|
||||
*/
|
||||
async function scanIntegrityBatch(
|
||||
limit: number,
|
||||
typeFilter?: string,
|
||||
): Promise<IntegrityScanResult> {
|
||||
const sql = db.getConnection();
|
||||
const typeCondition = typeFilter ? sql`AND slug LIKE ${typeFilter + '/%'}` : sql``;
|
||||
// Boolean validate is the documented contract; stringly-typed 'false' (quoted
|
||||
// YAML) diverges from the sequential path's strict === false check. Intentional
|
||||
// — gbrain lint should reject stringly-typed validate at write time.
|
||||
const validateCondition = sql`AND (frontmatter->>'validate' IS NULL OR frontmatter->>'validate' != 'false')`;
|
||||
|
||||
// DISTINCT ON (slug) mirrors getAllSlugs()'s Set<string> semantics: multi-source
|
||||
// brains can have the same slug under multiple source_ids (UNIQUE(source_id, slug)
|
||||
// since v0.18.0); we want one scan per slug, not one per row.
|
||||
const rows = await sql`
|
||||
SELECT DISTINCT ON (slug) slug, compiled_truth, frontmatter
|
||||
FROM pages
|
||||
WHERE 1=1 ${typeCondition} ${validateCondition}
|
||||
ORDER BY slug
|
||||
LIMIT ${limit}
|
||||
`;
|
||||
|
||||
const bareHits: BareTweetHit[] = [];
|
||||
const externalHits: ExternalLinkHit[] = [];
|
||||
|
||||
for (const row of rows) {
|
||||
const slug = row.slug as string;
|
||||
const compiledTruth = row.compiled_truth as string;
|
||||
bareHits.push(...findBareTweetHits(compiledTruth, slug));
|
||||
externalHits.push(...findExternalLinks(compiledTruth, slug));
|
||||
}
|
||||
|
||||
const byPage = new Map<string, number>();
|
||||
for (const h of bareHits) byPage.set(h.slug, (byPage.get(h.slug) ?? 0) + 1);
|
||||
const topPages = [...byPage.entries()]
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 10)
|
||||
.map(([slug, count]) => ({ slug, count }));
|
||||
|
||||
return { pagesScanned: rows.length, bareHits, externalHits, topPages };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// auto — three-bucket repair
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
+3
-15
@@ -1,19 +1,7 @@
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { startMcpServer } from '../mcp/server.ts';
|
||||
import { startHttpTransport } from '../mcp/http-transport.ts';
|
||||
|
||||
export async function runServe(engine: BrainEngine, args: string[] = []) {
|
||||
const useHttp = args.includes('--http');
|
||||
const portIdx = args.indexOf('--port');
|
||||
const port = portIdx >= 0 ? parseInt(args[portIdx + 1]) || 8787 : 8787;
|
||||
|
||||
if (useHttp) {
|
||||
console.error(`Starting GBrain MCP server (HTTP on port ${port})...`);
|
||||
await startHttpTransport({ port, engine });
|
||||
// Keep alive
|
||||
await new Promise(() => {});
|
||||
} else {
|
||||
console.error('Starting GBrain MCP server (stdio)...');
|
||||
await startMcpServer(engine);
|
||||
}
|
||||
export async function runServe(engine: BrainEngine) {
|
||||
console.error('Starting GBrain MCP server (stdio)...');
|
||||
await startMcpServer(engine);
|
||||
}
|
||||
|
||||
+5
-20
@@ -12,7 +12,6 @@ import {
|
||||
recordSyncFailures,
|
||||
unacknowledgedSyncFailures,
|
||||
acknowledgeSyncFailures,
|
||||
formatCodeBreakdown,
|
||||
} from '../core/sync.ts';
|
||||
import { estimateTokens, CHUNKER_VERSION } from '../core/chunkers/code.ts';
|
||||
import { EMBEDDING_MODEL, estimateEmbeddingCostUsd } from '../core/embedding.ts';
|
||||
@@ -523,13 +522,9 @@ export async function performSync(engine: BrainEngine, opts: SyncOpts): Promise<
|
||||
// current set, --retry-failed re-parses before running the normal sync.
|
||||
if (failedFiles.length > 0) {
|
||||
recordSyncFailures(failedFiles, headCommit);
|
||||
// Emit structured summary grouped by error code so the operator
|
||||
// can see *why* files failed, not just how many.
|
||||
const codeBreakdown = formatCodeBreakdown(failedFiles);
|
||||
if (!opts.skipFailed) {
|
||||
console.error(
|
||||
`\nSync blocked: ${failedFiles.length} file(s) failed to parse:\n` +
|
||||
`${codeBreakdown}\n\n` +
|
||||
`\nSync blocked: ${failedFiles.length} file(s) failed to parse. ` +
|
||||
`Fix the YAML frontmatter in the files above and re-run, or use ` +
|
||||
`'gbrain sync --skip-failed' to acknowledge and move on.`,
|
||||
);
|
||||
@@ -552,11 +547,8 @@ export async function performSync(engine: BrainEngine, opts: SyncOpts): Promise<
|
||||
}
|
||||
// --skip-failed: acknowledge the now-recorded set and proceed.
|
||||
const acked = acknowledgeSyncFailures();
|
||||
if (acked.count > 0) {
|
||||
console.error(
|
||||
` Acknowledged ${acked.count} failure(s) and advancing past them:\n` +
|
||||
`${formatCodeBreakdown(acked.summary)}`,
|
||||
);
|
||||
if (acked > 0) {
|
||||
console.error(` Acknowledged ${acked} failure(s) and advancing past them.`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -664,11 +656,9 @@ async function performFullSync(
|
||||
// the sync module owns the last_commit write. Respect the same gate.
|
||||
if (result.failures.length > 0) {
|
||||
recordSyncFailures(result.failures, headCommit);
|
||||
const codeBreakdown = formatCodeBreakdown(result.failures);
|
||||
if (!opts.skipFailed) {
|
||||
console.error(
|
||||
`\nFull sync blocked: ${result.failures.length} file(s) failed:\n` +
|
||||
`${codeBreakdown}\n\n` +
|
||||
`\nFull sync blocked: ${result.failures.length} file(s) failed. ` +
|
||||
`Fix the YAML in those files and re-run, or use '--skip-failed'.`,
|
||||
);
|
||||
await engine.setConfig('sync.last_run', new Date().toISOString());
|
||||
@@ -685,12 +675,7 @@ async function performFullSync(
|
||||
};
|
||||
}
|
||||
const acked = acknowledgeSyncFailures();
|
||||
if (acked.count > 0) {
|
||||
console.error(
|
||||
` Acknowledged ${acked.count} failure(s) and advancing past them:\n` +
|
||||
`${formatCodeBreakdown(acked.summary)}`,
|
||||
);
|
||||
}
|
||||
if (acked > 0) console.error(` Acknowledged ${acked} failure(s) and advancing past them.`);
|
||||
}
|
||||
|
||||
// Persist sync state so next sync is incremental (C1 fix: was missing).
|
||||
|
||||
@@ -2,7 +2,6 @@ import postgres from 'postgres';
|
||||
import { GBrainError, type EngineConfig } from './types.ts';
|
||||
import { SCHEMA_SQL } from './schema-embedded.ts';
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
import { verifySchema } from './schema-verify.ts';
|
||||
|
||||
let sql: ReturnType<typeof postgres> | null = null;
|
||||
let connectedUrl: string | null = null;
|
||||
@@ -238,8 +237,6 @@ export async function initSchema(): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
export { verifySchema } from './schema-verify.ts';
|
||||
|
||||
export async function withTransaction<T>(fn: (tx: ReturnType<typeof postgres>) => Promise<T>): Promise<T> {
|
||||
const conn = getConnection();
|
||||
return conn.begin(async (tx) => {
|
||||
|
||||
@@ -811,14 +811,6 @@ export const MIGRATIONS: Migration[] = [
|
||||
RAISE NOTICE 'v24: RLS backfill complete (role % has BYPASSRLS)', current_user;
|
||||
END $$;
|
||||
`,
|
||||
// PGLite has no RLS engine and is intrinsically single-tenant (local file).
|
||||
// The 8 ALTER TABLE ... ENABLE ROW LEVEL SECURITY statements above also
|
||||
// target tables that may not exist on PGLite (subagent_*, minion_inbox),
|
||||
// since pglite-schema.ts is the canonical PGLite schema source. No-op
|
||||
// override keeps PGLite upgrades unwedged and the version bump intact.
|
||||
sqlFor: {
|
||||
pglite: '',
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 25,
|
||||
|
||||
@@ -86,16 +86,6 @@ export class PGLiteEngine implements BrainEngine {
|
||||
}
|
||||
|
||||
async initSchema(): Promise<void> {
|
||||
// Pre-schema bootstrap: add forward-referenced state the embedded schema
|
||||
// blob requires but that older brains don't have yet. Without this, a
|
||||
// pre-v0.18 brain hits `CREATE INDEX idx_pages_source_id ON pages(source_id)`
|
||||
// (issues #366/#375/#378/#396) or a pre-v0.13 brain hits
|
||||
// `CREATE INDEX idx_links_source ON links(link_source)` (#266/#357), and
|
||||
// initSchema crashes before runMigrations gets a chance to apply the
|
||||
// missing column. Bootstrap is structurally idempotent and a no-op on
|
||||
// fresh installs and modern brains.
|
||||
await this.applyForwardReferenceBootstrap();
|
||||
|
||||
await this.db.exec(PGLITE_SCHEMA_SQL);
|
||||
|
||||
const { applied } = await runMigrations(this);
|
||||
@@ -104,111 +94,6 @@ export class PGLiteEngine implements BrainEngine {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bootstrap state that PGLITE_SCHEMA_SQL forward-references but that older
|
||||
* brains don't have yet. Currently covers:
|
||||
*
|
||||
* - `sources` table + default seed (FK target of pages.source_id) — v0.18
|
||||
* - `pages.source_id` column (indexed by `idx_pages_source_id`) — v0.18
|
||||
* - `links.link_source` column (indexed by `idx_links_source`) — v0.13
|
||||
* - `links.origin_page_id` column (indexed by `idx_links_origin`) — v0.13
|
||||
* - `content_chunks.symbol_name` column (indexed by `idx_chunks_symbol_name`) — v0.19
|
||||
* - `content_chunks.language` column (indexed by `idx_chunks_language`) — v0.19
|
||||
*
|
||||
* **Maintenance contract:** when a future migration adds a column-with-index
|
||||
* or new-table-with-FK referenced by PGLITE_SCHEMA_SQL, extend this method
|
||||
* AND `test/schema-bootstrap-coverage.test.ts`'s `REQUIRED_BOOTSTRAP_COVERAGE`.
|
||||
* The coverage test fails loudly if the bootstrap drifts behind the schema.
|
||||
*/
|
||||
private async applyForwardReferenceBootstrap(): Promise<void> {
|
||||
// Single round-trip probe for every forward-reference target.
|
||||
const { rows } = await this.db.query(`
|
||||
SELECT
|
||||
EXISTS (SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema='public' AND table_name='pages') AS pages_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema='public' AND table_name='pages' AND column_name='source_id') AS source_id_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema='public' AND table_name='links') AS links_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema='public' AND table_name='links' AND column_name='link_source') AS link_source_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema='public' AND table_name='links' AND column_name='origin_page_id') AS origin_page_id_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema='public' AND table_name='content_chunks') AS chunks_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema='public' AND table_name='content_chunks' AND column_name='symbol_name') AS symbol_name_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema='public' AND table_name='content_chunks' AND column_name='language') AS language_exists
|
||||
`);
|
||||
const probe = rows[0] as {
|
||||
pages_exists: boolean;
|
||||
source_id_exists: boolean;
|
||||
links_exists: boolean;
|
||||
link_source_exists: boolean;
|
||||
origin_page_id_exists: boolean;
|
||||
chunks_exists: boolean;
|
||||
symbol_name_exists: boolean;
|
||||
language_exists: boolean;
|
||||
};
|
||||
|
||||
const needsPagesBootstrap = probe.pages_exists && !probe.source_id_exists;
|
||||
const needsLinksBootstrap = probe.links_exists
|
||||
&& (!probe.link_source_exists || !probe.origin_page_id_exists);
|
||||
const needsChunksBootstrap = probe.chunks_exists
|
||||
&& (!probe.symbol_name_exists || !probe.language_exists);
|
||||
|
||||
// Fresh installs (no tables yet) and modern brains both no-op.
|
||||
if (!needsPagesBootstrap && !needsLinksBootstrap && !needsChunksBootstrap) return;
|
||||
|
||||
console.log(' Pre-v0.21 brain detected, applying forward-reference bootstrap');
|
||||
|
||||
if (needsPagesBootstrap) {
|
||||
// Mirror schema-embedded.ts shape for `sources` so the subsequent
|
||||
// PGLITE_SCHEMA_SQL CREATE TABLE IF NOT EXISTS is a true no-op.
|
||||
await this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS sources (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
local_path TEXT,
|
||||
last_commit TEXT,
|
||||
last_sync_at TIMESTAMPTZ,
|
||||
config JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
INSERT INTO sources (id, name, config)
|
||||
VALUES ('default', 'default', '{"federated": true}'::jsonb)
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
ALTER TABLE pages ADD COLUMN IF NOT EXISTS source_id TEXT
|
||||
NOT NULL DEFAULT 'default' REFERENCES sources(id) ON DELETE CASCADE;
|
||||
`);
|
||||
}
|
||||
|
||||
if (needsLinksBootstrap) {
|
||||
// v11 (links_provenance_columns) is responsible for the CHECK constraint
|
||||
// and backfill. The bootstrap only adds enough state for SCHEMA_SQL's
|
||||
// `CREATE INDEX idx_links_source/origin` not to crash. v11 runs later
|
||||
// via runMigrations and is idempotent (`IF NOT EXISTS` everywhere).
|
||||
await this.db.exec(`
|
||||
ALTER TABLE links ADD COLUMN IF NOT EXISTS link_source TEXT;
|
||||
ALTER TABLE links ADD COLUMN IF NOT EXISTS origin_page_id INTEGER
|
||||
REFERENCES pages(id) ON DELETE SET NULL;
|
||||
`);
|
||||
}
|
||||
|
||||
if (needsChunksBootstrap) {
|
||||
// v26 (content_chunks_code_metadata) adds the full code-chunk metadata
|
||||
// surface (language, symbol_name, symbol_type, start_line, end_line).
|
||||
// The bootstrap only adds the two columns the schema blob's partial
|
||||
// indexes reference (idx_chunks_symbol_name, idx_chunks_language).
|
||||
// v26 runs later via runMigrations and adds the rest idempotently.
|
||||
await this.db.exec(`
|
||||
ALTER TABLE content_chunks ADD COLUMN IF NOT EXISTS language TEXT;
|
||||
ALTER TABLE content_chunks ADD COLUMN IF NOT EXISTS symbol_name TEXT;
|
||||
`);
|
||||
}
|
||||
}
|
||||
|
||||
async withReservedConnection<T>(fn: (conn: ReservedConnection) => Promise<T>): Promise<T> {
|
||||
// PGLite has no connection pool. The single backing connection is
|
||||
// always effectively reserved — pass it through.
|
||||
|
||||
+1
-134
@@ -3,7 +3,6 @@ import type { BrainEngine, LinkBatchInput, TimelineBatchInput, ReservedConnectio
|
||||
import { MAX_SEARCH_LIMIT, clampSearchLimit } from './engine.ts';
|
||||
import { runMigrations } from './migrate.ts';
|
||||
import { SCHEMA_SQL } from './schema-embedded.ts';
|
||||
import { verifySchema } from './schema-verify.ts';
|
||||
import type {
|
||||
Page, PageInput, PageFilters, PageType,
|
||||
Chunk, ChunkInput, StaleChunkRow,
|
||||
@@ -99,26 +98,9 @@ export class PostgresEngine implements BrainEngine {
|
||||
async initSchema(): Promise<void> {
|
||||
const conn = this.sql;
|
||||
// Advisory lock prevents concurrent initSchema() calls from deadlocking
|
||||
// on DDL statements (DROP TRIGGER + CREATE TRIGGER acquire AccessExclusiveLock).
|
||||
//
|
||||
// Honest limitation: pg_advisory_lock(42) is session-scoped to this pooled
|
||||
// connection. runMigrations() below uses engine.transaction() and
|
||||
// withReservedConnection() which may hop to a different backend in the
|
||||
// pool. Cross-process serialization of initSchema is best-effort, not a
|
||||
// correctness guarantee. Pre-existing concern; the bootstrap doesn't
|
||||
// change it.
|
||||
// on DDL statements (DROP TRIGGER + CREATE TRIGGER acquire AccessExclusiveLock)
|
||||
await conn`SELECT pg_advisory_lock(42)`;
|
||||
try {
|
||||
// Pre-schema bootstrap: add forward-referenced state the embedded schema
|
||||
// blob requires but that older brains don't have yet. Without this, a
|
||||
// pre-v0.18 brain hits `CREATE INDEX idx_pages_source_id ON pages(source_id)`
|
||||
// (issues #366/#375/#378/#396), or a pre-v0.13 brain hits
|
||||
// `CREATE INDEX idx_links_source ON links(link_source)` (#266/#357), and
|
||||
// SCHEMA_SQL crashes before runMigrations gets a chance to apply the
|
||||
// missing column. Bootstrap is structurally idempotent and a no-op on
|
||||
// fresh installs and modern brains.
|
||||
await this.applyForwardReferenceBootstrap();
|
||||
|
||||
await conn.unsafe(SCHEMA_SQL);
|
||||
|
||||
// Run any pending migrations automatically
|
||||
@@ -126,126 +108,11 @@ export class PostgresEngine implements BrainEngine {
|
||||
if (applied > 0) {
|
||||
console.log(` ${applied} migration(s) applied`);
|
||||
}
|
||||
|
||||
// Post-migration schema verification: catches columns that migrations
|
||||
// defined but PgBouncer transaction-mode silently failed to create.
|
||||
// Self-heals missing columns via ALTER TABLE ADD COLUMN IF NOT EXISTS.
|
||||
const verify = await verifySchema(this);
|
||||
if (verify.healed.length > 0) {
|
||||
console.log(` Schema verify: self-healed ${verify.healed.length} missing column(s)`);
|
||||
}
|
||||
} finally {
|
||||
await conn`SELECT pg_advisory_unlock(42)`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bootstrap state that SCHEMA_SQL forward-references but that older brains
|
||||
* don't have yet. Mirror of `PGLiteEngine#applyForwardReferenceBootstrap`
|
||||
* in shape and intent. Currently covers:
|
||||
*
|
||||
* - `sources` table + default seed (FK target of pages.source_id) — v0.18
|
||||
* - `pages.source_id` column (indexed by `idx_pages_source_id`) — v0.18
|
||||
* - `links.link_source` column (indexed by `idx_links_source`) — v0.13
|
||||
* - `links.origin_page_id` column (indexed by `idx_links_origin`) — v0.13
|
||||
* - `content_chunks.symbol_name` column (indexed by `idx_chunks_symbol_name`) — v0.19
|
||||
* - `content_chunks.language` column (indexed by `idx_chunks_language`) — v0.19
|
||||
*
|
||||
* Keep this in sync with the PGLite version; covered by
|
||||
* `test/schema-bootstrap-coverage.test.ts` (PGLite side) and
|
||||
* `test/e2e/postgres-bootstrap.test.ts` (Postgres side).
|
||||
*/
|
||||
private async applyForwardReferenceBootstrap(): Promise<void> {
|
||||
const conn = this.sql;
|
||||
|
||||
// Single round-trip probe for every forward-reference target.
|
||||
// current_schema() resolves to whatever search_path the connection uses,
|
||||
// which matches schema-embedded.ts's `public.` references.
|
||||
const probeRows = await conn<{
|
||||
pages_exists: boolean;
|
||||
source_id_exists: boolean;
|
||||
links_exists: boolean;
|
||||
link_source_exists: boolean;
|
||||
origin_page_id_exists: boolean;
|
||||
chunks_exists: boolean;
|
||||
symbol_name_exists: boolean;
|
||||
language_exists: boolean;
|
||||
}[]>`
|
||||
SELECT
|
||||
EXISTS (SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = current_schema() AND table_name = 'pages') AS pages_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = 'pages' AND column_name = 'source_id') AS source_id_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = current_schema() AND table_name = 'links') AS links_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = 'links' AND column_name = 'link_source') AS link_source_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = 'links' AND column_name = 'origin_page_id') AS origin_page_id_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = current_schema() AND table_name = 'content_chunks') AS chunks_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = 'content_chunks' AND column_name = 'symbol_name') AS symbol_name_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = 'content_chunks' AND column_name = 'language') AS language_exists
|
||||
`;
|
||||
const probe = probeRows[0]!;
|
||||
|
||||
const needsPagesBootstrap = probe.pages_exists && !probe.source_id_exists;
|
||||
const needsLinksBootstrap = probe.links_exists
|
||||
&& (!probe.link_source_exists || !probe.origin_page_id_exists);
|
||||
const needsChunksBootstrap = probe.chunks_exists
|
||||
&& (!probe.symbol_name_exists || !probe.language_exists);
|
||||
|
||||
if (!needsPagesBootstrap && !needsLinksBootstrap && !needsChunksBootstrap) return;
|
||||
|
||||
console.log(' Pre-v0.21 brain detected, applying forward-reference bootstrap');
|
||||
|
||||
if (needsPagesBootstrap) {
|
||||
// Mirror schema-embedded.ts's `sources` shape so the subsequent
|
||||
// SCHEMA_SQL CREATE TABLE IF NOT EXISTS is a true no-op.
|
||||
await conn.unsafe(`
|
||||
CREATE TABLE IF NOT EXISTS sources (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
local_path TEXT,
|
||||
last_commit TEXT,
|
||||
last_sync_at TIMESTAMPTZ,
|
||||
config JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
INSERT INTO sources (id, name, config)
|
||||
VALUES ('default', 'default', '{"federated": true}'::jsonb)
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
ALTER TABLE pages ADD COLUMN IF NOT EXISTS source_id TEXT
|
||||
NOT NULL DEFAULT 'default' REFERENCES sources(id) ON DELETE CASCADE;
|
||||
`);
|
||||
}
|
||||
|
||||
if (needsLinksBootstrap) {
|
||||
// v11 (links_provenance_columns) handles the CHECK constraint, the
|
||||
// UNIQUE swap, and the backfill. The bootstrap only adds enough state
|
||||
// for SCHEMA_SQL's `CREATE INDEX idx_links_source/origin` not to crash.
|
||||
// v11 runs later via runMigrations and is idempotent.
|
||||
await conn.unsafe(`
|
||||
ALTER TABLE links ADD COLUMN IF NOT EXISTS link_source TEXT;
|
||||
ALTER TABLE links ADD COLUMN IF NOT EXISTS origin_page_id INTEGER
|
||||
REFERENCES pages(id) ON DELETE SET NULL;
|
||||
`);
|
||||
}
|
||||
|
||||
if (needsChunksBootstrap) {
|
||||
// v26 (content_chunks_code_metadata) adds the full code-chunk metadata
|
||||
// surface. The bootstrap only adds the two columns the schema blob's
|
||||
// partial indexes reference (idx_chunks_symbol_name, idx_chunks_language).
|
||||
// v26 runs later via runMigrations and adds the rest idempotently.
|
||||
await conn.unsafe(`
|
||||
ALTER TABLE content_chunks ADD COLUMN IF NOT EXISTS language TEXT;
|
||||
ALTER TABLE content_chunks ADD COLUMN IF NOT EXISTS symbol_name TEXT;
|
||||
`);
|
||||
}
|
||||
}
|
||||
|
||||
async transaction<T>(fn: (engine: BrainEngine) => Promise<T>): Promise<T> {
|
||||
const conn = this._sql || db.getConnection();
|
||||
return conn.begin(async (tx) => {
|
||||
|
||||
@@ -1,282 +0,0 @@
|
||||
/**
|
||||
* Post-migration schema verification with self-healing.
|
||||
*
|
||||
* PgBouncer transaction-mode poolers can silently swallow ALTER TABLE
|
||||
* statements: the SQL doesn't error, but the column never gets created.
|
||||
* The migration system increments the schema version counter anyway, so
|
||||
* gbrain thinks it's on v29 but the actual table is missing columns.
|
||||
*
|
||||
* This module parses the canonical CREATE TABLE definitions in
|
||||
* schema-embedded.ts and diffs them against information_schema.columns.
|
||||
* Missing columns are self-healed via ALTER TABLE ADD COLUMN IF NOT EXISTS.
|
||||
*
|
||||
* Called at the end of initSchema(), after all migrations complete.
|
||||
*/
|
||||
|
||||
import { SCHEMA_SQL } from './schema-embedded.ts';
|
||||
import type { BrainEngine } from './engine.ts';
|
||||
|
||||
/** A column expected to exist in the database. */
|
||||
export interface ExpectedColumn {
|
||||
table: string;
|
||||
column: string;
|
||||
/** The full column definition (type + constraints) from the CREATE TABLE. */
|
||||
definition: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse CREATE TABLE statements from SCHEMA_SQL to extract expected columns.
|
||||
*
|
||||
* This is a best-effort parser that handles the gbrain schema conventions:
|
||||
* - Standard column definitions with types and constraints
|
||||
* - Skips CONSTRAINT lines, CHECK lines, and UNIQUE lines
|
||||
* - Handles multi-line definitions
|
||||
*
|
||||
* Returns only tables and columns — not constraints, indexes, or triggers.
|
||||
*/
|
||||
export function parseExpectedColumns(): ExpectedColumn[] {
|
||||
const results: ExpectedColumn[] = [];
|
||||
|
||||
// Match CREATE TABLE IF NOT EXISTS <name> ( ... );
|
||||
const tableRegex = /CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\s+(\w+)\s*\(([\s\S]*?)\);/gi;
|
||||
|
||||
const SQL_KEYWORDS = new Set(['constraint', 'unique', 'check', 'primary', 'foreign', 'exclude']);
|
||||
|
||||
function processLine(tableName: string, line: string) {
|
||||
line = line.trim().replace(/,\s*$/, '');
|
||||
if (!line) return;
|
||||
|
||||
// Skip CONSTRAINT, UNIQUE, CHECK, PRIMARY KEY lines
|
||||
if (/^\s*(CONSTRAINT|UNIQUE|CHECK|PRIMARY\s+KEY)/i.test(line)) return;
|
||||
|
||||
const colMatch = line.match(/^\s*(\w+)\s+(.+)$/);
|
||||
if (colMatch) {
|
||||
const colName = colMatch[1].toLowerCase();
|
||||
if (SQL_KEYWORDS.has(colName)) return;
|
||||
|
||||
results.push({
|
||||
table: tableName,
|
||||
column: colName,
|
||||
definition: colMatch[2].trim(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = tableRegex.exec(SCHEMA_SQL)) !== null) {
|
||||
const tableName = match[1];
|
||||
const body = match[2];
|
||||
|
||||
const lines = body.split('\n');
|
||||
let currentLine = '';
|
||||
|
||||
for (const rawLine of lines) {
|
||||
const trimmed = rawLine.trim();
|
||||
|
||||
// Skip empty lines and comments
|
||||
if (!trimmed || trimmed.startsWith('--')) {
|
||||
// If we have accumulated content and hit a blank/comment line,
|
||||
// the accumulated content is a complete line
|
||||
if (currentLine.trim()) {
|
||||
processLine(tableName, currentLine);
|
||||
currentLine = '';
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
currentLine += ' ' + trimmed;
|
||||
|
||||
// If line ends with comma, it's a complete column definition
|
||||
if (trimmed.endsWith(',')) {
|
||||
processLine(tableName, currentLine);
|
||||
currentLine = '';
|
||||
}
|
||||
}
|
||||
|
||||
// Handle any remaining accumulated line (last column before closing paren)
|
||||
if (currentLine.trim()) {
|
||||
processLine(tableName, currentLine);
|
||||
}
|
||||
}
|
||||
|
||||
// Also parse ALTER TABLE ... ADD COLUMN IF NOT EXISTS statements.
|
||||
// These are used for columns added outside CREATE TABLE blocks
|
||||
// (e.g., pages.search_vector, files.source_id).
|
||||
const alterRegex = /ALTER\s+TABLE\s+(\w+)\s+ADD\s+COLUMN\s+IF\s+NOT\s+EXISTS\s+(\w+)\s+([^;,]+)/gi;
|
||||
let alterMatch: RegExpExecArray | null;
|
||||
const seen = new Set(results.map(r => `${r.table}.${r.column}`));
|
||||
while ((alterMatch = alterRegex.exec(SCHEMA_SQL)) !== null) {
|
||||
const table = alterMatch[1];
|
||||
const column = alterMatch[2].toLowerCase();
|
||||
const definition = alterMatch[3].trim().replace(/,\s*$/, '');
|
||||
const key = `${table}.${column}`;
|
||||
if (!seen.has(key)) {
|
||||
seen.add(key);
|
||||
results.push({ table, column, definition });
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a simplified type expression suitable for ALTER TABLE ADD COLUMN.
|
||||
*
|
||||
* Strips inline REFERENCES, CHECK, UNIQUE, and complex constraints that
|
||||
* can't be used in ADD COLUMN IF NOT EXISTS. Preserves NOT NULL, DEFAULT,
|
||||
* and the base type.
|
||||
*/
|
||||
export function simplifyColumnDef(definition: string): string {
|
||||
let def = definition;
|
||||
|
||||
// Remove REFERENCES ... (with optional ON DELETE/UPDATE clauses)
|
||||
def = def.replace(/REFERENCES\s+\w+\([^)]*\)(\s+ON\s+(DELETE|UPDATE)\s+\w+(\s+\w+)?)*\s*/gi, '');
|
||||
|
||||
// Remove CHECK constraints (handle nested parens)
|
||||
def = def.replace(/CHECK\s*\((?:[^()]*|\([^()]*\))*\)/gi, '');
|
||||
|
||||
// Remove inline UNIQUE
|
||||
def = def.replace(/\bUNIQUE\b/gi, '');
|
||||
|
||||
// Remove trailing commas and whitespace
|
||||
def = def.replace(/,\s*$/, '').trim();
|
||||
|
||||
// Collapse multiple spaces
|
||||
def = def.replace(/\s+/g, ' ').trim();
|
||||
|
||||
return def;
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the database for actual columns in the public schema.
|
||||
* Returns a Set of "table.column" strings for fast lookup.
|
||||
*/
|
||||
async function getActualColumns(engine: BrainEngine): Promise<Set<string>> {
|
||||
const rows = await engine.executeRaw<{ table_name: string; column_name: string }>(
|
||||
`SELECT table_name, column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public'`
|
||||
);
|
||||
const set = new Set<string>();
|
||||
for (const row of rows) {
|
||||
set.add(`${row.table_name}.${row.column_name}`);
|
||||
}
|
||||
return set;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the set of tables that actually exist in the database.
|
||||
*/
|
||||
async function getActualTables(engine: BrainEngine): Promise<Set<string>> {
|
||||
const rows = await engine.executeRaw<{ table_name: string }>(
|
||||
`SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_type = 'BASE TABLE'`
|
||||
);
|
||||
return new Set(rows.map(r => r.table_name));
|
||||
}
|
||||
|
||||
export interface VerifyResult {
|
||||
/** Total columns checked */
|
||||
checked: number;
|
||||
/** Columns that were missing */
|
||||
missing: Array<{ table: string; column: string }>;
|
||||
/** Columns successfully self-healed */
|
||||
healed: Array<{ table: string; column: string }>;
|
||||
/** Columns that failed to self-heal */
|
||||
failed: Array<{ table: string; column: string; error: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify that every column defined in schema-embedded.ts actually exists
|
||||
* in the database. Self-heals missing columns via ALTER TABLE ADD COLUMN.
|
||||
*
|
||||
* Should be called after initSchema() + runMigrations() complete.
|
||||
*
|
||||
* @returns VerifyResult with details of what was checked and fixed.
|
||||
* @throws Error if any columns could not be healed (after attempting all).
|
||||
*/
|
||||
export async function verifySchema(engine: BrainEngine): Promise<VerifyResult> {
|
||||
const expected = parseExpectedColumns();
|
||||
const actualColumns = await getActualColumns(engine);
|
||||
const actualTables = await getActualTables(engine);
|
||||
|
||||
const result: VerifyResult = {
|
||||
checked: 0,
|
||||
missing: [],
|
||||
healed: [],
|
||||
failed: [],
|
||||
};
|
||||
|
||||
// Group expected columns by table for cleaner logging
|
||||
for (const col of expected) {
|
||||
// Skip tables that don't exist yet — they'll be created by schema.sql
|
||||
// on the next initSchema() call. We only verify columns on tables that
|
||||
// DO exist (the failure mode is: table exists, migration ran, but ALTER
|
||||
// TABLE silently failed).
|
||||
if (!actualTables.has(col.table)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
result.checked++;
|
||||
|
||||
const key = `${col.table}.${col.column}`;
|
||||
if (!actualColumns.has(key)) {
|
||||
result.missing.push({ table: col.table, column: col.column });
|
||||
}
|
||||
}
|
||||
|
||||
if (result.missing.length === 0) {
|
||||
return result;
|
||||
}
|
||||
|
||||
// Log missing columns
|
||||
console.warn(`\n⚠️ Schema verification found ${result.missing.length} missing column(s):`);
|
||||
for (const m of result.missing) {
|
||||
console.warn(` ${m.table}.${m.column}`);
|
||||
}
|
||||
console.warn(' Attempting self-heal via ALTER TABLE ADD COLUMN...\n');
|
||||
|
||||
// Build a map from table.column -> definition for self-healing
|
||||
const defMap = new Map<string, string>();
|
||||
for (const col of expected) {
|
||||
defMap.set(`${col.table}.${col.column}`, col.definition);
|
||||
}
|
||||
|
||||
// Attempt to add each missing column
|
||||
for (const m of result.missing) {
|
||||
const rawDef = defMap.get(`${m.table}.${m.column}`);
|
||||
if (!rawDef) {
|
||||
result.failed.push({ ...m, error: 'No definition found in schema' });
|
||||
continue;
|
||||
}
|
||||
|
||||
const simpleDef = simplifyColumnDef(rawDef);
|
||||
|
||||
try {
|
||||
const sql = `ALTER TABLE ${m.table} ADD COLUMN IF NOT EXISTS ${m.column} ${simpleDef}`;
|
||||
await engine.runMigration(0, sql);
|
||||
result.healed.push({ table: m.table, column: m.column });
|
||||
console.log(` ✓ Added ${m.table}.${m.column}`);
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
result.failed.push({ ...m, error: msg });
|
||||
console.error(` ✗ Failed to add ${m.table}.${m.column}: ${msg}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (result.healed.length > 0) {
|
||||
console.log(`\n Schema self-heal: ${result.healed.length}/${result.missing.length} column(s) recovered.`);
|
||||
}
|
||||
|
||||
if (result.failed.length > 0) {
|
||||
const failList = result.failed.map(f => `${f.table}.${f.column}: ${f.error}`).join('\n ');
|
||||
throw new Error(
|
||||
`Schema verification failed: ${result.failed.length} column(s) could not be added:\n ${failList}\n` +
|
||||
'This usually means PgBouncer transaction-mode silently dropped ALTER TABLE statements.\n' +
|
||||
'Fix: connect directly to Postgres (not through PgBouncer) and run: gbrain apply-migrations --yes'
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -183,11 +183,7 @@ function acquireLock(workspace: string, opts: InstallOptions): void {
|
||||
const existing = readLock(workspace);
|
||||
const staleMs = opts.lockStaleMs ?? DEFAULT_LOCK_STALE_MS;
|
||||
if (existing) {
|
||||
// Clamp to 0. On Linux ext4, statSync().mtimeMs has sub-ms precision;
|
||||
// Date.now() is integer ms. A file written microseconds ago can report
|
||||
// a negative age here, which would break the staleMs:0 "any age is stale"
|
||||
// contract the force-unlock path relies on (CI passes, local macOS masks it).
|
||||
const age = Math.max(0, Date.now() - existing.mtimeMs);
|
||||
const age = Date.now() - existing.mtimeMs;
|
||||
// `staleMs: 0` in tests means "any age counts as stale". Use >=
|
||||
// so a just-written lock qualifies when the threshold is 0.
|
||||
// Negative age (mtime in the future) happens on fast CI filesystems
|
||||
|
||||
+6
-103
@@ -307,8 +307,6 @@ import { createHash as _createHash } from 'crypto';
|
||||
export interface SyncFailure {
|
||||
path: string;
|
||||
error: string;
|
||||
/** Structured error code extracted from the error message. */
|
||||
code?: string;
|
||||
commit: string;
|
||||
line?: number;
|
||||
ts: string;
|
||||
@@ -316,86 +314,6 @@ export interface SyncFailure {
|
||||
acknowledged_at?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort extraction of a structured error code from a sync failure
|
||||
* message. Matches known ParseValidationCode patterns (SLUG_MISMATCH,
|
||||
* YAML_PARSE, etc.) and common DB / timeout errors. Returns 'UNKNOWN'
|
||||
* when no pattern matches.
|
||||
*
|
||||
* Order matters: DB-layer errors are checked BEFORE YAML-layer ones so
|
||||
* Postgres `duplicate key value violates unique constraint` doesn't get
|
||||
* mislabeled as a YAML duplicate-key. Frontmatter patterns key off the
|
||||
* canonical messages emitted by `collectValidationErrors()` in markdown.ts.
|
||||
*/
|
||||
export function classifyErrorCode(errorMsg: string): string {
|
||||
// SLUG_MISMATCH: thrown by importFromFile() at src/core/import-file.ts:374.
|
||||
if (/slug.*does not match|SLUG_MISMATCH/i.test(errorMsg)) return 'SLUG_MISMATCH';
|
||||
|
||||
// DB-layer errors come BEFORE the YAML duplicate-key check. Postgres unique-
|
||||
// constraint violations contain "duplicate key" but are not a YAML problem.
|
||||
if (/duplicate key value violates unique constraint|DB_DUPLICATE_KEY/i.test(errorMsg)) {
|
||||
return 'DB_DUPLICATE_KEY';
|
||||
}
|
||||
if (/canceling statement due to statement timeout|STATEMENT_TIMEOUT/i.test(errorMsg)) {
|
||||
return 'STATEMENT_TIMEOUT';
|
||||
}
|
||||
|
||||
// YAML / frontmatter patterns. These match either the canonical message
|
||||
// strings in src/core/markdown.ts (collectValidationErrors) or the literal
|
||||
// ParseValidationCode token, so they fire whether the caller stores the
|
||||
// message or just the code.
|
||||
if (/YAML parse failed|YAML_PARSE/i.test(errorMsg)) return 'YAML_PARSE';
|
||||
if (/YAMLException|duplicated mapping key|YAML_DUPLICATE_KEY/i.test(errorMsg)) {
|
||||
return 'YAML_DUPLICATE_KEY';
|
||||
}
|
||||
if (/File is empty or whitespace-only|Frontmatter must start with ---|MISSING_OPEN/i.test(errorMsg)) {
|
||||
return 'MISSING_OPEN';
|
||||
}
|
||||
if (/No closing --- delimiter|Heading at line .* found inside frontmatter|MISSING_CLOSE/i.test(errorMsg)) {
|
||||
return 'MISSING_CLOSE';
|
||||
}
|
||||
if (/Frontmatter block is empty|EMPTY_FRONTMATTER/i.test(errorMsg)) return 'EMPTY_FRONTMATTER';
|
||||
if (/Content contains null bytes|NULL_BYTES|null byte/i.test(errorMsg)) return 'NULL_BYTES';
|
||||
if (/Nested double quotes|NESTED_QUOTES/i.test(errorMsg)) return 'NESTED_QUOTES';
|
||||
|
||||
// Generic fallbacks.
|
||||
if (/invalid UTF-?8|INVALID_UTF8/i.test(errorMsg)) return 'INVALID_UTF8';
|
||||
return 'UNKNOWN';
|
||||
}
|
||||
|
||||
/** Group failures by error code and return a sorted summary. */
|
||||
export function summarizeFailuresByCode(
|
||||
failures: Array<{ error: string; code?: string }>,
|
||||
): Array<{ code: string; count: number }> {
|
||||
const counts: Record<string, number> = {};
|
||||
for (const f of failures) {
|
||||
const code = f.code ?? classifyErrorCode(f.error);
|
||||
counts[code] = (counts[code] ?? 0) + 1;
|
||||
}
|
||||
return Object.entries(counts)
|
||||
.sort(([, a], [, b]) => b - a)
|
||||
.map(([code, count]) => ({ code, count }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a code-grouped summary as a human-readable multi-line string for
|
||||
* stderr / doctor output. Accepts either raw failures (which are summarized
|
||||
* internally) or an already-summarized `{code, count}[]` shape (the return
|
||||
* value of `summarizeFailuresByCode` or `AcknowledgeResult.summary`).
|
||||
* Returns an empty string when the input is empty.
|
||||
*/
|
||||
export function formatCodeBreakdown(
|
||||
input: Array<{ error: string; code?: string }> | Array<{ code: string; count: number }>,
|
||||
): string {
|
||||
// Distinguish by shape: summary entries have a numeric `count`. Empty array
|
||||
// returns '' from either branch — both paths produce a 0-length join.
|
||||
const summary =
|
||||
input.length > 0 && typeof (input[0] as { count?: unknown }).count === 'number'
|
||||
? (input as Array<{ code: string; count: number }>)
|
||||
: summarizeFailuresByCode(input as Array<{ error: string; code?: string }>);
|
||||
return summary.map(s => ` ${s.code}: ${s.count}`).join('\n');
|
||||
}
|
||||
|
||||
function _failuresDir(): string {
|
||||
return _joinPath(_homedir(), '.gbrain');
|
||||
}
|
||||
@@ -452,7 +370,6 @@ export function recordSyncFailures(
|
||||
const entry: SyncFailure = {
|
||||
path: f.path,
|
||||
error: f.error,
|
||||
code: classifyErrorCode(f.error),
|
||||
commit,
|
||||
line: f.line,
|
||||
ts: now,
|
||||
@@ -463,42 +380,28 @@ export function recordSyncFailures(
|
||||
}
|
||||
}
|
||||
|
||||
export interface AcknowledgeResult {
|
||||
count: number;
|
||||
summary: Array<{ code: string; count: number }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark all unacknowledged failures as acknowledged. Used by
|
||||
* `gbrain sync --skip-failed`. Returns count and a structured summary
|
||||
* grouped by error code so the operator can see *why* files were skipped.
|
||||
* `gbrain sync --skip-failed`. Returns the number newly acknowledged.
|
||||
*
|
||||
* We do not delete — acknowledged entries stay as historical record so
|
||||
* doctor can still show them under a "previously skipped" bucket.
|
||||
*/
|
||||
export function acknowledgeSyncFailures(): AcknowledgeResult {
|
||||
export function acknowledgeSyncFailures(): number {
|
||||
const entries = loadSyncFailures();
|
||||
if (entries.length === 0) return { count: 0, summary: [] };
|
||||
if (entries.length === 0) return 0;
|
||||
const now = new Date().toISOString();
|
||||
let changed = 0;
|
||||
const newlyAcked: SyncFailure[] = [];
|
||||
const updated = entries.map(e => {
|
||||
if (e.acknowledged) return e;
|
||||
changed++;
|
||||
// Backfill code for entries that predate the code field.
|
||||
const code = e.code ?? classifyErrorCode(e.error);
|
||||
const acked = { ...e, code, acknowledged: true, acknowledged_at: now };
|
||||
newlyAcked.push(acked);
|
||||
return acked;
|
||||
return { ...e, acknowledged: true, acknowledged_at: now };
|
||||
});
|
||||
if (changed === 0) return { count: 0, summary: [] };
|
||||
if (changed === 0) return 0;
|
||||
_mkdirSync(_failuresDir(), { recursive: true });
|
||||
const fd = require('fs').writeFileSync;
|
||||
fd(syncFailuresPath(), updated.map(e => JSON.stringify(e)).join('\n') + '\n');
|
||||
return {
|
||||
count: changed,
|
||||
summary: summarizeFailuresByCode(newlyAcked),
|
||||
};
|
||||
return changed;
|
||||
}
|
||||
|
||||
/** Return only unacknowledged failures. */
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
/**
|
||||
* Shared MCP tool-call dispatch — single source of truth for stdio + HTTP transports.
|
||||
*
|
||||
* Both transports validate the same params, build the same OperationContext shape,
|
||||
* and serialize errors identically. Drift between transports caused PR #483's reversed-args
|
||||
* + missing-context bugs; this module exists to prevent that recurring.
|
||||
*/
|
||||
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { operations, OperationError } from '../core/operations.ts';
|
||||
import type { Operation, OperationContext } from '../core/operations.ts';
|
||||
import { loadConfig } from '../core/config.ts';
|
||||
|
||||
export interface ToolResult {
|
||||
content: { type: 'text'; text: string }[];
|
||||
isError?: boolean;
|
||||
}
|
||||
|
||||
export interface DispatchOpts {
|
||||
/** Defaults to true (remote/untrusted). Local CLI callers (`gbrain call`) pass false. */
|
||||
remote?: boolean;
|
||||
/** Override the default stderr logger (e.g. CLI uses console.* directly). */
|
||||
logger?: OperationContext['logger'];
|
||||
}
|
||||
|
||||
/** Validate required params exist and have the expected type. Returns null on success, error message on failure. */
|
||||
export function validateParams(op: Operation, params: Record<string, unknown>): string | null {
|
||||
for (const [key, def] of Object.entries(op.params)) {
|
||||
if (def.required && (params[key] === undefined || params[key] === null)) {
|
||||
return `Missing required parameter: ${key}`;
|
||||
}
|
||||
if (params[key] !== undefined && params[key] !== null) {
|
||||
const val = params[key];
|
||||
const expected = def.type;
|
||||
if (expected === 'string' && typeof val !== 'string') return `Parameter "${key}" must be a string`;
|
||||
if (expected === 'number' && typeof val !== 'number') return `Parameter "${key}" must be a number`;
|
||||
if (expected === 'boolean' && typeof val !== 'boolean') return `Parameter "${key}" must be a boolean`;
|
||||
if (expected === 'object' && (typeof val !== 'object' || Array.isArray(val))) return `Parameter "${key}" must be an object`;
|
||||
if (expected === 'array' && !Array.isArray(val)) return `Parameter "${key}" must be an array`;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const stderrLogger: OperationContext['logger'] = {
|
||||
info: (msg: string) => process.stderr.write(`[info] ${msg}\n`),
|
||||
warn: (msg: string) => process.stderr.write(`[warn] ${msg}\n`),
|
||||
error: (msg: string) => process.stderr.write(`[error] ${msg}\n`),
|
||||
};
|
||||
|
||||
export function buildOperationContext(
|
||||
engine: BrainEngine,
|
||||
params: Record<string, unknown>,
|
||||
opts: DispatchOpts = {},
|
||||
): OperationContext {
|
||||
return {
|
||||
engine,
|
||||
config: loadConfig() || { engine: 'postgres' },
|
||||
logger: opts.logger || stderrLogger,
|
||||
dryRun: !!params.dry_run,
|
||||
remote: opts.remote ?? true,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve operation, validate params, build context, invoke handler, format result.
|
||||
*
|
||||
* Returns a `ToolResult` with the same shape both MCP transports need:
|
||||
* `{ content: [{ type: 'text', text }], isError?: boolean }`.
|
||||
*/
|
||||
export async function dispatchToolCall(
|
||||
engine: BrainEngine,
|
||||
name: string,
|
||||
params: Record<string, unknown> | undefined,
|
||||
opts: DispatchOpts = {},
|
||||
): Promise<ToolResult> {
|
||||
const op = operations.find(o => o.name === name);
|
||||
if (!op) {
|
||||
return { content: [{ type: 'text', text: `Error: Unknown tool: ${name}` }], isError: true };
|
||||
}
|
||||
|
||||
const safeParams = params || {};
|
||||
const validationError = validateParams(op, safeParams);
|
||||
if (validationError) {
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify({ error: 'invalid_params', message: validationError }, null, 2) }],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
const ctx = buildOperationContext(engine, safeParams, opts);
|
||||
|
||||
try {
|
||||
const result = await op.handler(ctx, safeParams);
|
||||
return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };
|
||||
} catch (e: unknown) {
|
||||
if (e instanceof OperationError) {
|
||||
return { content: [{ type: 'text', text: JSON.stringify(e.toJSON(), null, 2) }], isError: true };
|
||||
}
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
return { content: [{ type: 'text', text: `Error: ${msg}` }], isError: true };
|
||||
}
|
||||
}
|
||||
@@ -1,354 +0,0 @@
|
||||
/**
|
||||
* HTTP transport for `gbrain serve --http`.
|
||||
*
|
||||
* Postgres-only. PGLite users get a clear fail-fast at startup (the access_tokens
|
||||
* table doesn't exist on PGLite per pglite-schema.ts).
|
||||
*
|
||||
* Security model:
|
||||
* - Every request must include `Authorization: Bearer <token>` (except /health)
|
||||
* - Tokens are validated against SHA-256 hashes in the access_tokens table
|
||||
* - Create/manage tokens with auth.ts (gbrain auth create/list/revoke)
|
||||
* - No open OAuth, no client_credentials, no self-service tokens
|
||||
*
|
||||
* Hardening:
|
||||
* - CORS default-deny: allowlist via GBRAIN_HTTP_CORS_ORIGIN (comma-separated)
|
||||
* - Rate limit: per-IP pre-auth (protects DB from brute-force load) + per-token-id post-auth
|
||||
* (limits runaway clients). Default 30 req/min per IP, 60 req/min per token. Bounded LRU
|
||||
* so attacker-controlled keys can't grow memory unbounded.
|
||||
* - Body cap: 1 MiB default (GBRAIN_HTTP_MAX_BODY_BYTES). Stream-counted, not buffered —
|
||||
* chunked transfers without Content-Length are still capped.
|
||||
* - last_used_at debounce: only one UPDATE per token per 60s (SQL-level WHERE clause).
|
||||
* - mcp_request_log: one row per request with token_name + operation + status + latency.
|
||||
*
|
||||
* Replaces the standalone HTTP+OAuth wrapper that was vulnerable to unauthenticated
|
||||
* client registration (see SECURITY.md).
|
||||
*/
|
||||
|
||||
import { createHash } from 'crypto';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { buildToolDefs } from './tool-defs.ts';
|
||||
import { operations } from '../core/operations.ts';
|
||||
import { VERSION } from '../version.ts';
|
||||
import { dispatchToolCall } from './dispatch.ts';
|
||||
import { buildDefaultLimiters, type RateLimiter } from './rate-limit.ts';
|
||||
|
||||
const DEFAULT_BODY_CAP = 1024 * 1024; // 1 MiB
|
||||
|
||||
function hashToken(token: string): string {
|
||||
return createHash('sha256').update(token).digest('hex');
|
||||
}
|
||||
|
||||
function envInt(name: string, fallback: number): number {
|
||||
const v = process.env[name];
|
||||
if (!v) return fallback;
|
||||
const n = parseInt(v, 10);
|
||||
return Number.isFinite(n) && n > 0 ? n : fallback;
|
||||
}
|
||||
|
||||
function parseCorsAllowlist(): Set<string> | null {
|
||||
const v = process.env.GBRAIN_HTTP_CORS_ORIGIN;
|
||||
if (!v) return null;
|
||||
return new Set(v.split(',').map(s => s.trim()).filter(Boolean));
|
||||
}
|
||||
|
||||
interface HttpTransportOptions {
|
||||
port: number;
|
||||
engine: BrainEngine;
|
||||
/** Override limiters (for tests). Defaults to env-driven buildDefaultLimiters. */
|
||||
limiters?: { ip: RateLimiter; token: RateLimiter };
|
||||
}
|
||||
|
||||
interface AuthResult {
|
||||
ok: boolean;
|
||||
tokenId?: string;
|
||||
tokenName?: string;
|
||||
}
|
||||
|
||||
/** Read up to `cap` bytes off req.body. Returns null if cap exceeded. */
|
||||
async function readBodyWithCap(req: Request, cap: number): Promise<string | null> {
|
||||
const cl = req.headers.get('content-length');
|
||||
if (cl) {
|
||||
const n = parseInt(cl, 10);
|
||||
if (Number.isFinite(n) && n > cap) return null;
|
||||
}
|
||||
const reader = req.body?.getReader();
|
||||
if (!reader) return '';
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
if (!value) continue;
|
||||
total += value.byteLength;
|
||||
if (total > cap) {
|
||||
try { await reader.cancel(); } catch { /* noop */ }
|
||||
return null;
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
// Concatenate without Buffer to keep this Node-vs-Bun-portable.
|
||||
const merged = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const c of chunks) {
|
||||
merged.set(c, offset);
|
||||
offset += c.byteLength;
|
||||
}
|
||||
return new TextDecoder().decode(merged);
|
||||
}
|
||||
|
||||
/** Resolve client IP. Honors X-Forwarded-For only when GBRAIN_HTTP_TRUST_PROXY=1. */
|
||||
function resolveClientIp(req: Request, server: { requestIP: (r: Request) => { address: string } | null }): string {
|
||||
if (process.env.GBRAIN_HTTP_TRUST_PROXY === '1') {
|
||||
const xff = req.headers.get('x-forwarded-for');
|
||||
if (xff) {
|
||||
const first = xff.split(',')[0]?.trim();
|
||||
if (first) return first;
|
||||
}
|
||||
const xRealIp = req.headers.get('x-real-ip');
|
||||
if (xRealIp) return xRealIp.trim();
|
||||
}
|
||||
const sock = server.requestIP(req);
|
||||
return sock?.address || 'unknown';
|
||||
}
|
||||
|
||||
export async function startHttpTransport(opts: HttpTransportOptions) {
|
||||
const { port, engine } = opts;
|
||||
|
||||
// Fail-fast: HTTP transport requires Postgres because access_tokens / mcp_request_log
|
||||
// only exist in the Postgres schema (see src/core/pglite-schema.ts:5-6).
|
||||
if ((engine as { kind?: string }).kind !== 'postgres') {
|
||||
console.error('Error: gbrain serve --http requires a Postgres engine for remote auth tokens.');
|
||||
console.error('PGLite is local-only by design (access_tokens table is Postgres-only).');
|
||||
console.error('Either:');
|
||||
console.error(' - Use stdio: gbrain serve');
|
||||
console.error(' - Migrate to Postgres: gbrain migrate --to supabase');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const sql = (engine as unknown as { sql: any }).sql;
|
||||
if (!sql) {
|
||||
console.error('Error: Postgres engine has no .sql client. Engine may not be connected.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const limiters = opts.limiters || buildDefaultLimiters();
|
||||
const bodyCap = envInt('GBRAIN_HTTP_MAX_BODY_BYTES', DEFAULT_BODY_CAP);
|
||||
const corsAllowlist = parseCorsAllowlist();
|
||||
const tools = buildToolDefs(operations);
|
||||
|
||||
function corsHeaders(origin: string | null, extra: Record<string, string> = {}): Record<string, string> {
|
||||
const headers: Record<string, string> = { ...extra };
|
||||
if (corsAllowlist && origin && corsAllowlist.has(origin)) {
|
||||
headers['Access-Control-Allow-Origin'] = origin;
|
||||
headers['Vary'] = 'Origin';
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
function corsPreflightHeaders(origin: string | null): Record<string, string> {
|
||||
const headers: Record<string, string> = {
|
||||
'Access-Control-Allow-Methods': 'POST, GET, OPTIONS',
|
||||
'Access-Control-Allow-Headers': 'Content-Type, Authorization, Accept',
|
||||
};
|
||||
if (corsAllowlist && origin && corsAllowlist.has(origin)) {
|
||||
headers['Access-Control-Allow-Origin'] = origin;
|
||||
headers['Vary'] = 'Origin';
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
async function validateToken(authHeader: string | null): Promise<AuthResult> {
|
||||
if (!authHeader?.startsWith('Bearer ')) return { ok: false };
|
||||
const token = authHeader.slice(7);
|
||||
const hash = hashToken(token);
|
||||
try {
|
||||
const [row] = await sql`
|
||||
SELECT id, name FROM access_tokens
|
||||
WHERE token_hash = ${hash} AND revoked_at IS NULL
|
||||
`;
|
||||
if (!row) return { ok: false };
|
||||
// Debounced last_used_at update — only writes once per token per 60s.
|
||||
// SQL-level WHERE clause keeps this race-tolerant even under concurrent requests.
|
||||
sql`UPDATE access_tokens
|
||||
SET last_used_at = now()
|
||||
WHERE id = ${row.id}
|
||||
AND (last_used_at IS NULL OR last_used_at < now() - interval '60 seconds')`
|
||||
.catch(() => { /* fire-and-forget */ });
|
||||
return { ok: true, tokenId: row.id, tokenName: row.name };
|
||||
} catch {
|
||||
return { ok: false };
|
||||
}
|
||||
}
|
||||
|
||||
function logRequest(tokenName: string | null, operation: string, status: string, latencyMs: number) {
|
||||
sql`INSERT INTO mcp_request_log (token_name, operation, latency_ms, status)
|
||||
VALUES (${tokenName}, ${operation}, ${latencyMs}, ${status})`
|
||||
.catch(() => { /* best-effort */ });
|
||||
}
|
||||
|
||||
const server = Bun.serve({
|
||||
port,
|
||||
async fetch(req, server) {
|
||||
const startedMs = Date.now();
|
||||
const url = new URL(req.url);
|
||||
const path = url.pathname;
|
||||
const origin = req.headers.get('origin');
|
||||
|
||||
// CORS preflight
|
||||
if (req.method === 'OPTIONS') {
|
||||
return new Response(null, { headers: corsPreflightHeaders(origin) });
|
||||
}
|
||||
|
||||
// Health check — no auth, no rate limit. Probes the DB so orchestration
|
||||
// doesn't see "ok" while clients are getting misleading 401s during a DB outage.
|
||||
if (path === '/health') {
|
||||
try {
|
||||
await sql`SELECT 1`;
|
||||
return Response.json(
|
||||
{ status: 'ok', version: VERSION, transport: 'http', db: 'ok' },
|
||||
{ headers: corsHeaders(origin) },
|
||||
);
|
||||
} catch (e: any) {
|
||||
return Response.json(
|
||||
{ status: 'unhealthy', version: VERSION, transport: 'http', db: 'unreachable', error: e?.message ?? 'unknown' },
|
||||
{ status: 503, headers: corsHeaders(origin) },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (path !== '/mcp') {
|
||||
return Response.json({ error: 'not_found' }, { status: 404, headers: corsHeaders(origin) });
|
||||
}
|
||||
if (req.method !== 'POST') {
|
||||
return Response.json({ error: 'method_not_allowed' }, { status: 405, headers: corsHeaders(origin) });
|
||||
}
|
||||
|
||||
const ip = resolveClientIp(req, server);
|
||||
|
||||
// Pre-auth IP rate limit. Fires BEFORE the DB lookup so we actually limit brute-force load.
|
||||
const ipCheck = limiters.ip.check(ip);
|
||||
if (!ipCheck.allowed) {
|
||||
logRequest(null, 'unknown', 'rate_limited', Date.now() - startedMs);
|
||||
return Response.json(
|
||||
{ error: 'rate_limited', message: 'Too many requests' },
|
||||
{
|
||||
status: 429,
|
||||
headers: corsHeaders(origin, { 'Retry-After': String(ipCheck.retryAfter ?? 60) }),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// Body cap (stream-counted; chunked transfers caught here, not at req.json).
|
||||
const bodyText = await readBodyWithCap(req, bodyCap);
|
||||
if (bodyText === null) {
|
||||
logRequest(null, 'unknown', 'body_too_large', Date.now() - startedMs);
|
||||
return Response.json(
|
||||
{ error: 'payload_too_large', message: `Request body exceeds ${bodyCap} bytes` },
|
||||
{ status: 413, headers: corsHeaders(origin) },
|
||||
);
|
||||
}
|
||||
|
||||
// Auth.
|
||||
const auth = await validateToken(req.headers.get('Authorization'));
|
||||
if (!auth.ok) {
|
||||
logRequest(null, 'unknown', 'auth_failed', Date.now() - startedMs);
|
||||
return Response.json(
|
||||
{ error: 'invalid_token', message: 'Bearer token required. Create one: gbrain auth create <name>' },
|
||||
{ status: 401, headers: corsHeaders(origin) },
|
||||
);
|
||||
}
|
||||
|
||||
// Post-auth token-id rate limit. Limits runaway authed clients.
|
||||
const tokCheck = limiters.token.check(auth.tokenId!);
|
||||
if (!tokCheck.allowed) {
|
||||
logRequest(auth.tokenName!, 'unknown', 'rate_limited', Date.now() - startedMs);
|
||||
return Response.json(
|
||||
{ error: 'rate_limited', message: 'Too many requests for this token' },
|
||||
{
|
||||
status: 429,
|
||||
headers: corsHeaders(origin, { 'Retry-After': String(tokCheck.retryAfter ?? 60) }),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// Parse JSON-RPC body.
|
||||
let body: { method?: string; params?: any; id?: any };
|
||||
try {
|
||||
body = JSON.parse(bodyText);
|
||||
} catch (e: any) {
|
||||
logRequest(auth.tokenName!, 'unknown', 'parse_error', Date.now() - startedMs);
|
||||
return Response.json(
|
||||
{ error: 'parse_error', message: e?.message ?? 'invalid JSON' },
|
||||
{ status: 400, headers: corsHeaders(origin) },
|
||||
);
|
||||
}
|
||||
|
||||
const { method, params, id } = body;
|
||||
|
||||
// initialize
|
||||
if (method === 'initialize') {
|
||||
logRequest(auth.tokenName!, 'initialize', 'success', Date.now() - startedMs);
|
||||
return Response.json(
|
||||
{
|
||||
result: {
|
||||
protocolVersion: '2025-03-26',
|
||||
serverInfo: { name: 'gbrain', version: VERSION },
|
||||
capabilities: { tools: {} },
|
||||
},
|
||||
jsonrpc: '2.0',
|
||||
id,
|
||||
},
|
||||
{ headers: corsHeaders(origin) },
|
||||
);
|
||||
}
|
||||
|
||||
// notifications/initialized — acknowledge with 204
|
||||
if (method === 'notifications/initialized') {
|
||||
return new Response(null, { status: 204, headers: corsHeaders(origin) });
|
||||
}
|
||||
|
||||
// tools/list
|
||||
if (method === 'tools/list') {
|
||||
logRequest(auth.tokenName!, 'tools/list', 'success', Date.now() - startedMs);
|
||||
return Response.json(
|
||||
{ result: { tools }, jsonrpc: '2.0', id },
|
||||
{ headers: corsHeaders(origin) },
|
||||
);
|
||||
}
|
||||
|
||||
// tools/call — dispatch through shared dispatch.ts (parity with stdio)
|
||||
if (method === 'tools/call') {
|
||||
const toolName: string = params?.name ?? 'unknown';
|
||||
const args: Record<string, unknown> = params?.arguments ?? {};
|
||||
const result = await dispatchToolCall(engine, toolName, args, { remote: true });
|
||||
const status = result.isError ? 'error' : 'success';
|
||||
logRequest(auth.tokenName!, `tools/call:${toolName}`, status, Date.now() - startedMs);
|
||||
return Response.json(
|
||||
{ result, jsonrpc: '2.0', id },
|
||||
{ headers: corsHeaders(origin) },
|
||||
);
|
||||
}
|
||||
|
||||
logRequest(auth.tokenName!, method ?? 'unknown', 'unknown_method', Date.now() - startedMs);
|
||||
return Response.json(
|
||||
{ error: 'unknown_method', message: `Unknown method: ${method}` },
|
||||
{ status: 400, headers: corsHeaders(origin) },
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
console.error(`GBrain HTTP MCP server running on port ${port}`);
|
||||
console.error(` Health: http://localhost:${port}/health`);
|
||||
console.error(` MCP: http://localhost:${port}/mcp`);
|
||||
console.error(` Auth: Bearer token required (create with: gbrain auth create <name>)`);
|
||||
if (!corsAllowlist) {
|
||||
console.error(' CORS: default-deny. Set GBRAIN_HTTP_CORS_ORIGIN=https://your.app to allow browser clients.');
|
||||
} else {
|
||||
console.error(` CORS: allowlist = ${[...corsAllowlist].join(', ')}`);
|
||||
}
|
||||
console.error('');
|
||||
console.error('⚠️ Do NOT use open OAuth registration for remote MCP access.');
|
||||
console.error(' Tokens are managed via: gbrain auth create/list/revoke');
|
||||
|
||||
return server;
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
/**
|
||||
* Rate limiter for `gbrain serve --http`.
|
||||
*
|
||||
* Token-bucket per key, stored in a bounded LRU map so attacker-controlled keys
|
||||
* can't grow memory unbounded. TTL prune on every access (entries older than
|
||||
* 2× window are evicted) so abandoned keys don't sit around forever.
|
||||
*
|
||||
* Two buckets in the request pipeline (see http-transport.ts):
|
||||
* 1. Pre-auth IP bucket — fires BEFORE the DB lookup so we actually limit
|
||||
* brute-force load against access_tokens, not just response codes.
|
||||
* 2. Post-auth token-id bucket — fires after auth so legitimate-but-runaway
|
||||
* clients get throttled at the right principal.
|
||||
*
|
||||
* Both buckets behave identically; only the key differs.
|
||||
*/
|
||||
|
||||
export interface RateLimitOpts {
|
||||
/** Maximum requests in the window. */
|
||||
limit: number;
|
||||
/** Window length in milliseconds. */
|
||||
windowMs: number;
|
||||
/** LRU cap on distinct keys. Evicts least-recently-used on overflow. */
|
||||
lruCap: number;
|
||||
}
|
||||
|
||||
export interface RateLimitResult {
|
||||
allowed: boolean;
|
||||
/** Seconds until next request would be allowed (only set when !allowed). */
|
||||
retryAfter?: number;
|
||||
/** Tokens remaining in the bucket after this check. */
|
||||
remaining: number;
|
||||
}
|
||||
|
||||
interface Bucket {
|
||||
tokens: number;
|
||||
/** Used for refill math: tokens accrue based on elapsed time since this. */
|
||||
lastRefillMs: number;
|
||||
/** Used for TTL eviction: time of last check, regardless of refill. Prevents bucket-reset attack
|
||||
* where an exhausted key would otherwise get TTL-evicted and recreated fresh. */
|
||||
lastTouchedMs: number;
|
||||
}
|
||||
|
||||
/** Clock function — defaults to Date.now, overridable for tests. */
|
||||
type Clock = () => number;
|
||||
|
||||
export class RateLimiter {
|
||||
readonly opts: RateLimitOpts;
|
||||
private readonly buckets: Map<string, Bucket> = new Map();
|
||||
private readonly clock: Clock;
|
||||
|
||||
constructor(opts: RateLimitOpts, clock: Clock = Date.now) {
|
||||
if (opts.limit <= 0) throw new Error('RateLimiter: limit must be > 0');
|
||||
if (opts.windowMs <= 0) throw new Error('RateLimiter: windowMs must be > 0');
|
||||
if (opts.lruCap <= 0) throw new Error('RateLimiter: lruCap must be > 0');
|
||||
this.opts = opts;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
check(key: string): RateLimitResult {
|
||||
const now = this.clock();
|
||||
this.prune(now);
|
||||
|
||||
let bucket = this.buckets.get(key);
|
||||
if (!bucket) {
|
||||
bucket = { tokens: this.opts.limit, lastRefillMs: now, lastTouchedMs: now };
|
||||
} else {
|
||||
// Refill: tokens accrue continuously over the window. limit/windowMs tokens per ms.
|
||||
const elapsed = now - bucket.lastRefillMs;
|
||||
const refilled = Math.floor((elapsed * this.opts.limit) / this.opts.windowMs);
|
||||
if (refilled > 0) {
|
||||
bucket.tokens = Math.min(this.opts.limit, bucket.tokens + refilled);
|
||||
bucket.lastRefillMs = now;
|
||||
}
|
||||
bucket.lastTouchedMs = now;
|
||||
// LRU bookkeeping: re-insert to move to end (Map iteration order = insertion).
|
||||
this.buckets.delete(key);
|
||||
}
|
||||
|
||||
if (bucket.tokens > 0) {
|
||||
bucket.tokens -= 1;
|
||||
this.buckets.set(key, bucket);
|
||||
this.evictIfOver();
|
||||
return { allowed: true, remaining: bucket.tokens };
|
||||
}
|
||||
|
||||
// No tokens. Compute Retry-After from when the next token will accrue.
|
||||
const msPerToken = this.opts.windowMs / this.opts.limit;
|
||||
const msUntilNext = msPerToken - (now - bucket.lastRefillMs);
|
||||
const retryAfter = Math.max(1, Math.ceil(msUntilNext / 1000));
|
||||
this.buckets.set(key, bucket);
|
||||
this.evictIfOver();
|
||||
return { allowed: false, retryAfter, remaining: 0 };
|
||||
}
|
||||
|
||||
/** Evict TTL-expired entries (older than 2× window since last touch). Cheap: O(n) but n is bounded by lruCap.
|
||||
* Uses lastTouchedMs (not lastRefillMs) so an attacker can't reset their bucket by hammering an exhausted key
|
||||
* past the TTL — every check updates lastTouchedMs even when refill produces 0 tokens. */
|
||||
private prune(now: number): void {
|
||||
const ttl = this.opts.windowMs * 2;
|
||||
for (const [key, bucket] of this.buckets) {
|
||||
if (now - bucket.lastTouchedMs > ttl) {
|
||||
this.buckets.delete(key);
|
||||
} else {
|
||||
// Map iteration is in insertion order; once we hit a fresh entry, the rest are also fresh
|
||||
// ONLY if we maintain insertion-order = recency. That holds because check() does delete+set on every call.
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private evictIfOver(): void {
|
||||
while (this.buckets.size > this.opts.lruCap) {
|
||||
// Map iteration starts at oldest (first-inserted). Delete it.
|
||||
const oldestKey = this.buckets.keys().next().value;
|
||||
if (oldestKey === undefined) break;
|
||||
this.buckets.delete(oldestKey);
|
||||
}
|
||||
}
|
||||
|
||||
/** Test helper: current key count. */
|
||||
get size(): number {
|
||||
return this.buckets.size;
|
||||
}
|
||||
}
|
||||
|
||||
/** Parse a positive integer env var, falling back to default. */
|
||||
function envInt(name: string, fallback: number): number {
|
||||
const v = process.env[name];
|
||||
if (!v) return fallback;
|
||||
const n = parseInt(v, 10);
|
||||
return Number.isFinite(n) && n > 0 ? n : fallback;
|
||||
}
|
||||
|
||||
/** Build limiters from env. Keep this lazy — tests can construct RateLimiter directly. */
|
||||
export function buildDefaultLimiters(clock: Clock = Date.now): { ip: RateLimiter; token: RateLimiter } {
|
||||
const lruCap = envInt('GBRAIN_HTTP_RATE_LIMIT_LRU', 10000);
|
||||
const windowMs = 60_000;
|
||||
return {
|
||||
ip: new RateLimiter({ limit: envInt('GBRAIN_HTTP_RATE_LIMIT_IP', 30), windowMs, lruCap }, clock),
|
||||
token: new RateLimiter({ limit: envInt('GBRAIN_HTTP_RATE_LIMIT_TOKEN', 60), windowMs, lruCap }, clock),
|
||||
};
|
||||
}
|
||||
+66
-13
@@ -2,10 +2,30 @@ import { Server } from '@modelcontextprotocol/sdk/server/index.js';
|
||||
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
||||
import { ListToolsRequestSchema, CallToolRequestSchema } from '@modelcontextprotocol/sdk/types.js';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
import { operations } from '../core/operations.ts';
|
||||
import { operations, OperationError } from '../core/operations.ts';
|
||||
import type { Operation, OperationContext } from '../core/operations.ts';
|
||||
import { loadConfig } from '../core/config.ts';
|
||||
import { VERSION } from '../version.ts';
|
||||
import { buildToolDefs } from './tool-defs.ts';
|
||||
import { dispatchToolCall, validateParams, buildOperationContext } from './dispatch.ts';
|
||||
|
||||
/** Validate required params exist and have the expected type */
|
||||
function validateParams(op: Operation, params: Record<string, unknown>): string | null {
|
||||
for (const [key, def] of Object.entries(op.params)) {
|
||||
if (def.required && (params[key] === undefined || params[key] === null)) {
|
||||
return `Missing required parameter: ${key}`;
|
||||
}
|
||||
if (params[key] !== undefined && params[key] !== null) {
|
||||
const val = params[key];
|
||||
const expected = def.type;
|
||||
if (expected === 'string' && typeof val !== 'string') return `Parameter "${key}" must be a string`;
|
||||
if (expected === 'number' && typeof val !== 'number') return `Parameter "${key}" must be a number`;
|
||||
if (expected === 'boolean' && typeof val !== 'boolean') return `Parameter "${key}" must be a boolean`;
|
||||
if (expected === 'object' && (typeof val !== 'object' || Array.isArray(val))) return `Parameter "${key}" must be an object`;
|
||||
if (expected === 'array' && !Array.isArray(val)) return `Parameter "${key}" must be an array`;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function startMcpServer(engine: BrainEngine) {
|
||||
const server = new Server(
|
||||
@@ -20,21 +40,50 @@ export async function startMcpServer(engine: BrainEngine) {
|
||||
tools: buildToolDefs(operations),
|
||||
}));
|
||||
|
||||
// Dispatch tool calls via shared dispatch.ts (parity with HTTP transport).
|
||||
// MCP stdio callers are remote/untrusted; dispatch defaults remote=true.
|
||||
// The MCP SDK's response type widened in 1.29 to allow a managed-task wrapper;
|
||||
// gbrain ops are synchronous, so we return the legacy `{ content, isError? }`
|
||||
// shape and cast through `any` (the SDK accepts it via the ServerResult union).
|
||||
server.setRequestHandler(CallToolRequestSchema, async (request: any): Promise<any> => {
|
||||
// Dispatch tool calls to operation handlers
|
||||
server.setRequestHandler(CallToolRequestSchema, async (request: any) => {
|
||||
const { name, arguments: params } = request.params;
|
||||
return dispatchToolCall(engine, name, params, { remote: true });
|
||||
const op = operations.find(o => o.name === name);
|
||||
if (!op) {
|
||||
return { content: [{ type: 'text', text: `Error: Unknown tool: ${name}` }], isError: true };
|
||||
}
|
||||
|
||||
const ctx: OperationContext = {
|
||||
engine,
|
||||
config: loadConfig() || { engine: 'postgres' },
|
||||
logger: {
|
||||
info: (msg: string) => process.stderr.write(`[info] ${msg}\n`),
|
||||
warn: (msg: string) => process.stderr.write(`[warn] ${msg}\n`),
|
||||
error: (msg: string) => process.stderr.write(`[error] ${msg}\n`),
|
||||
},
|
||||
dryRun: !!(params?.dry_run),
|
||||
// MCP stdio callers are remote/untrusted; enforce strict file confinement.
|
||||
remote: true,
|
||||
};
|
||||
|
||||
const safeParams = params || {};
|
||||
const validationError = validateParams(op, safeParams);
|
||||
if (validationError) {
|
||||
return { content: [{ type: 'text', text: JSON.stringify({ error: 'invalid_params', message: validationError }, null, 2) }], isError: true };
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await op.handler(ctx, safeParams);
|
||||
return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };
|
||||
} catch (e: unknown) {
|
||||
if (e instanceof OperationError) {
|
||||
return { content: [{ type: 'text', text: JSON.stringify(e.toJSON(), null, 2) }], isError: true };
|
||||
}
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
return { content: [{ type: 'text', text: `Error: ${msg}` }], isError: true };
|
||||
}
|
||||
});
|
||||
|
||||
const transport = new StdioServerTransport();
|
||||
await server.connect(transport);
|
||||
}
|
||||
|
||||
// Backward compat: used by `gbrain call` command (trusted local path).
|
||||
// Backward compat: used by `gbrain call` command
|
||||
export async function handleToolCall(
|
||||
engine: BrainEngine,
|
||||
tool: string,
|
||||
@@ -46,10 +95,14 @@ export async function handleToolCall(
|
||||
const validationError = validateParams(op, params);
|
||||
if (validationError) throw new Error(validationError);
|
||||
|
||||
const ctx = buildOperationContext(engine, params, {
|
||||
remote: false,
|
||||
const ctx: OperationContext = {
|
||||
engine,
|
||||
config: loadConfig() || { engine: 'postgres' },
|
||||
logger: { info: console.log, warn: console.warn, error: console.error },
|
||||
});
|
||||
dryRun: !!(params?.dry_run),
|
||||
// Backing path for `gbrain call` CLI command — trusted local invocation.
|
||||
remote: false,
|
||||
};
|
||||
|
||||
return op.handler(ctx, params);
|
||||
}
|
||||
|
||||
@@ -1,192 +0,0 @@
|
||||
/**
|
||||
* PGLite forward-reference bootstrap tests.
|
||||
*
|
||||
* Validates the contract of `PGLiteEngine#applyForwardReferenceBootstrap`:
|
||||
* given a brain that lacks the schema-blob's forward-referenced state, the
|
||||
* bootstrap adds enough state for PGLITE_SCHEMA_SQL to replay safely.
|
||||
*
|
||||
* The bootstrap covers the wedge incidents from issues
|
||||
* #239/#266/#357/#366/#374/#375/#378/#396 — every gbrain release that added
|
||||
* a column-with-index in the schema blob without a corresponding bootstrap
|
||||
* triggered the same wedge family.
|
||||
*
|
||||
* Honest limitation: test 4 simulates a v20 brain by dropping known forward
|
||||
* state from a fresh-LATEST instance. This is the same down-mutation pattern
|
||||
* codex flagged as "weak simulation" — it can't simulate every possible
|
||||
* historical state. Acceptable here because the bootstrap's contract is
|
||||
* narrow ("given a brain that lacks the specific forward-references,
|
||||
* initSchema produces a brain at LATEST"), and that contract is exactly
|
||||
* what this test exercises.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { LATEST_VERSION } from '../src/core/migrate.ts';
|
||||
|
||||
describe('PGLiteEngine#applyForwardReferenceBootstrap', () => {
|
||||
test('no-op on fresh install (no pages or links table)', async () => {
|
||||
const engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
try {
|
||||
// Don't call initSchema — verify bootstrap alone does nothing on empty DB
|
||||
await (engine as any).applyForwardReferenceBootstrap();
|
||||
const { rows } = await (engine as any).db.query(`
|
||||
SELECT COUNT(*)::int AS c FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
`);
|
||||
expect(rows[0].c).toBe(0);
|
||||
} finally {
|
||||
await engine.disconnect();
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
test('idempotent: calling twice produces same result', async () => {
|
||||
const engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
try {
|
||||
await engine.initSchema();
|
||||
const db = (engine as any).db;
|
||||
|
||||
// Mutate to pre-v0.18 shape: drop source_id and the sources FK target
|
||||
await db.exec(`
|
||||
ALTER TABLE pages DROP CONSTRAINT IF EXISTS pages_source_slug_key;
|
||||
ALTER TABLE pages ADD CONSTRAINT pages_slug_key UNIQUE (slug);
|
||||
DROP INDEX IF EXISTS idx_pages_source_id;
|
||||
ALTER TABLE pages DROP COLUMN IF EXISTS source_id;
|
||||
DROP TABLE IF EXISTS sources CASCADE;
|
||||
`);
|
||||
|
||||
// First call: applies bootstrap
|
||||
await (engine as any).applyForwardReferenceBootstrap();
|
||||
// Second call: must not error, must not duplicate state
|
||||
await (engine as any).applyForwardReferenceBootstrap();
|
||||
|
||||
const { rows: cols } = await db.query(`
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'pages' AND column_name = 'source_id'
|
||||
`);
|
||||
expect(cols).toHaveLength(1);
|
||||
|
||||
const { rows: src } = await db.query(`SELECT COUNT(*)::int AS c FROM sources`);
|
||||
expect(src[0].c).toBe(1); // 'default' seed not duplicated
|
||||
} finally {
|
||||
await engine.disconnect();
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
test('no-op on modern brain (source_id and links provenance already present)', async () => {
|
||||
const engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
try {
|
||||
await engine.initSchema();
|
||||
const db = (engine as any).db;
|
||||
|
||||
const before = await db.query(`SELECT COUNT(*)::int AS c FROM sources`);
|
||||
await (engine as any).applyForwardReferenceBootstrap();
|
||||
const after = await db.query(`SELECT COUNT(*)::int AS c FROM sources`);
|
||||
|
||||
// Bootstrap probe should detect the brain is modern and skip the seed insert
|
||||
expect(after.rows[0].c).toBe(before.rows[0].c);
|
||||
} finally {
|
||||
await engine.disconnect();
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
test('full path: pre-v0.18 brain reaches LATEST_VERSION via initSchema', async () => {
|
||||
const engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
try {
|
||||
await engine.initSchema();
|
||||
const db = (engine as any).db;
|
||||
|
||||
// Mutate to pre-v0.18 shape: strip the forward-referenced state.
|
||||
// Match the shape from #399's regression fixture; constraints first
|
||||
// (so dropping columns succeeds).
|
||||
await db.exec(`
|
||||
ALTER TABLE pages DROP CONSTRAINT IF EXISTS pages_source_slug_key;
|
||||
ALTER TABLE pages ADD CONSTRAINT pages_slug_key UNIQUE (slug);
|
||||
DROP INDEX IF EXISTS idx_pages_source_id;
|
||||
ALTER TABLE pages DROP COLUMN IF EXISTS source_id;
|
||||
DROP TABLE IF EXISTS sources CASCADE;
|
||||
ALTER TABLE links DROP CONSTRAINT IF EXISTS links_resolution_type_check;
|
||||
ALTER TABLE links DROP COLUMN IF EXISTS resolution_type;
|
||||
`);
|
||||
await engine.setConfig('version', '20');
|
||||
|
||||
// Path under test: bootstrap → SCHEMA_SQL → runMigrations
|
||||
await engine.initSchema();
|
||||
|
||||
expect(await engine.getConfig('version')).toBe(String(LATEST_VERSION));
|
||||
|
||||
const { rows: srcCol } = await db.query(`
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'pages' AND column_name = 'source_id'
|
||||
`);
|
||||
expect(srcCol).toHaveLength(1);
|
||||
|
||||
const { rows: defaultSrc } = await db.query(`SELECT id FROM sources WHERE id = 'default'`);
|
||||
expect(defaultSrc).toHaveLength(1);
|
||||
} finally {
|
||||
await engine.disconnect();
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
test('fresh install regression: initSchema on empty DB produces LATEST', async () => {
|
||||
// The bootstrap's table-existence probe must not mis-classify "no table"
|
||||
// as "pre-v0.18 brain." Without the table-existence guard, the bootstrap
|
||||
// would call runMigrations against an empty DB and crash on
|
||||
// `relation "config" does not exist`. Regression test for that path.
|
||||
const engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
try {
|
||||
await engine.initSchema();
|
||||
expect(await engine.getConfig('version')).toBe(String(LATEST_VERSION));
|
||||
|
||||
const db = (engine as any).db;
|
||||
const pages = await db.query(`SELECT 1 FROM pages LIMIT 0`);
|
||||
const sources = await db.query(`SELECT 1 FROM sources LIMIT 0`);
|
||||
const config = await db.query(`SELECT 1 FROM config LIMIT 0`);
|
||||
expect(pages).toBeDefined();
|
||||
expect(sources).toBeDefined();
|
||||
expect(config).toBeDefined();
|
||||
} finally {
|
||||
await engine.disconnect();
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
test('pre-v0.13 links shape: bootstrap adds link_source + origin_page_id', async () => {
|
||||
// Issues #266 / #357 — pre-v0.13 brains had `links` without
|
||||
// `link_source` / `origin_page_id`. Schema blob's
|
||||
// `CREATE INDEX idx_links_source` would crash before v11 ran.
|
||||
const engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
try {
|
||||
await engine.initSchema();
|
||||
const db = (engine as any).db;
|
||||
|
||||
await db.exec(`
|
||||
DROP INDEX IF EXISTS idx_links_source;
|
||||
DROP INDEX IF EXISTS idx_links_origin;
|
||||
ALTER TABLE links DROP CONSTRAINT IF EXISTS links_from_to_type_source_origin_unique;
|
||||
ALTER TABLE links DROP COLUMN IF EXISTS link_source;
|
||||
ALTER TABLE links DROP COLUMN IF EXISTS origin_page_id;
|
||||
`);
|
||||
|
||||
await (engine as any).applyForwardReferenceBootstrap();
|
||||
|
||||
const { rows: lsCol } = await db.query(`
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'links' AND column_name = 'link_source'
|
||||
`);
|
||||
expect(lsCol).toHaveLength(1);
|
||||
|
||||
const { rows: opCol } = await db.query(`
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'links' AND column_name = 'origin_page_id'
|
||||
`);
|
||||
expect(opCol).toHaveLength(1);
|
||||
} finally {
|
||||
await engine.disconnect();
|
||||
}
|
||||
}, 30000);
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, test, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
|
||||
import { describe, expect, test, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, rmSync, writeFileSync, readFileSync, existsSync, mkdirSync, symlinkSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
@@ -9,7 +9,6 @@ import {
|
||||
BrainWriterError,
|
||||
} from '../src/core/brain-writer.ts';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
|
||||
const fence = '---';
|
||||
|
||||
@@ -116,24 +115,15 @@ describe('scanBrainSources (PGLite)', () => {
|
||||
let tmp: string;
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
// One PGLite per file — beforeEach wipes data only. PGLite cold-start is
|
||||
// ~20s on CI; sharing one engine across 6 tests in this block saves ~2 min.
|
||||
beforeAll(async () => {
|
||||
beforeEach(async () => {
|
||||
tmp = mkdtempSync(join(tmpdir(), 'brain-writer-scan-'));
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
afterEach(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
tmp = mkdtempSync(join(tmpdir(), 'brain-writer-scan-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
|
||||
@@ -1,233 +0,0 @@
|
||||
/**
|
||||
* E2E tests for src/mcp/http-transport.ts against real Postgres.
|
||||
*
|
||||
* Catches schema drift (column-name typos that would slip past the unit suite's
|
||||
* stubbed engine.sql) and proves the F1+F2+F3 dispatch pipeline works against a
|
||||
* real handler doing real DB work. Also exercises the SQL-level last_used_at
|
||||
* debounce against real Postgres semantics.
|
||||
*
|
||||
* Run: DATABASE_URL=... bun test test/e2e/http-transport.test.ts
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { createHash, randomBytes } from 'crypto';
|
||||
import { startHttpTransport } from '../../src/mcp/http-transport.ts';
|
||||
import { hasDatabase, setupDB, teardownDB, getEngine, getConn } from './helpers.ts';
|
||||
|
||||
const skip = !hasDatabase();
|
||||
const describeE2E = skip ? describe.skip : describe;
|
||||
|
||||
if (skip) {
|
||||
console.log('Skipping E2E http-transport tests (DATABASE_URL not set)');
|
||||
}
|
||||
|
||||
interface ServerHandle {
|
||||
port: number;
|
||||
stop: () => Promise<void>;
|
||||
}
|
||||
|
||||
function generateToken(): string {
|
||||
return 'gbrain_test_' + randomBytes(16).toString('hex');
|
||||
}
|
||||
|
||||
function hashToken(token: string): string {
|
||||
return createHash('sha256').update(token).digest('hex');
|
||||
}
|
||||
|
||||
async function startServer(): Promise<ServerHandle> {
|
||||
const engine = getEngine();
|
||||
const server = await startHttpTransport({ port: 0, engine: engine as any });
|
||||
return {
|
||||
port: (server as any).port,
|
||||
stop: async () => { (server as any).stop(true); },
|
||||
};
|
||||
}
|
||||
|
||||
function rpc(method: string, params?: unknown, id: number = 1) {
|
||||
return JSON.stringify({ jsonrpc: '2.0', id, method, ...(params !== undefined ? { params } : {}) });
|
||||
}
|
||||
|
||||
describeE2E('http-transport E2E (real Postgres)', () => {
|
||||
let srv: ServerHandle;
|
||||
let validToken: string;
|
||||
let revokedToken: string;
|
||||
let validTokenName: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
await setupDB();
|
||||
const conn = getConn();
|
||||
|
||||
// Seed a valid + revoked token directly via SQL (mirrors auth.ts's create path).
|
||||
validToken = generateToken();
|
||||
validTokenName = 'e2e-valid-' + randomBytes(4).toString('hex');
|
||||
await conn.unsafe(
|
||||
'INSERT INTO access_tokens (name, token_hash) VALUES ($1, $2)',
|
||||
[validTokenName, hashToken(validToken)],
|
||||
);
|
||||
revokedToken = generateToken();
|
||||
await conn.unsafe(
|
||||
'INSERT INTO access_tokens (name, token_hash, revoked_at) VALUES ($1, $2, now())',
|
||||
['e2e-revoked-' + randomBytes(4).toString('hex'), hashToken(revokedToken)],
|
||||
);
|
||||
|
||||
srv = await startServer();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (srv) await srv.stop();
|
||||
await teardownDB();
|
||||
});
|
||||
|
||||
test('1. /health → 200 with expected JSON shape', async () => {
|
||||
const r = await fetch(`http://localhost:${srv.port}/health`);
|
||||
expect(r.status).toBe(200);
|
||||
const body = await r.json();
|
||||
expect(body.status).toBe('ok');
|
||||
expect(body.transport).toBe('http');
|
||||
expect(body.version).toBeString();
|
||||
});
|
||||
|
||||
test('2. /mcp tools/list with valid Bearer → 200 + ops list', async () => {
|
||||
const r = await fetch(`http://localhost:${srv.port}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${validToken}`, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/list'),
|
||||
});
|
||||
expect(r.status).toBe(200);
|
||||
const body = await r.json();
|
||||
expect(body.result.tools).toBeArray();
|
||||
expect(body.result.tools.length).toBeGreaterThan(5);
|
||||
expect(r.headers.get('content-type')).toContain('application/json');
|
||||
});
|
||||
|
||||
test('3. /mcp tools/call (real op: list_pages) round-trips successfully — F1+F2+F3 guard', async () => {
|
||||
const r = await fetch(`http://localhost:${srv.port}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${validToken}`, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/call', { name: 'list_pages', arguments: { limit: 5 } }),
|
||||
});
|
||||
expect(r.status).toBe(200);
|
||||
const body = await r.json();
|
||||
expect(body.jsonrpc).toBe('2.0');
|
||||
expect(body.result.content).toBeArray();
|
||||
// Should NOT be an error — handler ran successfully against the real engine.
|
||||
expect(body.result.isError).toBeUndefined();
|
||||
// Result text should parse as JSON (list_pages returns an object/array)
|
||||
const resultText = body.result.content[0].text;
|
||||
const parsed = JSON.parse(resultText);
|
||||
expect(parsed).toBeDefined();
|
||||
});
|
||||
|
||||
test('4. revoked token → 401', async () => {
|
||||
const r = await fetch(`http://localhost:${srv.port}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${revokedToken}`, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/list'),
|
||||
});
|
||||
expect(r.status).toBe(401);
|
||||
});
|
||||
|
||||
test('5. last_used_at debounce: two consecutive valid calls → only one UPDATE within 60s', async () => {
|
||||
const conn = getConn();
|
||||
|
||||
// Reset last_used_at to NULL so the first call definitely updates
|
||||
await conn.unsafe('UPDATE access_tokens SET last_used_at = NULL WHERE name = $1', [validTokenName]);
|
||||
|
||||
// First request — should update last_used_at
|
||||
await fetch(`http://localhost:${srv.port}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${validToken}`, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/list'),
|
||||
});
|
||||
// Give the fire-and-forget UPDATE a moment to land
|
||||
await new Promise(r => setTimeout(r, 50));
|
||||
|
||||
const [row1] = await conn.unsafe(
|
||||
'SELECT last_used_at FROM access_tokens WHERE name = $1',
|
||||
[validTokenName],
|
||||
) as { last_used_at: Date | null }[];
|
||||
expect(row1.last_used_at).not.toBeNull();
|
||||
const firstUpdate = row1.last_used_at;
|
||||
|
||||
// Second request immediately — should NOT trigger another UPDATE (debounced by SQL WHERE)
|
||||
await fetch(`http://localhost:${srv.port}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${validToken}`, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/list'),
|
||||
});
|
||||
await new Promise(r => setTimeout(r, 50));
|
||||
|
||||
const [row2] = await conn.unsafe(
|
||||
'SELECT last_used_at FROM access_tokens WHERE name = $1',
|
||||
[validTokenName],
|
||||
) as { last_used_at: Date | null }[];
|
||||
// Same timestamp = same UPDATE = debounce held
|
||||
expect(row2.last_used_at?.getTime()).toBe(firstUpdate?.getTime());
|
||||
});
|
||||
|
||||
test('6. last_used_at debounce: simulating 65s gap → second request DOES update', async () => {
|
||||
const conn = getConn();
|
||||
|
||||
// Set last_used_at to 65 seconds ago — simulates the time gap without waiting in real time
|
||||
await conn.unsafe(
|
||||
`UPDATE access_tokens SET last_used_at = now() - interval '65 seconds' WHERE name = $1`,
|
||||
[validTokenName],
|
||||
);
|
||||
const [before] = await conn.unsafe(
|
||||
'SELECT last_used_at FROM access_tokens WHERE name = $1',
|
||||
[validTokenName],
|
||||
) as { last_used_at: Date | null }[];
|
||||
|
||||
await fetch(`http://localhost:${srv.port}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${validToken}`, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/list'),
|
||||
});
|
||||
await new Promise(r => setTimeout(r, 50));
|
||||
|
||||
const [after] = await conn.unsafe(
|
||||
'SELECT last_used_at FROM access_tokens WHERE name = $1',
|
||||
[validTokenName],
|
||||
) as { last_used_at: Date | null }[];
|
||||
expect(after.last_used_at?.getTime()).toBeGreaterThan(before.last_used_at!.getTime());
|
||||
});
|
||||
|
||||
test('7. mcp_request_log gets a row per request', async () => {
|
||||
const conn = getConn();
|
||||
const beforeRows = await conn.unsafe('SELECT count(*)::int AS n FROM mcp_request_log') as { n: number }[];
|
||||
const beforeN = beforeRows[0].n;
|
||||
|
||||
await fetch(`http://localhost:${srv.port}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${validToken}`, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/list'),
|
||||
});
|
||||
// Fire-and-forget audit insert — give it a tick
|
||||
await new Promise(r => setTimeout(r, 100));
|
||||
|
||||
const afterRows = await conn.unsafe('SELECT count(*)::int AS n FROM mcp_request_log') as { n: number }[];
|
||||
expect(afterRows[0].n).toBeGreaterThan(beforeN);
|
||||
|
||||
const [row] = await conn.unsafe(
|
||||
`SELECT token_name, operation, status, latency_ms FROM mcp_request_log
|
||||
WHERE token_name = $1 ORDER BY created_at DESC LIMIT 1`,
|
||||
[validTokenName],
|
||||
) as { token_name: string; operation: string; status: string; latency_ms: number }[];
|
||||
expect(row.token_name).toBe(validTokenName);
|
||||
expect(row.operation).toBe('tools/list');
|
||||
expect(row.status).toBe('success');
|
||||
expect(row.latency_ms).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
test('8. tools/call with malformed params → isError result with invalid_params', async () => {
|
||||
const r = await fetch(`http://localhost:${srv.port}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${validToken}`, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/call', { name: 'get_page', arguments: { slug: 42 } }),
|
||||
});
|
||||
expect(r.status).toBe(200);
|
||||
const body = await r.json();
|
||||
expect(body.result.isError).toBe(true);
|
||||
expect(body.result.content[0].text).toContain('invalid_params');
|
||||
});
|
||||
});
|
||||
@@ -1,168 +0,0 @@
|
||||
/**
|
||||
* E2E parity tests — scanIntegrity batch path vs sequential path.
|
||||
*
|
||||
* The batch path (Postgres-only fast path added in v0.20.x) and the sequential
|
||||
* path (engine.getAllSlugs + getPage loop) MUST return the same result for
|
||||
* every supported case, otherwise gbrain doctor reports different numbers
|
||||
* depending on engine type or whether batch was attempted.
|
||||
*
|
||||
* Codex review of the original perf commit caught a multi-source dedup
|
||||
* regression: the batch SQL scanned raw (source_id, slug) rows while
|
||||
* sequential's getAllSlugs() returned a Set<string>. v0.22.7 adds
|
||||
* SELECT DISTINCT ON (slug) to the batch SQL; these tests prove parity.
|
||||
*
|
||||
* Run: DATABASE_URL=... bun test test/e2e/integrity-batch.test.ts
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { hasDatabase, setupDB, teardownDB, getEngine, getConn } from './helpers.ts';
|
||||
import { scanIntegrity } from '../../src/commands/integrity.ts';
|
||||
|
||||
const skip = !hasDatabase();
|
||||
const describeE2E = skip ? describe.skip : describe;
|
||||
|
||||
if (skip) {
|
||||
console.log('Skipping E2E integrity batch parity tests (DATABASE_URL not set)');
|
||||
}
|
||||
|
||||
describeE2E('scanIntegrity batch parity (E2E, Postgres-only)', () => {
|
||||
beforeAll(async () => {
|
||||
await setupDB();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await teardownDB();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clean slate per case so fixtures don't leak across describes.
|
||||
const conn = getConn();
|
||||
await conn.unsafe(`TRUNCATE pages CASCADE`);
|
||||
});
|
||||
|
||||
describe('dedup', () => {
|
||||
test('multi-source duplicate slugs scan once, not once-per-source', async () => {
|
||||
const engine = getEngine();
|
||||
const conn = getConn();
|
||||
|
||||
// Seed default-source page via the engine.
|
||||
await engine.putPage('people/alice', {
|
||||
type: 'person',
|
||||
title: 'Alice',
|
||||
compiled_truth: 'Alice writes about AI safety.',
|
||||
timeline: '',
|
||||
frontmatter: {},
|
||||
});
|
||||
|
||||
// Seed alt-source row via raw SQL — engine.putPage doesn't take a source_id,
|
||||
// and we specifically need to test that DISTINCT ON (slug) collapses
|
||||
// the multi-source rows into one scan.
|
||||
await conn.unsafe(`
|
||||
INSERT INTO sources (id, name) VALUES ('test-source-2', 'test-source-2')
|
||||
ON CONFLICT DO NOTHING
|
||||
`);
|
||||
await conn.unsafe(`
|
||||
INSERT INTO pages (source_id, slug, type, title, compiled_truth, timeline, frontmatter)
|
||||
VALUES ('test-source-2', 'people/alice', 'person', 'Alice (alt source)',
|
||||
'Alice from another source.', '', '{}'::jsonb)
|
||||
`);
|
||||
|
||||
const batchResult = await scanIntegrity(engine, { limit: 100, batchLoad: true });
|
||||
const seqResult = await scanIntegrity(engine, { limit: 100, batchLoad: false });
|
||||
|
||||
// Both paths must report the same number of distinct slugs scanned.
|
||||
// Pre-fix: batch reported 2 (one per source row), sequential reported 1.
|
||||
expect(batchResult.pagesScanned).toBe(seqResult.pagesScanned);
|
||||
expect(batchResult.pagesScanned).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('hits', () => {
|
||||
test('bareHits and externalHits arrays match between paths', async () => {
|
||||
const engine = getEngine();
|
||||
|
||||
await engine.putPage('people/alice', {
|
||||
type: 'person',
|
||||
title: 'Alice',
|
||||
compiled_truth: 'Alice tweeted about AI safety last week.',
|
||||
timeline: '',
|
||||
frontmatter: {},
|
||||
});
|
||||
await engine.putPage('people/bob', {
|
||||
type: 'person',
|
||||
title: 'Bob',
|
||||
compiled_truth: 'Bob wrote at [example](https://example.com/bob).',
|
||||
timeline: '',
|
||||
frontmatter: {},
|
||||
});
|
||||
|
||||
const batchResult = await scanIntegrity(engine, { limit: 100, batchLoad: true });
|
||||
const seqResult = await scanIntegrity(engine, { limit: 100, batchLoad: false });
|
||||
|
||||
expect(batchResult.bareHits.length).toBe(seqResult.bareHits.length);
|
||||
expect(batchResult.externalHits.length).toBe(seqResult.externalHits.length);
|
||||
expect(batchResult.bareHits.map(h => h.slug).sort()).toEqual(
|
||||
seqResult.bareHits.map(h => h.slug).sort(),
|
||||
);
|
||||
expect(batchResult.externalHits.map(h => h.slug).sort()).toEqual(
|
||||
seqResult.externalHits.map(h => h.slug).sort(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('validate', () => {
|
||||
test('validate:false (boolean) page is skipped on both paths', async () => {
|
||||
const engine = getEngine();
|
||||
|
||||
await engine.putPage('people/alice', {
|
||||
type: 'person',
|
||||
title: 'Alice',
|
||||
compiled_truth: 'Alice tweeted about something.',
|
||||
timeline: '',
|
||||
frontmatter: {},
|
||||
});
|
||||
await engine.putPage('people/legacy', {
|
||||
type: 'person',
|
||||
title: 'Legacy',
|
||||
compiled_truth: 'Legacy tweeted about old stuff.',
|
||||
timeline: '',
|
||||
frontmatter: { validate: false },
|
||||
});
|
||||
|
||||
const batchResult = await scanIntegrity(engine, { limit: 100, batchLoad: true });
|
||||
const seqResult = await scanIntegrity(engine, { limit: 100, batchLoad: false });
|
||||
|
||||
expect(batchResult.pagesScanned).toBe(seqResult.pagesScanned);
|
||||
expect(batchResult.pagesScanned).toBe(1);
|
||||
expect(batchResult.bareHits.map(h => h.slug)).not.toContain('people/legacy');
|
||||
expect(seqResult.bareHits.map(h => h.slug)).not.toContain('people/legacy');
|
||||
});
|
||||
});
|
||||
|
||||
describe('topPages', () => {
|
||||
test('topPages ordering matches between paths', async () => {
|
||||
const engine = getEngine();
|
||||
|
||||
// Alice has 2 bare-tweet hits; Bob has 1.
|
||||
await engine.putPage('people/alice', {
|
||||
type: 'person',
|
||||
title: 'Alice',
|
||||
compiled_truth: 'Alice tweeted today. Alice tweeted yesterday too.',
|
||||
timeline: '',
|
||||
frontmatter: {},
|
||||
});
|
||||
await engine.putPage('people/bob', {
|
||||
type: 'person',
|
||||
title: 'Bob',
|
||||
compiled_truth: 'Bob tweeted once.',
|
||||
timeline: '',
|
||||
frontmatter: {},
|
||||
});
|
||||
|
||||
const batchResult = await scanIntegrity(engine, { limit: 100, batchLoad: true });
|
||||
const seqResult = await scanIntegrity(engine, { limit: 100, batchLoad: false });
|
||||
|
||||
expect(batchResult.topPages).toEqual(seqResult.topPages);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -43,7 +43,7 @@ beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({}); // in-memory PGLite
|
||||
await engine.initSchema(); // installs pages, minion_jobs, config, etc.
|
||||
}, 30000);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
/**
|
||||
* E2E test for PostgresEngine forward-reference bootstrap.
|
||||
*
|
||||
* Codex caught that `test/e2e/helpers.ts:74` uses the standalone
|
||||
* `db.initSchema()` from `src/core/db.ts`, which only runs SCHEMA_SQL and
|
||||
* never calls runMigrations(). A test using that helper would NOT exercise
|
||||
* `PostgresEngine.initSchema()`'s reordered path, producing false-positive
|
||||
* coverage. This test deliberately bypasses the standard helper and
|
||||
* instantiates `PostgresEngine` directly, calling `engine.initSchema()` so
|
||||
* the bootstrap → SCHEMA_SQL → runMigrations sequence runs end-to-end.
|
||||
*
|
||||
* Covers issues #366, #375, #378 — Postgres-side wedges where pre-v0.18
|
||||
* brains crashed on `column "source_id" does not exist`.
|
||||
*
|
||||
* NOTE: snapshot-based historical state simulation is out of scope for this
|
||||
* wave (would require maintaining historical schema dumps). The test
|
||||
* mutates a fresh-LATEST brain to a pre-v0.18 shape; codex flagged this as
|
||||
* approximate. Acceptable here because the bootstrap's contract is narrow:
|
||||
* "given a brain that lacks the specific forward-references, initSchema
|
||||
* produces a brain at LATEST." The test exercises exactly that contract.
|
||||
*
|
||||
* Run: DATABASE_URL=postgresql://... bun run test:e2e test/e2e/postgres-bootstrap.test.ts
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { PostgresEngine } from '../../src/core/postgres-engine.ts';
|
||||
import { LATEST_VERSION } from '../../src/core/migrate.ts';
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
const skip = !DATABASE_URL;
|
||||
|
||||
describe.skipIf(skip)('PostgresEngine forward-reference bootstrap (E2E)', () => {
|
||||
let engine: PostgresEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PostgresEngine();
|
||||
await engine.connect({ database_url: DATABASE_URL! });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
test('PostgresEngine.initSchema applies bootstrap → SCHEMA_SQL → migrations on pre-v0.18 brain', async () => {
|
||||
// First call: bring the test DB to LATEST shape so we have something to mutate.
|
||||
await engine.initSchema();
|
||||
|
||||
// Clear data from prior tests in the suite. Adding a UNIQUE(slug)
|
||||
// constraint below would fail if multi-source fixtures left rows with
|
||||
// duplicate slugs across sources (which is valid under the composite
|
||||
// UNIQUE this test is undoing).
|
||||
const conn = (engine as any).sql;
|
||||
await conn.unsafe(`TRUNCATE pages, content_chunks, links, tags, raw_data, timeline_entries, page_versions, ingest_log RESTART IDENTITY CASCADE`);
|
||||
|
||||
// Mutate to pre-v0.18 shape: drop source_id and the sources table.
|
||||
// The advisory lock is released between initSchema calls, so this
|
||||
// direct DDL won't deadlock.
|
||||
await conn.unsafe(`
|
||||
ALTER TABLE pages DROP CONSTRAINT IF EXISTS pages_source_slug_key;
|
||||
ALTER TABLE pages ADD CONSTRAINT pages_slug_key UNIQUE (slug);
|
||||
DROP INDEX IF EXISTS idx_pages_source_id;
|
||||
ALTER TABLE pages DROP COLUMN IF EXISTS source_id CASCADE;
|
||||
DROP TABLE IF EXISTS sources CASCADE;
|
||||
`);
|
||||
await engine.setConfig('version', '20');
|
||||
|
||||
// The path under test: full PostgresEngine.initSchema() including the
|
||||
// bootstrap call, SCHEMA_SQL replay, and runMigrations chain.
|
||||
await engine.initSchema();
|
||||
|
||||
expect(await engine.getConfig('version')).toBe(String(LATEST_VERSION));
|
||||
|
||||
// Verify the forward-referenced column exists after upgrade.
|
||||
const colCheck = await conn`
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_schema = current_schema()
|
||||
AND table_name = 'pages'
|
||||
AND column_name = 'source_id'
|
||||
`;
|
||||
expect(colCheck).toHaveLength(1);
|
||||
|
||||
// Verify the default source row was seeded.
|
||||
const srcCheck = await conn`SELECT id FROM sources WHERE id = 'default'`;
|
||||
expect(srcCheck).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('PostgresEngine.initSchema is idempotent on a brain already at LATEST', async () => {
|
||||
// Fresh-LATEST brain. Calling initSchema again must not error and must
|
||||
// not regress the version.
|
||||
await engine.initSchema();
|
||||
expect(await engine.getConfig('version')).toBe(String(LATEST_VERSION));
|
||||
});
|
||||
});
|
||||
@@ -7,39 +7,28 @@
|
||||
*
|
||||
* All tests use PGLite/in-memory — no DB connection required.
|
||||
*/
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { runExtractCore } from '../src/commands/extract.ts';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import type { BrainEngine } from '../src/core/engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
|
||||
// One PGLite per file (beforeAll), wipe data per test (beforeEach).
|
||||
// PGLite cold-start dominates wall-time; sharing the engine across all tests
|
||||
// in this file cuts ~22s × 8 tests = ~3 min on CI.
|
||||
let engine: PGLiteEngine;
|
||||
let tempDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
beforeEach(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({ engine: 'pglite' });
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
tempDir = mkdtempSync(join(tmpdir(), 'gbrain-extract-test-'));
|
||||
mkdirSync(join(tempDir, 'people'), { recursive: true });
|
||||
mkdirSync(join(tempDir, 'companies'), { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
afterEach(async () => {
|
||||
await engine.disconnect();
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
/**
|
||||
* Wipe per-test data on a connected PGLite engine without dropping the schema.
|
||||
* Used by tests that share one engine across the file (beforeAll) and need a
|
||||
* clean slate per test (beforeEach).
|
||||
*
|
||||
* Why this exists: PGLite WASM cold-start + initSchema() is ~20s on CI runners.
|
||||
* Spinning up a fresh engine per test (the prior beforeEach pattern) multiplies
|
||||
* that across every test in every file. Sharing one engine and wiping data
|
||||
* is two orders of magnitude faster.
|
||||
*
|
||||
* Implementation:
|
||||
* 1. TRUNCATE every public table CASCADE, including `sources` (so tests
|
||||
* that register their own sources don't leak rows into the next test).
|
||||
* 2. Re-seed the default source row that pages.source_id's DEFAULT FKs
|
||||
* against. Without this, the next page insert would fail FK validation.
|
||||
* 3. Preserve `schema_version` — it carries the migration ledger that
|
||||
* initSchema() populates; wiping it would make migration helpers think
|
||||
* the brain is on v0.
|
||||
*
|
||||
* Identifier-quoted defensively against pathological table names.
|
||||
*/
|
||||
import type { PGLiteEngine } from '../../src/core/pglite-engine.ts';
|
||||
|
||||
const PRESERVE_TABLES = new Set(['schema_version']);
|
||||
|
||||
export async function resetPgliteState(engine: PGLiteEngine): Promise<void> {
|
||||
const rows = await engine.executeRaw<{ tablename: string }>(
|
||||
`SELECT tablename FROM pg_tables WHERE schemaname='public'`,
|
||||
);
|
||||
const targets = rows
|
||||
.map(r => r.tablename)
|
||||
.filter(name => !PRESERVE_TABLES.has(name));
|
||||
if (targets.length === 0) return;
|
||||
const quoted = targets.map(t => `"${t.replace(/"/g, '""')}"`).join(', ');
|
||||
await engine.executeRaw(`TRUNCATE ${quoted} RESTART IDENTITY CASCADE`);
|
||||
// Re-seed the default source row that initSchema() inserts. Mirrors the
|
||||
// INSERT in src/core/pglite-schema.ts so the FK target survives reset.
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO sources (id, name, config)
|
||||
VALUES ('default', 'default', '{"federated": true}'::jsonb)
|
||||
ON CONFLICT (id) DO NOTHING`,
|
||||
);
|
||||
}
|
||||
@@ -1,519 +0,0 @@
|
||||
/**
|
||||
* Unit tests for src/mcp/http-transport.ts.
|
||||
*
|
||||
* Covers:
|
||||
* - Auth path (valid, missing header, no Bearer prefix, unknown, revoked, /health bypass)
|
||||
* - F1+F2+F3 round-trip guards (handler arg order, full OperationContext, param validation)
|
||||
* - JSON-only response shape (no SSE)
|
||||
* - CORS default-deny + allowlist
|
||||
* - Body cap (Content-Length + chunked)
|
||||
* - Rate limit (token + IP buckets, LRU eviction, TTL prune, /health bypass)
|
||||
*
|
||||
* No DATABASE_URL needed — engine.sql is mocked. E2E coverage of the real Postgres
|
||||
* round-trip lives in test/e2e/http-transport.test.ts.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { createHash } from 'crypto';
|
||||
import { startHttpTransport } from '../src/mcp/http-transport.ts';
|
||||
import { RateLimiter } from '../src/mcp/rate-limit.ts';
|
||||
|
||||
type SqlResult = unknown[] | unknown;
|
||||
type SqlHandler = (query: string, values: unknown[]) => SqlResult | Promise<SqlResult>;
|
||||
|
||||
interface FakeEngine {
|
||||
kind: 'postgres';
|
||||
sql: ReturnType<typeof makeSqlTag>;
|
||||
audit: { token_name: string | null; operation: string; status: string; latency_ms: number }[];
|
||||
}
|
||||
|
||||
function makeSqlTag(handler: SqlHandler) {
|
||||
return (strings: TemplateStringsArray, ...values: unknown[]) => {
|
||||
let query = '';
|
||||
for (let i = 0; i < strings.length; i++) {
|
||||
query += strings[i];
|
||||
if (i < values.length) query += '?';
|
||||
}
|
||||
const result = handler(query.trim(), values);
|
||||
return Promise.resolve(result);
|
||||
};
|
||||
}
|
||||
|
||||
function hash(token: string): string {
|
||||
return createHash('sha256').update(token).digest('hex');
|
||||
}
|
||||
|
||||
interface FakeEngineConfig {
|
||||
validTokens?: Map<string, { id: string; name: string }>;
|
||||
/** Tokens that are present but revoked (revoked_at IS NOT NULL — query returns empty). */
|
||||
revokedTokens?: Set<string>;
|
||||
/** If true, every SELECT throws (simulating DB outage). */
|
||||
dbDown?: boolean;
|
||||
}
|
||||
|
||||
function makeFakeEngine(cfg: FakeEngineConfig = {}): FakeEngine {
|
||||
const validTokens = cfg.validTokens ?? new Map();
|
||||
const revokedTokens = cfg.revokedTokens ?? new Set();
|
||||
const audit: FakeEngine['audit'] = [];
|
||||
|
||||
const sql = makeSqlTag((query, values) => {
|
||||
if (cfg.dbDown && query.startsWith('SELECT')) throw new Error('db down');
|
||||
|
||||
if (query === 'SELECT 1') {
|
||||
// /health DB probe
|
||||
return [{ '?column?': 1 }];
|
||||
}
|
||||
|
||||
if (query.startsWith('SELECT id, name FROM access_tokens')) {
|
||||
const tokenHash = values[0] as string;
|
||||
if (revokedTokens.has(tokenHash)) return [];
|
||||
const row = validTokens.get(tokenHash);
|
||||
return row ? [row] : [];
|
||||
}
|
||||
|
||||
if (query.startsWith('UPDATE access_tokens')) {
|
||||
// last_used_at debounce — succeed silently
|
||||
return [];
|
||||
}
|
||||
|
||||
if (query.startsWith('INSERT INTO mcp_request_log')) {
|
||||
audit.push({
|
||||
token_name: values[0] as string | null,
|
||||
operation: values[1] as string,
|
||||
latency_ms: values[2] as number,
|
||||
status: values[3] as string,
|
||||
});
|
||||
return [];
|
||||
}
|
||||
|
||||
return [];
|
||||
});
|
||||
|
||||
return { kind: 'postgres', sql, audit };
|
||||
}
|
||||
|
||||
interface TestServer {
|
||||
url: string;
|
||||
stop: () => void;
|
||||
engine: FakeEngine;
|
||||
ipLimiter: RateLimiter;
|
||||
tokenLimiter: RateLimiter;
|
||||
}
|
||||
|
||||
let mockNow = 0;
|
||||
function freezeClock(at: number) { mockNow = at; }
|
||||
function advanceClock(deltaMs: number) { mockNow += deltaMs; }
|
||||
|
||||
async function startTest(cfg: FakeEngineConfig & { lruCap?: number; ipLimit?: number; tokenLimit?: number; corsOrigin?: string; bodyCap?: number; trustProxy?: boolean } = {}): Promise<TestServer> {
|
||||
if (cfg.corsOrigin) process.env.GBRAIN_HTTP_CORS_ORIGIN = cfg.corsOrigin;
|
||||
else delete process.env.GBRAIN_HTTP_CORS_ORIGIN;
|
||||
if (cfg.bodyCap) process.env.GBRAIN_HTTP_MAX_BODY_BYTES = String(cfg.bodyCap);
|
||||
else delete process.env.GBRAIN_HTTP_MAX_BODY_BYTES;
|
||||
if (cfg.trustProxy) process.env.GBRAIN_HTTP_TRUST_PROXY = '1';
|
||||
else delete process.env.GBRAIN_HTTP_TRUST_PROXY;
|
||||
|
||||
const engine = makeFakeEngine(cfg);
|
||||
const clock = () => mockNow || Date.now();
|
||||
const ipLimiter = new RateLimiter(
|
||||
{ limit: cfg.ipLimit ?? 1000, windowMs: 60_000, lruCap: cfg.lruCap ?? 10000 },
|
||||
clock,
|
||||
);
|
||||
const tokenLimiter = new RateLimiter(
|
||||
{ limit: cfg.tokenLimit ?? 1000, windowMs: 60_000, lruCap: cfg.lruCap ?? 10000 },
|
||||
clock,
|
||||
);
|
||||
const server = await startHttpTransport({
|
||||
port: 0,
|
||||
engine: engine as any,
|
||||
limiters: { ip: ipLimiter, token: tokenLimiter },
|
||||
});
|
||||
return {
|
||||
url: `http://localhost:${(server as any).port}`,
|
||||
stop: () => (server as any).stop(true),
|
||||
engine,
|
||||
ipLimiter,
|
||||
tokenLimiter,
|
||||
};
|
||||
}
|
||||
|
||||
function rpc(method: string, params?: unknown, id: number = 1) {
|
||||
return JSON.stringify({ jsonrpc: '2.0', id, method, ...(params !== undefined ? { params } : {}) });
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Auth path
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('http-transport: auth', () => {
|
||||
let srv: TestServer;
|
||||
const VALID_TOKEN = 'valid-token-abc';
|
||||
const REVOKED_TOKEN = 'revoked-token-xyz';
|
||||
|
||||
beforeAll(async () => {
|
||||
srv = await startTest({
|
||||
validTokens: new Map([[hash(VALID_TOKEN), { id: 'tok-1', name: 'test' }]]),
|
||||
revokedTokens: new Set([hash(REVOKED_TOKEN)]),
|
||||
});
|
||||
});
|
||||
afterAll(() => srv.stop());
|
||||
|
||||
test('1. valid token → 200 + tools/list returns ops', async () => {
|
||||
const r = await fetch(`${srv.url}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${VALID_TOKEN}`, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/list'),
|
||||
});
|
||||
expect(r.status).toBe(200);
|
||||
const body = await r.json();
|
||||
expect(body.result.tools).toBeArray();
|
||||
expect(body.result.tools.length).toBeGreaterThan(0);
|
||||
expect(body.jsonrpc).toBe('2.0');
|
||||
});
|
||||
|
||||
test('2. missing Authorization header → 401', async () => {
|
||||
const r = await fetch(`${srv.url}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/list'),
|
||||
});
|
||||
expect(r.status).toBe(401);
|
||||
});
|
||||
|
||||
test('3. header missing Bearer prefix → 401', async () => {
|
||||
const r = await fetch(`${srv.url}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': VALID_TOKEN, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/list'),
|
||||
});
|
||||
expect(r.status).toBe(401);
|
||||
});
|
||||
|
||||
test('4. unknown token → 401', async () => {
|
||||
const r = await fetch(`${srv.url}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': 'Bearer not-a-real-token', 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/list'),
|
||||
});
|
||||
expect(r.status).toBe(401);
|
||||
});
|
||||
|
||||
test('5. revoked token → 401', async () => {
|
||||
const r = await fetch(`${srv.url}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${REVOKED_TOKEN}`, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/list'),
|
||||
});
|
||||
expect(r.status).toBe(401);
|
||||
});
|
||||
|
||||
test('6. /health → 200 without auth, body has expected fields, probes DB', async () => {
|
||||
const r = await fetch(`${srv.url}/health`);
|
||||
expect(r.status).toBe(200);
|
||||
const body = await r.json();
|
||||
expect(body.status).toBe('ok');
|
||||
expect(body.transport).toBe('http');
|
||||
expect(body.version).toBeString();
|
||||
expect(body.db).toBe('ok');
|
||||
});
|
||||
|
||||
test('6b. /health → 503 when DB is unreachable', async () => {
|
||||
const dbDownSrv = await startTest({ dbDown: true });
|
||||
try {
|
||||
const r = await fetch(`${dbDownSrv.url}/health`);
|
||||
expect(r.status).toBe(503);
|
||||
const body = await r.json();
|
||||
expect(body.status).toBe('unhealthy');
|
||||
expect(body.db).toBe('unreachable');
|
||||
} finally { dbDownSrv.stop(); }
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// F1+F2+F3 regression guards (the actual existing-PR bugs)
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('http-transport: tools/call dispatch', () => {
|
||||
let srv: TestServer;
|
||||
const TOK = 'tok-fix';
|
||||
|
||||
beforeAll(async () => {
|
||||
srv = await startTest({ validTokens: new Map([[hash(TOK), { id: 'tok-fix-id', name: 'fix' }]]) });
|
||||
});
|
||||
afterAll(() => srv.stop());
|
||||
|
||||
test('7. tools/call with a real op (list_pages) round-trips successfully (F1+F2 guard)', async () => {
|
||||
// list_pages doesn't need real DB rows in this stub — it'll call engine methods we don't mock,
|
||||
// so we expect EITHER a successful tool-result OR an isError result with a meaningful message.
|
||||
// The point is that the handler IS invoked with (ctx, params) order — not (params, ctx).
|
||||
// If F1 regressed, the handler would receive {limit: 1} as ctx and crash trying to read ctx.engine.
|
||||
const r = await fetch(`${srv.url}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/call', { name: 'list_pages', arguments: { limit: 1 } }),
|
||||
});
|
||||
expect(r.status).toBe(200);
|
||||
const body = await r.json();
|
||||
expect(body.jsonrpc).toBe('2.0');
|
||||
expect(body.result).toBeDefined();
|
||||
expect(body.result.content).toBeArray();
|
||||
// Either success (handler ran) or a structured error (handler ran and returned an error)
|
||||
// — both prove dispatch reached the handler with the correct shape.
|
||||
});
|
||||
|
||||
test('8. tools/call with malformed params → 200 wrapping an isError result (F3 guard via dispatch.ts)', async () => {
|
||||
const r = await fetch(`${srv.url}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' },
|
||||
// get_page expects `slug` as required string; passing a number triggers validateParams
|
||||
body: rpc('tools/call', { name: 'get_page', arguments: { slug: 42 } }),
|
||||
});
|
||||
expect(r.status).toBe(200);
|
||||
const body = await r.json();
|
||||
expect(body.result.isError).toBe(true);
|
||||
const text = body.result.content[0].text;
|
||||
expect(text).toContain('invalid_params');
|
||||
});
|
||||
|
||||
test('9. /mcp response has Content-Type: application/json (not SSE)', async () => {
|
||||
const r = await fetch(`${srv.url}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/list'),
|
||||
});
|
||||
expect(r.headers.get('content-type')).toContain('application/json');
|
||||
expect(r.headers.get('content-type')).not.toContain('event-stream');
|
||||
});
|
||||
|
||||
test('9b. unknown tool name → 200 wrapping an isError result', async () => {
|
||||
const r = await fetch(`${srv.url}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' },
|
||||
body: rpc('tools/call', { name: 'definitely_not_a_real_tool', arguments: {} }),
|
||||
});
|
||||
expect(r.status).toBe(200);
|
||||
const body = await r.json();
|
||||
expect(body.result.isError).toBe(true);
|
||||
expect(body.result.content[0].text).toContain('Unknown tool');
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// CORS
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('http-transport: CORS', () => {
|
||||
test('10. no GBRAIN_HTTP_CORS_ORIGIN + browser request → no ACAO header', async () => {
|
||||
const srv = await startTest({});
|
||||
try {
|
||||
const r = await fetch(`${srv.url}/health`, { headers: { 'Origin': 'https://evil.example' } });
|
||||
expect(r.headers.get('access-control-allow-origin')).toBeNull();
|
||||
} finally { srv.stop(); }
|
||||
});
|
||||
|
||||
test('11. env set + matching Origin → ACAO echoes', async () => {
|
||||
const srv = await startTest({ corsOrigin: 'https://claude.ai' });
|
||||
try {
|
||||
const r = await fetch(`${srv.url}/health`, { headers: { 'Origin': 'https://claude.ai' } });
|
||||
expect(r.headers.get('access-control-allow-origin')).toBe('https://claude.ai');
|
||||
expect(r.headers.get('vary')).toBe('Origin');
|
||||
} finally { srv.stop(); }
|
||||
});
|
||||
|
||||
test('12. env set + non-matching Origin → no ACAO header', async () => {
|
||||
const srv = await startTest({ corsOrigin: 'https://claude.ai' });
|
||||
try {
|
||||
const r = await fetch(`${srv.url}/health`, { headers: { 'Origin': 'https://evil.example' } });
|
||||
expect(r.headers.get('access-control-allow-origin')).toBeNull();
|
||||
} finally { srv.stop(); }
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Body cap
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('http-transport: body cap', () => {
|
||||
const TOK = 'body-cap-tok';
|
||||
|
||||
test('13. Content-Length over cap → 413', async () => {
|
||||
const srv = await startTest({
|
||||
validTokens: new Map([[hash(TOK), { id: 'b-1', name: 'b' }]]),
|
||||
bodyCap: 100,
|
||||
});
|
||||
try {
|
||||
const big = 'x'.repeat(200);
|
||||
const r = await fetch(`${srv.url}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' },
|
||||
body: big,
|
||||
});
|
||||
expect(r.status).toBe(413);
|
||||
} finally { srv.stop(); }
|
||||
});
|
||||
|
||||
test('14. chunked transfer (no Content-Length) over cap → 413', async () => {
|
||||
const srv = await startTest({
|
||||
validTokens: new Map([[hash(TOK), { id: 'b-2', name: 'b' }]]),
|
||||
bodyCap: 100,
|
||||
});
|
||||
try {
|
||||
// Build a chunked body via a ReadableStream — Bun fetch sends without Content-Length.
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
for (let i = 0; i < 10; i++) controller.enqueue(new TextEncoder().encode('y'.repeat(50)));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
const r = await fetch(`${srv.url}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' },
|
||||
body: stream as any,
|
||||
// @ts-expect-error Bun fetch supports duplex for streaming bodies
|
||||
duplex: 'half',
|
||||
});
|
||||
expect(r.status).toBe(413);
|
||||
} finally { srv.stop(); }
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Rate limit
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('http-transport: rate limit', () => {
|
||||
const TOK = 'rl-tok';
|
||||
|
||||
test('15. token bucket: refill mechanic over time', async () => {
|
||||
freezeClock(1000);
|
||||
const srv = await startTest({
|
||||
validTokens: new Map([[hash(TOK), { id: 'rl-id', name: 'rl' }]]),
|
||||
tokenLimit: 2,
|
||||
ipLimit: 100,
|
||||
});
|
||||
try {
|
||||
// Use up 2 tokens
|
||||
const ok1 = await fetch(`${srv.url}/mcp`, { method: 'POST', headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' }, body: rpc('tools/list') });
|
||||
expect(ok1.status).toBe(200);
|
||||
const ok2 = await fetch(`${srv.url}/mcp`, { method: 'POST', headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' }, body: rpc('tools/list') });
|
||||
expect(ok2.status).toBe(200);
|
||||
|
||||
// Third should 429
|
||||
const blocked = await fetch(`${srv.url}/mcp`, { method: 'POST', headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' }, body: rpc('tools/list') });
|
||||
expect(blocked.status).toBe(429);
|
||||
|
||||
// Advance past the refill window (60s for 2 limit = 30s/token; advance 35s)
|
||||
advanceClock(35_000);
|
||||
const refilled = await fetch(`${srv.url}/mcp`, { method: 'POST', headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' }, body: rpc('tools/list') });
|
||||
expect(refilled.status).toBe(200);
|
||||
} finally { srv.stop(); freezeClock(0); }
|
||||
});
|
||||
|
||||
test('16. token bucket exhausted → 429 + Retry-After header', async () => {
|
||||
freezeClock(1000);
|
||||
const srv = await startTest({
|
||||
validTokens: new Map([[hash(TOK), { id: 'rl16', name: 'rl' }]]),
|
||||
tokenLimit: 1,
|
||||
ipLimit: 100,
|
||||
});
|
||||
try {
|
||||
await fetch(`${srv.url}/mcp`, { method: 'POST', headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' }, body: rpc('tools/list') });
|
||||
const r = await fetch(`${srv.url}/mcp`, { method: 'POST', headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' }, body: rpc('tools/list') });
|
||||
expect(r.status).toBe(429);
|
||||
expect(r.headers.get('retry-after')).not.toBeNull();
|
||||
expect(parseInt(r.headers.get('retry-after')!, 10)).toBeGreaterThan(0);
|
||||
} finally { srv.stop(); freezeClock(0); }
|
||||
});
|
||||
|
||||
test('17. LRU eviction at cap (insert > cap evicts LRU)', () => {
|
||||
let now = 0;
|
||||
const lim = new RateLimiter({ limit: 10, windowMs: 60_000, lruCap: 3 }, () => now);
|
||||
lim.check('a'); now += 1;
|
||||
lim.check('b'); now += 1;
|
||||
lim.check('c'); now += 1;
|
||||
expect(lim.size).toBe(3);
|
||||
lim.check('d'); now += 1;
|
||||
expect(lim.size).toBe(3);
|
||||
// 'a' should have been evicted (oldest by insertion). After re-checking 'a' it's a fresh bucket again.
|
||||
// Easiest verification: hammer 'a' should NOT be already exhausted — fresh bucket starts at limit.
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const r = lim.check('a');
|
||||
expect(r.allowed).toBe(true);
|
||||
}
|
||||
// 11th should fail (no refill since clock barely moved)
|
||||
expect(lim.check('a').allowed).toBe(false);
|
||||
});
|
||||
|
||||
test('18. TTL prune (entries older than 2× window evicted)', () => {
|
||||
let now = 1000;
|
||||
const lim = new RateLimiter({ limit: 10, windowMs: 1000, lruCap: 100 }, () => now);
|
||||
lim.check('stale'); // touched at t=1000
|
||||
expect(lim.size).toBe(1);
|
||||
now = 1000 + 2001; // advance past 2× window
|
||||
lim.check('fresh'); // triggers prune
|
||||
expect(lim.size).toBe(1); // 'stale' evicted, only 'fresh' remains
|
||||
});
|
||||
|
||||
test('19. pre-auth IP bucket fires BEFORE auth (DB not called when IP exhausted)', async () => {
|
||||
freezeClock(1000);
|
||||
const srv = await startTest({
|
||||
ipLimit: 1,
|
||||
tokenLimit: 100,
|
||||
validTokens: new Map([[hash(TOK), { id: 'rl19', name: 'rl' }]]),
|
||||
});
|
||||
try {
|
||||
// First request consumes IP token (will hit auth and succeed)
|
||||
const r1 = await fetch(`${srv.url}/mcp`, { method: 'POST', headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' }, body: rpc('tools/list') });
|
||||
expect(r1.status).toBe(200);
|
||||
// Second: IP bucket exhausted. We send WITHOUT auth header. Should be 429 (IP-limited),
|
||||
// not 401 (auth-failed) — proving IP check happened first.
|
||||
const r2 = await fetch(`${srv.url}/mcp`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: rpc('tools/list') });
|
||||
expect(r2.status).toBe(429);
|
||||
} finally { srv.stop(); freezeClock(0); }
|
||||
});
|
||||
|
||||
test('20. /health bypasses rate limit', async () => {
|
||||
freezeClock(1000);
|
||||
const srv = await startTest({ ipLimit: 1, tokenLimit: 1 });
|
||||
try {
|
||||
// Hammer health 5 times — none should 429
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const r = await fetch(`${srv.url}/health`);
|
||||
expect(r.status).toBe(200);
|
||||
}
|
||||
} finally { srv.stop(); freezeClock(0); }
|
||||
});
|
||||
});
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// mcp_request_log audit
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
describe('http-transport: mcp_request_log audit', () => {
|
||||
test('21. successful request → audit row with token_name + operation + status', async () => {
|
||||
const TOK = 'audit-tok';
|
||||
const srv = await startTest({ validTokens: new Map([[hash(TOK), { id: 'a-1', name: 'audit-test' }]]) });
|
||||
try {
|
||||
await fetch(`${srv.url}/mcp`, { method: 'POST', headers: { 'Authorization': `Bearer ${TOK}`, 'Content-Type': 'application/json' }, body: rpc('tools/list') });
|
||||
// Audit insert is fire-and-forget; give it a tick to land in the fake handler
|
||||
await new Promise(r => setTimeout(r, 10));
|
||||
expect(srv.engine.audit.length).toBeGreaterThanOrEqual(1);
|
||||
const row = srv.engine.audit[srv.engine.audit.length - 1];
|
||||
expect(row.token_name).toBe('audit-test');
|
||||
expect(row.operation).toBe('tools/list');
|
||||
expect(row.status).toBe('success');
|
||||
expect(row.latency_ms).toBeGreaterThanOrEqual(0);
|
||||
} finally { srv.stop(); }
|
||||
});
|
||||
|
||||
test('22. failed auth → audit row with null token_name + auth_failed status', async () => {
|
||||
const srv = await startTest({});
|
||||
try {
|
||||
await fetch(`${srv.url}/mcp`, { method: 'POST', headers: { 'Authorization': 'Bearer wrong', 'Content-Type': 'application/json' }, body: rpc('tools/list') });
|
||||
await new Promise(r => setTimeout(r, 10));
|
||||
expect(srv.engine.audit.length).toBeGreaterThanOrEqual(1);
|
||||
const row = srv.engine.audit[srv.engine.audit.length - 1];
|
||||
expect(row.token_name).toBeNull();
|
||||
expect(row.status).toBe('auth_failed');
|
||||
} finally { srv.stop(); }
|
||||
});
|
||||
});
|
||||
@@ -287,17 +287,6 @@ describe('migration v24 — rls_backfill_missing_tables', () => {
|
||||
test('LATEST_VERSION has caught up to 24', () => {
|
||||
expect(LATEST_VERSION).toBeGreaterThanOrEqual(24);
|
||||
});
|
||||
|
||||
// PGLite has no RLS engine and is intrinsically single-tenant. The 8 RLS
|
||||
// backfill ALTER statements target tables that may not exist on PGLite
|
||||
// (subagent_*, minion_inbox aren't always present in pglite-schema.ts).
|
||||
// sqlFor.pglite='' makes v24 a no-op on PGLite while still bumping the
|
||||
// version counter. Engine.kind discrimination in runMigrations selects
|
||||
// sqlFor[engine.kind] over m.sql. Issue #395.
|
||||
test('uses a PGLite no-op override so local brains skip Postgres-only RLS ALTER TABLEs', () => {
|
||||
const v24 = MIGRATIONS.find(m => m.version === 24);
|
||||
expect(v24?.sqlFor?.pglite).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -1,149 +0,0 @@
|
||||
/**
|
||||
* CI guard: PGLITE_SCHEMA_SQL must not forward-reference state that
|
||||
* `applyForwardReferenceBootstrap` doesn't know how to create.
|
||||
*
|
||||
* Background: gbrain ships an "embedded latest schema" blob
|
||||
* (`pglite-schema.ts`) for fast bootstraps, alongside a numbered migration
|
||||
* chain (`migrate.ts`) for incremental upgrades. Across 2 years and 6 schema
|
||||
* versions, every release that added a column-with-index in the schema blob
|
||||
* without a corresponding bootstrap addition has triggered the same wedge
|
||||
* incident class (#239, #243, #266, #266, #357, #366, #374, #375, #378,
|
||||
* #395, #396).
|
||||
*
|
||||
* The bootstrap is the structural fix. This test enforces the contract:
|
||||
* for every "forward reference" the schema blob makes (FK or indexed column
|
||||
* defined later than its reference site, or any column that older brains
|
||||
* lack), the bootstrap MUST add enough state so that running the schema
|
||||
* blob is replay-safe on a brain that lacks every member of
|
||||
* `REQUIRED_BOOTSTRAP_COVERAGE`.
|
||||
*
|
||||
* **When you add a new schema-blob forward reference:**
|
||||
* 1. Extend `applyForwardReferenceBootstrap` in pglite-engine.ts +
|
||||
* postgres-engine.ts to add the new state.
|
||||
* 2. Add an entry to `REQUIRED_BOOTSTRAP_COVERAGE` below.
|
||||
* 3. This test will pass.
|
||||
*
|
||||
* If you add a forward reference but skip step 1, this test fails. If you
|
||||
* skip step 2, this test passes but the bootstrap silently drifts behind
|
||||
* the schema. The eng-review polish notes recommended layered coverage
|
||||
* (per-engine integration tests in `test/bootstrap.test.ts` +
|
||||
* `test/e2e/postgres-bootstrap.test.ts`) to catch step 2 oversights.
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
|
||||
// Forward-reference targets that PGLITE_SCHEMA_SQL requires.
|
||||
// When you add a new one, extend this list AND the bootstrap.
|
||||
type ForwardReference =
|
||||
| { kind: 'table'; name: string }
|
||||
| { kind: 'column'; table: string; column: string };
|
||||
|
||||
const REQUIRED_BOOTSTRAP_COVERAGE: ForwardReference[] = [
|
||||
// Forward-referenced by `pages.source_id REFERENCES sources(id)` and the
|
||||
// `INSERT INTO sources (id, name, config) VALUES ('default', ...)` seed.
|
||||
{ kind: 'table', name: 'sources' },
|
||||
// Forward-referenced by `CREATE INDEX idx_pages_source_id ON pages(source_id)`.
|
||||
{ kind: 'column', table: 'pages', column: 'source_id' },
|
||||
// Forward-referenced by `CREATE INDEX idx_links_source ON links(link_source)`.
|
||||
{ kind: 'column', table: 'links', column: 'link_source' },
|
||||
// Forward-referenced by `CREATE INDEX idx_links_origin ON links(origin_page_id)`.
|
||||
{ kind: 'column', table: 'links', column: 'origin_page_id' },
|
||||
// v0.19+ — forward-referenced by `CREATE INDEX idx_chunks_symbol_name
|
||||
// ON content_chunks(symbol_name) WHERE symbol_name IS NOT NULL`.
|
||||
{ kind: 'column', table: 'content_chunks', column: 'symbol_name' },
|
||||
// v0.19+ — forward-referenced by `CREATE INDEX idx_chunks_language
|
||||
// ON content_chunks(language) WHERE language IS NOT NULL`.
|
||||
{ kind: 'column', table: 'content_chunks', column: 'language' },
|
||||
];
|
||||
|
||||
test('applyForwardReferenceBootstrap covers every forward reference declared in REQUIRED_BOOTSTRAP_COVERAGE', async () => {
|
||||
const engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
try {
|
||||
await engine.initSchema();
|
||||
const db = (engine as any).db;
|
||||
|
||||
// Strip every required forward-reference target so the brain looks like
|
||||
// it pre-dates the migrations that introduced these objects. Drop columns
|
||||
// before the table-level constraints that depend on them.
|
||||
await db.exec(`
|
||||
ALTER TABLE pages DROP CONSTRAINT IF EXISTS pages_source_slug_key;
|
||||
ALTER TABLE pages ADD CONSTRAINT pages_slug_key UNIQUE (slug);
|
||||
DROP INDEX IF EXISTS idx_pages_source_id;
|
||||
ALTER TABLE pages DROP COLUMN IF EXISTS source_id;
|
||||
DROP TABLE IF EXISTS sources CASCADE;
|
||||
|
||||
DROP INDEX IF EXISTS idx_links_source;
|
||||
DROP INDEX IF EXISTS idx_links_origin;
|
||||
ALTER TABLE links DROP CONSTRAINT IF EXISTS links_from_to_type_source_origin_unique;
|
||||
ALTER TABLE links DROP COLUMN IF EXISTS link_source;
|
||||
ALTER TABLE links DROP COLUMN IF EXISTS origin_page_id;
|
||||
|
||||
DROP INDEX IF EXISTS idx_chunks_symbol_name;
|
||||
DROP INDEX IF EXISTS idx_chunks_language;
|
||||
ALTER TABLE content_chunks DROP COLUMN IF EXISTS symbol_name;
|
||||
ALTER TABLE content_chunks DROP COLUMN IF EXISTS language;
|
||||
`);
|
||||
|
||||
// Run bootstrap in isolation (NOT initSchema). This is what we're testing.
|
||||
await (engine as any).applyForwardReferenceBootstrap();
|
||||
|
||||
// Assert every required forward-reference target now satisfies the
|
||||
// schema-blob's expectations.
|
||||
for (const ref of REQUIRED_BOOTSTRAP_COVERAGE) {
|
||||
if (ref.kind === 'table') {
|
||||
const { rows } = await db.query(
|
||||
`SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name = $1`,
|
||||
[ref.name],
|
||||
);
|
||||
expect(rows.length).toBeGreaterThan(0);
|
||||
} else {
|
||||
const { rows } = await db.query(
|
||||
`SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = $1 AND column_name = $2`,
|
||||
[ref.table, ref.column],
|
||||
);
|
||||
expect(rows.length).toBeGreaterThan(0);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await engine.disconnect();
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
test('after bootstrap, PGLITE_SCHEMA_SQL replays without crashing on missing forward references', async () => {
|
||||
// End-to-end contract: bootstrap → SCHEMA_SQL must succeed even on a brain
|
||||
// that lacks every forward-referenced target. This catches the case where
|
||||
// REQUIRED_BOOTSTRAP_COVERAGE drifts behind PGLITE_SCHEMA_SQL — if the
|
||||
// schema blob added a new index on a column the bootstrap doesn't create,
|
||||
// the SCHEMA_SQL exec below would crash even though the per-target asserts
|
||||
// above pass.
|
||||
const engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
try {
|
||||
await engine.initSchema();
|
||||
const db = (engine as any).db;
|
||||
|
||||
await db.exec(`
|
||||
ALTER TABLE pages DROP CONSTRAINT IF EXISTS pages_source_slug_key;
|
||||
ALTER TABLE pages ADD CONSTRAINT pages_slug_key UNIQUE (slug);
|
||||
DROP INDEX IF EXISTS idx_pages_source_id;
|
||||
ALTER TABLE pages DROP COLUMN IF EXISTS source_id;
|
||||
DROP TABLE IF EXISTS sources CASCADE;
|
||||
DROP INDEX IF EXISTS idx_links_source;
|
||||
DROP INDEX IF EXISTS idx_links_origin;
|
||||
ALTER TABLE links DROP CONSTRAINT IF EXISTS links_from_to_type_source_origin_unique;
|
||||
ALTER TABLE links DROP COLUMN IF EXISTS link_source;
|
||||
ALTER TABLE links DROP COLUMN IF EXISTS origin_page_id;
|
||||
`);
|
||||
|
||||
// Bootstrap, then schema replay. Either step crashing fails the test.
|
||||
const { PGLITE_SCHEMA_SQL } = await import('../src/core/pglite-schema.ts');
|
||||
await (engine as any).applyForwardReferenceBootstrap();
|
||||
await db.exec(PGLITE_SCHEMA_SQL);
|
||||
} finally {
|
||||
await engine.disconnect();
|
||||
}
|
||||
}, 30000);
|
||||
@@ -1,108 +0,0 @@
|
||||
import { describe, it, expect } from 'bun:test';
|
||||
import { parseExpectedColumns, simplifyColumnDef } from '../src/core/schema-verify.ts';
|
||||
|
||||
describe('parseExpectedColumns', () => {
|
||||
it('extracts columns from all major tables', () => {
|
||||
const columns = parseExpectedColumns();
|
||||
|
||||
// Should find columns from known tables
|
||||
const tables = new Set(columns.map(c => c.table));
|
||||
expect(tables.has('pages')).toBe(true);
|
||||
expect(tables.has('content_chunks')).toBe(true);
|
||||
expect(tables.has('links')).toBe(true);
|
||||
expect(tables.has('sources')).toBe(true);
|
||||
expect(tables.has('minion_jobs')).toBe(true);
|
||||
expect(tables.has('files')).toBe(true);
|
||||
|
||||
// Should find specific columns that have historically been missed by PgBouncer
|
||||
const columnKeys = new Set(columns.map(c => `${c.table}.${c.column}`));
|
||||
expect(columnKeys.has('content_chunks.symbol_type')).toBe(true);
|
||||
expect(columnKeys.has('content_chunks.start_line')).toBe(true);
|
||||
expect(columnKeys.has('content_chunks.end_line')).toBe(true);
|
||||
expect(columnKeys.has('content_chunks.parent_symbol_path')).toBe(true);
|
||||
expect(columnKeys.has('content_chunks.doc_comment')).toBe(true);
|
||||
expect(columnKeys.has('content_chunks.symbol_name_qualified')).toBe(true);
|
||||
expect(columnKeys.has('content_chunks.search_vector')).toBe(true);
|
||||
|
||||
// pages columns
|
||||
expect(columnKeys.has('pages.slug')).toBe(true);
|
||||
expect(columnKeys.has('pages.source_id')).toBe(true);
|
||||
expect(columnKeys.has('pages.page_kind')).toBe(true);
|
||||
expect(columnKeys.has('pages.search_vector')).toBe(true);
|
||||
|
||||
// sources columns
|
||||
expect(columnKeys.has('sources.chunker_version')).toBe(true);
|
||||
});
|
||||
|
||||
it('does not include CONSTRAINT lines as columns', () => {
|
||||
const columns = parseExpectedColumns();
|
||||
const colNames = columns.map(c => c.column);
|
||||
|
||||
// These are constraint names, not column names
|
||||
expect(colNames).not.toContain('constraint');
|
||||
expect(colNames).not.toContain('unique');
|
||||
expect(colNames).not.toContain('check');
|
||||
expect(colNames).not.toContain('primary');
|
||||
expect(colNames).not.toContain('foreign');
|
||||
});
|
||||
|
||||
it('returns non-empty definitions for all columns', () => {
|
||||
const columns = parseExpectedColumns();
|
||||
for (const col of columns) {
|
||||
expect(col.definition.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('simplifyColumnDef', () => {
|
||||
it('strips REFERENCES clauses', () => {
|
||||
const result = simplifyColumnDef(
|
||||
"TEXT NOT NULL DEFAULT 'default' REFERENCES sources(id) ON DELETE CASCADE"
|
||||
);
|
||||
expect(result).toBe("TEXT NOT NULL DEFAULT 'default'");
|
||||
});
|
||||
|
||||
it('strips CHECK constraints', () => {
|
||||
const result = simplifyColumnDef(
|
||||
"TEXT NOT NULL DEFAULT 'markdown' CHECK (page_kind IN ('markdown','code'))"
|
||||
);
|
||||
expect(result).toBe("TEXT NOT NULL DEFAULT 'markdown'");
|
||||
});
|
||||
|
||||
it('preserves simple type + NOT NULL + DEFAULT', () => {
|
||||
const result = simplifyColumnDef("INTEGER NOT NULL DEFAULT 0");
|
||||
expect(result).toBe("INTEGER NOT NULL DEFAULT 0");
|
||||
});
|
||||
|
||||
it('strips UNIQUE keyword', () => {
|
||||
const result = simplifyColumnDef("TEXT NOT NULL UNIQUE");
|
||||
expect(result).toBe("TEXT NOT NULL");
|
||||
});
|
||||
|
||||
it('handles complex REFERENCES with ON DELETE and ON UPDATE', () => {
|
||||
const result = simplifyColumnDef(
|
||||
"INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE"
|
||||
);
|
||||
expect(result).toBe("INTEGER NOT NULL");
|
||||
});
|
||||
|
||||
it('handles bare type', () => {
|
||||
const result = simplifyColumnDef("TEXT");
|
||||
expect(result).toBe("TEXT");
|
||||
});
|
||||
|
||||
it('handles vector type', () => {
|
||||
const result = simplifyColumnDef("vector(1536)");
|
||||
expect(result).toBe("vector(1536)");
|
||||
});
|
||||
|
||||
it('handles TSVECTOR type', () => {
|
||||
const result = simplifyColumnDef("TSVECTOR");
|
||||
expect(result).toBe("TSVECTOR");
|
||||
});
|
||||
|
||||
it('handles array types', () => {
|
||||
const result = simplifyColumnDef("TEXT[]");
|
||||
expect(result).toBe("TEXT[]");
|
||||
});
|
||||
});
|
||||
@@ -11,7 +11,6 @@ import {
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
utimesSync,
|
||||
writeFileSync,
|
||||
} from 'fs';
|
||||
import { dirname, join } from 'path';
|
||||
@@ -345,31 +344,6 @@ describe('planInstall + applyInstall', () => {
|
||||
expect(result.summary.wroteNew).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('D-CX-11: --force-unlock works when lock mtime is sub-ms ahead of Date.now (Linux fs jitter)', () => {
|
||||
// Regression guard: on Linux ext4, statSync().mtimeMs has sub-ms precision
|
||||
// while Date.now() is integer ms, so a just-written lock can report a
|
||||
// negative age. If acquireLock does not clamp, stale=false and the
|
||||
// forceUnlock path is unreachable. Simulate deterministically by pushing
|
||||
// the lock's mtime 10ms into the future.
|
||||
const { gbrainRoot } = scratchGbrain();
|
||||
const { workspace, skillsDir } = scratchTarget();
|
||||
const lockFile = join(workspace, '.gbrain-skillpack.lock');
|
||||
writeFileSync(lockFile, '99999');
|
||||
const future = (Date.now() + 10) / 1000;
|
||||
utimesSync(lockFile, future, future);
|
||||
const opts = {
|
||||
gbrainRoot,
|
||||
targetWorkspace: workspace,
|
||||
targetSkillsDir: skillsDir,
|
||||
skillSlug: 'alpha',
|
||||
forceUnlock: true,
|
||||
lockStaleMs: 0,
|
||||
};
|
||||
const plan = planInstall(opts);
|
||||
const result = applyInstall(plan, opts);
|
||||
expect(result.summary.wroteNew).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('managed block is written atomically (tmp then rename)', () => {
|
||||
const { gbrainRoot } = scratchGbrain();
|
||||
const { workspace, skillsDir } = scratchTarget();
|
||||
|
||||
+4
-276
@@ -76,19 +76,18 @@ describe('Bug 9 — sync-failures JSONL helpers', () => {
|
||||
{ path: 'b.md', error: 'err2' },
|
||||
], 'commit1');
|
||||
|
||||
const result = acknowledgeSyncFailures();
|
||||
expect(result.count).toBe(2);
|
||||
expect(result.summary.length).toBeGreaterThan(0);
|
||||
const n = acknowledgeSyncFailures();
|
||||
expect(n).toBe(2);
|
||||
const after = loadSyncFailures();
|
||||
expect(after.every(e => e.acknowledged === true)).toBe(true);
|
||||
expect(after.every(e => typeof e.acknowledged_at === 'string')).toBe(true);
|
||||
|
||||
// Second ack: nothing new to mark.
|
||||
expect(acknowledgeSyncFailures().count).toBe(0);
|
||||
expect(acknowledgeSyncFailures()).toBe(0);
|
||||
|
||||
// Adding a fresh failure then ack: only the new one flips.
|
||||
recordSyncFailures([{ path: 'c.md', error: 'err3' }], 'commit2');
|
||||
expect(acknowledgeSyncFailures().count).toBe(1);
|
||||
expect(acknowledgeSyncFailures()).toBe(1);
|
||||
expect(loadSyncFailures().length).toBe(3);
|
||||
expect(loadSyncFailures().every(e => e.acknowledged === true)).toBe(true);
|
||||
});
|
||||
@@ -159,274 +158,3 @@ describe('Bug 9 — sync.ts CLI flag wiring', () => {
|
||||
expect(source).toContain('recordSyncFailures');
|
||||
});
|
||||
});
|
||||
|
||||
describe('classifyErrorCode — error message to code mapping', () => {
|
||||
test('classifies SLUG_MISMATCH from error message', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode(
|
||||
'Frontmatter slug "my-friend-mike" does not match path-derived slug "2008-03-20-my-friend-mike"'
|
||||
)).toBe('SLUG_MISMATCH');
|
||||
});
|
||||
|
||||
test('classifies YAML_PARSE from error message', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode('YAML parse failed: unexpected colon in title')).toBe('YAML_PARSE');
|
||||
});
|
||||
|
||||
test('classifies YAML_DUPLICATE_KEY', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode('YAMLException: duplicated mapping key')).toBe('YAML_DUPLICATE_KEY');
|
||||
});
|
||||
|
||||
test('classifies STATEMENT_TIMEOUT', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode('canceling statement due to statement timeout')).toBe('STATEMENT_TIMEOUT');
|
||||
});
|
||||
|
||||
test('classifies NULL_BYTES', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode('invalid UTF-8: null byte at position 3770')).toBe('NULL_BYTES');
|
||||
});
|
||||
|
||||
test('classifies INVALID_UTF8', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode('invalid UTF-8 sequence at position 500')).toBe('INVALID_UTF8');
|
||||
});
|
||||
|
||||
test('returns UNKNOWN for unrecognized errors', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode('something completely different')).toBe('UNKNOWN');
|
||||
});
|
||||
});
|
||||
|
||||
describe('summarizeFailuresByCode — grouped summary', () => {
|
||||
test('groups failures by classified code', async () => {
|
||||
const { summarizeFailuresByCode } = await import('../src/core/sync.ts');
|
||||
const summary = summarizeFailuresByCode([
|
||||
{ error: 'Frontmatter slug "a" does not match path-derived slug "b"' },
|
||||
{ error: 'Frontmatter slug "c" does not match path-derived slug "d"' },
|
||||
{ error: 'YAML parse failed: bad colon' },
|
||||
{ error: 'something unknown' },
|
||||
]);
|
||||
expect(summary).toEqual([
|
||||
{ code: 'SLUG_MISMATCH', count: 2 },
|
||||
{ code: 'YAML_PARSE', count: 1 },
|
||||
{ code: 'UNKNOWN', count: 1 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('respects pre-classified code field', async () => {
|
||||
const { summarizeFailuresByCode } = await import('../src/core/sync.ts');
|
||||
const summary = summarizeFailuresByCode([
|
||||
{ error: 'anything', code: 'SLUG_MISMATCH' },
|
||||
{ error: 'anything', code: 'SLUG_MISMATCH' },
|
||||
{ error: 'anything', code: 'YAML_PARSE' },
|
||||
]);
|
||||
expect(summary).toEqual([
|
||||
{ code: 'SLUG_MISMATCH', count: 2 },
|
||||
{ code: 'YAML_PARSE', count: 1 },
|
||||
]);
|
||||
});
|
||||
|
||||
test('returns empty array for no failures', async () => {
|
||||
const { summarizeFailuresByCode } = await import('../src/core/sync.ts');
|
||||
expect(summarizeFailuresByCode([])).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('acknowledgeSyncFailures — structured return', () => {
|
||||
test('returns count and code summary', async () => {
|
||||
const { recordSyncFailures, acknowledgeSyncFailures } = await import('../src/core/sync.ts');
|
||||
recordSyncFailures([
|
||||
{ path: 'a.md', error: 'Frontmatter slug "x" does not match path-derived slug "y"' },
|
||||
{ path: 'b.md', error: 'Frontmatter slug "p" does not match path-derived slug "q"' },
|
||||
{ path: 'c.md', error: 'YAML parse failed: bad' },
|
||||
], 'commit1');
|
||||
|
||||
const result = acknowledgeSyncFailures();
|
||||
expect(result.count).toBe(3);
|
||||
expect(result.summary).toEqual([
|
||||
{ code: 'SLUG_MISMATCH', count: 2 },
|
||||
{ code: 'YAML_PARSE', count: 1 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('recordSyncFailures — code field', () => {
|
||||
test('records classified code alongside error message', async () => {
|
||||
const { recordSyncFailures, loadSyncFailures } = await import('../src/core/sync.ts');
|
||||
recordSyncFailures([
|
||||
{ path: 'a.md', error: 'Frontmatter slug "x" does not match path-derived slug "y"' },
|
||||
], 'commit1');
|
||||
|
||||
const entries = loadSyncFailures();
|
||||
expect(entries[0].code).toBe('SLUG_MISMATCH');
|
||||
});
|
||||
});
|
||||
|
||||
// classifyErrorCode disambiguates Postgres unique-constraint errors from
|
||||
// YAML duplicate-key errors. Pre-fix, every "duplicate.*key" string mapped
|
||||
// to YAML_DUPLICATE_KEY, which mislabels DB-layer failures during sync.
|
||||
describe('classifyErrorCode — DB vs YAML duplicate-key disambiguation', () => {
|
||||
test('Postgres unique-constraint violation classifies as DB_DUPLICATE_KEY', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode(
|
||||
'duplicate key value violates unique constraint "pages_slug_key"'
|
||||
)).toBe('DB_DUPLICATE_KEY');
|
||||
});
|
||||
|
||||
test('YAML duplicated mapping key still classifies as YAML_DUPLICATE_KEY', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode('YAMLException: duplicated mapping key "title"'))
|
||||
.toBe('YAML_DUPLICATE_KEY');
|
||||
});
|
||||
|
||||
test('DB pattern is checked BEFORE YAML so DB errors are not mislabeled', async () => {
|
||||
// Both patterns historically matched /duplicate.*key/i — order matters now.
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode(
|
||||
'duplicate key value violates unique constraint on table "pages"'
|
||||
)).toBe('DB_DUPLICATE_KEY');
|
||||
expect(classifyErrorCode(
|
||||
'duplicate key value violates unique constraint on table "pages"'
|
||||
)).not.toBe('YAML_DUPLICATE_KEY');
|
||||
});
|
||||
});
|
||||
|
||||
// classifyErrorCode matches the canonical messages emitted by
|
||||
// collectValidationErrors() in src/core/markdown.ts. Pre-fix, the regexes
|
||||
// keyed off "missing open" / "missing close" / "empty frontmatter" — none
|
||||
// of which are produced upstream. Today these all classify correctly.
|
||||
describe('classifyErrorCode — canonical message coverage', () => {
|
||||
test('MISSING_OPEN matches "File is empty or whitespace-only"', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode(
|
||||
'File is empty or whitespace-only; expected frontmatter starting with ---'
|
||||
)).toBe('MISSING_OPEN');
|
||||
});
|
||||
|
||||
test('MISSING_OPEN matches "Frontmatter must start with ---"', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode(
|
||||
'Frontmatter must start with --- on the first non-empty line'
|
||||
)).toBe('MISSING_OPEN');
|
||||
});
|
||||
|
||||
test('MISSING_CLOSE matches "No closing --- delimiter"', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode('No closing --- delimiter found')).toBe('MISSING_CLOSE');
|
||||
});
|
||||
|
||||
test('MISSING_CLOSE matches "Heading at line N found inside frontmatter"', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode(
|
||||
'Heading at line 5 found inside frontmatter zone (closing --- comes after)'
|
||||
)).toBe('MISSING_CLOSE');
|
||||
});
|
||||
|
||||
test('EMPTY_FRONTMATTER matches "Frontmatter block is empty"', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode('Frontmatter block is empty')).toBe('EMPTY_FRONTMATTER');
|
||||
});
|
||||
|
||||
test('NULL_BYTES matches "Content contains null bytes"', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode('Content contains null bytes (likely binary corruption)'))
|
||||
.toBe('NULL_BYTES');
|
||||
});
|
||||
|
||||
test('NESTED_QUOTES matches "Nested double quotes"', async () => {
|
||||
const { classifyErrorCode } = await import('../src/core/sync.ts');
|
||||
expect(classifyErrorCode('Nested double quotes in YAML value at line 3'))
|
||||
.toBe('NESTED_QUOTES');
|
||||
});
|
||||
});
|
||||
|
||||
// acknowledgeSyncFailures backfills `code` on legacy entries that were
|
||||
// recorded before the code field existed (~/.gbrain/sync-failures.jsonl
|
||||
// from pre-PR brains). Without this branch, upgraded users see "UNKNOWN"
|
||||
// for every previously-recorded failure even when the message is parseable.
|
||||
describe('acknowledgeSyncFailures — backfill on legacy entries', () => {
|
||||
test('backfills code on entries that predate the code field', async () => {
|
||||
const { acknowledgeSyncFailures, loadSyncFailures, syncFailuresPath } =
|
||||
await import('../src/core/sync.ts');
|
||||
|
||||
// Hand-write a legacy entry with no `code` field. Mimics a pre-PR
|
||||
// ~/.gbrain/sync-failures.jsonl row that exists on real upgrades.
|
||||
const { mkdirSync } = await import('fs');
|
||||
const { dirname } = await import('path');
|
||||
mkdirSync(dirname(syncFailuresPath()), { recursive: true });
|
||||
writeFileSync(
|
||||
syncFailuresPath(),
|
||||
JSON.stringify({
|
||||
path: 'a.md',
|
||||
error: 'Frontmatter slug "x" does not match path-derived slug "y"',
|
||||
commit: 'old',
|
||||
ts: '2025-01-01T00:00:00Z',
|
||||
}) + '\n',
|
||||
);
|
||||
|
||||
const result = acknowledgeSyncFailures();
|
||||
expect(result.count).toBe(1);
|
||||
expect(result.summary).toEqual([{ code: 'SLUG_MISMATCH', count: 1 }]);
|
||||
|
||||
const after = loadSyncFailures();
|
||||
expect(after).toHaveLength(1);
|
||||
expect(after[0].code).toBe('SLUG_MISMATCH');
|
||||
expect(after[0].acknowledged).toBe(true);
|
||||
});
|
||||
|
||||
test('preserves existing code field; never reclassifies', async () => {
|
||||
const { acknowledgeSyncFailures, loadSyncFailures, syncFailuresPath } =
|
||||
await import('../src/core/sync.ts');
|
||||
|
||||
const { mkdirSync } = await import('fs');
|
||||
const { dirname } = await import('path');
|
||||
mkdirSync(dirname(syncFailuresPath()), { recursive: true });
|
||||
// Pre-classified entry — should NOT be re-run through classifier.
|
||||
writeFileSync(
|
||||
syncFailuresPath(),
|
||||
JSON.stringify({
|
||||
path: 'a.md',
|
||||
error: 'some message that would otherwise classify as UNKNOWN',
|
||||
code: 'CUSTOM_CODE',
|
||||
commit: 'x',
|
||||
ts: '2025-01-01T00:00:00Z',
|
||||
}) + '\n',
|
||||
);
|
||||
|
||||
const result = acknowledgeSyncFailures();
|
||||
expect(result.summary).toEqual([{ code: 'CUSTOM_CODE', count: 1 }]);
|
||||
expect(loadSyncFailures()[0].code).toBe('CUSTOM_CODE');
|
||||
});
|
||||
});
|
||||
|
||||
// formatCodeBreakdown is the DRY helper used by both the failures-array
|
||||
// path (sync.ts blocked-by-failures + full-sync stderr) and the pre-summarized
|
||||
// AcknowledgeResult.summary path (--skip-failed ack message). One renderer,
|
||||
// two input shapes.
|
||||
describe('formatCodeBreakdown — dual input shape', () => {
|
||||
test('renders raw failures by classifying internally', async () => {
|
||||
const { formatCodeBreakdown } = await import('../src/core/sync.ts');
|
||||
const out = formatCodeBreakdown([
|
||||
{ error: 'Frontmatter slug "a" does not match path-derived slug "b"' },
|
||||
{ error: 'Frontmatter slug "c" does not match path-derived slug "d"' },
|
||||
{ error: 'YAML parse failed: bad' },
|
||||
]);
|
||||
expect(out).toBe(' SLUG_MISMATCH: 2\n YAML_PARSE: 1');
|
||||
});
|
||||
|
||||
test('renders pre-summarized {code, count} input directly', async () => {
|
||||
const { formatCodeBreakdown } = await import('../src/core/sync.ts');
|
||||
const out = formatCodeBreakdown([
|
||||
{ code: 'SLUG_MISMATCH', count: 5 },
|
||||
{ code: 'YAML_PARSE', count: 2 },
|
||||
]);
|
||||
expect(out).toBe(' SLUG_MISMATCH: 5\n YAML_PARSE: 2');
|
||||
});
|
||||
|
||||
test('returns empty string for empty input', async () => {
|
||||
const { formatCodeBreakdown } = await import('../src/core/sync.ts');
|
||||
expect(formatCodeBreakdown([])).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
+4
-13
@@ -1,11 +1,10 @@
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { buildSyncManifest, isSyncable, pathToSlug } from '../src/core/sync.ts';
|
||||
import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
import { tmpdir } from 'os';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
|
||||
describe('buildSyncManifest', () => {
|
||||
test('parses A/M/D entries from single commit', () => {
|
||||
@@ -205,20 +204,11 @@ describe('performSync dry-run never writes', () => {
|
||||
let engine: PGLiteEngine;
|
||||
let repoPath: string;
|
||||
|
||||
// One PGLite per file — beforeEach wipes data only. Each test still gets a
|
||||
// fresh git repo via mkdtempSync, but skips the ~20s PGLite cold-start.
|
||||
beforeAll(async () => {
|
||||
beforeEach(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
repoPath = mkdtempSync(join(tmpdir(), 'gbrain-sync-dryrun-'));
|
||||
execSync('git init', { cwd: repoPath, stdio: 'pipe' });
|
||||
execSync('git config user.email "test@test.com"', { cwd: repoPath, stdio: 'pipe' });
|
||||
@@ -243,7 +233,8 @@ describe('performSync dry-run never writes', () => {
|
||||
execSync('git add -A && git commit -m "initial"', { cwd: repoPath, stdio: 'pipe' });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
afterEach(async () => {
|
||||
await engine.disconnect();
|
||||
if (repoPath) rmSync(repoPath, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user